From 17872018cc037e699b1f6e1589d0ad05e0883cca Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 15 Jul 2026 17:52:39 +0200 Subject: [PATCH 001/171] feat(nsjail): make python/ansible rlimit_as configurable per worker (GIT-921) (#10138) nsjail caps a jailed job's virtual address space at rlimit_as (4096 MiB for python3 and ansible). JIT runtimes (Bun/JavaScriptCore, the JVM) reserve large virtual ranges up front, so a subprocess spawned from a jailed Python/Ansible job can crash against this cap even when its physical memory use is modest (e.g. the Bun-compiled claude CLI hitting JSC/pthread allocation failures). Most other language protos already run with disable_rl: true (unlimited); python3 and ansible are the outliers with an explicit rlimit_as. This exposes that cap via a per-language env var (NSJAIL_PY_RLIMIT_AS_MB, NSJAIL_ANSIBLE_RLIMIT_AS_MB) so operators can raise or lift it on a dedicated worker pool without a source patch/rebuild and without weakening the mount/PID/user-namespace isolation that provides the real security boundary. Only the address-space limit changes; cpu/fsize/nofile rlimits are untouched. Value is in MiB, or unlimited/none/inf/0 to uncap (rlimit_as_type: INF). Unset keeps the historical 4096 default. Co-authored-by: Claude Opus 4.8 (1M context) --- .../nsjail/run.ansible.config.proto | 2 +- .../nsjail/run.python3.config.proto | 2 +- .../windmill-worker/src/ansible_executor.rs | 11 ++- backend/windmill-worker/src/common.rs | 82 +++++++++++++++++++ .../windmill-worker/src/python_executor.rs | 15 ++-- backend/windmill-worker/src/worker.rs | 8 ++ 6 files changed, 110 insertions(+), 10 deletions(-) diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index 2c731e34f9..afaf066f17 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -5,7 +5,7 @@ hostname: "ansible" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 53d5a6c64d..269ae1f0f8 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -5,7 +5,7 @@ hostname: "python" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 292246c62c..92bffe0cf0 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -31,13 +31,14 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, - start_child_process, transform_json, OccupancyMetrics, + read_and_check_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_ANSIBLE_RLIMIT_AS_MB, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1659,6 +1660,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_ANSIBLE_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index a88ba67bb7..6618ac7906 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1124,6 +1124,45 @@ pub async fn resolve_nsjail_timeout( (duration.as_secs() + 15).to_string() } +/// Render the `rlimit_as` line for an nsjail run config, honoring a per-language +/// env-var override. +/// +/// nsjail caps a jailed job's virtual address space at `rlimit_as` MiB. JIT +/// runtimes (Bun/JavaScriptCore, the JVM) reserve large virtual ranges up front, +/// so a subprocess spawned from a jailed Python/Ansible job can crash against this +/// cap even when its physical memory use is modest. Lifting it lets operators run +/// such workloads on a dedicated worker pool (set the env var only there) without +/// giving up the mount/PID/user-namespace isolation that provides the real +/// security boundary. Only the address-space limit is affected; the other rlimits +/// (cpu/fsize/nofile) in the proto are untouched. +/// +/// `env_override` is the raw value of the language's `NSJAIL_*_RLIMIT_AS_MB` env var: +/// - unset/empty -> historical default (`rlimit_as: {default_mb}`) +/// - `unlimited`/`none`/`inf`/`0` -> `rlimit_as_type: INF` (address space uncapped) +/// - a positive integer (MiB) -> `rlimit_as: {n}` +pub fn render_nsjail_rlimit_as(env_override: Option<&str>, default_mb: u32) -> String { + match env_override.map(str::trim) { + None | Some("") => format!("rlimit_as: {default_mb}"), + Some(v) + if v.eq_ignore_ascii_case("unlimited") + || v.eq_ignore_ascii_case("none") + || v.eq_ignore_ascii_case("inf") + || v == "0" => + { + "rlimit_as_type: INF".to_string() + } + Some(v) => match v.parse::() { + Ok(mb) => format!("rlimit_as: {mb}"), + Err(_) => { + tracing::warn!( + "Invalid nsjail rlimit_as override {v:?}, using default {default_mb}MiB" + ); + format!("rlimit_as: {default_mb}") + } + }, + } +} + /// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes, /// used when the `nsjail_tmpfs_size_mb` instance setting is unset. pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000; @@ -1233,6 +1272,49 @@ pub(crate) async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String { bind_mount_block(&jail_tmp) } +#[cfg(test)] +mod nsjail_rlimit_as_tests { + use super::render_nsjail_rlimit_as; + + #[test] + fn unset_uses_default() { + assert_eq!(render_nsjail_rlimit_as(None, 4096), "rlimit_as: 4096"); + assert_eq!(render_nsjail_rlimit_as(Some(" "), 4096), "rlimit_as: 4096"); + } + + #[test] + fn numeric_override_is_used() { + assert_eq!( + render_nsjail_rlimit_as(Some("16384"), 4096), + "rlimit_as: 16384" + ); + assert_eq!( + render_nsjail_rlimit_as(Some(" 8192 "), 4096), + "rlimit_as: 8192" + ); + } + + #[test] + fn unlimited_keywords_emit_inf() { + for v in ["unlimited", "UNLIMITED", "none", "inf", "0"] { + assert_eq!( + render_nsjail_rlimit_as(Some(v), 4096), + "rlimit_as_type: INF", + "value {v:?}" + ); + } + } + + #[test] + fn invalid_falls_back_to_default() { + assert_eq!( + render_nsjail_rlimit_as(Some("abc"), 4096), + "rlimit_as: 4096" + ); + assert_eq!(render_nsjail_rlimit_as(Some("-1"), 4096), "rlimit_as: 4096"); + } +} + #[cfg(test)] mod nsjail_tmp_mount_tests { use super::*; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9ca79c6852..a3ae036894 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -155,16 +155,17 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, + read_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, StreamNotifier, + DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, - UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, NSJAIL_PY_RLIMIT_AS_MB, PATH_ENV, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, + TZ_ENV, UV_CACHE_DIR, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, }; use windmill_common::client::AuthedClient; @@ -1077,6 +1078,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_PY_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{JOB_DIR}", job_dir) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a4672145bb..69642cd4ea 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -342,6 +342,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + /// Per-language override for the nsjail `rlimit_as` (virtual address space) cap. + /// Value is in MiB, or `unlimited`/`none`/`inf`/`0` to uncap. Unset keeps the + /// historical default baked into the proto. See `render_nsjail_rlimit_as`. + pub static ref NSJAIL_PY_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_PY_RLIMIT_AS_MB").ok(); + pub static ref NSJAIL_ANSIBLE_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_ANSIBLE_RLIMIT_AS_MB").ok(); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() From 8bfe5c93404ba3f137394f16d0571a06d891dc3b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 15 Jul 2026 17:53:09 +0200 Subject: [PATCH 002/171] fix(ai): stop sending the AI agent system prompt twice for OpenAI (#10126) * fix(ai): stop sending the AI agent system prompt twice for OpenAI Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ai): document collect_system_prompt precedence and trim duplicate comments Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ai): hoist only the leading system prompt for OpenAI Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-ai/src/providers/anthropic.rs | 101 ++++++++++++-- backend/windmill-ai/src/providers/openai.rs | 131 +++++++++++++++++- backend/windmill-ai/src/utils.rs | 108 ++++++++++++++- 3 files changed, 325 insertions(+), 15 deletions(-) diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index d71de2f35f..a255f47a6d 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -6,7 +6,10 @@ use crate::{ query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, should_use_structured_output_tool, AI_HTTP_HEADERS}, + utils::{ + collect_system_prompt, extract_text_content, should_use_structured_output_tool, + AI_HTTP_HEADERS, + }, }; use async_trait::async_trait; use http::Method; @@ -209,7 +212,7 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec { - // Skip - handled via args.system_prompt in build_text_request + // Lifted into the request's top-level `system` field by build_text_request } "user" => { // Convert user messages @@ -601,19 +604,17 @@ impl AnthropicQueryBuilder { } } - // Build system content from system_prompt, but None if system_prompt is empty string - let system = match args.system_prompt { - Some(s) if !s.is_empty() => Some(vec![AnthropicSystemContent { + let system = collect_system_prompt(&prepared_messages, args.system_prompt).map(|text| { + vec![AnthropicSystemContent { r#type: "text".to_string(), - text: s.to_string(), + text, cache_control: if self.is_vertex() { None } else { Some(CacheControl::ephemeral()) }, - }]), - _ => None, - }; + }] + }); // Check if we need to force tool usage for structured output let has_output_properties = args @@ -842,6 +843,88 @@ mod tests { } } + const SYSTEM_PROMPT: &str = "You are a helpful assistant"; + + fn authed_client() -> AuthedClient { + AuthedClient::new( + "http://localhost:8000".to_string(), + "test-workspace".to_string(), + "token".to_string(), + None, + ) + } + + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + let args = BuildRequestArgs { + messages, + tools: None, + model: "claude-sonnet-4", + temperature: None, + reasoning_effort: None, + max_tokens: None, + output_schema: None, + output_type: &OutputType::Text, + system_prompt, + user_message: "hello", + attachments: None, + has_websearch: false, + }; + + AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard) + .build_request(&args, &authed_client(), "test-workspace") + .await + .unwrap() + } + + /// The worker prepends the system prompt as a system message *and* passes it as + /// `system_prompt`; the request must still carry it exactly once. + #[tokio::test] + async fn sends_system_prompt_only_in_system_field() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["text"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let sent = request["messages"].as_array().unwrap(); + assert!(sent.iter().all(|message| message["role"] != "system")); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0]["role"], "user"); + } + + /// Manual-memory conversations supply their own system messages without a + /// `system_prompt` arg: those must still reach the model. + #[tokio::test] + async fn lifts_manual_system_messages_into_system_field() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["text"], "be terse"); + assert_eq!(request["messages"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn omits_system_without_a_system_prompt() { + let messages = vec![message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(request.get("system").is_none()); + } + fn has_header(headers: &[(String, String)], name: &str, value: &str) -> bool { headers .iter() diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index bdb882679b..8a90d84658 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -6,7 +6,7 @@ use crate::{ query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, sse::{OpenAIResponsesSSEParser, SSEParser}, types::*, - utils::extract_text_content, + utils::{collect_system_prompt, extract_text_content}, }; use async_trait::async_trait; use serde::{Deserialize, Serialize}; @@ -206,7 +206,7 @@ pub struct ResponsesApiRequest<'a> { pub model: &'a str, pub input: Vec, #[serde(skip_serializing_if = "Option::is_none")] - pub instructions: Option<&'a str>, + pub instructions: Option, pub tools: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub stream: Option, @@ -371,9 +371,19 @@ impl OpenAIQueryBuilder { let prepared_messages = prepare_messages_for_api(args.messages, client, workspace_id).await?; + // Only the system prompt leading the conversation moves to `instructions`; echoing it in + // `input` as well would send it twice. This API accepts system messages anywhere in + // `input`, so any later one stays where the caller put it, position and content intact. + let leading_system = prepared_messages + .iter() + .take_while(|message| message.role == "system") + .count(); + let instructions = + collect_system_prompt(&prepared_messages[..leading_system], args.system_prompt); + // Convert full message history to Responses API input format // (following frontend pattern from openai-responses.ts) - let input_items = convert_messages_to_responses_input(&prepared_messages); + let input_items = convert_messages_to_responses_input(&prepared_messages[leading_system..]); // Build tools array using typed structs let mut tools: Vec = Vec::new(); @@ -416,7 +426,7 @@ impl OpenAIQueryBuilder { let request = ResponsesApiRequest { model: args.model, input: input_items, - instructions: args.system_prompt, // System prompt goes to instructions field + instructions, tools, stream: Some(true), temperature: args.temperature, @@ -474,7 +484,7 @@ impl OpenAIQueryBuilder { let request = ResponsesApiRequest { model: args.model, input: vec![ResponsesApiInputItem::InputMessage { role: "user".to_string(), content }], - instructions: args.system_prompt, + instructions: args.system_prompt.map(str::to_string), tools, stream: None, // Image generation doesn't use streaming temperature: args.temperature, @@ -582,3 +592,114 @@ impl QueryBuilder for OpenAIQueryBuilder { vec![("Authorization", format!("Bearer {}", api_key))] } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::query_builder::QueryBuilder; + + const SYSTEM_PROMPT: &str = "You are a helpful assistant"; + + fn client() -> AuthedClient { + AuthedClient::new( + "http://localhost:8000".to_string(), + "test-workspace".to_string(), + "token".to_string(), + None, + ) + } + + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + let args = BuildRequestArgs { + messages, + tools: None, + model: "gpt-5", + temperature: None, + reasoning_effort: None, + max_tokens: None, + output_schema: None, + output_type: &OutputType::Text, + system_prompt, + user_message: "hello", + attachments: None, + has_websearch: false, + }; + + OpenAIQueryBuilder::new(AIProvider::OpenAI) + .build_request(&args, &client(), "test-workspace") + .await + .unwrap() + } + + /// The worker prepends the system prompt as a system message *and* passes it as + /// `system_prompt`; the request must still carry it exactly once. + #[tokio::test] + async fn sends_system_prompt_only_in_instructions() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let input = request["input"].as_array().unwrap(); + assert!(input.iter().all(|item| item["role"] != "system")); + assert_eq!(input.len(), 1); + assert_eq!(input[0]["role"], "user"); + } + + /// This API takes system messages anywhere in `input`, so a late steering message keeps + /// its position instead of being hoisted into `instructions`. + #[tokio::test] + async fn keeps_a_mid_conversation_system_message_in_place() { + let messages = vec![ + message("system", SYSTEM_PROMPT), + message("user", "hi"), + message("system", "answer in one word"), + message("user", "and now?"), + ]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let input = request["input"].as_array().unwrap(); + assert_eq!(input.len(), 3); + assert_eq!(input[1]["role"], "system"); + assert_eq!(input[1]["content"][0]["text"], "answer in one word"); + } + + /// Manual-memory conversations supply their own system messages without a + /// `system_prompt` arg: those must still reach the model. + #[tokio::test] + async fn lifts_manual_system_messages_into_instructions() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], "be terse"); + assert_eq!(request["input"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn omits_instructions_without_a_system_prompt() { + let messages = vec![message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(request.get("instructions").is_none()); + } +} diff --git a/backend/windmill-ai/src/utils.rs b/backend/windmill-ai/src/utils.rs index fab1644255..66b68e9b6f 100644 --- a/backend/windmill-ai/src/utils.rs +++ b/backend/windmill-ai/src/utils.rs @@ -1,6 +1,6 @@ use crate::{ ai_providers::AIProvider, - ai_types::{ContentPart, OpenAIContent}, + ai_types::{ContentPart, OpenAIContent, OpenAIMessage}, }; use windmill_common::utils::configure_client; @@ -69,6 +69,37 @@ pub fn should_use_structured_output_tool(provider: &AIProvider, model: &str) -> model.contains("claude") || provider == &AIProvider::AWSBedrock } +/// Collect the system prompt for providers that take it in a dedicated top-level field +/// (Anthropic's `system`, OpenAI's `instructions`) instead of inline in the message list. +/// +/// Every system message in `messages` is joined, since manual-memory conversations can carry +/// system messages of their own alongside the one the caller prepends from `system_prompt`. +/// `system_prompt` is a fallback used only when `messages` holds no system message, for callers +/// that pass it without prepending it. Only text content survives, so pass just the messages the +/// provider cannot render inline: Anthropic's API takes no system role at all and hands over +/// everything, while OpenAI's accepts system messages inside `input` and hands over only the +/// leading ones. Whatever is passed here must be left out of the message list the provider +/// sends, or the same prompt goes over the wire twice. +pub fn collect_system_prompt( + messages: &[OpenAIMessage], + system_prompt: Option<&str>, +) -> Option { + let from_messages = messages + .iter() + .filter(|message| message.role == "system") + .filter_map(|message| message.content.as_ref().map(extract_text_content)) + .filter(|text| !text.is_empty()) + .collect::>(); + + if from_messages.is_empty() { + system_prompt + .filter(|prompt| !prompt.is_empty()) + .map(str::to_string) + } else { + Some(from_messages.join("\n\n")) + } +} + /// Extract text content from OpenAIContent, joining parts with space if multiple pub fn extract_text_content(content: &OpenAIContent) -> String { match content { @@ -93,6 +124,81 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + #[test] + fn joins_every_system_message() { + let messages = vec![ + message("system", "be helpful"), + message("user", "hi"), + message("system", "be terse"), + ]; + + assert_eq!( + collect_system_prompt(&messages, Some("be helpful")), + Some("be helpful\n\nbe terse".to_string()) + ); + } + + /// A dedicated system field is text-only, so non-text parts cannot be carried over. + #[test] + fn keeps_only_text_parts_of_a_system_message() { + let messages = vec![OpenAIMessage { + role: "system".to_string(), + content: Some(OpenAIContent::Parts(vec![ + ContentPart::Text { text: "be terse".to_string() }, + ContentPart::ImageUrl { + image_url: crate::ai_types::ImageUrlData { + url: "data:image/png;base64,x".to_string(), + }, + }, + ])), + ..Default::default() + }]; + + assert_eq!( + collect_system_prompt(&messages, None), + Some("be terse".to_string()) + ); + } + + /// The argument is a fallback, not an extra source: system messages win outright. + #[test] + fn prefers_system_messages_over_the_argument() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + assert_eq!( + collect_system_prompt(&messages, Some("unused fallback")), + Some("be terse".to_string()) + ); + } + + #[test] + fn falls_back_to_the_system_prompt_argument() { + let messages = vec![message("user", "hi")]; + + assert_eq!( + collect_system_prompt(&messages, Some("be helpful")), + Some("be helpful".to_string()) + ); + } + + #[test] + fn treats_empty_prompts_as_absent() { + assert_eq!( + collect_system_prompt(&[message("user", "hi")], Some("")), + None + ); + assert_eq!(collect_system_prompt(&[message("user", "hi")], None), None); + assert_eq!(collect_system_prompt(&[message("system", "")], None), None); + } + /// Regression test for GHSA-5q4v-c4v3-v7wr: `AI_HTTP_CLIENT` must not follow redirects. #[tokio::test] async fn ai_http_client_does_not_follow_redirects() { From 2fe999f66cd15acd81850f970ada31e9892abff2 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:55:01 +0200 Subject: [PATCH 003/171] fix(frontend): treat a displaced draft save as superseded, not failed (#10094) --- frontend/src/lib/coalescingRunner.svelte.ts | 22 ++- frontend/src/lib/coalescingRunner.test.ts | 81 +++++++++++ frontend/src/lib/userDraftDbSyncer.svelte.ts | 41 ++++-- .../src/lib/userDraftDisplacedSave.test.ts | 133 ++++++++++++++++++ 4 files changed, 264 insertions(+), 13 deletions(-) create mode 100644 frontend/src/lib/userDraftDisplacedSave.test.ts diff --git a/frontend/src/lib/coalescingRunner.svelte.ts b/frontend/src/lib/coalescingRunner.svelte.ts index 0f687d4829..61849c3e36 100644 --- a/frontend/src/lib/coalescingRunner.svelte.ts +++ b/frontend/src/lib/coalescingRunner.svelte.ts @@ -30,6 +30,9 @@ export type CoalescingKeyedRunner = { cancel(key: string): boolean /** Reactively whether `key`'s chain is running (SvelteSet-backed). */ isRunning(key: string): boolean + /** Resolves once `key`'s chain has drained (nothing running, nothing + * pending), immediately if it's idle. Never rejects. */ + settled(key: string): Promise } type PendingTask = { @@ -51,6 +54,8 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { // Reactive mirror of keys with a running chain, kept in lock-step with // `state` (SvelteSet for per-key `isRunning` subscriptions). const runningKeys = new SvelteSet() + // Live chain promise per key, backing `settled`. + const chains = new Map>() async function chain(key: string, first: PendingTask): Promise { let current: PendingTask | undefined = first @@ -71,6 +76,7 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { } state.delete(key) runningKeys.delete(key) + chains.delete(key) } /** Set `task` pending for `key`, displacing (and rejecting) any prior @@ -84,7 +90,15 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { } state.set(key, { pending: undefined }) runningKeys.add(key) - void chain(key, task) + // Register the chain promise BEFORE the first task runs. `chain` invokes + // the task synchronously, so a task that calls `settled(key)` (or that + // throws synchronously, running cleanup) would otherwise race ahead of a + // `chains.set(key, chain(...))` and leave the map wrong. A separate + // deferred sidesteps that: it's live before the task starts and resolves + // when the chain drains. + let done!: () => void + chains.set(key, new Promise((resolve) => (done = resolve))) + void chain(key, task).finally(done) } function submit(key: string, fn: CoalescingTask): void { @@ -114,5 +128,9 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { return runningKeys.has(key) } - return { submit, submitAndWait, cancel, isRunning } + function settled(key: string): Promise { + return chains.get(key) ?? Promise.resolve() + } + + return { submit, submitAndWait, cancel, isRunning, settled } } diff --git a/frontend/src/lib/coalescingRunner.test.ts b/frontend/src/lib/coalescingRunner.test.ts index 49cca4688e..1ae97e7db1 100644 --- a/frontend/src/lib/coalescingRunner.test.ts +++ b/frontend/src/lib/coalescingRunner.test.ts @@ -115,6 +115,87 @@ describe('createCoalescingKeyedRunner', () => { expect(runner.cancel('k')).toBe(false) }) + it('settled resolves immediately for an idle key', async () => { + const runner = createCoalescingKeyedRunner() + await expect(runner.settled('k')).resolves.toBeUndefined() + }) + + it('settled resolves once the chain drains, including the displacing task', async () => { + const runner = createCoalescingKeyedRunner() + const d = deferred() + const last = deferred() + const h = vi.fn(() => last.promise) + + runner.submit('k', () => d.promise) // in flight + void runner.submitAndWait('k', () => Promise.resolve()).catch(() => {}) // displaced below + runner.submit('k', h) + + let drained = false + void runner.settled('k').then(() => (drained = true)) + + d.resolve() + await d.promise + await Promise.resolve() + await Promise.resolve() + expect(h).toHaveBeenCalledTimes(1) + expect(drained).toBe(false) // h still running + + last.resolve() + await runner.settled('k') + expect(drained).toBe(true) + expect(runner.isRunning('k')).toBe(false) + }) + + it('settled called synchronously from within the first task does not resolve early', async () => { + const runner = createCoalescingKeyedRunner() + const d = deferred() + let settledEarly = false + let settledResolved = false + runner.submit('k', () => { + // Re-entrant: the task is invoked synchronously as the chain starts. + const p = runner.settled('k') + void p.then(() => (settledResolved = true)) + // Give the microtask a tick to (wrongly) resolve if the entry is missing. + void Promise.resolve().then(() => { + if (settledResolved) settledEarly = true + }) + return d.promise + }) + await Promise.resolve() + await Promise.resolve() + expect(settledEarly).toBe(false) + expect(settledResolved).toBe(false) // still running + + d.resolve() + await runner.settled('k') + expect(settledResolved).toBe(true) + }) + + it('a synchronously-throwing first task leaves no stale chain entry', async () => { + const runner = createCoalescingKeyedRunner() + const err = vi.spyOn(console, 'error').mockImplementation(() => {}) + runner.submit('k', () => { + throw new Error('sync boom') + }) + // Chain drained synchronously; the key must be idle and settled a no-op. + expect(runner.isRunning('k')).toBe(false) + await expect(runner.settled('k')).resolves.toBeUndefined() + // A fresh submit still starts a new chain (map wasn't left stale). + const ran = vi.fn(() => Promise.resolve()) + runner.submit('k', ran) + expect(ran).toHaveBeenCalledTimes(1) + err.mockRestore() + }) + + it('settled ignores a task failure (the chain survives it)', async () => { + const runner = createCoalescingKeyedRunner() + const err = vi.spyOn(console, 'error').mockImplementation(() => {}) + runner.submit('k', () => Promise.reject(new Error('boom'))) + await expect(runner.settled('k')).resolves.toBeUndefined() + expect(runner.isRunning('k')).toBe(false) + err.mockRestore() + }) + it('does not abort the in-flight task on cancel', async () => { const runner = createCoalescingKeyedRunner() const d = deferred() diff --git a/frontend/src/lib/userDraftDbSyncer.svelte.ts b/frontend/src/lib/userDraftDbSyncer.svelte.ts index a75f6b6837..77e2eecc88 100644 --- a/frontend/src/lib/userDraftDbSyncer.svelte.ts +++ b/frontend/src/lib/userDraftDbSyncer.svelte.ts @@ -1,7 +1,7 @@ import { SvelteMap } from 'svelte/reactivity' import { DraftService, type UserDraftItemKind } from './gen' import { OpenAPI } from './gen/core/OpenAPI' -import { createCoalescingKeyedRunner } from './coalescingRunner.svelte' +import { createCoalescingKeyedRunner, CoalescingDisplacedError } from './coalescingRunner.svelte' import { createDebouncerByKey } from './debouncerByKey.svelte' import { setLocalDraftHint } from './localDraftHints.svelte' @@ -92,10 +92,16 @@ export type UserDraftDbSyncerSaveOpts = { value: unknown | null /** Bypass the debouncer: cancel any pending autosave for this key (it * would otherwise overwrite what we send), route through the coalescing - * runner to preserve ordering against an in-flight POST, and resolve - * the returned promise only once the POST lands. Use for + * runner to preserve ordering against an in-flight POST, and resolve only + * once the key's save chain has drained. Use for * `await save(...); read-the-server` flows where a fire-and-forget save - * would race the next read. */ + * would race the next read. + * + * Resolving means "the key is settled", NOT "your payload won": a newer + * save can displace this one (it then carries the later state), and — as + * with every other `save` — `postSave` routes a rejected or failed POST to + * `conflicts` / `failures` rather than throwing. Read those to know what + * actually landed. */ immediate?: boolean /** Skip the optimistic-concurrency check and overwrite the server row. * Used by the conflict-resolution UI ("Overwrite the remote"). Default @@ -441,10 +447,19 @@ export const UserDraftDbSyncer = { pendingSaveOpts.set(key, opts) if (opts.immediate) { // Drop the queued autosave — firing it after our POST would - // re-save the pre-delete value. + // re-save the pre-delete value. `submitAndWait` displaces the + // runner's own pending task, so no `runner.cancel` needed. debouncer.cancel(key) - runner.cancel(key) - await runner.submitAndWait(key, () => postSave(opts)) + try { + await runner.submitAndWait(key, () => postSave(opts)) + } catch (e) { + // Displacement is not a failure: a newer save took our slot, so + // re-POSTing ours would undo it. Wait for the chain instead — + // callers await this to know the key is settled, not to know + // their own payload won. + if (!(e instanceof CoalescingDisplacedError)) throw e + await runner.settled(key) + } return } // Auto-save off: opts stay parked (above) for an explicit flush but @@ -563,8 +578,10 @@ export const UserDraftDbSyncer = { /** * Force-save: bypass the `last_sync` check and overwrite the server row - * (conflict modal's "Overwrite the remote"). Resolves only after the - * POST lands so the caller can `await` before navigating / refetching. + * (conflict modal's "Overwrite the remote"). Resolves once the key's save + * chain drains — see `immediate`; resolution means the chain settled, not + * that this force payload won (a later save can displace it). Callers + * `await` before navigating / refetching. */ async overwrite(opts: Omit): Promise { await this.save({ ...opts, immediate: true, force: true }) @@ -572,8 +589,10 @@ export const UserDraftDbSyncer = { /** * Flush the draft's queued autosave NOW (explicit Ctrl/Cmd+S). Re-submits - * the parked opts with `immediate: true` and resolves only after the POST - * lands, so callers can `await flush(...); show "Saved"`. + * the parked opts with `immediate: true` and resolves once the key's save + * chain drains (see `immediate` — the parked payload may be displaced by a + * later save carrying newer state), so callers can `await flush(...); show + * "Saved"`. * * No-op when nothing is pending. "No pending" does NOT mean "nothing to * save" — Monaco may hold unmaterialized text; flush the editor diff --git a/frontend/src/lib/userDraftDisplacedSave.test.ts b/frontend/src/lib/userDraftDisplacedSave.test.ts new file mode 100644 index 0000000000..50e24ba5c6 --- /dev/null +++ b/frontend/src/lib/userDraftDisplacedSave.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect, afterEach, vi } from 'vitest' + +// Mocked so a test can hold a POST in flight — that window is what makes a +// queued save displaceable. +const updateDraft = vi.fn(async (..._args: any[]) => ({ + status: 'saved' as const, + current_timestamp: '2020-01-01T00:00:00Z' +})) + +vi.mock('./gen', () => ({ + DraftService: { updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) } +})) +vi.mock('./gen/core/OpenAPI', () => ({ OpenAPI: { BASE: '' } })) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) + +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' + +function deferred() { + let resolve!: (v: T) => void + const promise = new Promise((res) => (resolve = res)) + return { promise, resolve } +} + +afterEach(() => { + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' }) +}) + +/** + * Deploying queues several saves for one draft key back-to-back (mirror write, + * post-deploy delete, unmount flush), so the runner displaces one of them. A + * displaced save must read as "superseded", never as a failure. + */ +describe('UserDraftDbSyncer immediate save displacement', () => { + it('resolves a displaced immediate save once the superseding save lands', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_a' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + // Queues behind `first`, then gets displaced by the delete below. + const displaced = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + + inFlight.resolve() + await expect(displaced).resolves.toBeUndefined() + await Promise.all([first, deleting]) + + // The displaced task never POSTed — the delete carries the later state. + expect(updateDraft).toHaveBeenCalledTimes(2) + expect(updateDraft.mock.calls.map((c: any[]) => c[0].requestBody.value)).toEqual([ + { content: '1' }, + null + ]) + }) + + it('does not resolve a displaced save before the superseding POST lands', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_b' } + const inFlight = deferred() + const deletePost = deferred() + updateDraft + .mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + .mockImplementationOnce(async () => { + await deletePost.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:01Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + const displaced = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + + let displacedSettled = false + void displaced.then(() => (displacedSettled = true)) + + inFlight.resolve() + await vi.waitFor(() => expect(updateDraft).toHaveBeenCalledTimes(2)) + // Delete still in flight: callers that `await save()` before invalidating + // must not read the server yet. + expect(displacedSettled).toBe(false) + + deletePost.resolve() + await Promise.all([first, displaced, deleting]) + expect(displacedSettled).toBe(true) + }) + + it('resolves a pending save dropped by lockSync without POSTing it', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_lock' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + const dropped = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + // Another user's draft was loaded: this value must never reach the server. + UserDraftDbSyncer.lockSync(q) + + inFlight.resolve() + // Resolves like every other save on a locked key — the lock's whole point + // is that the write is dropped, so the caller has nothing to wait for. + await expect(dropped).resolves.toBeUndefined() + await first + expect(updateDraft).toHaveBeenCalledTimes(1) + expect(updateDraft.mock.calls[0][0].requestBody.value).toEqual({ content: '1' }) + UserDraftDbSyncer.unlockSync(q) + }) + + it('resolves a flush displaced by a later immediate save', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_c' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + // Park opts (the reactive mirror's autosave) so `flush` has something to send. + void UserDraftDbSyncer.save({ ...q, value: { content: 'typed' }, auto: true }) + const first = UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, immediate: true }) + const flushed = UserDraftDbSyncer.flush(q) // pending behind `first` + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) // displaces it + + inFlight.resolve() + await expect(flushed).resolves.toBeUndefined() + await Promise.all([first, deleting]) + }) +}) From af177cefe07e6037e33cc90757088a98fb63a49f Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 15 Jul 2026 17:55:21 +0200 Subject: [PATCH 004/171] fix(frontend): graceful small-screen timeframe picker on the runs page (#10073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): prevent runs timeframe calendar popover overflow on small screens The Runs page timeframe picker rendered its popover as a wide 3-column row (preset list + two side-by-side calendars). With the right-aligned trigger and a center-anchored `bottom` placement, the popup ran off the right edge on narrow viewports. Anchor the popover to the right edge (`placement="bottom-end"`) and make its content reflow to a vertical stack below the `sm` breakpoint, capped at `max-w-[calc(100vw-2rem)] max-h-[80vh] overflow-auto` so it can never exceed the viewport. The desktop side-by-side layout is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(frontend): compact runs timeframe picker with a Start/End toggle on small screens The two-calendar desktop popover needs ~780px (two min-w-9 grids + presets + popover padding); below that it overflows. Under 800px, show a single calendar with a Start/End toggle picking which bound it edits, using set-start/set-end so each bound keeps its date and HH:MM time inputs — the same precision the desktop start/end pair offers. On short/landscape viewports the compact panel is scroll-contained within the popover's fitViewport height (contentClasses overflow-y-auto, scoped to the small layout) so its lower controls stay reachable. The desktop two-calendar layout is unchanged. Presets are shared between both layouts via a snippet, and the active range is preserved across the breakpoint since both branches drive the same value. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(frontend): let InlineCalendarInput month/year selects portal, use in compact timeframe picker Add an opt-in `portalSelects` prop to InlineCalendarInput that portals the month/year dropdowns to the body (default keeps them in-flow, so existing consumers are unchanged). The compact runs timeframe picker enables it so the dropdowns escape its scroll-contained (overflow-y-auto) popover instead of being clipped. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../common/InlineCalendarInput.svelte | 15 ++- .../components/runs/TimeframeSelect.svelte | 123 +++++++++++++----- 2 files changed, 100 insertions(+), 38 deletions(-) diff --git a/frontend/src/lib/components/common/InlineCalendarInput.svelte b/frontend/src/lib/components/common/InlineCalendarInput.svelte index 2ef2e4bd2f..7c20af8e74 100644 --- a/frontend/src/lib/components/common/InlineCalendarInput.svelte +++ b/frontend/src/lib/components/common/InlineCalendarInput.svelte @@ -63,9 +63,18 @@ type Props = (DateProps | RangeProps) & { class?: string + // Portal the month/year dropdowns to the body so they escape an `overflow` + // ancestor (e.g. a scroll-contained popover). Off by default: in-flow. + portalSelects?: boolean } - let { mode = 'date', value = $bindable(), class: className, ...rest }: Props = $props() + let { + mode = 'date', + value = $bindable(), + class: className, + portalSelects = false, + ...rest + }: Props = $props() const onClickBehavior = $derived( mode === 'range' ? ((rest as RangeProps).onClickBehavior ?? 'set-range') : 'set-range' @@ -425,14 +434,14 @@ (viewYear = parseInt(val) || viewYear)} items={YEAR_LIST.map((year) => ({ label: year.toString(), value: year }))} diff --git a/frontend/src/lib/components/runs/TimeframeSelect.svelte b/frontend/src/lib/components/runs/TimeframeSelect.svelte index cc5a99f70c..f8d2b0df71 100644 --- a/frontend/src/lib/components/runs/TimeframeSelect.svelte +++ b/frontend/src/lib/components/runs/TimeframeSelect.svelte @@ -78,6 +78,8 @@ + + +{#snippet presetButtons()} + {#each items as item (item.label)} + + {/each} +{/snippet} +
- {/each} + {#if isSmall} +
+
+ {@render presetButtons()} +
+
+ + {#snippet children({ item })} + + + {/snippet} + + range, + (v) => + onManualInput( + smallBound === 'end' + ? { maxTs: fromCalendarDate(v.end)?.toISOString() ?? null } + : { minTs: fromCalendarDate(v.start)?.toISOString() ?? null } + ) + } + /> +
- range, - (v) => onManualInput({ minTs: fromCalendarDate(v.start)?.toISOString() ?? null }) - } - /> - range, - (v) => onManualInput({ maxTs: fromCalendarDate(v.end)?.toISOString() ?? null }) - } - /> -
+ {:else} +
+
+ {@render presetButtons()} +
+ range, + (v) => onManualInput({ minTs: fromCalendarDate(v.start)?.toISOString() ?? null }) + } + /> + range, + (v) => onManualInput({ maxTs: fromCalendarDate(v.end)?.toISOString() ?? null }) + } + /> +
+ {/if} {/snippet} From 9705d602848966f850613233d6e23b73a753c259 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 15 Jul 2026 17:56:19 +0200 Subject: [PATCH 005/171] fix(frontend): keep session-exit URL clean by syncing new_draft strip with the router (#10101) * fix(frontend): keep session-exit URL clean by syncing new_draft strip with the router Co-Authored-By: Claude Opus 4.8 (1M context) * chore(frontend): correct replaceState comment and test-mock wording per review Co-Authored-By: Claude Fable 5 * docs(frontend): correct replaceState comment and drop drafting-history phrasing Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 4.8 (1M context) --- frontend/src/lib/newDraftFlag.test.ts | 15 +++++++++++++++ frontend/src/lib/newDraftFlag.ts | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/newDraftFlag.test.ts b/frontend/src/lib/newDraftFlag.test.ts index 95063aa34c..4a2141817e 100644 --- a/frontend/src/lib/newDraftFlag.test.ts +++ b/frontend/src/lib/newDraftFlag.test.ts @@ -17,6 +17,18 @@ vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn(), getLocalDraftHint: () => hints.value })) +// `stripNewDraftFlag` rewrites the URL through SvelteKit's `replaceState` and +// refreshes the session-switch's remembered nav route. Mock those so the strip +// is observable via `window.location.href` (mirroring jsdom) and the remembered +// route can be asserted. +const rememberNavRoute = vi.hoisted(() => vi.fn()) +vi.mock('$app/navigation', () => ({ + replaceState: (url: URL | string, _state: unknown) => { + window.location.href = new URL(url, window.location.href).toString() + } +})) +vi.mock('$app/state', () => ({ page: { state: {} } })) +vi.mock('$lib/components/sessions/sessionSwitch.svelte', () => ({ rememberNavRoute })) import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' import { stripNewDraftFlagOnSave, shouldSeedNewDraft } from './newDraftFlag' @@ -108,6 +120,9 @@ describe('stripNewDraftFlagOnSave', () => { expect(window.location.href).not.toContain('new_draft') // Sibling seeding params are preserved. expect(window.location.href).toContain('template=foo') + // The remembered nav route is refreshed to the stripped URL so exiting an + // AI session returns here without re-adding ?new_draft. + expect(rememberNavRoute).toHaveBeenCalledWith('/scripts/edit/u/me/draft_d?template=foo') }) it('does not strip on a delete save', async () => { diff --git a/frontend/src/lib/newDraftFlag.ts b/frontend/src/lib/newDraftFlag.ts index 20f45cf0e9..88e58f030a 100644 --- a/frontend/src/lib/newDraftFlag.ts +++ b/frontend/src/lib/newDraftFlag.ts @@ -1,16 +1,27 @@ +import { page } from '$app/state' +import { replaceState } from '$app/navigation' +import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { UserDraftDbSyncer, type UserDraftLastSyncQuery } from '$lib/userDraftDbSyncer.svelte' import { getLocalDraftHint } from '$lib/localDraftHints.svelte' import type { UserDraftItemKind } from '$lib/gen' /** Drop `?new_draft=true` from the current URL (preserving every other param), * mutating the address bar without a navigation. No-op when the flag is absent - * or `window` is unavailable (SSR). */ + * or `window` is unavailable (SSR). + * + * Uses SvelteKit's `replaceState` (not raw `history.replaceState`, which the + * router warns conflicts with it) so the history entry keeps the router's + * bookkeeping and `page.state`. Also refreshes the remembered nav route: + * `afterNavigate` never observes this in-place rewrite, so without it + * `exitSessionMode` would restore the pre-strip URL — still carrying + * `?new_draft=true` — and re-enter the seed-empty branch. */ export function stripNewDraftFlag(): void { if (typeof window === 'undefined') return const url = new URL(window.location.href) if (url.searchParams.get('new_draft') !== 'true') return url.searchParams.delete('new_draft') - window.history.replaceState(window.history.state, '', url.toString()) + replaceState(url, page.state) + rememberNavRoute(url.pathname + url.search) } /** From 2092155191401474b6791dd02e2d15183aad3c60 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 15 Jul 2026 18:00:53 +0200 Subject: [PATCH 006/171] chore(main): release 1.760.0 (#10128) * chore(main): release 1.760.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 22 +++ backend/Cargo.lock | 164 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 144 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ca0d1da5b..0e6a13c300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.760.0](https://github.com/windmill-labs/windmill/compare/v1.759.0...v1.760.0) (2026-07-15) + + +### Features + +* **nsjail:** make python/ansible rlimit_as configurable per worker (GIT-921) ([#10138](https://github.com/windmill-labs/windmill/issues/10138)) ([1787201](https://github.com/windmill-labs/windmill/commit/17872018cc037e699b1f6e1589d0ad05e0883cca)) + + +### Bug Fixes + +* **ai:** disable redirects on worker AI provider client (GHSA-5q4v) ([#10122](https://github.com/windmill-labs/windmill/issues/10122)) ([27ead8d](https://github.com/windmill-labs/windmill/commit/27ead8d0848cceacaf0c49fed0e8896472b851e3)) +* **ai:** stop sending the AI agent system prompt twice for OpenAI ([#10126](https://github.com/windmill-labs/windmill/issues/10126)) ([8bfe5c9](https://github.com/windmill-labs/windmill/commit/8bfe5c93404ba3f137394f16d0571a06d891dc3b)) +* **apps:** invalidate cached app policy on change or deletion (GHSA-r5v4-cxh9-7qhq) ([#10121](https://github.com/windmill-labs/windmill/issues/10121)) ([f7eb5c4](https://github.com/windmill-labs/windmill/commit/f7eb5c460d78792c24297e20cb062638522a4f68)) +* **bash:** normalize CRLF line endings before running scripts ([#10131](https://github.com/windmill-labs/windmill/issues/10131)) ([6407d9f](https://github.com/windmill-labs/windmill/commit/6407d9ff5ce51e71ff8b8fc503d89a2bdc2e1761)) +* **cli-image:** patch fixable CRITICAL CVEs in windmill-cli image (GIT-922) ([#10135](https://github.com/windmill-labs/windmill/issues/10135)) ([5626768](https://github.com/windmill-labs/windmill/commit/56267684718944ef4d4ecd3b610bad81132e1990)) +* **frontend:** graceful small-screen timeframe picker on the runs page ([#10073](https://github.com/windmill-labs/windmill/issues/10073)) ([af177ce](https://github.com/windmill-labs/windmill/commit/af177cefe07e6037e33cc90757088a98fb63a49f)) +* **frontend:** keep session-exit URL clean by syncing new_draft strip with the router ([#10101](https://github.com/windmill-labs/windmill/issues/10101)) ([9705d60](https://github.com/windmill-labs/windmill/commit/9705d602848966f850613233d6e23b73a753c259)) +* **frontend:** only carry custom-tag overrides on 'Run again' ([#10137](https://github.com/windmill-labs/windmill/issues/10137)) ([bd3adc9](https://github.com/windmill-labs/windmill/commit/bd3adc9781d8e77928c9feb03d0e05b63a1aaf7c)) +* **frontend:** treat a displaced draft save as superseded, not failed ([#10094](https://github.com/windmill-labs/windmill/issues/10094)) ([2fe999f](https://github.com/windmill-labs/windmill/commit/2fe999f66cd15acd81850f970ada31e9892abff2)) +* reject git URL fragment/query SSRF bypass (GHSA-p5cj-8cfh-mjv6) ([#10120](https://github.com/windmill-labs/windmill/issues/10120)) ([73c8d7f](https://github.com/windmill-labs/windmill/commit/73c8d7f08ad55cd2323d1bb9438f00a8ac30e046)) +* **security:** enforce variables:write scope on resource-delete var cascade (GHSA-xmr2-98m6-cjf7) ([#10123](https://github.com/windmill-labs/windmill/issues/10123)) ([188647a](https://github.com/windmill-labs/windmill/commit/188647a942a0cb496d63c71bf3e0f7a3136ec217)) + ## [1.759.0](https://github.com/windmill-labs/windmill/compare/v1.758.0...v1.759.0) (2026-07-15) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 489390e83a..ea2a6eedde 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -5212,9 +5212,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "globset" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" dependencies = [ "aho-corasick", "bstr", @@ -10978,9 +10978,9 @@ checksum = "8003eb09806ff2ae4661dd0dca27cbd9f65ba85de06cc0302c364b0d661ba368" [[package]] name = "snap" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" [[package]] name = "socket2" @@ -13870,7 +13870,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-nats", @@ -13952,7 +13952,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.759.0" +version = "1.760.0" dependencies = [ "async-stream", "async-trait", @@ -13985,7 +13985,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13998,7 +13998,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "argon2", @@ -14136,7 +14136,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14159,7 +14159,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14174,7 +14174,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14200,7 +14200,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.759.0" +version = "1.760.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14210,7 +14210,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14227,7 +14227,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14249,7 +14249,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14272,7 +14272,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14288,7 +14288,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14309,7 +14309,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14330,7 +14330,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14344,7 +14344,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-nats", @@ -14379,7 +14379,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14404,7 +14404,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14422,7 +14422,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14444,7 +14444,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14464,7 +14464,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14501,7 +14501,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14529,7 +14529,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.759.0" +version = "1.760.0" dependencies = [ "lazy_static", "serde", @@ -14541,7 +14541,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.759.0" +version = "1.760.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14566,7 +14566,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14580,7 +14580,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.759.0" +version = "1.760.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14615,7 +14615,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.759.0" +version = "1.760.0" dependencies = [ "chrono", "lazy_static", @@ -14629,7 +14629,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14648,7 +14648,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.759.0" +version = "1.760.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14750,7 +14750,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.759.0" +version = "1.760.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14769,7 +14769,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.759.0" +version = "1.760.0" dependencies = [ "regex", "serde", @@ -14784,7 +14784,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14808,7 +14808,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "futures", @@ -14825,7 +14825,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.759.0" +version = "1.760.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14841,7 +14841,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -14862,7 +14862,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -14893,7 +14893,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "arc-swap", @@ -14918,7 +14918,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-stream", @@ -14952,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "futures", @@ -14970,7 +14970,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.759.0" +version = "1.760.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14979,7 +14979,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -14991,7 +14991,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -15003,7 +15003,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "gosyn", @@ -15015,7 +15015,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -15027,7 +15027,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -15039,7 +15039,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "nu-parser", @@ -15050,7 +15050,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15061,7 +15061,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15073,7 +15073,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15084,7 +15084,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-recursion", @@ -15106,7 +15106,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -15118,7 +15118,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -15132,7 +15132,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15149,7 +15149,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -15162,7 +15162,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde", @@ -15174,7 +15174,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -15192,7 +15192,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15208,7 +15208,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15224,7 +15224,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde", @@ -15235,7 +15235,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-recursion", @@ -15274,7 +15274,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "const_format", @@ -15314,7 +15314,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.759.0" +version = "1.760.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15325,7 +15325,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-recursion", @@ -15359,7 +15359,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15383,7 +15383,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15416,7 +15416,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15449,7 +15449,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15469,7 +15469,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15503,7 +15503,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15539,7 +15539,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15562,7 +15562,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15586,7 +15586,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-nats", @@ -15610,7 +15610,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15645,7 +15645,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15673,7 +15673,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-trait", @@ -15698,7 +15698,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15717,7 +15717,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-once-cell", @@ -15827,7 +15827,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.759.0" +version = "1.760.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index acaef26b20..d367cde7eb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.759.0" +version = "1.760.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.759.0" +version = "1.760.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 222292b087..46885ae98d 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.759.0" +version = "1.760.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.759.0" +version = "1.760.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.759.0" +version = "1.760.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.759.0" +version = "1.760.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 4a2ad50f4c..90cbce6838 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.759.0" +version = "1.760.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 70bb50aa45..96b3f96bd6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.759.0 + version: 1.760.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 1cc713ae60..81c57ac9d5 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.759.0"; +export const VERSION = "v1.760.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 6cd9394f19..991c0bd5b7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.759.0"; +export const VERSION = "1.760.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8e4d5c8cbd..3fbc6b8126 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.760.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.760.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 0015541e04..8445d86221 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.760.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 62e6aea3c0..6ddaff60b0 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.759.0" +wmill = ">=1.760.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 1b5d938026..9973653550 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.759.0 + version: 1.760.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index ba13ab2a42..d137711c36 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.759.0' + ModuleVersion = '1.760.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 7a6a4ccf02..1ba4a31515 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.759.0" +version = "1.760.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6839da6bba..f454d94cc6 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.759.0", + "version": "1.760.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index aa2e068397..a90b8241ac 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.759.0", + "version": "1.760.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 67fbbd4999..920e44d956 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.759.0 +1.760.0 From 8c725d9e44e38bf3b35f41b847cdf8745e07942e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 15 Jul 2026 21:18:15 +0200 Subject: [PATCH 007/171] fix(apps): honor presigned S3 signature on app display/preview routes (#10141) The app provenance gate short-circuits on a valid presigned signature, but only the raw download_s3_file route parsed it. The parquet/csv/table-count/file-preview/metadata routes discarded sig/exp and always fell through to the provenance gate, so a presigned S3 object rendered as a table showed "File restricted" for any viewer who did not produce it. Thread sig/exp through every apps_u S3 display route and forward the presigned bearer from ParqetCsvTableRenderer/DisplayResult. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/Cargo.lock | 2 + backend/Cargo.toml | 2 + backend/tests/app_s3_onbehalf.rs | 125 ++++++++++++++++++ backend/windmill-api/openapi.yaml | 24 ++++ backend/windmill-api/src/apps.rs | 49 ++++--- .../src/lib/components/DisplayResult.svelte | 2 + .../components/ParqetCsvTableRenderer.svelte | 29 +++- 7 files changed, 214 insertions(+), 19 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ea2a6eedde..21e390f4ea 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -13885,6 +13885,8 @@ dependencies = [ "futures", "gethostname", "git-version", + "hex", + "hmac", "lazy_static", "once_cell", "opentelemetry 0.30.0", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d367cde7eb..85b129ea0b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -326,6 +326,8 @@ async-nats.workspace = true aws-sdk-sqs.workspace = true aws-config.workspace = true aws-credential-types.workspace = true +hmac.workspace = true +hex.workspace = true [workspace.dependencies] diff --git a/backend/tests/app_s3_onbehalf.rs b/backend/tests/app_s3_onbehalf.rs index 2e1085edd3..9dd96e30f1 100644 --- a/backend/tests/app_s3_onbehalf.rs +++ b/backend/tests/app_s3_onbehalf.rs @@ -144,6 +144,131 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: Ok(()) } +/// Mint a presigned bearer (`exp=..&sig=..`) exactly as `sign_s3_objects` does: +/// `HMAC-SHA256(workspace_key, "file_key={s3}&exp={exp}")` (no storage param, since +/// these routes send none). `validate_s3_signature` is `private`-gated, so this test +/// only runs with the `private` feature. +#[cfg(feature = "private")] +fn mint_presigned(workspace_key: &str, s3: &str, exp: i64) -> String { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let mut mac = Hmac::::new_from_slice(workspace_key.as_bytes()).unwrap(); + mac.update(format!("file_key={s3}&exp={exp}").as_bytes()); + let sig = hex::encode(mac.finalize().into_bytes()); + format!("exp={exp}&sig={sig}") +} + +/// A presigned S3 object (bearer minted by `signS3Objects`) bypasses the provenance +/// gate on EVERY app-scoped display route, not just the raw `download_s3_file` +/// download: a valid signature clears the gate on preview/count/metadata/csv routes, +/// while an unsigned key stays denied and a forged/expired signature is rejected. +#[cfg(feature = "private")] +#[sqlx::test(fixtures("base"))] +async fn test_deployed_app_s3_presigned_bypasses_gate(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP, + "summary": "s3 presigned test", + "value": {}, + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "allowed_s3_keys": [{ "s3_path": DECLARED }] + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); + + let workspace_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = 'test-workspace' AND kind = 'cloud'", + ) + .fetch_one(&db) + .await?; + let exp = chrono::Utc::now().timestamp() + 3600; + let presigned = mint_presigned(&workspace_key, NON_PROVENANCE, exp); + + let get = |route: String, token: &'static str| { + let url = format!("{ws}/apps_u/{route}"); + authed(client().get(url), token).send() + }; + let denied = |body: &str| body.contains("File restricted"); + + // Control: NON_PROVENANCE without a signature is denied by the gate. + let body = get( + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"), + USER_TOKEN, + ) + .await? + .text() + .await?; + assert!( + denied(&body), + "unsigned non-provenance key must be denied: {body}" + ); + + // Every display route: a valid presigned key clears the gate (falls through to + // the storage read, which fails with a storage error, never "File restricted"). + // `read_bytes_*` are required on load_file_preview. + let routes = [ + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{presigned}"), + format!("load_table_count/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + format!("load_csv_preview/{APP}?file_key={NON_PROVENANCE}&limit=5&offset=0&{presigned}"), + format!("load_parquet_preview/{APP}?file_key={NON_PROVENANCE}&limit=5&offset=0&{presigned}"), + format!("load_file_metadata/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + format!( + "load_file_preview/{APP}?file_key={NON_PROVENANCE}&read_bytes_from=0&read_bytes_length=4096&{presigned}" + ), + format!("download_s3_parquet_file_as_csv/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + ]; + for route in routes { + let body = get(route.clone(), USER_TOKEN).await?.text().await?; + assert!( + !denied(&body), + "presigned key must bypass the gate on {route}: {body}" + ); + } + + // A tampered signature must NOT bypass: presence of `sig` commits to validation, + // so a wrong sig is rejected outright ("Invalid signature") rather than falling + // back to the provenance gate. + let forged = format!("exp={exp}&sig={}", "00".repeat(32)); + let body = get( + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{forged}"), + USER_TOKEN, + ) + .await? + .text() + .await?; + assert!( + body.contains("Invalid signature"), + "forged signature must be rejected: {body}" + ); + + // An expired-but-valid signature is rejected on expiry, not bypassed. + let past = chrono::Utc::now().timestamp() - 10; + let expired = mint_presigned(&workspace_key, NON_PROVENANCE, past); + let body = get( + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{expired}"), + USER_TOKEN, + ) + .await? + .text() + .await?; + assert!( + body.contains("Signature expired"), + "expired signature must be rejected: {body}" + ); + + Ok(()) +} + /// Seed a completed job whose result carries an s3 object. `app_trigger` sets the /// app-origination marker exactly as `execute_component` stamps it: `Some(app_path)` /// => `trigger_kind = 'app'` + `trigger = ` (an app-launched run); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 96b3f96bd6..4df03ae467 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12139,6 +12139,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: FileMetadata @@ -12191,6 +12193,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: FilePreview @@ -12241,6 +12245,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Parquet Preview @@ -12293,6 +12299,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Csv Preview @@ -12325,6 +12333,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Table count @@ -12354,6 +12364,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: The downloaded file @@ -22660,6 +22672,18 @@ components: required: true schema: type: string + S3Sig: + name: sig + in: query + description: HMAC signature of a presigned S3 object (bypasses the app provenance gate) + schema: + type: string + S3Exp: + name: exp + in: query + description: Expiry timestamp of a presigned S3 object signature + schema: + type: string CustomPath: name: custom_path in: path diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 8169a42677..fe10229af3 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -3896,6 +3896,10 @@ async fn check_if_allowed_to_access_s3_file_from_app( windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) }); + // A valid presigned bearer is a self-authorizing capability, so it short-circuits + // the provenance gate. OSS builds cannot validate signatures (no workspace-key + // HMAC), so there the bearer is ignored and the request falls through to the + // checks below — the same path these routes took before presigning. if file_query.sig.is_some() { #[cfg(feature = "private")] { @@ -3908,13 +3912,11 @@ async fn check_if_allowed_to_access_s3_file_from_app( &db, ) .await?; - Ok(()) + return Ok(()); } - #[cfg(not(feature = "private"))] - return Err(Error::InternalErr( - "Internal error: signature validation is not supported in open source mode".to_string(), - )); - } else if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed { + } + + if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed { // Viewer mode: the on-behalf identity IS the viewer, so the downstream // get_workspace_s3_resource_and_check_paths already bounds the read by // their own perms — no provenance gate (it would over-restrict). Embed @@ -4051,14 +4053,25 @@ async fn download_s3_file_from_app( .await } +// Presigned bearer params (`exp=..&sig=..`) extracted as a second `Query` so the +// app-scoped preview/count/metadata routes honor a presigned key the same way the +// raw `download_s3_file` route does. #[cfg(feature = "parquet")] -fn app_s3_file_query(s3: String, storage: Option) -> AppS3FileQuery { +#[derive(Deserialize)] +struct AppS3Sig { + sig: Option, + #[cfg(feature = "private")] + exp: Option, +} + +#[cfg(feature = "parquet")] +fn app_s3_file_query(s3: String, storage: Option, sig: AppS3Sig) -> AppS3FileQuery { AppS3FileQuery { s3, storage, - sig: None, + sig: sig.sig, #[cfg(feature = "private")] - exp: None, + exp: sig.exp, } } @@ -4154,9 +4167,10 @@ async fn app_download_s3_parquet_file_as_csv( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; crate::job_helpers_oss::download_s3_parquet_file_as_csv_internal( @@ -4179,9 +4193,10 @@ async fn app_load_file_metadata( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = @@ -4195,9 +4210,10 @@ async fn app_load_file_preview( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = @@ -4211,10 +4227,11 @@ async fn app_load_table_count( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = @@ -4229,10 +4246,11 @@ async fn app_load_parquet_preview( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( @@ -4248,10 +4266,11 @@ async fn app_load_csv_preview( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 4e0c5ef8aa..ae44015cb8 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -1060,6 +1060,7 @@ {appPath} s3resource={s3object?.s3} storage={s3object?.storage} + presigned={s3object?.presigned} /> {/key} {:else if s3object?.s3?.endsWith('.png') || s3object?.s3?.endsWith('.jpeg') || s3object?.s3?.endsWith('.jpg') || s3object?.s3?.endsWith('.webp')} @@ -1116,6 +1117,7 @@ {appPath} s3resource={s3object?.s3} storage={s3object?.storage} + presigned={s3object?.presigned} />{:else} - {/if} - {#if !isTerminal(job.status)} - - {/if} - + {/if} + {#if !isTerminal(job.status)} + + {/if} + {/snippet} + + + {#if canPreview} + (showSource = v === 'source')} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} + + + +
+ {#if source} + + {#key `${artifact.id}:${artifact.updatedAt}`} + + {/key} + {:else} + +
+
+ +
+ {/if} +
+ diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte new file mode 100644 index 0000000000..96412d9b3c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte @@ -0,0 +1,64 @@ + + + a.id} + rowTitle={(a) => a.name} + onPick={(a: PersistedArtifact) => aiChatManager.openArtifact?.(a.id, a.name)} +> + {#snippet row(a)} + {a.name} + + {a.kind} + + + + + {/snippet} + {#snippet actions(a)} + + {:else if s.phase === 'draft'} + + {:else if s.phase === 'under_review'} + + {/if} + + + {#if s.phase === 'predeploy'} +
+ + Bundling creates a draft project on the Hub from the selected scripts, flows and + apps of {s.selectedFolder}/. + {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected. + +
+
+ + Resource dependencies + {#if s.detectingResources} + + {:else} + ({s.dependencyTypes.length}) + {/if} + + Resource types the selected items depend on (whether passed as inputs or + referenced by a hardcoded path). Synced to the Hub so a fork knows what + credentials it needs to fill. + + + {#if s.dependencyTypes.length === 0} + + No resource references detected in the current selection. + + {:else} + {#each s.dependencyTypes as r (r.resource_type)} + + {r.resource_type} + + {/each} + + {/if} +
+
+ + Data table dependencies + {#if s.detectingDatatables} + + {:else} + ({s.datatableUsage.size}) + {/if} + + Data tables the selected items read or write. A best-effort CREATE TABLE + migration for these is generated in the bundle step and shipped with the + project, so a fork can recreate the tables it needs. + + + {#if s.datatableUsage.size === 0} + + No data table usage detected in the current selection. + + {:else} + {#each [...s.datatableUsage] as [dt, tables] (dt)} + + {dt} + {#if tables.size > 0} + ×{tables.size} + {/if} + + {/each} + {/if} +
+ {/if} + {#if s.phase === 'draft'} +
+ + A recording captures one real run of a script or flow — inputs, logs, step outputs + and result — replayable on the Hub so visitors see it work before forking. Public + apps can also be shared as live iframes. Optional, but recommended. + +
+ {/if} + {#if s.phase === 'predeploy'} +
+ + Triggers + {#if s.triggersLoading} + + {:else} + ({s.relevantTriggers.length}) + {/if} + + {#if s.triggerDiscoveryFailed} + + Some trigger kinds could not be listed — publishing is disabled so triggers + aren't silently left out of the bundle. + + + {:else if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} + + {TRIGGER_KINDS[kind].badge} + ×{triggers.length} + + {/each} + + {/if} +
+ {/if} + {#if s.phase === 'under_review'} +
+ +
+ Locked while under review + + The Windmill team is reviewing this submission. Editing, recording, and sharing + actions are disabled. Estimated turnaround: 1-2 business days. + +
+
+ {/if} + {#if s.phase === 'draft'} + {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} + {@const pct = s.recordableItems.length + ? Math.round((recordedCount / s.recordableItems.length) * 100) + : 0} +
+ {recordedCount}/{s.recordableItems.length} +
+
+
+ + {s.allRecorded ? 'Full recordings' : 'Recordings recommended'} + +
+ {/if} + + {/snippet} + + {#snippet itemSummary(item)} + {@const it = item as DeployItem} + + {it.summary?.trim() || it.path} + + {/snippet} + + {#snippet itemActions(item)} + {@const it = item as DeployItem} + {#if s.phase !== 'predeploy' && canRecord(it.kind)} + {#if it.rec === 'recorded'} + + Recorded + + {#if s.recordings[it.key]} + + See recording + + {/if} + {#if s.phase === 'draft'} + + {/if} + {:else if s.phase === 'draft'} + No recording + + {:else} + No recording + {/if} + {/if} + {#if s.phase !== 'predeploy' && canShareAsIframe(it)} + {#if it.published} + + Public + + {#if it.publicUrl} + + Open + + + {:else if s.phase !== 'under_review'} + + + {/if} + {#if s.phase !== 'under_review'} + + {/if} + {:else if s.phase !== 'under_review'} + + {/if} + {/if} + {/snippet} + + {#snippet footer()} +
+ {#if s.phase === 'predeploy'} + + Select the items to include — all selected by default. + + {:else if s.phase === 'draft'} + + {#if s.allRecorded} + All scripts and flows have a recording — best chance of approval and featuring. + {:else} + {s.recordableItems.filter((i) => i.rec === 'recorded').length} of {s + .recordableItems.length} + recorded. Bundles with full recordings get approved faster and featured on the public + Hub. + {/if} + + {:else if s.phase === 'under_review'} + + Waiting for the Windmill team to review the submission. + + {:else} + Iterate further by starting a new draft. + {/if} +
+ {/snippet} + + + + + recordDrawer?.closeDrawer()} + > +
+

+ Run this {s.recordTarget?.kind} once with the inputs below. The full execution — args, logs, + intermediate step outputs and final result — is saved as a replayable recording + shown on the Hub page. Visitors can step through it to see how the {s.recordTarget + ?.kind} works without running anything themselves. +

+ + {#if s.runState !== 'idle'} +
+
+ {#if s.runState === 'running'} + + Running… + {:else if s.runState === 'success'} + + + Execution succeeded + + {:else} + + Execution failed + {/if} + {#if s.runJobId} + + Open job + + {/if} +
+ {#if s.runState === 'success' && s.runResult !== undefined} +
+ Result preview: +
{JSON.stringify(s.runResult, null, 2)}
+
+ {:else if s.runState === 'failed' && s.runError} +
{s.runError}
+ {/if} + {#if s.runState === 'success'} +
+ + Looks good? Save this run as the Hub recording. + + +
+ {:else if s.runState === 'failed'} + + Fix inputs and try again. Only successful runs can be saved as a recording. + + {/if} +
+ {/if} + + {#if s.recordSchemaLoading} + Loading schema… + {:else} + + {/if} +
+ {#snippet actions()} + {#if s.runState === 'success'} + + + {:else} + + {/if} + {/snippet} +
+
+ + + publishDrawer?.closeDrawer()} + > +
+

+ Expose {s.publishTarget?.path} at a public URL + so it can be embedded as an iframe (e.g. on the Hub, a docs page, or your own site). Anyone + with the URL will be able to interact with it. +

+ +
+
+ + Rate limit (workspace-wide) + + Caps public app executions per minute per server. Applies to all public apps in this + workspace. + +
+ {#if s.workspaceRateLimit && s.workspaceRateLimit > 0} + + Currently {s.workspaceRateLimit} executions + / minute / server. + + {:else} + + No rate limit configured — anyone with the URL can hit this app at any rate. + + {/if} + publishDrawer?.closeDrawer()} + > + Edit in Workspace settings → Apps + +
+
+ {#snippet actions()} + + + {/snippet} +
+
+ + + resourceDrawer?.closeDrawer()}> +
+

+ Resource types the selected items depend on. Each is synced to the Hub so a fork knows + what credentials it needs to fill. Input means the + item takes the resource as a parameter; + hardcoded path means the item pins a specific resource + path in its code. +

+ {#if s.dependencyTypes.length === 0} + No resource references in the current selection. + {:else} + {#each s.dependencyTypes as r (r.resource_type)} +
+
+ + {r.resource_type} + + + {r.usages.length} usage{r.usages.length > 1 ? 's' : ''} + +
+
+ {#each r.usages as u, ui (ui)} + {#if u.role === 'trigger'} +
+ + {u.label} + + {TRIGGER_KINDS[u.triggerKind].badge} trigger + +
+ {:else} + {@const itemUrl = s.itemUrl(u.kind, u.itemPath)} +
+
+ {#if u.kind === 'script'} + + {:else if u.kind === 'flow'} + + {:else} + + {/if} + {u.label} + + {u.role === 'hardcoded' ? 'hardcoded path' : 'input'} + {#if u.role === 'hardcoded'} + + + {#snippet text()} +
+ + This {u.kind} references the resource by a hardcoded path + $res:{u.path}. + + + For portability, prefer taking the resource as an input — a + fork won't have this exact path. It's relocated into the + project on publish, but converting it to an input keeps the + item reusable. + +
+ {/snippet} +
+ {/if} +
+ {#if itemUrl} + + + + {/if} +
+
+ {/if} + {/each} +
+
+ {/each} + {/if} +
+
+
+ + + triggerDrawer?.closeDrawer()}> +
+

+ Triggers attached to the selected scripts and flows. Synced to the Hub as + disabled stubs. Recipients review and enable each one + manually after importing. External hooks (Slack/Discord webhooks, message-queue + subscriptions, etc.) must be re-registered against the importing instance. +

+ {#if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} +
+
+ + {TRIGGER_KINDS[kind].badge} + + + {triggers.length} trigger{triggers.length > 1 ? 's' : ''} + + {#if TRIGGER_KINDS[kind].note} + + + + {#snippet text()} +
+ {TRIGGER_KINDS[kind].note} +
+ {/snippet} +
+
+ {/if} +
+
+ {#each triggers as t (t.path)} + {@const runnableSummary = s.runnableSummaryByPath.get( + `${t.is_flow ? 'flow' : 'script'}:${t.script_path}` + )} + {@const details = triggerDetails(t)} + {@const cfg = t.config as any} + {@const previewKey = + t.kind === 'schedule' ? `${cfg.schedule}|${cfg.timezone}` : ''} + {@const preview = + t.kind === 'schedule' ? s.schedulePreviews[previewKey] : undefined} + {@const triggerUrl = s.triggerListUrl(t.kind)} +
+
+ {#if t.is_flow} + + {:else} + + {/if} + + {runnableSummary || t.script_path} + + + {t.is_flow ? 'flow' : 'script'} + + {#if triggerUrl} + + + + {/if} +
+ {#if details.length > 0} +
+ {#each details as d (d.label)} +
{d.label}
+
{d.value}
+ {/each} + {#if t.kind === 'schedule'} +
Next runs
+
+ {#if preview && preview.length > 0} +
+ {#each preview as date (date)} + {displayDate(date)} + {/each} +
+ {:else if preview && preview.length === 0} + No upcoming run + {:else} + Loading… + {/if} +
+ {/if} +
+ {/if} +
+ {/each} +
+
+ {/each} + {/if} +
+
+
+ + + bundleDrawer?.closeDrawer()}> +
+

+ Name and document your bundle. The readme can be updated later, but a clear one speeds + up the Windmill team's review. +

+ {#if s.bundlePreview && s.bundlePreview.unresolved.length > 0} +
+ {s.bundlePreview.unresolved.length} unresolved reference(s) — cannot publish + + These items or resources couldn't be resolved, so the bundle would ship broken + references. Deselect or fix them, then retry: + +
    + {#each s.bundlePreview.unresolved as u (u)} +
  • {u}
  • + {/each} +
+
+ {/if} + +
+ Project slug + + {s.effectiveSlug || sanitizeSlug(s.hubName) || '—'} + + + {#if s.effectiveSlug} + Locked — items live under f/{s.effectiveSlug}/. + {:else if s.hubName.trim() && !isValidSlug(sanitizeSlug(s.hubName))} + + The name yields an invalid slug. Use at least 3 letters/digits. + + {:else} + Auto-generated from the name. Once project forked, items will live under + f/{sanitizeSlug(s.hubName) || ''}/. + {/if} + +
+ + +
+
+ + Data table migrations +
+ {#if s.migrationsGenerating} +
+ + Detecting data tables used by this project… +
+ {:else if s.migrationDrafts.length === 0} + + No data table usage detected in this project's scripts, flows, or raw apps. + + {:else} + + We detected these data tables. When included, the migration recreates their tables + on import. Best-effort — review and edit before publishing. + + {#each s.migrationDrafts as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ +
+ {/each} + {/if} +
+
+ {#snippet actions()} + + {/snippet} +
+
+ {/key} +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte new file mode 100644 index 0000000000..76e3da3e07 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte @@ -0,0 +1,40 @@ + + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts new file mode 100644 index 0000000000..11926c804b --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -0,0 +1,1635 @@ +import { untrack } from 'svelte' +import { base } from '$lib/base' +import { + AppService, + FlowService, + JobService, + RawAppService, + ResourceService, + ScriptService, + WorkspaceService, + ScheduleService +} from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { sleep, emptySchema } from '$lib/utils' +import { computeSecretUrl } from '$lib/components/apps/editor/appDeploy.svelte' +import { + buildProjectBundle, + buildPathMap, + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + rewriteTriggerConfig, + rewriteVarRefsInValue, + type BundleDeps, + type BundledItem, + type FetchedItem, + type ItemKind, + type ItemRef, + type ProjectBundle +} from './projectBundle' +import { + detectDatatableTables, + generateDatatableMigrations, + type GeneratedMigration +} from './projectMigrations' +import type { Kind } from '$lib/utils_deployable' +import { + TRIGGER_KINDS, + listAllWorkspaceTriggers, + triggerResourcePath, + triggerHandlerRefs, + portableTriggerConfig, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' + +export type Phase = 'predeploy' | 'draft' | 'under_review' | 'live' +export type RecStatus = 'none' | 'recorded' +export interface DeployItem { + key: string + path: string + kind: Kind + summary?: string + rec: RecStatus + published?: boolean + publicUrl?: string + [k: string]: unknown +} + +export const canRecord = (k: Kind) => k === 'script' || k === 'flow' +// Legacy raw apps live only in the `raw_app` table, but the iframe share flow +// drives AppService (the `app` table), so it can only target apps stored there. +export const canShareAsIframe = (it: DeployItem): boolean => + it.kind === 'app' || (it.kind === 'raw_app' && it.appTable === true) + +// Hub rehydration only carries draft membership, not the live share state of an +// app. Copy the public-execution flag, public URL, and app-table origin from the +// loaded workspace items onto matching draft items so a still-public app keeps its +// Public badge, Unpublish, and iframe controls after its draft is reopened. Returns +// the original array unchanged when nothing needs merging (stable reference). +export function mergeShareState( + draftItems: DeployItem[], + workspaceItems: DeployItem[] +): DeployItem[] { + if (draftItems.length === 0 || workspaceItems.length === 0) return draftItems + const byKey = new Map(workspaceItems.map((w) => [w.key, w])) + let changed = false + const merged = draftItems.map((d) => { + const w = byKey.get(d.key) + if (!w) return d + if (w.published !== d.published || w.publicUrl !== d.publicUrl || w.appTable !== d.appTable) { + changed = true + return { ...d, published: w.published, publicUrl: w.publicUrl, appTable: w.appTable } + } + return d + }) + return changed ? merged : draftItems +} + +export function sanitizeSlug(s: string): string { + return s + .toLowerCase() + .replace(/[_\s]+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50) + .replace(/-+$/g, '') +} +const SLUG_RE = /^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$/ +export function isValidSlug(s: string): boolean { + return SLUG_RE.test(s) +} + +export type RunState = 'idle' | 'running' | 'success' | 'failed' + +const ITEM_KIND_ROUTE: Record = { + script: 'scripts/get', + flow: 'flows/get', + app: 'apps/get', + raw_app: 'apps_raw/get' +} + +const HIDDEN_RESOURCE_TYPES = new Set(['app_theme', 'state', 'cache']) + +function typesFromSchema(schema: any): string[] { + const out = new Set() + const props = schema?.properties + if (props && typeof props === 'object') { + for (const key of Object.keys(props)) { + const fmt = props[key]?.format + if (typeof fmt === 'string' && fmt.startsWith('resource-')) { + out.add(fmt.slice('resource-'.length)) + } + } + } + return [...out] +} + +type DependencyUsage = + | { role: 'input'; label: string; kind: ItemKind; itemPath: string } + | { role: 'hardcoded'; label: string; kind: ItemKind; path: string; itemPath: string } + | { role: 'trigger'; label: string; triggerKind: WorkspaceTriggerKind; path: string } +export interface DependencyType { + resource_type: string + hasHardcoded: boolean + usages: DependencyUsage[] +} + +interface SessionDeps { + hasEeLicense: () => boolean +} + +/** + * All state and async operations for one Deploy-to-Hub surface, bound to an + * immutable (workspace, folder) pair. A workspace or folder change never mutates + * a session — `useDeployToHubSession` replaces the instance, so in-flight async + * work keeps writing to the discarded object and cannot leak into the new scope. + * The only invalidation tokens left are intra-session (competing calls on the + * same session), not lifecycle guards. + */ +export class DeployToHubSession { + readonly workspace: string + readonly folder: string + /** `f/`-prefixed folder path the project is scoped to. */ + readonly selectedFolder: string + + #disposed = false + #deps: SessionDeps + + phase = $state('predeploy') + workspaceItems = $state([]) + draftItems = $state([]) + workspaceTriggers = $state([]) + triggersLoading = $state(false) + // True when a trigger kind's discovery failed (not a feature-gated 404): + // the trigger list may be incomplete, so publishing is blocked until a + // retry succeeds. + triggerDiscoveryFailed = $state(false) + schedulePreviews = $state>({}) + manualDeselected = $state>(new Set()) + loading = $state(false) + workspaceRateLimit = $state(undefined) + deploymentStatus = $state< + Record + >({}) + deploying = $state(false) + + recordTarget = $state() + recordArgs = $state>({}) + recordValid = $state(true) + recordSchema = $state>(emptySchema()) + recordSchemaLoading = $state(false) + runState = $state('idle') + runJobId = $state(undefined) + runResult = $state(undefined) + runError = $state(undefined) + recordings = $state>({}) + + publishTarget = $state() + publishing = $state(false) + + hubName = $state('') + hubSummary = $state('') + hubReadme = $state('') + effectiveSlug = $state('') + hubItemIds = $state>({}) + + // Best-effort data table migrations for the bundle, editable in the drawer and + // pushed on deploy. Regenerated when the bundle drawer opens. + migrationDrafts = $state([]) + migrationsGenerating = $state(false) + // Bumped whenever the drafts are (re)generated, to re-key the Monaco editors so + // they pick up the fresh SQL (Monaco doesn't sync external `code` changes). + migrationsGeneration = $state(0) + + bundlePreview = $state(undefined) + detectingResources = $state(false) + // Data tables (→ tables) the current selection reads/writes, detected off the + // same bundle preview. Drives the predeploy "Data table dependencies" summary; + // the editable migration itself is generated in the bundle drawer. + datatableUsage = $state>>(new Map()) + detectingDatatables = $state(false) + + submitting = $state(false) + syncing = $state(false) + + // Intra-session tokens: latest call wins among competing calls on this session. + #triggerLoadTok = 0 + #recordRunTok = 0 + #migrationsTok = 0 + #schedulePreviewsInFlight = new Set() + // Preview-only cache: toggling checkboxes re-runs the closure walk, but item + // contents don't change mid-session. deployAll bypasses this and fetches fresh. + #previewItemCache = new Map>() + #previewTypeCache = new Map>() + + constructor(workspace: string, folder: string, deps: SessionDeps) { + this.workspace = workspace + this.folder = folder + this.selectedFolder = `f/${folder}` + this.#deps = deps + } + + dispose() { + this.#disposed = true + } + + load() { + void this.#loadWorkspace() + void this.#loadTriggers() + void this.rehydrateFromHub() + } + + filteredWorkspaceItems = $derived( + this.workspaceItems.filter((i) => i.path.startsWith(this.selectedFolder + '/')) + ) + // Derived (not merged at load time) so it settles regardless of which of the + // racing loads (#loadWorkspace / rehydrateFromHub) finishes last. + draftItemsWithLocalState = $derived(mergeShareState(this.draftItems, this.workspaceItems)) + items = $derived( + this.phase === 'predeploy' ? this.filteredWorkspaceItems : this.draftItemsWithLocalState + ) + selectedItems = $derived( + this.phase === 'predeploy' + ? this.filteredWorkspaceItems.filter((i) => !this.manualDeselected.has(i.key)) + : [] + ) + selectedItemKeys = $derived(this.selectedItems.map((i) => i.key)) + allSelected = $derived( + this.phase === 'predeploy' && + this.selectedItemKeys.length === this.filteredWorkspaceItems.length + ) + recordableItems = $derived(this.items.filter((i) => canRecord(i.kind))) + allRecorded = $derived( + this.recordableItems.length > 0 && this.recordableItems.every((i) => i.rec === 'recorded') + ) + hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) + + relevantTriggers = $derived.by(() => { + const selectedScripts = new Set( + this.selectedItems.filter((i) => i.kind === 'script').map((i) => i.path) + ) + const selectedFlows = new Set( + this.selectedItems.filter((i) => i.kind === 'flow').map((i) => i.path) + ) + return this.workspaceTriggers.filter((t) => + t.is_flow ? selectedFlows.has(t.script_path) : selectedScripts.has(t.script_path) + ) + }) + + triggersByKind = $derived.by(() => { + const out = new Map() + for (const t of this.relevantTriggers) { + const arr = out.get(t.kind) ?? [] + arr.push(t) + out.set(t.kind, arr) + } + return Array.from(out.entries()).sort((a, b) => a[0].localeCompare(b[0])) + }) + + runnableSummaryByPath = $derived.by(() => { + const m = new Map() + for (const it of this.workspaceItems) { + if (it.kind === 'script' || it.kind === 'flow') { + m.set(`${it.kind}:${it.path}`, it.summary) + } + } + return m + }) + + // `hasHardcoded` = pinned via $res: path (relocated as a stub); else input-only. + dependencyTypes = $derived.by(() => { + const b = this.bundlePreview + if (!b) return [] as DependencyType[] + const stubByNewPath = new Map(b.resourceStubs.map((s) => [s.newPath, s])) + const byType = new Map() + const ensure = (rt: string) => { + let e = byType.get(rt) + if (!e) { + e = { resource_type: rt, hasHardcoded: false, usages: [] } + byType.set(rt, e) + } + return e + } + for (const it of b.items) { + const label = (it.summary?.trim() || it.path) ?? it.path + const refs = + it.kind === 'flow' + ? extractFlowRefs(it.value).filter((r) => r.kind === 'resource') + : it.kind === 'app' + ? extractAppRefs(it.value) + : extractScriptRefs(it.content ?? '') + for (const r of refs) { + const stub = stubByNewPath.get(r.path) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + const e = ensure(stub.resource_type) + e.hasHardcoded = true + e.usages.push({ + role: 'hardcoded', + label, + kind: it.kind, + path: stub.originalPath, + itemPath: it.path + }) + } + for (const t of typesFromSchema(it.schema)) { + if (HIDDEN_RESOURCE_TYPES.has(t)) continue + ensure(t).usages.push({ role: 'input', label, kind: it.kind, itemPath: it.path }) + } + } + // Resources referenced only by a trigger (no item uses them in code) — + // its kind resource field or any `$res:` token in its config. + const stubByOriginal = new Map(b.resourceStubs.map((s) => [s.originalPath, s])) + for (const t of this.relevantTriggers) { + const refs = new Set( + extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config)) + ) + const rp = triggerResourcePath(t) + if (rp) refs.add(rp) + for (const ref of refs) { + const stub = stubByOriginal.get(ref) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + ensure(stub.resource_type).usages.push({ + role: 'trigger', + label: t.summary?.trim() || t.path, + triggerKind: t.kind, + path: stub.originalPath + }) + } + } + return [...byType.values()].sort((a, b) => a.resource_type.localeCompare(b.resource_type)) + }) + + toggleItem = (item: { key: string }) => { + const next = new Set(this.manualDeselected) + if (next.has(item.key)) next.delete(item.key) + else next.add(item.key) + this.manualDeselected = next + } + selectAll = () => { + this.manualDeselected = new Set() + } + deselectAll = () => { + this.manualDeselected = new Set(this.filteredWorkspaceItems.map((i) => i.key)) + } + + #folderQs(): string { + return `?folder=${encodeURIComponent(this.folder)}` + } + + itemUrl(kind: ItemKind, path: string): string | undefined { + if (!path) return undefined + return `${base}/${ITEM_KIND_ROUTE[kind]}/${path}?workspace=${this.workspace}` + } + triggerListUrl(kind: WorkspaceTriggerKind): string { + return `${base}/${TRIGGER_KINDS[kind].route}?workspace=${this.workspace}` + } + + #patchItem(key: string, patch: Partial) { + this.workspaceItems = this.workspaceItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + this.draftItems = this.draftItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + } + + async #listAllPages( + fetcher: (params: { perPage: number; page: number }) => Promise + ): Promise { + const perPage = 100 + const out: T[] = [] + for (let page = 1; page <= 1000; page++) { + const batch = await fetcher({ perPage, page }) + out.push(...batch) + if (batch.length < perPage) return out + } + return out + } + + async #loadWorkspace() { + const workspace = this.workspace + this.loading = true + try { + const [apps, rawApps, flows, scripts, settings] = await Promise.all([ + this.#listAllPages((p) => AppService.listApps({ workspace, ...p })), + this.#listAllPages((p) => RawAppService.listRawApps({ workspace, ...p })), + this.#listAllPages((p) => FlowService.listFlows({ workspace, ...p })), + this.#listAllPages((p) => ScriptService.listScripts({ workspace, ...p })), + WorkspaceService.getSettings({ workspace }).catch(() => undefined) + ]) + if (this.#disposed) return + + this.workspaceRateLimit = settings?.public_app_execution_limit_per_minute + + const next: DeployItem[] = [] + const publicApps = apps.filter((a) => a.execution_mode === 'anonymous') + const publicUrls = await Promise.all(publicApps.map((a) => this.#resolvePublicUrl(a.path))) + const publicUrlByPath = new Map(publicApps.map((a, i) => [a.path, publicUrls[i]])) + for (const a of apps) { + const isPublic = a.execution_mode === 'anonymous' + // Raw apps live in the `app` table (value = files/runnables) but must be + // published to the Hub as raw apps, not low-code apps. + const isRaw = (a as any).raw_app === true + next.push({ + key: `${isRaw ? 'raw_app' : 'app'}:${a.path}`, + path: a.path, + kind: isRaw ? 'raw_app' : 'app', + appTable: isRaw || undefined, + summary: a.summary, + rec: 'none', + published: isPublic, + publicUrl: isPublic ? publicUrlByPath.get(a.path) : undefined + }) + } + for (const a of rawApps) { + next.push({ + key: `raw_app:${a.path}`, + path: a.path, + kind: 'raw_app', + summary: a.summary, + rec: 'none' + }) + } + for (const f of flows) { + next.push({ + key: `flow:${f.path}`, + path: f.path, + kind: 'flow', + summary: f.summary, + rec: 'none' + }) + } + for (const s of scripts) { + next.push({ + key: `script:${s.path}`, + path: s.path, + kind: 'script', + summary: s.summary, + rec: 'none' + }) + } + if (this.#disposed) return + this.workspaceItems = next + } catch (e: any) { + if (!this.#disposed) { + sendUserToast(`Failed to load project items: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed) this.loading = false + } + } + + /** Re-fetch triggers, e.g. after the EE license hydrates late. */ + reloadTriggers() { + void this.#loadTriggers() + } + + async #loadTriggers() { + const tok = ++this.#triggerLoadTok + this.triggersLoading = true + try { + const { triggers, failedKinds } = await listAllWorkspaceTriggers(this.workspace, { + includeEeOnly: this.#deps.hasEeLicense(), + onError: (message) => { + if (!this.#disposed) sendUserToast(message, true) + } + }) + if (this.#disposed || tok !== this.#triggerLoadTok) return + this.workspaceTriggers = triggers + this.triggerDiscoveryFailed = failedKinds.length > 0 + } finally { + if (!this.#disposed && tok === this.#triggerLoadTok) this.triggersLoading = false + } + } + + async #resolvePublicUrl(path: string): Promise { + try { + const secret = await AppService.getPublicSecretOfApp({ workspace: this.workspace, path }) + return computeSecretUrl(secret) + } catch { + return undefined + } + } + + async rehydrateFromHub() { + try { + const res = await fetch(`/api/w/${this.workspace}/hub/project${this.#folderQs()}`, { + credentials: 'include', + headers: { accept: 'application/json' } + }) + if (this.#disposed) return + if (!res.ok) return // 404 = no project published for this folder yet + const p = JSON.parse(await res.text()) + if (this.#disposed || !p?.slug) return + this.effectiveSlug = p.slug + this.hubName = p.name ?? '' + this.hubSummary = p.summary ?? '' + this.hubReadme = p.readme ?? '' + this.phase = + p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' + const ids: Record = {} + this.draftItems = (p.items ?? []).map((it: any) => { + const wpath = it.source_path ?? it.path + const key = `${it.kind}:${wpath}` + if (typeof it.hub_id === 'number') ids[key] = it.hub_id + return { + key, + path: wpath, + kind: it.kind as Kind, + summary: it.summary ?? undefined, + rec: it.has_recording ? 'recorded' : 'none' + } satisfies DeployItem + }) + this.hubItemIds = ids + } catch {} + } + + /** Kick off schedule-preview fetches for any relevant schedule trigger missing one. */ + ensureSchedulePreviews() { + for (const t of this.relevantTriggers) { + if (t.kind !== 'schedule') continue + const c = t.config as any + const key = `${c.schedule}|${c.timezone}` + if (this.schedulePreviews[key] || this.#schedulePreviewsInFlight.has(key)) continue + this.#schedulePreviewsInFlight.add(key) + ScheduleService.previewSchedule({ + requestBody: { + schedule: c.schedule, + timezone: c.timezone, + cron_version: c.cron_version ?? 'v2' + } + }) + .then((dates) => { + this.schedulePreviews = { ...this.schedulePreviews, [key]: dates.slice(0, 3) } + }) + .catch(() => {}) + .finally(() => this.#schedulePreviewsInFlight.delete(key)) + } + } + + /** + * Rebuild the predeploy bundle preview (resource + data table dependency + * summaries), debounced so rapid checkbox toggles coalesce into one walk. + * Reads its reactive inputs synchronously and returns a cancel function, so + * it can be driven from an `$effect` with proper cleanup. + */ + queueBundlePreview(): (() => void) | undefined { + if (this.phase !== 'predeploy') { + this.bundlePreview = undefined + this.datatableUsage = new Map() + return undefined + } + this.detectingResources = true + this.detectingDatatables = true + const slug = this.hubSlug + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, slug) + ] + const triggerResources = this.#triggerResourcePaths(this.relevantTriggers) + const triggerVars = this.#triggerVarPaths(this.relevantTriggers) + let cancelled = false + const timer = setTimeout(() => { + buildProjectBundle(seed, slug, this.#cachedBundleDeps(), triggerResources, triggerVars) + .then((b) => { + if (cancelled) return + this.bundlePreview = b + // Detect data table usage off the same fetched items. + detectDatatableTables(b.items) + .then((usage) => { + if (!cancelled) this.datatableUsage = usage + }) + .finally(() => { + if (!cancelled) this.detectingDatatables = false + }) + }) + .finally(() => { + if (!cancelled) this.detectingResources = false + }) + }, 250) + return () => { + cancelled = true + clearTimeout(timer) + } + } + + #buildBundleDeps(): BundleDeps { + const workspace = this.workspace + return { + fetchItem: async (ref: ItemRef): Promise => { + try { + if (ref.kind === 'script') { + const s = await ScriptService.getScriptByPath({ workspace, path: ref.path }) + return { + kind: 'script', + path: ref.path, + summary: s.summary, + description: s.description ?? undefined, + content: s.content, + language: s.language, + schema: s.schema, + lock: s.lock ?? undefined, + scriptKind: typeof s.kind === 'string' ? s.kind.toLowerCase() : 'script' + } + } else if (ref.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace, path: ref.path }) + return { + kind: 'flow', + path: ref.path, + summary: f.summary, + description: f.description ?? undefined, + value: f.value, + schema: f.schema + } + } else if (ref.kind === 'app') { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + return { kind: 'app', path: ref.path, summary: a.summary, value: a.value } + } else if (ref.kind === 'raw_app') { + // Modern raw apps live in the `app` table: fetch source files + + // runnables + the compiled bundle, and shape them into the `raw` + // payload the Hub's RawAppView expects (JSON is valid YAML). + const isModern = this.workspaceItems.some( + (i) => i.kind === 'raw_app' && i.path === ref.path && i.appTable + ) + if (isModern) { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ + workspace, + path: ref.path + }) + // The compiled JS bundle is required; a missing one means the app + // was never built/deployed, so fail loudly instead of pushing a blank app. + const [jsRes, cssRes] = await Promise.all([ + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.js`, { + credentials: 'include' + }), + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.css`, { + credentials: 'include' + }) + ]) + if (!jsRes.ok) { + throw new Error(`raw app ${ref.path} has no compiled bundle — deploy it first`) + } + const js = await jsRes.text() + const css = cssRes.ok ? await cssRes.text() : '' + const v: any = a.value ?? {} + const content = JSON.stringify({ + files: { ...(v.files ?? {}), '/bundle.js': js, '/bundle.css': css }, + runnables: v.runnables ?? {}, + // Preserve the full-code app's explicit data table declaration so it + // survives publish/import and feeds migration detection. + ...(v.data !== undefined ? { data: v.data } : {}), + ...(v.datatables !== undefined ? { datatables: v.datatables } : {}) + }) + return { kind: 'raw_app', path: ref.path, summary: a.summary, content } + } + const r = await fetch(`/api/w/${workspace}/raw_apps/get_data/0/${ref.path}`, { + credentials: 'include' + }) + if (!r.ok) return undefined + return { kind: 'raw_app', path: ref.path, content: await r.text() } + } + } catch (e: any) { + return undefined + } + return undefined + }, + resolveResourceType: async (path: string): Promise => { + try { + const r = await ResourceService.getResource({ workspace, path }) + return r.resource_type ?? undefined + } catch (e: any) { + return undefined + } + } + } + } + + #cachedBundleDeps(): BundleDeps { + const deps = this.#buildBundleDeps() + // Memoize only successful lookups: a miss (undefined) is likely transient, so + // evict it once it resolves. Otherwise a fixed/retried dependency can never + // clear `bundlePreview.unresolved` until the whole session is recreated. + const memoize = ( + cache: Map>, + key: string, + run: () => Promise + ) => { + let p = cache.get(key) + if (!p) { + p = run() + cache.set(key, p) + void p.then((r) => { + if (r === undefined && cache.get(key) === p) cache.delete(key) + }) + } + return p + } + return { + fetchItem: (ref) => + memoize(this.#previewItemCache, `${ref.kind}:${ref.path}`, () => deps.fetchItem(ref)), + resolveResourceType: (path) => + memoize(this.#previewTypeCache, path, () => deps.resolveResourceType(path)) + } + } + + async #postHub(path: string, body: unknown): Promise | undefined> { + const res = await fetch(`/api/w/${this.workspace}${path}${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(body) + }) + const text = await res.text() + if (!res.ok) throw new Error(text) + try { + return JSON.parse(text) + } catch { + return undefined + } + } + + async regenerateMigrations() { + const tok = ++this.#migrationsTok + this.migrationsGenerating = true + try { + // Same handler-augmented seed as deployAll: a data table used only by a + // bundled trigger handler must still get its migration. + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, this.hubSlug || 'project') + ] + // Detection is independent of the final slug (data table refs aren't + // relocated), so any placeholder slug works for this throwaway bundle. + const bundle = await buildProjectBundle( + seed, + this.hubSlug || 'project', + this.#buildBundleDeps(), + [] + ) + const usage = await detectDatatableTables(bundle.items) + const drafts = await generateDatatableMigrations(this.workspace, usage) + if (this.#disposed || tok !== this.#migrationsTok) return + this.migrationDrafts = drafts + this.migrationsGeneration++ + } catch (e: any) { + if (!this.#disposed && tok === this.#migrationsTok) { + this.migrationDrafts = [] + this.migrationsGeneration++ + // Toast so a genuine failure isn't mistaken for "no data table usage". + sendUserToast(`Could not generate data table migrations: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed && tok === this.#migrationsTok) this.migrationsGenerating = false + } + } + + /** Prefill bundle metadata and start migration detection (bundle drawer opening). */ + prepareBundle() { + this.hubName = this.hubName || this.folder + void this.regenerateMigrations() + } + + /** + * Create the Hub draft then push the full bundle. `deploying` is set + * synchronously before the first request so a double-click cannot start a + * second publish, and the whole run is refused while triggers are still + * loading — an incomplete `relevantTriggers` snapshot would permanently + * omit triggers (and their handlers and migrations) from the draft. + * `onDraftCreated` fires once the draft exists (the bundle drawer closes + * there while items continue publishing). + */ + async publishBundle(onDraftCreated?: () => void): Promise { + if (this.deploying || this.triggersLoading || this.triggerDiscoveryFailed) return + this.deploying = true + try { + if (!(await this.#createDraft())) return + onDraftCreated?.() + await this.#deployAll() + } finally { + this.deploying = false + } + } + + /** + * Create the Hub draft project. Returns true when the draft exists and + * publishing can proceed. + */ + async #createDraft(): Promise { + this.hubName = this.hubName.trim() + this.hubSummary = this.hubSummary.trim() + this.hubReadme = this.hubReadme.trim() + try { + const res = await fetch(`/api/w/${this.workspace}/hub/publish_draft${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + slug: this.hubSlug, + name: this.hubName, + summary: this.hubSummary || this.hubName, + readme: this.hubReadme || undefined + }) + }) + const text = await res.text() + if (!res.ok) { + sendUserToast(`Hub draft creation failed: ${text}`, true) + return false + } + // Abort if Hub didn't echo a slug — guessing here lands items under + // a folder the Hub never locked. + let returnedSlug: string | undefined + try { + const parsed = JSON.parse(text) + if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + } catch {} + if (!returnedSlug) { + sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) + return false + } + // Session replaced mid-request (workspace/folder switch): publishing now + // would push another scope's items into this draft. Abort. + if (this.#disposed) { + sendUserToast(`Workspace changed during publish — aborted to avoid mixing items.`, true) + return false + } + this.effectiveSlug = returnedSlug + return true + } catch (e: any) { + sendUserToast(`Hub draft creation failed: ${e?.message ?? e}`, true) + return false + } + } + + async #pushBundledItem(slug: string, it: BundledItem): Promise { + const key = `${it.kind}:${it.path}` + if (it.kind === 'script') { + const resp = await this.#postHub('/hub/scripts', { + summary: it.summary || it.newPath, + app: slug, + description: it.description ?? '', + kind: it.scriptKind ?? 'script', + content: it.content, + language: it.language, + schema: it.schema ?? undefined, + lockfile: it.lock ?? undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'flow') { + const resp = await this.#postHub('/hub/flows', { + flow: { + summary: it.summary || it.newPath, + description: it.description ?? undefined, + value: it.value, + schema: it.schema ?? undefined + }, + apps: [], + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'app') { + await this.#postHub('/hub/apps', { + app: it.value, + apps: [], + summary: it.summary || it.newPath, + description: undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + } else if (it.kind === 'raw_app') { + const resp = await this.#postHub('/hub/raw_apps', { + raw: it.content ?? '', + apps: [], + summary: it.summary || it.newPath, + path: it.newPath, + source_path: it.path, + description: undefined, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } + } + + // Handler runnables (trigger error handlers, schedule on_* handlers) ship + // with the bundle like the primary runnables do; hub refs stay external. + #triggerHandlerSeed(triggers: WorkspaceTrigger[], slug: string): ItemRef[] { + return triggers.flatMap(triggerHandlerRefs).filter((r) => classifyPath(r.path, slug) !== 'hub') + } + + // Every resource a trigger's exported config references: the kind-specific + // broker/auth field plus any `$res:` token nested in it (schedule args, + // handler extra args, …) — all must enter the bundle path map. + #triggerResourcePaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + const rp = triggerResourcePath(t) + if (rp) out.add(rp) + for (const p of extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config))) { + out.add(p) + } + } + return [...out] + } + + // Every whole-string `$var:`/`$jsonvar:` value a trigger's config resolves (SQS + // queue_url, schedule args, …) — relocated through the bundle map like item vars. + #triggerVarPaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + for (const p of extractVarRefsFromValue(portableTriggerConfig(t.kind, t.config))) out.add(p) + } + return [...out] + } + + async #pushTriggers( + slug: string, + resourcePathMap: Map, + relevant: WorkspaceTrigger[] + ): Promise { + const pathMap = buildPathMap( + relevant.map((t) => t.path), + slug + ) + const triggers: Array> = [] + const skipped: string[] = [] + for (const t of relevant) { + const itemKind: ItemKind = t.is_flow ? 'flow' : 'script' + const runnableKey = `${itemKind}:${t.script_path}` + const hubId = this.hubItemIds[runnableKey] + if (!hubId) { + skipped.push(t.path) + continue + } + // Full-config remap: resource paths, error-handler paths, schedule on_* + // handler refs and whole-string `$var:` values all relocate through the map. + const config = rewriteVarRefsInValue( + rewriteTriggerConfig(portableTriggerConfig(t.kind, t.config), resourcePathMap), + resourcePathMap + ) + triggers.push({ + path: pathMap.get(t.path) ?? t.path, + kind: t.kind, + summary: t.summary ?? null, + description: (t.config as any)?.description ?? null, + config, + script_ask_id: t.is_flow ? null : hubId, + flow_id: t.is_flow ? hubId : null + }) + } + if (skipped.length > 0) { + sendUserToast( + `Skipped ${skipped.length} trigger(s) whose runnable did not publish: ${skipped.join(', ')}`, + true + ) + } + // Full-set sync: always push (an empty list clears the Hub's triggers on a + // re-deploy), so removing every trigger doesn't leave stale ones on the Hub. + await this.#postHub('/hub/triggers', { triggers, project_slug: slug }) + } + + // Builtin types (git_repository, ...) aren't in resource_type — push with empty schema. + async #pushResourceTypes(slug: string, types: string[]): Promise { + const results = await Promise.all( + types.map(async (name) => { + let schema: unknown = undefined + let description: string | undefined = undefined + try { + const rt = await ResourceService.getResourceType({ + workspace: this.workspace, + path: name + }) + schema = rt.schema ?? undefined + description = rt.description ?? undefined + } catch (e: any) {} + try { + await this.#postHub('/hub/resource_types', { + name, + schema, + description, + project_slug: slug + }) + return 0 + } catch (e: any) { + sendUserToast(`Resource type ${name} push failed: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + return results.reduce((a: number, b) => a + b, 0) + } + + async #deployAll() { + const slug = this.hubSlug + // Snapshot the selection up-front: `selectedItems`/`relevantTriggers` are + // derived from live workspace data and `migrationDrafts` is edited in the + // drawer — the deploy must publish exactly what the user confirmed. + const itemsSnapshot = this.selectedItems.slice() + const triggersSnapshot = this.relevantTriggers.slice() + const migrationsSnapshot = this.migrationDrafts.slice() + this.hubItemIds = {} + this.deploymentStatus = {} + let failures = 0 + try { + const seed: ItemRef[] = [ + ...itemsSnapshot + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(triggersSnapshot, slug) + ] + const triggerResources = this.#triggerResourcePaths(triggersSnapshot) + const triggerVars = this.#triggerVarPaths(triggersSnapshot) + const bundle = await buildProjectBundle( + seed, + slug, + this.#buildBundleDeps(), + triggerResources, + triggerVars + ) + // Full path map (incl. unresolved) so a trigger's resource path is always + // relocated — never leaks the publisher's original private path to the Hub. + const resourcePathMap = bundle.pathMap + + // A dangling reference (a selected root or transitive runnable that failed + // to fetch, or a resource whose type can't be resolved) means the bundle + // doesn't close: the root would silently vanish, or a published item would + // still point at the publisher's private source-workspace path. Refuse to + // publish until every reference resolves rather than ship a broken project. + if (bundle.unresolved.length > 0) { + sendUserToast( + `Cannot publish: ${bundle.unresolved.length} unresolved reference(s): ${bundle.unresolved.join(', ')}. Deselect or fix them, then retry.`, + true + ) + return + } + + // Bundle building is slow — bail before the first Hub write if the session + // was replaced (workspace/folder switch) in the meantime. + if (this.#disposed) return + + // Types come from $res: stubs AND schema inputs (resource-). + const inputTypes = bundle.items + .flatMap((i) => typesFromSchema(i.schema)) + .filter((t) => !HIDDEN_RESOURCE_TYPES.has(t)) + const types = [ + ...new Set([...bundle.resourceStubs.map((s) => s.resource_type), ...inputTypes]) + ] + const depFailures = await this.#pushResourceTypes(slug, types) + + // Input-type deps with no path get a conventional f// stub. + const stubsByPath = new Map() + for (const s of bundle.resourceStubs) + stubsByPath.set(s.newPath, { path: s.newPath, resource_type: s.resource_type }) + for (const t of inputTypes) { + const path = `f/${slug}/${t}` + if (!stubsByPath.has(path)) stubsByPath.set(path, { path, resource_type: t }) + } + const stubs = [...stubsByPath.values()] + if (stubs.length > 0) { + try { + await this.#postHub('/hub/resources', { resources: stubs, project_slug: slug }) + } catch (e: any) { + sendUserToast(`Resource sync failed: ${e?.message ?? e}`, true) + failures++ + } + } + failures += depFailures + if (failures > 0) { + sendUserToast( + `Resource dependency sync failed — items not published to avoid broken references.`, + true + ) + return + } + + for (const it of bundle.items) { + // Stop writing item status / Hub IDs once the session is replaced — + // continuing would publish into a project the user has moved away from. + if (this.#disposed) return + const key = `${it.kind}:${it.path}` + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'loading' } } + try { + await this.#pushBundledItem(slug, it) + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'deployed' } } + } catch (e: any) { + failures++ + this.deploymentStatus = { + ...this.deploymentStatus, + [key]: { status: 'failed', error: e?.message ?? String(e) } + } + } + } + // A re-bundle clears the Hub-side embed (idempotent replace), so re-push it + // for any raw app that is already public — keeps the live iframe in sync + // without forcing an unpublish/share round-trip. Updates by hub id, safe in parallel. + const embedResults = await Promise.all( + bundle.items + .filter((it) => it.kind === 'raw_app') + .map(async (it) => { + const hubId = this.hubItemIds[`${it.kind}:${it.path}`] + const src = itemsSnapshot.find((i) => i.kind === 'raw_app' && i.path === it.path) + if (!hubId || !src?.published) return 0 + // The re-bundle cleared the embed; a public raw app with no resolved URL + // can't have its iframe restored, so it's an incomplete publish too — + // count it (like a push failure) so the draft can't become submit-ready. + if (!src.publicUrl) { + sendUserToast(`Cannot restore the iframe for ${it.path}: missing public URL`, true) + return 1 + } + try { + await this.#pushRawAppEmbed(hubId, src.publicUrl) + return 0 + } catch (e: any) { + sendUserToast(`Failed to sync iframe for ${it.path}: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + failures += embedResults.reduce((a: number, b) => a + b, 0) + if (this.#disposed) return + try { + await this.#pushTriggers(slug, resourcePathMap, triggersSnapshot) + } catch (e: any) { + sendUserToast(`Trigger sync failed: ${e?.message ?? e}`, true) + failures++ + } + + // Full-set sync: always push (an empty list clears the Hub's migrations on + // a re-deploy). The Hub drops empty-SQL entries, so disabled placeholders + // don't persist. + try { + await this.#postHub('/hub/migrations', { + migrations: migrationsSnapshot.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down, + enabled: m.enabled + })), + project_slug: slug + }) + } catch (e: any) { + sendUserToast(`Data table migration sync failed: ${e?.message ?? e}`, true) + failures++ + } + + await sleep(150) + if (this.#disposed) return + // An incomplete push must never become submittable: a failed transitive item + // can leave a pushed runnable pointing at content that never landed. Stay in + // predeploy (deploymentStatus keeps the failed items visible) so re-publishing + // retries every write — createDraft and the item pushes are idempotent. + if (failures > 0) { + sendUserToast( + `Publish incomplete: ${failures} write(s) failed. Nothing was submitted — fix them and re-publish.`, + true + ) + return + } + this.deploymentStatus = {} + this.recordings = {} + // Deterministic baseline so a transient Hub read failure can't leave the + // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. + this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + } finally { + this.deploying = false + } + } + + submitForReview = async () => { + const slug = this.hubSlug + if (!slug) return + this.submitting = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/submit${this.#folderQs()}`, + { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + } + ) + if (!res.ok) { + sendUserToast(`Submit for review failed: ${await res.text()}`, true) + return + } + this.phase = 'under_review' + sendUserToast('Submitted for review by the Windmill team.') + } finally { + this.submitting = false + } + } + + syncWithHub = async () => { + this.syncing = true + try { + if (this.phase === 'draft') { + await this.#loadWorkspace() + const prev = new Map(this.draftItems.map((i) => [i.key, { rec: i.rec }])) + this.draftItems = this.workspaceItems + .filter((i) => prev.has(i.key)) + .map((i) => ({ ...i, rec: prev.get(i.key)?.rec ?? 'none' })) + } else { + // under_review / live: re-fetch the Hub project to pick up an + // admin status change (under_review -> live). + const before = this.phase + await this.rehydrateFromHub() + sendUserToast( + this.phase === before + ? 'Still waiting for review.' + : this.phase === 'live' + ? 'Approved — your project is now live.' + : `Status updated: ${this.phase}.` + ) + } + } catch (e: any) { + sendUserToast(`Sync failed: ${e?.message ?? e}`, true) + } finally { + this.syncing = false + } + } + + startNewDraft = () => { + this.draftItems = [] + this.recordings = {} + this.phase = 'predeploy' + } + + /** Reset record-drawer state and load the target's schema. */ + async openRecord(it: DeployItem) { + const tok = ++this.#recordRunTok + this.recordTarget = it + this.recordArgs = {} + this.recordValid = true + this.recordSchema = emptySchema() + this.recordSchemaLoading = true + this.runState = 'idle' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + if (it.kind === 'script') { + const s = await ScriptService.getScriptByPath({ + workspace: this.workspace, + path: it.path + }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (s.schema as Record) ?? emptySchema() + } else if (it.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace: this.workspace, path: it.path }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (f.schema as Record) ?? emptySchema() + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + sendUserToast(`Failed to load schema: ${e?.message ?? e}`, true) + } finally { + if (tok === this.#recordRunTok) this.recordSchemaLoading = false + } + } + + /** Invalidate any in-flight record run/poll (record drawer closed). */ + cancelRecordRun = () => { + this.#recordRunTok++ + } + + runJob = async () => { + const it = this.recordTarget + if (!it) return + const tok = ++this.#recordRunTok + this.runState = 'running' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + let jobId: string + if (it.kind === 'script') { + jobId = await JobService.runScriptByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else if (it.kind === 'flow') { + jobId = await JobService.runFlowByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else { + if (tok === this.#recordRunTok) this.runState = 'idle' + return + } + if (tok !== this.#recordRunTok) return + this.runJobId = jobId + await this.#pollJobUntilComplete(jobId, tok) + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Failed to start: ${e?.message ?? e}` + } + } + + async #pollJobUntilComplete(jobId: string, tok: number) { + // First check immediately (fast scripts complete in ms), then back off to 2s. + const deadline = Date.now() + 5 * 60_000 + let interval = 250 + while (Date.now() < deadline) { + if (tok !== this.#recordRunTok) return + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId + }) + if (tok !== this.#recordRunTok) return + if (r.completed) { + this.runResult = r.result + if (r.success) { + this.runState = 'success' + } else { + this.runState = 'failed' + this.runError = typeof r.result === 'string' ? r.result : JSON.stringify(r.result) + } + return + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Polling failed: ${e?.message ?? e}` + return + } + await sleep(interval) + interval = Math.min(interval * 2, 2000) + } + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = 'Timed out after 5 minutes' + } + + async #buildScriptRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const s = await ScriptService.getScriptByPath({ workspace, path: it.path }) + const job = await JobService.getCompletedJob({ workspace, id: jobId }) + const initial_job = { ...(job as any), type: 'CompletedJob' } + const events = [{ t: 0, data: { completed: true, job: initial_job } }] + const duration = (initial_job.duration_ms as number) ?? 0 + return { + version: 1, + type: 'script' as const, + recorded_at: new Date().toISOString(), + script_path: it.path, + total_duration_ms: duration, + code: s.content, + language: s.language, + args: (job.args ?? {}) as Record, + schema: s.schema, + job: { initial_job, events } + } + } + + async #buildFlowRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const f = await FlowService.getFlowByPath({ workspace, path: it.path }) + const root = (await JobService.getCompletedJob({ workspace, id: jobId })) as any + const jobs: Record = {} + const collect = async (j: any) => { + const stamped = { ...j, type: 'CompletedJob' } + jobs[j.id] = { + initial_job: stamped, + events: [{ t: 0, data: { completed: true, job: stamped } }] + } + const modules = (j.flow_status?.modules ?? []).filter( + (m: any) => m.job && typeof m.job === 'string' + ) + // Sub-jobs at the same level are independent reads. + await Promise.all( + modules.map(async (m: any) => { + try { + const sub = (await JobService.getCompletedJob({ workspace, id: m.job })) as any + await collect(sub) + } catch { + /* sub-job missing — skip */ + } + }) + ) + } + await collect(root) + return { + version: 1, + recorded_at: new Date().toISOString(), + flow_path: it.path, + total_duration_ms: (root.duration_ms as number) ?? 0, + flow: { + path: it.path, + value: f.value, + schema: f.schema ?? { type: 'object', properties: {}, required: [] }, + summary: f.summary ?? '', + archived: false, + edited_at: '', + edited_by: '', + extra_perms: {} + }, + jobs + } + } + + /** Save the current successful run as the Hub recording. Returns true on success. */ + async saveRecording(): Promise { + const it = this.recordTarget + if (!it || !this.runJobId || this.runState !== 'success') return false + const hubId = this.hubItemIds[it.key] + if (!hubId) { + sendUserToast(`Push the bundle to the Hub first before saving recordings`, true) + return false + } + if (it.kind !== 'script' && it.kind !== 'flow') { + sendUserToast(`Recordings only supported for script/flow`, true) + return false + } + try { + const recording = + it.kind === 'script' + ? await this.#buildScriptRecording(it, this.runJobId) + : await this.#buildFlowRecording(it, this.runJobId) + const path = it.kind === 'script' ? 'scripts' : 'flows' + await this.#postHub(`/hub/${path}/${hubId}/recording`, { + recording, + project_slug: this.hubSlug + }) + this.recordings = { ...this.recordings, [it.key]: this.runJobId } + this.#patchItem(it.key, { rec: 'recorded' }) + sendUserToast(`Recording saved — job ${this.runJobId}`) + return true + } catch (e: any) { + sendUserToast(`Failed to save recording: ${e?.message ?? e}`, true) + return false + } + } + + // Set the Hub raw app's live-iframe URL (or clear it with null). The Hub renders + // from external_embed_url; project_slug scopes ownership. + async #pushRawAppEmbed(hubId: number, url: string | null) { + await this.#postHub(`/hub/raw_apps/${hubId}/embed`, { + external_embed_url: url, + project_slug: this.hubSlug + }) + } + + // Flip an app/raw app between public (anonymous) and private (publisher) and keep + // the Hub raw-app iframe in sync. Returns the resolved public URL when shared. + async #setAppShared(it: DeployItem, shared: boolean): Promise { + const workspace = this.workspace + const hubId = it.kind === 'raw_app' ? this.hubItemIds[it.key] : undefined + // Sharing a raw app as an iframe needs its Hub item to wire the embed. Fail + // before flipping the app public so it can't be left anonymous with no embed. + if (shared && it.kind === 'raw_app' && !hubId) { + throw new Error('Push the bundle to the Hub first to share the live iframe') + } + const app = await AppService.getAppByPath({ workspace, path: it.path }) + const prevMode = (app.policy?.execution_mode ?? 'publisher') as 'anonymous' | 'publisher' + const nextMode = (shared ? 'anonymous' : 'publisher') as 'anonymous' | 'publisher' + const setMode = (mode: 'anonymous' | 'publisher', message: string) => + AppService.updateApp({ + workspace, + path: it.path, + requestBody: { + policy: { ...(app.policy ?? {}), execution_mode: mode }, + deployment_message: message + } + }) + // Undo the policy flip so the app's public state stays consistent when a later + // step of the share fails. Best-effort: a revert failure must not mask the cause. + const rollback = () => setMode(prevMode, 'Revert iframe share').catch(() => {}) + await setMode(nextMode, shared ? 'Share as iframe' : 'Unshare iframe') + const url = shared ? ((await this.#resolvePublicUrl(it.path)) ?? null) : null + // A share with no resolvable public URL is incomplete (no embeddable link, no + // Unpublish control); don't leave the app anonymous while reporting success. + if (shared && url === null) { + await rollback() + throw new Error(`Could not resolve the public URL for ${it.path}`) + } + if (hubId && it.kind === 'raw_app' && (!shared || url)) { + try { + await this.#pushRawAppEmbed(hubId, shared ? url : null) + } catch (e) { + await rollback() + throw e + } + } + return url + } + + /** Make the publish target public. Returns true on success. */ + async confirmPublish(): Promise { + const it = this.publishTarget + if (!it || !canShareAsIframe(it)) return false + this.publishing = true + try { + const url = await this.#setAppShared(it, true) + this.#patchItem(it.key, { published: true, publicUrl: url ?? undefined }) + sendUserToast(`${it.path} is now public`) + return true + } catch (e: any) { + sendUserToast(`Failed to publish: ${e?.message ?? e}`, true) + return false + } finally { + this.publishing = false + } + } + + unpublishApp = async (it: DeployItem) => { + if (!canShareAsIframe(it)) return + try { + await this.#setAppShared(it, false) + this.#patchItem(it.key, { published: false, publicUrl: undefined }) + sendUserToast('App unpublished') + } catch (e: any) { + sendUserToast(`Failed to unpublish: ${e?.message ?? e}`, true) + } + } +} + +/** + * Owns the session lifecycle: a new `DeployToHubSession` is created whenever the + * (workspace, folder) identity actually changes — a spurious same-value store + * emit reuses the live session — and the previous one is disposed, which is the + * single mechanism invalidating its in-flight work. Also hosts the reactive + * plumbing the session itself can't (license-hydration reload, schedule + * previews, debounced bundle preview). + */ +export function useDeployToHubSession(args: { + workspace: () => string | undefined + folder: () => string + hasEeLicense: () => boolean +}) { + let session = $state() + + $effect(() => { + const workspace = args.workspace() + const folder = args.folder() + if (!workspace) return + untrack(() => { + if (session && session.workspace === workspace && session.folder === folder) return + session?.dispose() + const next = new DeployToHubSession(workspace, folder, { + hasEeLicense: args.hasEeLicense + }) + session = next + next.load() + }) + }) + + // The EE license hydrates async; if it lands after a license-less trigger load, + // EE kinds stay empty. Re-fetch on false→true (the session reads the license + // getter at call time). + let prevHadLicense: boolean | undefined = undefined + $effect(() => { + const hasLicense = args.hasEeLicense() + untrack(() => { + if (hasLicense && prevHadLicense === false) session?.reloadTriggers() + prevHadLicense = hasLicense + }) + }) + + // Leaving/entering predeploy invalidates manual selection tweaks. + $effect(() => { + const s = session + if (!s) return + s.phase + untrack(() => { + s.manualDeselected = new Set() + }) + }) + + // Schedule previews for relevant schedule triggers (deduped in the session). + $effect(() => { + session?.ensureSchedulePreviews() + }) + + // Debounced predeploy bundle preview; the session reads its reactive inputs + // synchronously and returns the cancel function used as effect cleanup. + $effect(() => { + const s = session + if (!s) return + return s.queueBundlePreview() + }) + + return { + get session() { + return session + } + } +} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts new file mode 100644 index 0000000000..7b61251fcc --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest' +import { canShareAsIframe, mergeShareState, type DeployItem } from './deployToHubSession.svelte' + +function item(over: Partial & Pick): DeployItem { + return { rec: 'none', ...over } +} + +describe('canShareAsIframe', () => { + it('allows low-code apps and app-table raw apps', () => { + expect(canShareAsIframe(item({ key: 'app:f/a', path: 'f/a', kind: 'app' }))).toBe(true) + expect( + canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })) + ).toBe(true) + }) + it('hides the action for legacy raw apps (raw_app table only)', () => { + // Legacy entries from RawAppService carry no appTable flag; AppService can't load them. + expect(canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' }))).toBe(false) + }) + it('never offers the action for flows or scripts', () => { + expect(canShareAsIframe(item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' }))).toBe(false) + }) +}) + +describe('mergeShareState', () => { + it('carries live public-share state from workspace items onto matching drafts', () => { + const drafts = [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })] + const workspace = [ + item({ + key: 'app:f/a', + path: 'f/a', + kind: 'app', + published: true, + publicUrl: 'https://x/app' + }) + ] + const merged = mergeShareState(drafts, workspace) + expect(merged[0].published).toBe(true) + expect(merged[0].publicUrl).toBe('https://x/app') + }) + it('restores the app-table origin so app-table raw apps stay shareable', () => { + const drafts = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' })] + const workspace = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })] + expect(canShareAsIframe(mergeShareState(drafts, workspace)[0])).toBe(true) + }) + it('returns the same reference when nothing changes', () => { + const drafts = [item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' })] + expect(mergeShareState(drafts, drafts)).toBe(drafts) + }) + it('leaves drafts without a workspace match untouched', () => { + const drafts = [item({ key: 'app:f/gone', path: 'f/gone', kind: 'app' })] + const merged = mergeShareState(drafts, [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })]) + expect(merged).toBe(drafts) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts new file mode 100644 index 0000000000..669c0a43de --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -0,0 +1,869 @@ +import { describe, it, expect } from 'vitest' +import { + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + buildPathMap, + rewriteContent, + rewriteTriggerConfig, + rewriteFlowValue, + rewriteAppValue, + extractRawAppRefs, + rewriteRawAppContent, + buildProjectBundle, + retargetProjectExport, + collectExportVarPaths, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + type ProjectExport, + type FetchedItem, + type ItemRef +} from './projectBundle' + +describe('classifyPath', () => { + it('internal for paths under the project folder', () => { + expect(classifyPath('f/proj/db', 'proj')).toBe('internal') + expect(classifyPath('f/proj', 'proj')).toBe('internal') + }) + it('hub for hub paths', () => { + expect(classifyPath('hub/16043/discord/send', 'proj')).toBe('hub') + }) + it('external for user and other folders', () => { + expect(classifyPath('u/admin/db', 'proj')).toBe('external') + expect(classifyPath('f/other/db', 'proj')).toBe('external') + }) + it('does not treat a prefix-only match as internal', () => { + expect(classifyPath('f/project2/db', 'proj')).toBe('external') + }) +}) + +describe('extractScriptRefs', () => { + it('finds $res: and res:// resource refs, deduped', () => { + const c = `const a = "$res:u/admin/db"; const b = "res://f/x/api"; const c2 = "$res:u/admin/db"` + expect(extractScriptRefs(c)).toEqual([ + { kind: 'resource', path: 'u/admin/db' }, + { kind: 'resource', path: 'f/x/api' } + ]) + }) + it('returns nothing when no refs', () => { + expect(extractScriptRefs('export async function main() {}')).toEqual([]) + }) +}) + +describe('extractFlowRefs', () => { + it('finds inline-code, static-input, and script-path refs', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { + other: { type: 'static', value: '$res:f/shared/api' }, + expr1: { type: 'javascript', expr: 'flow_input.x' } + } + } + }, + { + id: 'b', + value: { + type: 'branchone', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'd', value: { type: 'script', path: 'hub/123/x/y' } } + ] + } + ], + default: [{ id: 'e', value: { type: 'rawscript', content: 'no refs' } }] + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'f/shared/api' }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/my_script' }) + expect(refs).toContainEqual({ kind: 'script', path: 'hub/123/x/y' }) + // a javascript expr (flow_input) is not a hardcoded ref + expect(refs.filter((r) => r.path === 'flow_input.x')).toEqual([]) + }) + it('finds sub-flow refs from type: flow steps', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }, + { id: 'b', value: { type: 'flow', path: 'hub/9/x/y' } } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sub_flow' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'hub/9/x/y' }) + }) +}) + +describe('buildPathMap', () => { + it('reparents into the project folder keeping the leaf name', () => { + const m = buildPathMap(['u/admin/db', 'f/other/api'], 'proj') + expect(m.get('u/admin/db')).toBe('f/proj/db') + expect(m.get('f/other/api')).toBe('f/proj/api') + }) + it('suffixes collisions deterministically', () => { + const m = buildPathMap(['u/alice/db', 'f/shared/db', 'u/bob/db'], 'proj') + // sorted: f/shared/db, u/alice/db, u/bob/db + expect(m.get('f/shared/db')).toBe('f/proj/db') + expect(m.get('u/alice/db')).toBe('f/proj/db_2') + expect(m.get('u/bob/db')).toBe('f/proj/db_3') + }) + it('maps internal paths to themselves, preserving subfolder depth', () => { + const m = buildPathMap(['f/proj/api', 'f/proj/sub/deep/script'], 'proj') + expect(m.get('f/proj/api')).toBe('f/proj/api') + expect(m.get('f/proj/sub/deep/script')).toBe('f/proj/sub/deep/script') + }) + it('does not flatten two internal items sharing a leaf name', () => { + const m = buildPathMap(['f/proj/a/x', 'f/proj/b/x'], 'proj') + expect(m.get('f/proj/a/x')).toBe('f/proj/a/x') + expect(m.get('f/proj/b/x')).toBe('f/proj/b/x') + }) + it('relocates an external onto a suffix when its leaf collides with an internal path', () => { + const m = buildPathMap(['f/proj/db', 'u/admin/db'], 'proj') + expect(m.get('f/proj/db')).toBe('f/proj/db') + expect(m.get('u/admin/db')).toBe('f/proj/db_2') + }) +}) + +describe('rewriteContent', () => { + it('rewrites mapped refs and leaves unmapped ones', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + expect(rewriteContent('x = "$res:u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "res://u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "$res:hub/1/a/b"', map)).toBe('x = "$res:hub/1/a/b"') + }) + it('does not partial-match a longer path', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + // u/admin/db2 must not be rewritten by the u/admin/db entry + expect(rewriteContent('x = "$res:u/admin/db2"', map)).toBe('x = "$res:u/admin/db2"') + }) +}) + +describe('rewriteTriggerConfig', () => { + const map = new Map([ + ['f/proj/kafka', 'f/target/kafka'], + ['f/proj/script', 'f/target/script'] + ]) + it('remaps plain resource path fields', () => { + expect( + rewriteTriggerConfig({ kafka_resource_path: 'f/proj/kafka', group_id: 'g1' }, map) + ).toEqual({ kafka_resource_path: 'f/target/kafka', group_id: 'g1' }) + }) + it('remaps nested objects, arrays, and $res: tokens', () => { + expect( + rewriteTriggerConfig( + { + nested: { path: 'f/proj/script' }, + list: ['f/proj/kafka', 'unrelated'], + code: 'x = "$res:f/proj/kafka"' + }, + map + ) + ).toEqual({ + nested: { path: 'f/target/script' }, + list: ['f/target/kafka', 'unrelated'], + code: 'x = "$res:f/target/kafka"' + }) + }) + it('leaves non-matching strings and non-string values untouched', () => { + const config = { url: 'wss://example.com', port: 9092, enabled: true, extra: null } + expect(rewriteTriggerConfig(config, map)).toEqual(config) + }) +}) + +describe('rewriteFlowValue', () => { + it('rewrites inline code, static inputs, and script paths; clones input', () => { + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['f/shared/api', 'f/proj/api'], + ['u/admin/my_script', 'f/proj/my_script'] + ]) + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { other: { type: 'static', value: '$res:f/shared/api' } } + } + }, + { id: 'b', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'c', value: { type: 'script', path: 'hub/1/keep/me' } } + ] + } + const out = rewriteFlowValue(value, map) + expect(out.modules[0].value.content).toBe('const db = "$res:f/proj/pg"') + expect(out.modules[0].value.input_transforms.other.value).toBe('$res:f/proj/api') + expect(out.modules[1].value.path).toBe('f/proj/my_script') + expect(out.modules[2].value.path).toBe('hub/1/keep/me') + // original untouched (deep clone) + expect(value.modules[0].value.content).toBe('const db = "$res:u/admin/pg"') + }) +}) + +// A trimmed app value: a runnable-by-path component, a hub runnable, a $res in an +// inline script, and incidental `f/...` text that must NOT be rewritten. +const appValue = () => ({ + grid: [ + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'script', path: 'u/admin/charts' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'flow', path: 'f/shared/sync' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'hubscript', path: 'hub/1/keep' } + } + } + } + ], + hiddenInlineScripts: [ + { name: 'h', inlineScript: { content: 'x = "$res:u/admin/pg"', language: 'deno' } } + ], + someLabel: 'see docs at f/shared/sync for details' +}) + +describe('extractAppRefs', () => { + it('extracts runnable-by-path scripts/flows and $res resources, skips hub', () => { + const refs = extractAppRefs(appValue()) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/charts' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'f/shared/sync' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) +}) + +describe('rewriteAppValue', () => { + it('relocates runnable paths and $res, leaves hub refs and incidental text intact', () => { + const map = new Map([ + ['u/admin/charts', 'f/proj/charts'], + ['f/shared/sync', 'f/proj/sync'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const value = appValue() + const out = rewriteAppValue(value, map) + expect(out.grid[0].data.componentInput.runnable.path).toBe('f/proj/charts') + expect(out.grid[1].data.componentInput.runnable.path).toBe('f/proj/sync') + expect(out.grid[2].data.componentInput.runnable.path).toBe('hub/1/keep') + expect(out.hiddenInlineScripts[0].inlineScript.content).toBe('x = "$res:f/proj/pg"') + // incidental text untouched + expect(out.someLabel).toBe('see docs at f/shared/sync for details') + // original untouched (deep clone) + expect(value.grid[0].data.componentInput.runnable.path).toBe('u/admin/charts') + }) +}) + +describe('raw app (value.raw JSON string)', () => { + const rawContent = () => + JSON.stringify({ + runnables: { + a: { type: 'path', runType: 'flow', path: 'u/admin/sync' }, + b: { type: 'path', runType: 'script', path: 'f/shared/calc' }, + c: { type: 'path', runType: 'hubscript', path: 'hub/1/keep' } + }, + files: { '/bundle.js': 'const conn = "$res:u/admin/pg"' } + }) + + it('extractRawAppRefs sees nested runnables and $res, skips hub', () => { + const refs = extractRawAppRefs(rawContent()) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sync' }) + expect(refs).toContainEqual({ kind: 'script', path: 'f/shared/calc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) + + it('rewriteRawAppContent relocates nested runnable paths and $res', () => { + const map = new Map([ + ['u/admin/sync', 'f/proj/sync'], + ['f/shared/calc', 'f/proj/calc'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const out = JSON.parse(rewriteRawAppContent(rawContent(), map)) + expect(out.runnables.a.path).toBe('f/proj/sync') + expect(out.runnables.b.path).toBe('f/proj/calc') + expect(out.runnables.c.path).toBe('hub/1/keep') + expect(out.files['/bundle.js']).toBe('const conn = "$res:f/proj/pg"') + }) + + it('falls back to $res scan on non-JSON content', () => { + expect(extractRawAppRefs('x = "$res:u/admin/pg"')).toContainEqual({ + kind: 'resource', + path: 'u/admin/pg' + }) + expect( + rewriteRawAppContent('x = "$res:u/admin/pg"', new Map([['u/admin/pg', 'f/proj/pg']])) + ).toBe('x = "$res:f/proj/pg"') + }) +}) + +describe('buildProjectBundle', () => { + // A flow that calls an external script which itself hardcodes a resource. + const flow: FetchedItem = { + kind: 'flow', + path: 'u/admin/my_flow', + summary: 'Flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/helper' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:f/shared/api"', + input_transforms: {} + } + } + ] + } + } + const helper: FetchedItem = { + kind: 'script', + path: 'u/admin/helper', + summary: 'Helper', + language: 'bun', + content: 'const db = "$res:u/admin/pg"; export async function main(){}' + } + + const deps = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/my_flow') return flow + if (ref.path === 'u/admin/helper') return helper + return undefined + }, + resolveResourceType: async (path: string) => { + if (path === 'u/admin/pg') return 'postgresql' + if (path === 'f/shared/api') return 'http_api' + return undefined + } + } + + it('pulls in referenced scripts + resources and rewrites everything under the folder', async () => { + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/my_flow' }], + 'proj', + deps + ) + + // flow + transitively-pulled helper script are both bundled + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + expect(Object.keys(byPath).sort()).toEqual(['u/admin/helper', 'u/admin/my_flow']) + + // items relocated under f/proj/ + expect(byPath['u/admin/my_flow'].newPath).toBe('f/proj/my_flow') + expect(byPath['u/admin/helper'].newPath).toBe('f/proj/helper') + + // flow's script-path ref rewritten to the helper's new path + expect(byPath['u/admin/my_flow'].value.modules[0].value.path).toBe('f/proj/helper') + // flow inline + helper code resource refs rewritten + expect(byPath['u/admin/my_flow'].value.modules[1].value.content).toBe( + 'const x = "$res:f/proj/api"' + ) + expect(byPath['u/admin/helper'].content).toContain('"$res:f/proj/pg"') + + // resource stubs created at new paths with resolved types + const stubs = Object.fromEntries(bundle.resourceStubs.map((s) => [s.originalPath, s])) + expect(stubs['u/admin/pg'].newPath).toBe('f/proj/pg') + expect(stubs['u/admin/pg'].resource_type).toBe('postgresql') + expect(stubs['f/shared/api'].resource_type).toBe('http_api') + + expect(bundle.unresolved).toEqual([]) + }) + + it('pulls in a sub-flow referenced by a type: flow step and rewrites its path', async () => { + const parent: FetchedItem = { + kind: 'flow', + path: 'u/admin/parent_flow', + value: { modules: [{ id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }] } + } + const sub: FetchedItem = { + kind: 'flow', + path: 'u/admin/sub_flow', + value: { + modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/keep/me' } }] + } + } + const d = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/parent_flow') return parent + if (ref.path === 'u/admin/sub_flow') return sub + return undefined + }, + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/parent_flow' }], + 'proj', + d + ) + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + // both flows bundled + expect(Object.keys(byPath).sort()).toEqual(['u/admin/parent_flow', 'u/admin/sub_flow']) + // parent's type: flow ref rewritten to the sub-flow's new path + expect(byPath['u/admin/parent_flow'].value.modules[0].value.path).toBe('f/proj/sub_flow') + expect(byPath['u/admin/sub_flow'].newPath).toBe('f/proj/sub_flow') + // hub ref inside the sub-flow left untouched + expect(byPath['u/admin/sub_flow'].value.modules[0].value.path).toBe('hub/1/keep/me') + expect(bundle.unresolved).toEqual([]) + }) + + it('leaves hub script references untouched and does not fetch them', async () => { + const hubFlow: FetchedItem = { + kind: 'flow', + path: 'u/admin/hub_flow', + value: { modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/x/y' } }] } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/hub_flow' ? hubFlow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/hub_flow' }], 'proj', d) + expect(bundle.items.map((i) => i.path)).toEqual(['u/admin/hub_flow']) + expect(bundle.items[0].value.modules[0].value.path).toBe('hub/1/x/y') + expect(bundle.unresolved).toEqual([]) + }) + + it('reports a missing item and an unresolvable resource as unresolved', async () => { + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/gone' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:u/admin/untyped"', + input_transforms: {} + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved.sort()).toEqual(['u/admin/gone', 'u/admin/untyped']) + }) + + it('relocates $var:/$jsonvar: refs into the slug when it differs from the source folder', async () => { + const flow: FetchedItem = { + kind: 'flow', + path: 'f/source_folder/main', + value: { + flow_env: { CFG: '$jsonvar:f/source_folder/cfg' }, + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + // Whole-value ref is relocated; the inline literal is not. + content: 'return "$var:f/source_folder/key"', + input_transforms: { k: { type: 'static', value: '$var:f/source_folder/key' } } + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'f/source_folder/main' ? flow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'f/source_folder/main' }], + 'kit', + d + ) + const v = bundle.items[0].value + expect(v.modules[0].value.input_transforms.k.value).toBe('$var:f/kit/key') + expect(v.flow_env.CFG).toBe('$jsonvar:f/kit/cfg') + // Inline code literal is untouched. + expect(v.modules[0].value.content).toBe('return "$var:f/source_folder/key"') + }) + + it('dedupes a path missing as both a script and a flow', async () => { + // A missing script + flow sharing a path each push the bare path once; the + // list must stay unique so a keyed UI render of it can't collide. + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/dup' } }, + { id: 'b', value: { type: 'flow', path: 'u/admin/dup' } } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved).toEqual(['u/admin/dup']) + }) +}) + +describe('extractVarRefsFromValue', () => { + it('collects whole-value `$var:`/`$jsonvar:` refs, deduped, walking nested JSON', () => { + const value = { + flow_env: { API: '$var:u/admin/key' }, + modules: [ + { value: { input_transforms: { a: { type: 'static', value: '$var:f/proj/token' } } } }, + { value: { input_transforms: { b: { type: 'static', value: '$jsonvar:u/admin/cfg' } } } }, + { value: { input_transforms: { c: { type: 'static', value: '$var:u/admin/key' } } } } + ] + } + expect(extractVarRefsFromValue(value).sort()).toEqual([ + 'f/proj/token', + 'u/admin/cfg', + 'u/admin/key' + ]) + }) + it('ignores a `$var:` token embedded in inline code (not a whole value)', () => { + // The worker only substitutes a value that *is* the reference, so an inline + // script literal must not be treated as a variable arg. + const value = { + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/example/template"' } }] + } + expect(extractVarRefsFromValue(value)).toEqual([]) + }) +}) + +describe('retargetProjectExport', () => { + const baseExport = (): ProjectExport => ({ + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [ + { + path: 'f/proj/hello', + content: 'const r = "$res:f/proj/db"', + summary: 'hello' + } + ], + flows: [ + { + path: 'f/proj/main_flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'f/proj/hello', input_transforms: {} } } + ] + } + } + ], + apps: [ + { + path: 'f/proj/dashboard', + value: { grid: [{ data: { componentInput: { runnable: {} } } }] } + }, + { + path: 'f/proj/rawapp', + app_type: 'raw', + value: { raw: JSON.stringify({ files: {}, runnables: {} }) } + } + ], + resources: [{ path: 'f/proj/db', resource_type: 'postgresql' }], + triggers: [ + { + path: 'f/proj/every_day', + kind: 'schedule', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { schedule: '0 0 12 * * *' } + }, + { + path: 'f/proj/kafka_in', + kind: 'kafka', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { kafka_resource_path: 'f/proj/db' } + } + ] + }) + + it('returns the bundle unchanged when the folder matches the slug', () => { + const bundle = baseExport() + expect(retargetProjectExport(bundle, 'proj', 'proj')).toBe(bundle) + }) + + it('relocates every item path and internal reference into the target folder', () => { + const out = retargetProjectExport(baseExport(), 'proj', 'dest') + expect(out.scripts[0].path).toBe('f/dest/hello') + expect(out.scripts[0].content).toContain('$res:f/dest/db') + expect(out.flows[0].path).toBe('f/dest/main_flow') + expect(out.flows[0].value.modules[0].value.path).toBe('f/dest/hello') + expect(out.apps.map((a) => a.path)).toEqual(['f/dest/dashboard', 'f/dest/rawapp']) + expect(out.resources[0].path).toBe('f/dest/db') + expect(out.triggers[0].path).toBe('f/dest/every_day') + expect(out.triggers[0].runnable_path).toBe('f/dest/hello') + // Plain-string resource path in a trigger config is remapped too. + expect(out.triggers[1].config.kafka_resource_path).toBe('f/dest/db') + }) + + it('leaves external and hub paths untouched', () => { + const bundle = baseExport() + bundle.scripts[0].content = 'const a = "$res:u/admin/db"; const b = "$res:hub/1/x"' + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.scripts[0].content).toContain('$res:u/admin/db') + expect(out.scripts[0].content).toContain('$res:hub/1/x') + }) + + it('retargets internal $var:/$jsonvar: refs but leaves external ones', () => { + const bundle = baseExport() + bundle.flows[0].value.modules[0].value.input_transforms = { + key: { type: 'static', value: '$var:f/proj/api_key' }, + ext: { type: 'static', value: '$var:u/admin/personal' } + } + bundle.flows[0].value.flow_env = { CFG: '$jsonvar:f/proj/cfg' } + bundle.triggers[1].config.queue_url = '$var:f/proj/sqs' + const out = retargetProjectExport(bundle, 'proj', 'dest') + const it = out.flows[0].value.modules[0].value.input_transforms + expect(it.key.value).toBe('$var:f/dest/api_key') + expect(it.ext.value).toBe('$var:u/admin/personal') + expect(out.flows[0].value.flow_env.CFG).toBe('$jsonvar:f/dest/cfg') + expect(out.triggers[1].config.queue_url).toBe('$var:f/dest/sqs') + }) + + it('leaves an inert $var: literal embedded in inline code unchanged', () => { + const bundle = baseExport() + // Same path as a real runtime ref, but here it is a literal inside code: it + // must not be rewritten even once the path enters the retarget map. + bundle.flows[0].value.modules[0].value = { + type: 'rawscript', + content: 'return "$var:f/proj/api_key"', + input_transforms: { real: { type: 'static', value: '$var:f/proj/api_key' } } + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + const mod = out.flows[0].value.modules[0].value + expect(mod.content).toBe('return "$var:f/proj/api_key"') + expect(mod.input_transforms.real.value).toBe('$var:f/dest/api_key') + }) +}) + +describe('collectExportVarPaths', () => { + it('gathers variable refs from flows, apps, and triggers (deduped)', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [], + flows: [{ path: 'f/proj/f', value: { flow_env: { A: '$var:f/proj/a' }, modules: [] } }], + apps: [ + { + path: 'f/proj/raw', + app_type: 'raw', + value: { raw: JSON.stringify({ runnables: { r: { fields: { x: '$var:u/admin/b' } } } }) } + } + ], + triggers: [{ path: 'f/proj/t', kind: 'sqs', config: { queue_url: '$jsonvar:f/proj/a' } }], + resources: [] + } + expect(collectExportVarPaths(bundle).sort()).toEqual(['f/proj/a', 'u/admin/b']) + }) +}) + +describe('trigger handler relocation', () => { + it('rewriteTriggerConfig remaps script/- and flow/-prefixed handler refs', () => { + const map = new Map([ + ['u/admin/handler', 'f/proj/handler'], + ['u/admin/recovery_flow', 'f/proj/recovery_flow'] + ]) + const out = rewriteTriggerConfig( + { + error_handler_path: 'u/admin/handler', + on_failure: 'script/u/admin/handler', + on_recovery: 'flow/u/admin/recovery_flow', + on_success: 'script/u/admin/unmapped' + }, + map + ) + expect(out.error_handler_path).toBe('f/proj/handler') + expect(out.on_failure).toBe('script/f/proj/handler') + expect(out.on_recovery).toBe('flow/f/proj/recovery_flow') + expect(out.on_success).toBe('script/u/admin/unmapped') + }) + + it('remaps $script:/$flow: only in the url field, never in literal payloads', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + initial_messages: [{ raw_message: '$script:u/admin/builder' }] + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.initial_messages[0].raw_message).toBe('$script:u/admin/builder') + }) + + it('leaves literal handler-shaped strings in args untouched', () => { + const map = new Map([['f/proj/handler', 'f/dest/handler']]) + const out = rewriteTriggerConfig( + { + on_failure: 'script/f/proj/handler', + args: { note: 'script/f/proj/handler' } + }, + map + ) + expect(out.on_failure).toBe('script/f/dest/handler') + expect(out.args.note).toBe('script/f/proj/handler') + }) + + it('leaves nested url keys untouched, rewriting only the top-level websocket url', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + args: { url: '$script:u/admin/builder' } + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.args.url).toBe('$script:u/admin/builder') + }) + + it('extracts and relocates $res refs nested in static input transform JSON', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'script', + path: 'f/proj/step', + input_transforms: { + provider: { type: 'static', value: { resource: '$res:u/admin/openai' } }, + note: { type: 'static', value: 'plain text' } + } + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/openai' }) + const out = rewriteFlowValue(value, new Map([['u/admin/openai', 'f/proj/openai']])) + const it0 = out.modules[0].value.input_transforms + expect(it0.provider.value).toEqual({ resource: '$res:f/proj/openai' }) + expect(typeof it0.note.value).toBe('string') + }) + + it('retargetProjectExport remaps trigger error handlers with the bundle', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [{ path: 'f/proj/handler', content: '' }], + flows: [], + apps: [], + resources: [], + triggers: [ + { + path: 'f/proj/sched', + kind: 'schedule', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { schedule: '0 0 * * * *', on_failure: 'script/f/proj/handler' } + }, + { + path: 'f/proj/mq', + kind: 'mqtt', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { error_handler_path: 'f/proj/handler' } + } + ] + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.triggers[0].config.on_failure).toBe('script/f/dest/handler') + expect(out.triggers[1].config.error_handler_path).toBe('f/dest/handler') + }) +}) + +describe('extractTriggerConfigResourceRefs', () => { + it('collects $res: tokens nested anywhere in a trigger config', () => { + expect( + extractTriggerConfigResourceRefs({ + schedule: '0 0 * * * *', + args: { channel: '$res:u/admin/slack' }, + on_failure_extra_args: { db: 'res://f/other/pg' }, + error_handler_args: { nested: { deep: '$res:u/admin/slack' } } + }) + ).toEqual(['u/admin/slack', 'f/other/pg']) + }) +}) + +describe('flow_env and preprocessor_module', () => { + const flowValue = { + modules: [], + preprocessor_module: { + id: 'pre', + value: { type: 'script', path: 'u/admin/preproc', input_transforms: {} } + }, + flow_env: { SLACK: '$res:u/admin/slack', PLAIN: 'not-a-ref' } + } + + it('walks nested children of the failure module', () => { + const refs = extractFlowRefs({ + modules: [], + failure_module: { + id: 'failure', + value: { + type: 'forloopflow', + modules: [ + { id: 'f-a', value: { type: 'script', path: 'u/admin/cleanup', input_transforms: {} } } + ] + } + } + }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/cleanup' }) + }) + + it('extractFlowRefs sees preprocessor scripts and flow_env resources', () => { + const refs = extractFlowRefs(flowValue) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/preproc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/slack' }) + }) + + it('sees and relocates $res refs nested inside JSON flow_env values', () => { + const value = { + modules: [], + flow_env: { CFG: { db: '$res:u/admin/pg', opts: ['res://u/admin/s3'] } } + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/s3' }) + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['u/admin/s3', 'f/proj/s3'] + ]) + const out = rewriteFlowValue(value, map) + expect(out.flow_env.CFG.db).toBe('$res:f/proj/pg') + expect(out.flow_env.CFG.opts[0]).toBe('$res:f/proj/s3') + }) + + it('rewriteFlowValue relocates both', () => { + const map = new Map([ + ['u/admin/preproc', 'f/proj/preproc'], + ['u/admin/slack', 'f/proj/slack'] + ]) + const out = rewriteFlowValue(flowValue, map) + expect(out.preprocessor_module.value.path).toBe('f/proj/preproc') + expect(out.flow_env.SLACK).toBe('$res:f/proj/slack') + expect(out.flow_env.PLAIN).toBe('not-a-ref') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts new file mode 100644 index 0000000000..1fd62498d5 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -0,0 +1,652 @@ +// Pure logic for the "project = folder" Hub bundle. A project is one folder +// `f//...`. Bundling: collect the transitive closure, relocate external +// refs (`u//`, `f//` -> `f//`, `_2`/`_3`… +// on collision) and rewrite them. Hub refs stay external; runtime string-concat +// paths are out of scope. No API/Svelte deps so it's unit-testable. + +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { isRunnableByPath } from '$lib/components/apps/inputType' + +export type RefKind = 'resource' | 'script' | 'flow' + +export interface Ref { + kind: RefKind + /** Bare path, without the `$res:` / `res://` prefix for resources. */ + path: string +} + +export type PathClass = 'internal' | 'hub' | 'external' + +/** A single `$res:PATH` / `res://PATH` token (path captured in group 1). */ +const RES_TOKEN_RE = /(?:\$res:|res:\/\/)([\w\-./]+)/g + +// A whole-string `$var:PATH` / `$jsonvar:PATH` value. The worker substitutes these +// only when an argument value *is* the reference (walking nested JSON), never a +// token embedded in inline code, so the whole value must match. `_KIND` captures +// the prefix (group 1) and path (group 2) so a rewrite can preserve `var`/`jsonvar`. +const VAR_VALUE_RE = /^\$(?:json)?var:([\w\-./]+)$/ +const VAR_VALUE_RE_KIND = /^\$(var|jsonvar):([\w\-./]+)$/ + +// Variable paths a value will resolve at runtime (flow static inputs, flow_env, +// app runnable inputs, trigger config fields). Walk the parsed structure and match +// whole string values so inline code carrying a literal `$var:` string is ignored. +export function extractVarRefsFromValue(value: any): string[] { + const out = new Set() + const walk = (v: any) => { + if (typeof v === 'string') { + const m = VAR_VALUE_RE.exec(v) + if (m) out.add(m[1]) + } else if (Array.isArray(v)) { + for (const x of v) walk(x) + } else if (v && typeof v === 'object') { + for (const k of Object.keys(v)) walk(v[k]) + } + } + walk(value) + return [...out] +} + +export function classifyPath(path: string, slug: string): PathClass { + if (path.startsWith(`f/${slug}/`) || path === `f/${slug}`) return 'internal' + if (path.startsWith('hub/')) return 'hub' + return 'external' +} + +export function extractScriptRefs(content: string): Ref[] { + const out: Ref[] = [] + const seen = new Set() + let m: RegExpExecArray | null + RES_TOKEN_RE.lastIndex = 0 + while ((m = RES_TOKEN_RE.exec(content)) !== null) { + if (!seen.has(m[1])) { + seen.add(m[1]) + out.push({ kind: 'resource', path: m[1] }) + } + } + return out +} + +/** + * References inside a flow value: + * - inline rawscript code with `$res:` (resource) + * - static step inputs whose value is a `$res:` literal (resource) + * - `type: script` steps that reference a script by path (script) + * - `type: flow` steps that reference a sub-flow by path (flow) + */ +export function extractFlowRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + // getAllModules flattens the whole tree (loops, branches, aiagent tools, + // failure module) so each module only needs local inspection; the + // preprocessor module sits outside `modules` and is walked the same way. + for (const mod of allFlowModules(value)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if (v.type === 'script' && typeof v.path === 'string') add('script', v.path) + if (v.type === 'flow' && typeof v.path === 'string') add('flow', v.path) + if (typeof v.content === 'string') { + for (const r of extractScriptRefs(v.content)) add('resource', r.path) + } + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Static values can be a bare `$res:` string or arbitrary JSON with + // refs nested anywhere — the worker resolves both, so scan the full + // serialization. + if (t?.type === 'static' && t.value !== undefined) { + const text = typeof t.value === 'string' ? t.value : JSON.stringify(t.value) + for (const r of extractScriptRefs(text)) add('resource', r.path) + } + } + } + } + // flow_env values support `$res:path` references — as whole string values or + // nested inside JSON values (the worker resolves both), so scan the full + // serialization. + if (value?.flow_env && typeof value.flow_env === 'object') { + for (const r of extractScriptRefs(JSON.stringify(value.flow_env))) add('resource', r.path) + } + return out +} + +// Every module of a flow value: the tree under `modules`, the failure module, +// and the preprocessor module (which lives outside `modules`). Any walk over a +// flow's modules must go through this — a walk that misses a module class +// silently drops its dependencies from bundles or migrations. All three go in +// the root list (not getAllModules' failure_module parameter, which appends +// the module without expanding its descendants) so nested children of a +// failure or preprocessor module are walked too. +export function allFlowModules(value: any) { + return getAllModules([ + ...(value?.modules ?? []), + ...(value?.preprocessor_module ? [value.preprocessor_module] : []), + ...(value?.failure_module ? [value.failure_module] : []) + ]) +} + +// Visit every object node in an app value tree (JSON-safe, no cycles). +function walkAppNodes(value: any, visit: (node: Record) => void): void { + if (value == null || typeof value !== 'object') return + if (Array.isArray(value)) { + for (const v of value) walkAppNodes(v, visit) + return + } + visit(value) + for (const k of Object.keys(value)) walkAppNodes(value[k], visit) +} + +// `runnableByPath`/`path` nodes reference a workspace runnable by path. +function runnableRef(node: Record): Ref | undefined { + if (!isRunnableByPath(node as any) || typeof node.path !== 'string') return undefined + if (node.runType === 'flow') return { kind: 'flow', path: node.path } + if (node.runType === 'script') return { kind: 'script', path: node.path } + return undefined // hubscript -> external hub, ignored +} + +// App refs: `$res:` resources anywhere in the value, plus script/flow runnables +// referenced by path in components. +export function extractAppRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + walkAppNodes(value, (node) => { + const r = runnableRef(node) + if (r) add(r.kind, r.path) + }) + for (const r of extractScriptRefs(JSON.stringify(value ?? {}))) add('resource', r.path) + return out +} + +/** + * Build the relocation map. Internal paths (`f//...`) map to themselves + * and are reserved first; external paths relocate to `f//` (`_2`/`_3`… + * on collision). Input is sorted so suffix assignment is deterministic. + */ +export function buildPathMap(paths: Iterable, slug: string): Map { + const map = new Map() + const used = new Set() + const sorted = [...new Set(paths)].sort() + for (const p of sorted) { + if (classifyPath(p, slug) === 'internal') { + map.set(p, p) + used.add(p) + } + } + for (const old of sorted) { + if (map.has(old)) continue + const name = old.split('/').filter(Boolean).pop() ?? old + let candidate = `f/${slug}/${name}` + let n = 2 + while (used.has(candidate)) candidate = `f/${slug}/${name}_${n++}` + used.add(candidate) + map.set(old, candidate) + } + return map +} + +// Both ref forms normalize to `$res:` on rewrite. +export function rewriteContent(content: string, map: Map): string { + return content.replace(RES_TOKEN_RE, (whole, path) => { + const next = map.get(path) + return next ? `$res:${next}` : whole + }) +} + +// Structurally relocate whole-string `$var:`/`$jsonvar:` values — the only form the +// worker resolves. Walks the parsed value so an inert token embedded in inline code +// or arbitrary text is left untouched, unlike token replacement over serialized +// strings. Only paths present in the map move (the retarget map carries variables). +export function rewriteVarRefsInValue(value: any, map: Map): any { + if (typeof value === 'string') { + const m = VAR_VALUE_RE_KIND.exec(value) + if (m) { + const next = map.get(m[2]) + if (next) return `$${m[1]}:${next}` + } + return value + } + if (Array.isArray(value)) return value.map((v) => rewriteVarRefsInValue(v, map)) + if (value && typeof value === 'object') { + const out: Record = {} + for (const k of Object.keys(value)) out[k] = rewriteVarRefsInValue(value[k], map) + return out + } + return value +} + +/** + * `$res:`/`res://` tokens anywhere in a trigger config — schedule args, + * on_*_extra_args, error_handler_args, … (e.g. the built-in Slack handler + * stores its channel resource this way). These must enter the bundle path map + * so `rewriteTriggerConfig` relocates them and a stub is exported. + */ +export function extractTriggerConfigResourceRefs(config: any): string[] { + return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) +} + +/** + * Trigger configs reference resources as plain path strings (e.g. + * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting + * misses them. Deep-walk the config and remap any string that exact-matches a + * map key (map keys are full bundle paths, so an exact match is a reference), + * or a `script/`/`flow/` handler reference (schedules' on_failure + * et al.), falling back to `$res:` token rewriting for embedded refs. + */ +// Top-level config fields whose string values are prefixed runnable refs. +// Prefixed forms are remapped ONLY in these known positions: deciding meaning +// from string shape alone rewrote literal payloads that merely looked like +// refs. Bare-path exact matches and $res: tokens stay position-independent. +const HANDLER_REF_FIELDS = new Set(['on_failure', 'on_recovery', 'on_success']) + +export function rewriteTriggerConfig(config: any, map: Map, depth = 0): any { + if (typeof config === 'string') { + const direct = map.get(config) + if (direct) return direct + return rewriteContent(config, map) + } + if (Array.isArray(config)) return config.map((v) => rewriteTriggerConfig(v, map, depth + 1)) + if (config && typeof config === 'object') { + return Object.fromEntries( + Object.entries(config).map(([k, v]) => { + if (depth === 0 && typeof v === 'string') { + // Websocket url: $script: / $flow:. + if (k === 'url') { + const m = /^\$(script|flow):(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `$${m[1]}:${map.get(m[2])}`] + } + // Schedule handlers: script/ / flow/. + if (HANDLER_REF_FIELDS.has(k)) { + const m = /^(script|flow)\/(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `${m[1]}/${map.get(m[2])}`] + } + } + return [k, rewriteTriggerConfig(v, map, depth + 1)] + }) + ) + } + return config +} + +export function rewriteFlowValue(value: any, map: Map): any { + const cloned = JSON.parse(JSON.stringify(value ?? {})) + for (const mod of allFlowModules(cloned)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if ( + (v.type === 'script' || v.type === 'flow') && + typeof v.path === 'string' && + map.has(v.path) + ) { + v.path = map.get(v.path) + } + if (typeof v.content === 'string') v.content = rewriteContent(v.content, map) + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Mirror extraction: rewrite refs wherever they sit, preserving the + // value's type (a string stays a string, JSON round-trips). + if (t?.type === 'static' && t.value !== undefined) { + if (typeof t.value === 'string') { + t.value = rewriteContent(t.value, map) + } else { + t.value = JSON.parse(rewriteContent(JSON.stringify(t.value), map)) + } + } + } + } + } + if (cloned?.flow_env && typeof cloned.flow_env === 'object') { + // Tokens can sit inside nested JSON values, not just string values; the + // serialize→rewrite→parse round-trip reaches all of them (paths contain + // no characters that would break JSON string literals). + cloned.flow_env = JSON.parse(rewriteContent(JSON.stringify(cloned.flow_env), map)) + } + return cloned +} + +// Relocate `$res:` tokens (one round-trip, also produces a fresh clone) then +// runnable-by-path refs structurally. Incidental `f//` strings stay intact. +export function rewriteAppValue(value: any, map: Map): any { + if (value == null) return value + const cloned = JSON.parse(rewriteContent(JSON.stringify(value), map)) + walkAppNodes(cloned, (node) => { + if (runnableRef(node) && map.has(node.path)) node.path = map.get(node.path) + }) + return cloned +} + +// Raw/compiled apps store their structure as a JSON string (`{ runnables, files }`). +// Parse it so runnable-by-path refs in the runnables map are seen, reusing the +// same walk; fall back to plain `$res:` scanning if it isn't valid JSON. +export function extractRawAppRefs(content: string): Ref[] { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return extractScriptRefs(content) + } + return extractAppRefs(parsed) +} + +export function rewriteRawAppContent(content: string, map: Map): string { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return rewriteContent(content, map) + } + return JSON.stringify(rewriteAppValue(parsed, map)) +} + +// --------------------------------------------------------------------------- +// Hub project export format (what /projects/{slug}/export returns) and its +// retargeting into a destination folder. Kept here, next to the rewriters, +// so the bundle format is defined in one module for both publish and install. +// --------------------------------------------------------------------------- + +export type ExportItem = Record +export interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean +} +export interface ProjectExport { + project: { slug: string; name: string; summary: string; readme: string | null } + scripts: ExportItem[] + flows: ExportItem[] + apps: ExportItem[] + resources: ExportItem[] + triggers: ExportItem[] + migrations?: ProjectMigration[] +} + +// Map bundled paths `f//...` -> `f//...`. Only enumerated +// paths go in, so rewriters touch real refs, never incidental text. +export function buildRetargetMap( + bundle: ProjectExport, + fromSlug: string, + folder: string +): Map { + const map = new Map() + const prefix = `f/${fromSlug}/` + const add = (p: unknown) => { + if (typeof p === 'string' && p.startsWith(prefix)) { + map.set(p, `f/${folder}/${p.slice(prefix.length)}`) + } + } + for (const s of bundle.scripts) add(s.path) + for (const f of bundle.flows) add(f.path) + for (const a of bundle.apps) add(a.path) + for (const r of bundle.resources) add(r.path) + for (const t of bundle.triggers) { + add(t.path) + add(t.runnable_path) + } + // Variables aren't enumerated in the export; their `$var:`/`$jsonvar:` refs live + // inside item values. Relocate the internal ones so a renamed-folder import + // rewrites them into the target folder instead of retaining the old prefix. + for (const p of collectExportVarPaths(bundle)) add(p) + return map +} + +// Internal-or-external variable paths referenced by the export's flows, apps and +// triggers. Scripts carry no variable args. Raw apps hold their structure in the +// `value.raw` JSON string. +export function collectExportVarPaths(bundle: ProjectExport): string[] { + const out = new Set() + const collect = (value: any) => { + for (const p of extractVarRefsFromValue(value)) out.add(p) + } + for (const f of bundle.flows) collect(f.value) + for (const a of bundle.apps) collect(a.app_type === 'raw' ? safeParseRaw(a.value?.raw) : a.value) + for (const t of bundle.triggers) collect(t.config) + return [...out] +} + +function safeParseRaw(raw: unknown): any { + if (typeof raw !== 'string') return undefined + try { + return JSON.parse(raw) + } catch { + return undefined + } +} + +// Structural retarget: rewrite each item's path and its internal refs, +// leaving Hub refs and arbitrary content untouched. +export function retargetProjectExport( + bundle: ProjectExport, + fromSlug: string, + folder: string +): ProjectExport { + if (folder === fromSlug) return bundle + const map = buildRetargetMap(bundle, fromSlug, folder) + const remap = (p: unknown) => (typeof p === 'string' ? (map.get(p) ?? p) : p) + return { + ...bundle, + scripts: bundle.scripts.map((s) => ({ + ...s, + path: remap(s.path), + content: rewriteContent(s.content ?? '', map) + })), + flows: bundle.flows.map((f) => ({ + ...f, + path: remap(f.path), + value: rewriteVarRefsInValue(rewriteFlowValue(f.value, map), map) + })), + apps: bundle.apps.map((a) => ({ + ...a, + path: remap(a.path), + // Raw apps keep their structure in the `value.raw` JSON string. + value: + a.app_type === 'raw' + ? { + ...a.value, + raw: rewriteRawVarRefs(rewriteRawAppContent(a.value?.raw ?? '', map), map) + } + : rewriteVarRefsInValue(rewriteAppValue(a.value, map), map) + })), + resources: bundle.resources.map((r) => ({ ...r, path: remap(r.path) })), + triggers: bundle.triggers.map((t) => ({ + ...t, + path: remap(t.path), + runnable_path: remap(t.runnable_path), + // Configs hold `$res:` tokens, plain resource paths (kafka_resource_path + // etc.) and whole-string `$var:` values — rewrite all three. + config: t.config ? rewriteVarRefsInValue(rewriteTriggerConfig(t.config, map), map) : t.config + })) + } +} + +// Var relocation for a raw app's `value.raw` JSON string: parse, structurally +// rewrite whole-string var values, re-serialize; leave invalid JSON untouched. +function rewriteRawVarRefs(raw: string, map: Map): string { + const parsed = safeParseRaw(raw) + if (parsed === undefined) return raw + return JSON.stringify(rewriteVarRefsInValue(parsed, map)) +} + +export type ItemKind = 'script' | 'flow' | 'app' | 'raw_app' + +export interface ItemRef { + kind: ItemKind + path: string +} + +export interface FetchedItem { + kind: ItemKind + path: string + summary?: string + description?: string + /** scripts + raw_apps */ + content?: string + /** flows + apps */ + value?: any + /** scripts */ + language?: string + schema?: any + lock?: string + scriptKind?: string +} + +export interface BundleDeps { + /** Fetch a workspace item by ref, or undefined if it doesn't exist. */ + fetchItem: (ref: ItemRef) => Promise + /** Resolve a resource path to its type, or undefined if missing. */ + resolveResourceType: (path: string) => Promise +} + +export interface BundledItem extends FetchedItem { + /** Path the item takes inside the project folder. */ + newPath: string +} + +export interface ResourceStub { + originalPath: string + newPath: string + resource_type: string +} + +export interface ProjectBundle { + items: BundledItem[] + resourceStubs: ResourceStub[] + /** Original -> relocated path for every item and resource (incl. unresolved). */ + pathMap: Map + /** External paths we couldn't fetch/resolve (missing items or untyped resources). */ + unresolved: string[] +} + +function refsForFetched(item: FetchedItem): Ref[] { + if (item.kind === 'script') return extractScriptRefs(item.content ?? '') + if (item.kind === 'flow') return extractFlowRefs(item.value) + if (item.kind === 'app') return extractAppRefs(item.value) + if (item.kind === 'raw_app') return extractRawAppRefs(item.content ?? '') + return [] +} + +// Whole-string `$var:`/`$jsonvar:` paths an item resolves at runtime. Scripts carry +// no variable args; raw apps hold their structure in the `content` JSON string. +function varRefsForFetched(item: FetchedItem): string[] { + if (item.kind === 'flow' || item.kind === 'app') return extractVarRefsFromValue(item.value) + if (item.kind === 'raw_app') return extractVarRefsFromValue(safeParseRaw(item.content)) + return [] +} + +// Walks the transitive closure: scripts referenced by path are pulled in +// recursively, resources become empty stubs, hub refs stay external. +export async function buildProjectBundle( + seed: ItemRef[], + slug: string, + deps: BundleDeps, + extraResourcePaths: string[] = [], + extraVarPaths: string[] = [] +): Promise { + const fetched = new Map() + const queued = new Set() + const resourcePaths = new Set() + const varPaths = new Set() + const unresolved: string[] = [] + + // Resources and variables referenced by triggers (by config value, not `$res:` + // in code) — relocated through the same map so the export stays slug-relative. + for (const p of extraResourcePaths) { + if (classifyPath(p, slug) !== 'hub') resourcePaths.add(p) + } + for (const p of extraVarPaths) varPaths.add(p) + + // Key by `${kind}:${path}`, not bare path: a script and flow can share a path, + // and keying by path alone would silently drop one. + const refKey = (kind: string, path: string) => `${kind}:${path}` + + // Refs at the same BFS depth are independent: fetch each level concurrently. + let level: ItemRef[] = [] + for (const s of seed) { + const key = refKey(s.kind, s.path) + if (!queued.has(key)) { + queued.add(key) + level.push(s) + } + } + while (level.length > 0) { + const results = await Promise.all( + level.map(async (ref) => ({ ref, item: await deps.fetchItem(ref) })) + ) + const next: ItemRef[] = [] + for (const { ref, item } of results) { + if (!item) { + unresolved.push(ref.path) + continue + } + fetched.set(refKey(ref.kind, ref.path), item) + for (const r of refsForFetched(item)) { + if (classifyPath(r.path, slug) === 'hub') continue + if (r.kind === 'resource') { + resourcePaths.add(r.path) + } else if (r.kind === 'script' || r.kind === 'flow') { + const key = refKey(r.kind, r.path) + if (!queued.has(key)) { + queued.add(key) + next.push({ kind: r.kind, path: r.path }) + } + } + } + // Relocate the item's runtime variable refs into the project folder too, so + // the export is slug-relative regardless of the source folder (import then + // materializes them as placeholders). Variables are never hub-hosted. + for (const p of varRefsForFetched(item)) varPaths.add(p) + } + level = next + } + + const fetchedItems = [...fetched.values()] + const itemPaths = fetchedItems.map((it) => it.path) + const map = buildPathMap([...itemPaths, ...resourcePaths, ...varPaths], slug) + + const items: BundledItem[] = fetchedItems.map((it) => { + const rewritten: BundledItem = { ...it, newPath: map.get(it.path) ?? it.path } + if (it.kind === 'script') { + rewritten.content = rewriteContent(it.content ?? '', map) + } else if (it.kind === 'raw_app') { + rewritten.content = rewriteRawVarRefs(rewriteRawAppContent(it.content ?? '', map), map) + } else if (it.kind === 'flow') { + rewritten.value = rewriteVarRefsInValue(rewriteFlowValue(it.value, map), map) + } else if (it.kind === 'app') { + rewritten.value = rewriteVarRefsInValue(rewriteAppValue(it.value, map), map) + } + return rewritten + }) + + const resourceStubs: ResourceStub[] = [] + const resolved = await Promise.all( + [...resourcePaths].map(async (path) => ({ path, type: await deps.resolveResourceType(path) })) + ) + for (const { path, type } of resolved) { + if (!type) { + unresolved.push(path) + continue + } + resourceStubs.push({ originalPath: path, newPath: map.get(path) ?? path, resource_type: type }) + } + + // `unresolved` keys missing items by kind:path but stores the bare path, so a + // missing script and flow (or a runnable and resource) sharing a path can push + // the same string twice. Dedupe: callers use it as a display/blocker list where + // duplicate keys would break keyed rendering. + return { items, resourceStubs, pathMap: map, unresolved: [...new Set(unresolved)] } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts new file mode 100644 index 0000000000..a02d4e264d --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest' +import { refContainmentViolation, varContainmentViolation } from './projectInstall' +import type { Ref } from './projectBundle' + +describe('refContainmentViolation', () => { + const folder = 'proj' + const violation = (r: Ref) => refContainmentViolation([r], folder) + + it('allows references relocated into the target folder', () => { + expect(violation({ kind: 'resource', path: 'f/proj/db' })).toBeUndefined() + expect(violation({ kind: 'script', path: 'f/proj/helper' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'f/proj/sub' })).toBeUndefined() + }) + + it('allows hub script/flow references but never hub resources', () => { + expect(violation({ kind: 'script', path: 'hub/1/x/y' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'hub/1/a/b' })).toBeUndefined() + // Resources are not hub-hosted, so a hub/ resource path is still an escape. + expect(violation({ kind: 'resource', path: 'hub/1/x/y' })).toBeDefined() + }) + + it('rejects references bound to another namespace', () => { + // The crux: an in-folder runnable pointing its resource at an existing asset. + expect(violation({ kind: 'resource', path: 'u/admin/db' })).toContain('escapes') + expect(violation({ kind: 'script', path: 'f/other/helper' })).toContain('escapes') + expect(violation({ kind: 'flow', path: 'u/admin/sub' })).toContain('escapes') + }) + + it('does not treat a prefix-only folder match as internal', () => { + expect(violation({ kind: 'script', path: 'f/proj2/helper' })).toContain('escapes') + }) + + it('reports the first offending reference and passes a fully-contained set', () => { + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'script', path: 'hub/1/x/y' } + ], + folder + ) + ).toBeUndefined() + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'resource', path: 'u/admin/secret' } + ], + folder + ) + ).toContain('u/admin/secret') + }) +}) + +describe('varContainmentViolation', () => { + const folder = 'proj' + + it('allows in-folder variable references', () => { + expect(varContainmentViolation({ token: '$var:f/proj/token' }, folder)).toBeUndefined() + expect(varContainmentViolation({ x: 'no refs here' }, folder)).toBeUndefined() + }) + + it('rejects a `$var:` or `$jsonvar:` bound to another namespace', () => { + // The crux: a variable arg the ref extractors miss, resolved under the perms. + expect(varContainmentViolation({ queue_url: '$var:u/admin/token' }, folder)).toContain( + 'u/admin/token' + ) + expect(varContainmentViolation({ cfg: '$jsonvar:f/other/secret' }, folder)).toContain('escapes') + }) + + it('ignores a `$var:` literal embedded in inline code', () => { + const flowValue = { + flow_env: { API: '$var:f/proj/api_key' }, + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/admin/should_not_flag"' } }] + } + expect(varContainmentViolation(flowValue, folder)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts new file mode 100644 index 0000000000..1827e00283 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -0,0 +1,405 @@ +// Imports a Hub project export into a workspace: one importer per item kind, +// each item reported individually so one bad item never aborts the rest. +// UI-free — the install page owns folder choice and migration review. + +import { + AppService, + FlowService, + FolderService, + ResourceService, + ScriptService, + VariableService, + WorkspaceService +} from '$lib/gen' +import { + TRIGGER_KINDS, + createWorkspaceTriggerDisabled, + triggerHandlerRefs, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { + classifyPath, + collectExportVarPaths, + extractAppRefs, + extractFlowRefs, + extractRawAppRefs, + extractScriptRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + retargetProjectExport, + type ExportItem, + type ProjectExport, + type ProjectMigration, + type Ref +} from './projectBundle' + +export interface InstallResult { + path: string + ok: boolean + error?: string +} + +// Guarding an item's own path is not enough: the `$res:`/script/flow refs baked +// into its content are live bindings the backend acts on. A well-formed export +// relocates them all into f// (hub/ script refs stay external); anything +// else points a runnable at an existing asset in another namespace, so refuse the +// item rather than bind it there. Resources are never hub-hosted, so a hub/ path +// there is not a valid escape hatch. Mirrors the trigger-config containment. +export function refContainmentViolation(refs: Ref[], folder: string): string | undefined { + for (const r of refs) { + const cls = classifyPath(r.path, folder) + if (cls === 'internal') continue + if (cls === 'hub' && r.kind !== 'resource') continue + return `reference '${r.path}' escapes the target folder f/${folder}/ — skipped` + } + return undefined +} + +// `$var:`/`$jsonvar:` references (in flow static inputs, flow_env, app runnable +// inputs, trigger config) are resolved at runtime under the imported runnable's +// permissions and are never hub-hosted. Retargeting relocates a project's own refs +// into the target folder; anything still outside it points at another namespace, so +// reject those. Takes the parsed value so inline code carrying a literal is ignored. +export function varContainmentViolation(value: any, folder: string): string | undefined { + for (const p of extractVarRefsFromValue(value)) { + if (classifyPath(p, folder) !== 'internal') { + return `variable '${p}' escapes the target folder f/${folder}/ — skipped` + } + } + return undefined +} + +// Surface the backend's explanation: API errors carry the real message in +// `.body` (plain text for Windmill 4xx), while `.message` is the generic +// status text ("Bad Request"). Prefer the body so e.g. a path/route_path +// collision reads as the actual reason, not just "Bad Request". +function errorMessage(e: any): string { + const body = e?.body + if (typeof body === 'string' && body.trim() !== '') return body + if (body && typeof body === 'object') + return body.error?.message ?? body.message ?? JSON.stringify(body) + return e?.message ?? String(e) +} + +// Recompute an app's execution policy from its (retargeted) value, mirroring +// what the editor does on deploy. `triggerables_v2` is keyed by +// `:rawscript/`; retargeting rewrites that +// content, so a copied or empty policy would leave every inline runnable +// "forbidden by policy" at runtime. Default to publisher (auth required). +async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} +async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} + +function importScript(workspace: string, s: ExportItem): Promise { + return ScriptService.createScript({ + workspace, + requestBody: { + path: s.path, + summary: s.summary ?? '', + description: s.description ?? '', + content: s.content ?? '', + language: s.language, + schema: s.schema ?? undefined, + kind: s.kind ?? 'script', + lock: s.lockfile ?? undefined + } + }) +} + +function importFlow(workspace: string, f: ExportItem): Promise { + return FlowService.createFlow({ + workspace, + requestBody: { + path: f.path, + summary: f.summary ?? '', + description: f.description ?? '', + value: f.value, + schema: f.schema ?? undefined + } + }) +} + +// Stubs only: never overwrite an existing resource's value (updateIfExists +// stays false so a path collision is reported as a failed item instead). +function importResourceStub(workspace: string, r: ExportItem): Promise { + return ResourceService.createResource({ + workspace, + updateIfExists: false, + requestBody: { + path: r.path, + resource_type: r.resource_type, + value: {}, + description: 'Imported stub — fill in the value.' + } + }) +} + +// Variables hold secrets/config, so their values are never shipped. Create an empty +// secret placeholder for a project variable the importer must fill, mirroring the +// resource stubs. Conflict-safe: an already-present variable (the importer filled it, +// or a re-import) is left untouched rather than clobbered. +async function importVariablePlaceholder(workspace: string, path: string): Promise { + if (await VariableService.existsVariable({ workspace, path })) return + await VariableService.createVariable({ + workspace, + requestBody: { + path, + value: '', + is_secret: true, + description: 'Imported placeholder — fill in the value.' + } + }) +} + +async function importApp(workspace: string, a: ExportItem): Promise { + if (a.app_type === 'raw') { + let parsed: any + try { + parsed = JSON.parse(a.value?.raw ?? '{}') + } catch (e: any) { + throw new Error(`invalid raw app bundle: ${e?.message ?? String(e)}`) + } + const files = { ...(parsed.files ?? {}) } + const js = files['/bundle.js'] ?? '' + const css = files['/bundle.css'] ?? '' + delete files['/bundle.js'] + delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} + return AppService.createAppRaw({ + workspace, + formData: { + app: { + path: a.path, + summary: a.summary ?? '', + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) + }, + js, + css + } + }) + } + return AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }) +} + +// Apply one migration to the target data table. If the data table opted into +// migrations, record it (datatable_migrations + _wm_migrations, run only this +// version); otherwise run the SQL once as a preview job (unrecorded). +async function applyOneMigration( + workspace: string, + projectSlug: string, + m: ProjectMigration +): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${projectSlug}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } +} + +/** + * Install a project export into `workspace` under `f//`: create the + * folder, retarget every item, import kind by kind, then apply the (already + * reviewed) migrations. Each item's outcome is reported through `onResult`; + * failures never abort the remaining items. + */ +export async function installProject(args: { + workspace: string + exportData: ProjectExport + folder: string + migrations: ProjectMigration[] + hasEeLicense: boolean + onResult: (r: InstallResult) => void +}): Promise { + const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + + const record = (path: string, p: Promise): Promise => + p.then( + () => onResult({ path, ok: true }), + (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) + ) + + try { + await FolderService.createFolder({ workspace, requestBody: { name: folder } }) + } catch {} + + const proj = retargetProjectExport(exportData, exportData.project.slug, folder) + + // The export is remote input: every path it wants to write must stay inside + // the folder the user chose. Anything else (crafted export, or an export + // whose items weren't relocated into f// at publish) is refused + // per-item instead of being created in another namespace. + const prefix = `f/${folder}/` + const guard = (path: unknown, ...also: unknown[]): string | undefined => { + for (const p of [path, ...also]) { + if (typeof p !== 'string' || !p.startsWith(prefix)) { + return `path '${String(p)}' escapes the target folder ${prefix} — skipped` + } + } + return undefined + } + const checked = (path: unknown, run: () => Promise, ...also: unknown[]) => { + const violation = guard(path, ...also) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + // `refs` catches structured runnable/`$res:` refs; `varValue` is the parsed item + // walked for `$var:`/`$jsonvar:` argument refs (which the ref extractors miss). + const checkedItem = (path: unknown, refs: Ref[], varValue: any, run: () => Promise) => { + const violation = + guard(path) ?? + refContainmentViolation(refs, folder) ?? + varContainmentViolation(varValue, folder) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + for (const s of proj.scripts) { + // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), + // not in script source, so there is no variable arg to contain here. + await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => + importScript(workspace, s) + ) + } + for (const f of proj.flows) { + await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) + } + for (const r of proj.resources) { + await checked(r.path, () => importResourceStub(workspace, r)) + } + // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted + // into this folder). External refs are rejected per-item, so only stub in-folder + // ones; guard again in case an out-of-folder ref slipped through retargeting. + for (const p of collectExportVarPaths(proj)) { + if (!p.startsWith(prefix)) continue + await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) + } + for (const a of proj.apps) { + const isRaw = a.app_type === 'raw' + const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) + // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the + // walk sees the same structure the backend resolves. Malformed raw fails at import. + let varValue: any = a.value + if (isRaw) { + try { + varValue = JSON.parse(a.value?.raw ?? '{}') + } catch { + varValue = undefined + } + } + await checkedItem(a.path, refs, varValue, () => importApp(workspace, a)) + } + // A trigger's config is a live binding, not inert content: resource fields, + // handler runnables and $res: refs it names are acted on by the backend, so + // every one must stay inside the chosen folder (handlers may also point at + // hub/ scripts). Otherwise a crafted export could bind the trigger to + // existing assets in another namespace. + const triggerConfigViolation = (t: ExportItem): string | undefined => { + const cfg = (t.config ?? {}) as Record + for (const r of triggerHandlerRefs({ kind: t.kind, config: cfg } as WorkspaceTrigger)) { + if (!r.path.startsWith(prefix) && !r.path.startsWith('hub/')) { + return `handler '${r.path}' escapes the target folder ${prefix} — skipped` + } + } + const resourceRefs = new Set(extractTriggerConfigResourceRefs(cfg)) + const field = TRIGGER_KINDS[t.kind as WorkspaceTriggerKind]?.resourceField + const fieldValue = field ? cfg[field] : undefined + if (typeof fieldValue === 'string' && fieldValue !== '') resourceRefs.add(fieldValue) + for (const p of resourceRefs) { + if (!p.startsWith(prefix)) { + return `resource '${p}' escapes the target folder ${prefix} — skipped` + } + } + // Config fields (e.g. SQS queue_url) can carry `$var:`/`$jsonvar:` refs too. + return varContainmentViolation(cfg, folder) + } + for (const t of proj.triggers) { + const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) + await record( + String(t.path), + violation + ? Promise.reject(new Error(violation)) + : createWorkspaceTriggerDisabled( + workspace, + { + kind: t.kind, + path: t.path, + script_path: t.runnable_path, + is_flow: t.runnable_kind === 'flow', + summary: t.summary ?? null, + config: t.config ?? null + }, + { hasEeLicense } + ) + ) + } + + // Apply the reviewed data table migrations after items exist. + for (const m of migrations) { + await record( + `data table: ${m.datatable_name}`, + applyOneMigration(workspace, exportData.project.slug, m) + ) + } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..7d35c51650 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('collects datatable refs from the preprocessor module', async () => { + inferAssetsMock.mockResolvedValue({ status: 'ok', assets: [] }) + const items: FetchedItem[] = [ + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [], + preprocessor_module: { + id: 'pre', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/inbox' }] + } + } + } + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])]).toEqual(['inbox']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('leaves a qualified ref unresolved when its schema misses, never another schema\'s table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['sales.orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"sales.orders" is referenced but was not found') + expect(migrations[0].sql).not.toContain('CREATE TABLE "') + }) + + it('emits all CREATE TABLEs before any FK constraint so circular FKs work', async () => { + const cyclicSchema = { + public: { + a: { + name: 'a', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'b_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.b', + columns: [{ source_column: 'b_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + }, + b: { + name: 'b', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'a_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.a', + columns: [{ source_column: 'a_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(cyclicSchema) + const usage = new Map([['main', new Set(['a', 'b'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('"public"."a"') + expect(sql).toContain('"public"."b"') + // Both FK constraints present, and every CREATE TABLE precedes the first one. + expect(sql.match(/ADD CONSTRAINT/g)?.length).toBe(2) + const lastCreate = sql.lastIndexOf('CREATE TABLE IF NOT EXISTS') + const firstConstraint = sql.indexOf('DO $$') + expect(lastCreate).toBeGreaterThan(-1) + expect(firstConstraint).toBeGreaterThan(lastCreate) + }) + + it('guards FK creation so re-running on an existing table does not abort', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + // The ADD CONSTRAINT must be wrapped in a pg_constraint existence check. + expect(sql).toContain('DO $$') + expect(sql).toContain('SELECT 1 FROM pg_constraint') + expect(sql).toContain(`conrelid = '"public"."orders"'::regclass`) + // No unguarded ALTER TABLE ... ADD at the start of a line. + expect(/^ALTER TABLE .* ADD CONSTRAINT/m.test(sql)).toBe(false) + }) + + it('creates non-public schemas before their tables', async () => { + const appSchema = { + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(appSchema) + const usage = new Map([['main', new Set(['app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS "app";') + expect(sql.indexOf('CREATE SCHEMA IF NOT EXISTS "app";')).toBeLessThan( + sql.indexOf('CREATE TABLE IF NOT EXISTS "app"."customers"') + ) + expect(sql).not.toContain('CREATE SCHEMA IF NOT EXISTS "public"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..b4e72389c0 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,345 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { allFlowModules } from './projectBundle' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateAddedTableSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of allFlowModules(item.value)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). A schema-qualified ref that +// misses stays unresolved: falling back to a same-named table in another +// schema would generate a migration for an unrelated table while the code +// still references the missing one. +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + return schema[schemaName]?.[tableName] ? { schemaName, tableName } : undefined + } + // Bare name: find it across every schema, first match wins. + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][tableRef]) return { schemaName, tableName: tableRef } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateAddedTableSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + // Every CREATE TABLE is emitted before any FK constraint: circular FKs have + // no valid creation order, so constraints can only run once all tables exist. + const creates: string[] = [] + const constraints: string[] = [] + for (const t of ordered) { + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + const gen = generateAddedTableSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + if (!gen) continue + creates.push(gen.create) + constraints.push(...gen.constraints) + } + const statements = [...creates, ...constraints] + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index 42de9d38a5..343e7a3e48 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -14,7 +14,8 @@ import { sendUserToast } from '$lib/utils' import DataTable from '$lib/components/table/DataTable.svelte' import Cell from '$lib/components/table/Cell.svelte' - import { Pen, Trash, Plus } from 'lucide-svelte' + import { Pen, Trash, Plus, UploadCloud } from 'lucide-svelte' + import DeployToHub from '$lib/components/workspaceSettings/DeployToHub.svelte' import Head from '$lib/components/table/Head.svelte' import Row from '$lib/components/table/Row.svelte' import Badge from '$lib/components/common/badge/Badge.svelte' @@ -30,6 +31,8 @@ let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) let folderDrawer: Drawer | undefined = $state() + let hubDrawer: Drawer | undefined = $state() + let publishFolderName: string = $state('') async function loadFolders(): Promise { folders = (await FolderService.listFolders({ workspace: $workspaceStore! })).map((x) => { @@ -88,6 +91,22 @@ + + { + hubDrawer?.closeDrawer() + publishFolderName = '' + }} + > + {#if publishFolderName} + {#key publishFolderName} + + {/key} + {/if} + + + {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.folders} + {#if ambiguousNames.has(workspace.name)} +
+ {workspace.id} +
+ {/if} {#if isSelected} @@ -363,7 +385,9 @@ {item} > - {activeWorkspace?.name ?? $workspaceStore} settings + {(activeWorkspace && ambiguousNames.has(activeWorkspace.name) + ? activeWorkspace.id + : activeWorkspace?.name) ?? $workspaceStore} settings {/if} From 717e38a0c6b5bb2340e236a9d49645a4cebf4849 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 24 Jul 2026 09:58:27 +0200 Subject: [PATCH 146/171] feat: let a workspace fall back to the instance critical alert channels (#10292) * feat(alerts): let a workspace fall back to the instance critical alert channels A workspace with no error handler had no way to surface failed jobs, and the instance critical alert channels a superadmin already configured (Slack, Teams, email) were unreachable from a workspace: the workspace Slack error handler posts with the workspace's own bot token, not the instance one. Adds an opt-in workspace setting that reports failed jobs to those channels when, and only when, no workspace error handler is configured. The report is send-only: it skips the `alerts` table so workspace job failures never flood the instance-wide feed superadmins triage. Rejected on cloud (the channels belong to the instance operator, who is not the tenant) and on fork workspaces (throwaway copies of a parent's runnables). Settable from workspace settings and from the new-workspace screen. The opt-in and the existing `mute_critical_alerts` flag are folded into the query already behind WORKSPACE_ERROR_HANDLER_CACHE, so a failed job costs no extra round trip, and workspaces with neither a handler nor the opt-in return before the per-runnable mute lookup. * chore(sqlx): add offline query cache entries for the new settings queries * refactor(alerts): make instance alerts a destination tab and address review Instance alerts are a fifth error-handler destination rather than a separate toggle: the backend already treats them as mutually exclusive with a handler script, so one "where do failures go?" control matches the semantics and drops the inert-while-a-handler-is-set state. The tab is offered on the workspace error handler only, not on schedules or triggers. Review fixes: - the fork boundary is enforced at dispatch (join on parent_workspace_id), so a workspace attached as a fork/dev after opting in stops reporting; attaching also clears the stored flag, and the settings page never selects a tab it does not render, which would have submitted a value the API rejects on a fork - mute_critical_alerts no longer gates this path: it is the UI-feed mute, and this path writes no feed entry - cancellations are not reported: they are a human action, and this destination has no per-workspace mute of its own - per-workspace throttle with a rollup count, so a flapping runnable cannot turn into unbounded Slack/SMTP traffic on channels shared by the whole instance - log the dispatch, audit the flag, name the columns in the rename INSERT, drop the generated migration placeholders * chore(alerts): state the fork/cloud invariant on canUseInstanceAlerts * chore(sqlx): cache the attach_dev_workspace settings update --- ...0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json | 15 ++ ...234f6a2f295a9c46897f5e9aab8f7298ff0d7.json | 15 ++ ...76705eeda7d17d49cafb3e9e6285dee6804c9.json | 15 ++ ...67ad00ff6c13918d46dd474cc48824b12ccaf.json | 208 ++++++++++++++++++ ...e63954f39b291f525e36e6ece916b98b4d2c9.json | 15 ++ ...d8a46dc7320e7dcdd64befe808dec8cbe2265.json | 16 ++ ...6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json | 20 ++ ...ad84b0a1bb18b35e0f505ab008bff1310528b.json | 12 + ...ndler_fallback_to_instance_alerts.down.sql | 2 + ...handler_fallback_to_instance_alerts.up.sql | 2 + .../tests/workspaces.rs | 58 +++++ .../windmill-api-workspaces/src/workspaces.rs | 78 ++++++- .../src/workspaces_extra.rs | 2 +- backend/windmill-api/openapi.yaml | 11 + backend/windmill-common/src/utils.rs | 14 ++ backend/windmill-queue/src/jobs.rs | 178 ++++++++++----- .../components/ErrorOrRecoveryHandler.svelte | 35 ++- .../CreateWorkspaceInner.svelte | 29 ++- .../(logged)/workspace_settings/+page.svelte | 65 ++++-- 19 files changed, 703 insertions(+), 87 deletions(-) create mode 100644 backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json create mode 100644 backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json create mode 100644 backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json create mode 100644 backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json create mode 100644 backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json create mode 100644 backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json create mode 100644 backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json create mode 100644 backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json create mode 100644 backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql create mode 100644 backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql diff --git a/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json b/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json new file mode 100644 index 0000000000..9574582d5f --- /dev/null +++ b/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca" +} diff --git a/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json b/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json new file mode 100644 index 0000000000..8714711595 --- /dev/null +++ b/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7" +} diff --git a/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json b/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json new file mode 100644 index 0000000000..114712fc28 --- /dev/null +++ b/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9" +} diff --git a/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json b/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json new file mode 100644 index 0000000000..353920fdeb --- /dev/null +++ b/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json @@ -0,0 +1,208 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "teams_command_script", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "slack_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "deploy_to", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "ai_config", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 17, + "name": "datatable", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "ducklake", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "default_scripts", + "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 24, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 26, + "name": "git_app_installations", + "type_info": "Jsonb" + }, + { + "ordinal": 27, + "name": "auto_invite", + "type_info": "Jsonb" + }, + { + "ordinal": 28, + "name": "error_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 29, + "name": "success_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "public_app_execution_limit_per_minute", + "type_info": "Int4" + }, + { + "ordinal": 31, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf" +} diff --git a/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json b/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json new file mode 100644 index 0000000000..243c3f5fa7 --- /dev/null +++ b/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9" +} diff --git a/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json b/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json new file mode 100644 index 0000000000..7eef899aa7 --- /dev/null +++ b/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings\n (workspace_id, color, error_handler_fallback_to_instance_alerts)\n VALUES ($1, $2, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265" +} diff --git a/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json b/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json new file mode 100644 index 0000000000..21679c4ec2 --- /dev/null +++ b/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10" +} diff --git a/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json b/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json new file mode 100644 index 0000000000..13e31508a0 --- /dev/null +++ b/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b" +} diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql new file mode 100644 index 0000000000..a1e5abccc3 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS error_handler_fallback_to_instance_alerts; diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql new file mode 100644 index 0000000000..2ab4b8b673 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS error_handler_fallback_to_instance_alerts BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index c319a1b9ec..a681fd982e 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -901,6 +901,64 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let stored = || async { + sqlx::query_scalar!( + "SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "enable: {}", resp.text().await?); + assert!(stored().await?); + + // A client that predates the setting (the CLI pushing settings.yaml) omits the field and + // must not silently turn it back off. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "extra_args": null})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "omitted: {}", resp.text().await?); + assert!(stored().await?); + + sqlx::query!( + "UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "fork must be rejected"); + + // The settings page stops offering the option once the workspace is a fork, so its next save + // sends `false`: that must go through rather than lock the whole error handler behind a 400. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert!(!stored().await?); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_get_imports(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index b025261270..9cf80e5e39 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -302,6 +302,7 @@ pub struct WorkspaceSettings { pub success_handler: Option, #[serde(skip_serializing_if = "Option::is_none")] pub public_app_execution_limit_per_minute: Option, + pub error_handler_fallback_to_instance_alerts: bool, } /// Subset of `WorkspaceSettings` that is safe to return to any workspace @@ -451,6 +452,8 @@ struct CreateWorkspace { name: String, username: Option, color: Option, + #[serde(default)] + error_handler_fallback_to_instance_alerts: bool, } #[derive(Deserialize)] @@ -558,6 +561,9 @@ pub struct EditErrorHandlerNew { pub muted_on_cancel: bool, #[serde(default)] pub muted_on_user_path: bool, + /// Left as `None` by clients that predate the setting (the CLI among them), which must + /// keep the stored value rather than silently reset it on every settings push. + pub fallback_to_instance_alerts: Option, } // Legacy format for error handler (flat fields from old CLI) @@ -586,6 +592,7 @@ impl EditErrorHandler { extra_args: legacy.error_handler_extra_args, muted_on_cancel: legacy.error_handler_muted_on_cancel, muted_on_user_path: false, // Old format doesn't have this field + fallback_to_instance_alerts: None, }, } } @@ -973,7 +980,8 @@ async fn get_settings( auto_invite, error_handler, success_handler, - public_app_execution_limit_per_minute + public_app_execution_limit_per_minute, + error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE @@ -4182,6 +4190,19 @@ async fn edit_error_handler( let mut tx = db.begin().await?; + if let Some(fallback_to_instance_alerts) = ee.fallback_to_instance_alerts { + if fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &w_id).await?; + } + sqlx::query!( + "UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2", + fallback_to_instance_alerts, + &w_id + ) + .execute(&mut *tx) + .await?; + } + sqlx::query_as!( Group, "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", @@ -4260,7 +4281,16 @@ async fn edit_error_handler( ActionKind::Update, &w_id, Some(&authed.email), - Some([("error_handler", &format!("{:?}", ee.path)[..])].into()), + Some( + [ + ("error_handler", &format!("{:?}", ee.path)[..]), + ( + "fallback_to_instance_alerts", + &format!("{:?}", ee.fallback_to_instance_alerts)[..], + ), + ] + .into(), + ), ) .await?; tx.commit().await?; @@ -4737,6 +4767,36 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The instance critical alert channels belong to the instance operator, who on cloud is +/// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run +/// throwaway copies of their parent's runnables, so instance-wide operational alerting must +/// stay a property of the real workspace. +async fn ensure_instance_alert_fallback_allowed<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, +) -> Result<()> { + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels is not available on cloud" + .to_string(), + )); + } + let is_fork = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#, + w_id + ) + .fetch_optional(&mut **tx) + .await? + .unwrap_or(false); + if is_fork { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels cannot be enabled on a fork workspace" + .to_string(), + )); + } + Ok(()) +} + pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> { if w_id == "global" { return Err(windmill_common::error::Error::BadRequest( @@ -4913,12 +4973,16 @@ async fn create_workspace( ) .execute(&mut *tx) .await?; + if nw.error_handler_fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &nw.id).await?; + } sqlx::query!( "INSERT INTO workspace_settings - (workspace_id, color) - VALUES ($1, $2)", + (workspace_id, color, error_handler_fallback_to_instance_alerts) + VALUES ($1, $2, $3)", nw.id, nw.color, + nw.error_handler_fallback_to_instance_alerts, ) .execute(&mut *tx) .await?; @@ -6996,8 +7060,12 @@ async fn attach_dev_workspace( ) .execute(&mut *tx) .await?; + // Clearing the instance-alert opt-in here keeps the stored setting truthful for a workspace + // that becomes parent-managed: dispatch enforces the fork boundary on its own, but a lingering + // `true` would survive a later detach and would make the settings page submit a value the API + // rejects on a fork. sqlx::query!( - "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + "UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2", &prod_w_id, &dev_w_id ) diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index aa89f1ffd5..7998a769f9 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -108,7 +108,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8c082f5052..b67461de43 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3617,6 +3617,9 @@ paths: public_app_execution_limit_per_minute: type: integer description: Rate limit for public app executions per minute per server. NULL or 0 means disabled. + error_handler_fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. /w/{workspace}/workspaces/get_deploy_to: get: @@ -24232,6 +24235,9 @@ components: muted_on_user_path: type: boolean default: false + fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Omit to leave the stored value untouched. EditErrorHandlerLegacy: type: object @@ -26883,6 +26889,7 @@ components: - slack - teams - email + - instance_alerts NewSchedule: type: object @@ -29811,6 +29818,10 @@ components: type: string color: type: string + error_handler_fallback_to_instance_alerts: + type: boolean + default: false + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Not available on cloud or on fork workspaces. required: - id - name diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index efe6477ad3..e2597fdfce 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -563,6 +563,20 @@ pub async fn report_critical_error( } } +/// Route a workspace-level failure to the instance critical alert channels without +/// recording an `alerts` row: job failures are workspace noise and would otherwise flood +/// the instance-wide feed superadmins triage. The channels belong to the instance operator, +/// who on cloud is not the workspace owner, hence the hard stop there. Callers own the +/// per-workspace opt-in. +pub async fn send_workspace_error_to_instance_channels(_error_message: String, _db: &DB) -> () { + if *CLOUD_HOSTED { + return; + } + + #[cfg(feature = "enterprise")] + send_critical_alert(_error_message, _db, CriticalAlertKind::CriticalError, None).await; +} + pub async fn report_recovered_critical_error( message: String, _db: DB, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 4b2196bf63..f2d0425710 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -953,8 +953,13 @@ lazy_static::lazy_static! { static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), (bool, Option)> = Cache::new(10000); // Cache for workspace error handler settings with 60s TTL - // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, expiry_timestamp) - static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, bool, i64)> = Cache::new(1000); + // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, report_to_instance_alerts, expiry_timestamp) + static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, bool, bool, i64)> = Cache::new(1000); + + // Best-effort per-worker throttle for the instance-channel fallback: a flapping runnable + // would otherwise turn every failure into outbound Slack/SMTP traffic on channels shared by + // the whole instance. Key: workspace_id, Value: (last_sent_epoch, failures suppressed since) + static ref INSTANCE_ALERT_THROTTLE: Cache = Cache::new(1000); // Cache for workspace success handler settings with 60s TTL // Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp) @@ -962,6 +967,7 @@ lazy_static::lazy_static! { } const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60; +const INSTANCE_ALERT_COOLDOWN_SECONDS: i64 = 60; pub async fn add_completed_job( db: &Pool, @@ -2125,7 +2131,7 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( async fn fetch_error_handler_from_db( db: &Pool, w_id: &str, -) -> Result<(Option, Option>>, bool, bool), Error> { +) -> Result<(Option, Option>>, bool, bool, bool), Error> { sqlx::query_as::< _, ( @@ -2133,6 +2139,7 @@ async fn fetch_error_handler_from_db( Option>>, Option, Option, + bool, ), >( r#" @@ -2140,23 +2147,28 @@ async fn fetch_error_handler_from_db( error_handler->>'path', (error_handler->'extra_args')::text::json, (error_handler->>'muted_on_cancel')::boolean, - (error_handler->>'muted_on_user_path')::boolean - FROM workspace_settings - WHERE workspace_id = $1 + (error_handler->>'muted_on_user_path')::boolean, + ws.error_handler_fallback_to_instance_alerts AND w.parent_workspace_id IS NULL + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE ws.workspace_id = $1 "#, ) .bind(w_id) .fetch_optional(db) .await .context("fetching error handler info from workspace_settings")? - .map(|(path, extra_args, muted_on_cancel, muted_on_user_path)| { - ( - path, - extra_args, - muted_on_cancel.unwrap_or(false), - muted_on_user_path.unwrap_or(false), - ) - }) + .map( + |(path, extra_args, muted_on_cancel, muted_on_user_path, report_to_instance_alerts)| { + ( + path, + extra_args, + muted_on_cancel.unwrap_or(false), + muted_on_user_path.unwrap_or(false), + report_to_instance_alerts, + ) + }, + ) .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}"))) } @@ -2174,15 +2186,22 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, + report_to_instance_alerts, ) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { - if cached.4 > now { - (cached.0.clone(), cached.1.clone(), cached.2, cached.3) + if cached.5 > now { + ( + cached.0.clone(), + cached.1.clone(), + cached.2, + cached.3, + cached.4, + ) } else { let row = fetch_error_handler_from_db(db, w_id).await?; let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row.0.clone(), row.1.clone(), row.2, row.3, expiry), + (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry), ); row } @@ -2191,11 +2210,17 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row.0.clone(), row.1.clone(), row.2, row.3, expiry), + (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry), ); row }; + // Nothing to do for the vast majority of workspaces, and returning here keeps the + // per-runnable mute lookup below off the path of every failed job. + if error_handler.is_none() && !report_to_instance_alerts { + return Ok(()); + } + if is_canceled && error_handler_muted_on_cancel { return Ok(()); } @@ -2209,51 +2234,90 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> } } - if let Some(error_handler) = error_handler { - let ws_error_handler_muted: Option = match queued_job.kind { - JobKind::Script => { - sqlx::query_scalar!( + let ws_error_handler_muted: Option = match queued_job.kind { + JobKind::Script => { + sqlx::query_scalar!( "SELECT ws_error_handler_muted FROM script WHERE workspace_id = $1 AND hash = $2", queued_job.workspace_id, queued_job.runnable_id.map(|x| x.0), ) - .fetch_optional(db) - .await? - } - JobKind::Flow => { - sqlx::query_scalar!( - "SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2", - queued_job.workspace_id, - queued_job.runnable_path.clone(), - ) - .fetch_optional(db) - .await? - } - _ => None, - }; - - let muted = ws_error_handler_muted.unwrap_or(false); - if !muted { - tracing::info!("workspace error handled for job {}", &queued_job.id); - - push_error_handler( - db, - queued_job.id, - queued_job.schedule_path(), + .fetch_optional(db) + .await? + } + JobKind::Flow => { + sqlx::query_scalar!( + "SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2", + queued_job.workspace_id, queued_job.runnable_path.clone(), - queued_job.is_flow(), - &queued_job.workspace_id, - &error_handler, - result, - None, - queued_job.started_at, - error_handler_extra_args, - &queued_job.permissioned_as_email, - false, - false, - None, ) - .await?; + .fetch_optional(db) + .await? + } + _ => None, + }; + + if ws_error_handler_muted.unwrap_or(false) { + return Ok(()); + } + + if let Some(error_handler) = error_handler { + tracing::info!("workspace error handled for job {}", &queued_job.id); + + push_error_handler( + db, + queued_job.id, + queued_job.schedule_path(), + queued_job.runnable_path.clone(), + queued_job.is_flow(), + &queued_job.workspace_id, + &error_handler, + result, + None, + queued_job.started_at, + error_handler_extra_args, + &queued_job.permissioned_as_email, + false, + false, + None, + ) + .await?; + } else if !is_canceled { + // A cancellation is a human action rather than an operational failure, and unlike the + // handler path this one has no per-workspace toggle to opt out of reporting them. + let suppressed = match INSTANCE_ALERT_THROTTLE.get(w_id) { + Some((last_sent, suppressed)) + if now - last_sent < INSTANCE_ALERT_COOLDOWN_SECONDS => + { + INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (last_sent, suppressed + 1)); + None + } + entry => { + INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (now, 0)); + Some(entry.map(|(_, suppressed)| suppressed).unwrap_or(0)) + } + }; + if let Some(suppressed) = suppressed { + tracing::info!( + "reporting failed job {} to the instance critical alert channels", + &queued_job.id + ); + let base_url = windmill_common::BASE_URL.load(); + let rollup = if suppressed > 0 { + format!( + " (and {suppressed} more failure(s) in the preceding {INSTANCE_ALERT_COOLDOWN_SECONDS}s)" + ) + } else { + String::new() + }; + windmill_common::utils::send_workspace_error_to_instance_channels( + format!( + "Job {} failed in workspace {w_id} ({base_url}/run/{}?workspace={w_id}){rollup}", + queued_job.runnable_path.as_deref().unwrap_or("preview"), + queued_job.id + ), + db, + ) + .await; } } Ok(()) diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index 330275ee75..89e2554c01 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -85,6 +85,9 @@ * nav `$workspaceStore`; a trigger editor in a forked session passes its * acting workspace so the handler is resolved and saved there. */ workspace?: string + /** Offer the instance critical alert channels as a destination. Workspace-level + * error handling only: schedules and triggers have no such setting. */ + showInstanceAlerts?: boolean } let { @@ -99,7 +102,8 @@ customHandlerKind = $bindable('script'), customTabTooltip, noMargin = false, - workspace = undefined + workspace = undefined, + showInstanceAlerts = false }: Props = $props() let effectiveWorkspace = $derived(workspace ?? $workspaceStore) @@ -363,6 +367,14 @@ handlerPath = hubPaths.emailErrorHandler } }) + + // The instance channels are reached by having no workspace handler at all, so the tab + // owns an empty path rather than a handler script. + $effect(() => { + if (handlerSelected === 'instance_alerts') { + handlerPath = undefined + } + })
@@ -378,6 +390,15 @@ disabled={!isEditable} tooltip={customTabTooltip ? 'Custom error handler with script or flow' : undefined} /> + {#if showInstanceAlerts} + + {/if} {/snippet} @@ -652,6 +673,18 @@ {/if}
{/if} + {:else if handlerSelected === 'instance_alerts'} +
+ + Failed jobs are reported to the Slack, Teams and email channels configured at the instance + level. Those channels are managed in instance settings by a superadmin, not here, and the + report is sent without adding an entry to the instance critical alert feed. Canceled jobs + are not reported. + + + Configure the instance critical alert channels + +
{/if} diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte index b085649b89..23ed37caad 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -14,7 +14,13 @@ import { validateUsername } from '$lib/utils' import { logoutWithRedirect } from '$lib/logoutKit' import { page } from '$app/state' - import { superadmin, usersWorkspaceStore, userWorkspaces, workspaceStore } from '$lib/stores' + import { + enterpriseLicense, + superadmin, + usersWorkspaceStore, + userWorkspaces, + workspaceStore + } from '$lib/stores' import { workspaceIsFork, findWorkspaceRoot, @@ -204,6 +210,7 @@ let workspaceColor: string | undefined = $state(undefined) let colorEnabled = $state(false) + let errorHandlerFallbackToInstanceAlerts = $state(false) function generateRandomColor() { const randomColor = @@ -452,7 +459,8 @@ id, name, color: colorEnabled && workspaceColor ? workspaceColor : undefined, - username: automateUsernameCreation ? undefined : username + username: automateUsernameCreation ? undefined : username, + error_handler_fallback_to_instance_alerts: errorHandlerFallbackToInstanceAlerts } }) if (auto_invite) { @@ -784,6 +792,23 @@ {/if} + {#if !isFork && !isCloudHosted() && $enterpriseLicense} + + {/if} {#if isFork && canDesignateDevWorkspace}