From d382ea7c8b372471dd3393720ff93749fde898f5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 30 Sep 2025 14:45:30 +0000 Subject: [PATCH] feat: support esm mode for codebase bundles (#6709) * rawAppsS3 * make fn common * all * merge * nit * fix ingress * all * all * all * all --- backend/windmill-api/src/jobs.rs | 58 +---- backend/windmill-api/src/scripts.rs | 48 +--- backend/windmill-common/src/s3_helpers.rs | 61 +++++ backend/windmill-common/src/scripts.rs | 48 +++- backend/windmill-common/src/worker.rs | 7 +- backend/windmill-worker/src/bun_executor.rs | 48 ++-- backend/windmill-worker/src/worker.rs | 21 +- cli/src/commands/script/script.ts | 5 +- cli/src/core/conf.ts | 221 +++++++++++++----- frontend/package-lock.json | 15 -- frontend/src/lib/components/Dev.svelte | 7 +- .../apps/components/buttons/AppButton.svelte | 20 +- .../RawAppInlineScriptRunnable.svelte | 1 - 13 files changed, 352 insertions(+), 208 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e370896b52..48c89c2654 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -18,6 +18,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; use sqlx::Pool; +use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::ops::{Deref, DerefMut}; @@ -41,7 +42,6 @@ use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; -use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; use windmill_common::variables::get_workspace_key; use crate::triggers::trigger_helpers::ScriptId; @@ -3526,6 +3526,7 @@ struct Preview { tag: Option, dedicated_worker: Option, lock: Option, + format: Option } #[derive(Deserialize)] @@ -5590,6 +5591,7 @@ async fn run_wait_result_preview_script( return result; } + async fn run_bundle_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -5598,7 +5600,6 @@ async fn run_bundle_preview_script( Query(run_query): Query, mut multipart: axum::extract::Multipart, ) -> error::Result<(StatusCode, String)> { - use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH; if authed.is_operator { return Err(error::Error::NotAuthorized( @@ -5610,6 +5611,7 @@ async fn run_bundle_preview_script( let mut tx = None; let mut uploaded = false; let mut is_tar = false; + let mut format = BundleFormat::Cjs; while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); @@ -5617,6 +5619,7 @@ async fn run_bundle_preview_script( let data = data.map_err(to_anyhow)?; if name == "preview" { let preview: Preview = serde_json::from_slice(&data).map_err(to_anyhow)?; + format = preview.format.and_then(|s| BundleFormat::from_string(&s)).unwrap_or(BundleFormat::Cjs); let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); @@ -5637,11 +5640,7 @@ async fn run_bundle_preview_script( ltx, &w_id, JobPayload::Code(RawCode { - hash: if is_tar { - Some(PREVIEW_IS_TAR_CODEBASE_HASH) - } else { - Some(PREVIEW_IS_CODEBASE_HASH) - }, + hash: Some(windmill_common::scripts::codebase_to_hash(is_tar, format == BundleFormat::Esm)), content: preview.content.unwrap_or_default(), path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), @@ -5690,52 +5689,17 @@ async fn run_bundle_preview_script( // tracing::info!("is_tar 2: {is_tar}"); + if format == BundleFormat::Esm { + id = format!("{}.esm", id); + } if is_tar { id = format!("{}.tar", id); } uploaded = true; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::get_object_store().await; - - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let object_store: Option<()> = None; - - if &windmill_common::utils::MODE_AND_ADDONS.mode - == &windmill_common::utils::Mode::Standalone - && object_store.is_none() - { - std::fs::create_dir_all( - windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(), - )?; - windmill_common::worker::write_file_bytes( - &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - &id, - &data, - )?; - } else { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - return Err(Error::ExecutionErr("codebase is an EE feature".to_string())); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = object_store { - check_license_key_valid().await?; - - let path = windmill_common::s3_helpers::bundle(&w_id, &id); - if let Err(e) = os - .put(&object_store::path::Path::from(path.clone()), data.into()) - .await - { - tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); - return Err(Error::ExecutionErr(format!("Failed to put {path} to s3"))); - } - } else { - return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); - } - } + let path = windmill_common::s3_helpers::bundle(&w_id, &id); + upload_artifact_to_store(&path, data, &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR).await?; } // println!("Length of `{}` is {} bytes", name, data.len()); } diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index c321f16b1c..40657b4510 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -41,11 +41,7 @@ use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; use windmill_common::{ - assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, - error::to_anyhow, - scripts::hash_script, - utils::WarnAfterExt, - worker::CLOUD_HOSTED, + assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, s3_helpers::upload_artifact_to_store, scripts::hash_script, utils::WarnAfterExt, worker::CLOUD_HOSTED }; use windmill_common::{ @@ -421,45 +417,8 @@ async fn create_snapshot_script( uploaded = true; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::get_object_store().await; - - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let object_store: Option<()> = None; - - if &windmill_common::utils::MODE_AND_ADDONS.mode - == &windmill_common::utils::Mode::Standalone - && object_store.is_none() - { - std::fs::create_dir_all( - windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(), - )?; - windmill_common::worker::write_file_bytes( - &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - &hash, - &data, - )?; - } else { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - return Err(Error::ExecutionErr("codebase is an EE feature".to_string())); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = object_store { - let path = windmill_common::s3_helpers::bundle(&w_id, &hash); - - if let Err(e) = os - .put(&object_store::path::Path::from(path.clone()), data.into()) - .await - { - tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); - return Err(Error::ExecutionErr(format!("Failed to put {path} to s3"))); - } - } else { - return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); - } - } + let path = windmill_common::s3_helpers::bundle(&w_id, &hash); + upload_artifact_to_store(&path, data, &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR).await?; } // println!("Length of `{}` is {} bytes", name, data.len()); } @@ -479,6 +438,7 @@ async fn create_snapshot_script( return Ok((StatusCode::CREATED, format!("{}", script_hash.unwrap()))); } + async fn list_paths_from_workspace_runnable( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 1fed417382..3935593de6 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -526,6 +526,67 @@ pub async fn build_object_store_client( } } + +#[derive(PartialEq)] +pub enum BundleFormat { + Esm, + Cjs, +} + +impl BundleFormat { + pub fn from_string(s: &str) -> Option { + match s { + "esm" => Some(Self::Esm), + "cjs" => Some(Self::Cjs), + _ => None, + } + } +} + +pub async fn upload_artifact_to_store(path: &str, data: bytes::Bytes, standalone_dir: &str) -> error::Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let object_store = crate::s3_helpers::get_object_store().await; + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + let object_store: Option<()> = None; + Ok(if &crate::utils::MODE_AND_ADDONS.mode + == &crate::utils::Mode::Standalone + && object_store.is_none() + { + let path = format!("{}/{}", standalone_dir, path); + tracing::info!("Writing file to path {path}"); + + let split_path = path.split("/").collect::>(); + std::fs::create_dir_all( + split_path[..split_path.len() - 1].join("/"), + )?; + + crate::worker::write_file_bytes( + &path, + &data, + )?; + } else { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + return Err(error::Error::ExecutionErr("codebase is an EE feature".to_string())); + } + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = object_store { + + if let Err(e) = os + .put(&object_store::path::Path::from(path), data.into()) + .await + { + tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); + return Err(error::Error::ExecutionErr(format!("Failed to put {path} to s3"))); + } + } else { + return Err(error::Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); + } + }) +} + + #[cfg(feature = "parquet")] pub async fn attempt_fetch_bytes( client: Arc, diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index fcfd099c67..32ed0c3c2d 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -211,9 +211,53 @@ impl Display for ScriptKind { } } -pub const PREVIEW_IS_CODEBASE_HASH: i64 = -42; -pub const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; +const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; +pub fn is_special_codebase_hash(hash: i64) -> bool { + hash == PREVIEW_IS_CODEBASE_HASH || hash == PREVIEW_IS_TAR_CODEBASE_HASH || hash == PREVIEW_IS_ESM_CODEBASE_HASH || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH +} + +pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { + if is_tar { + if is_esm { + PREVIEW_IS_TAR_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_TAR_CODEBASE_HASH + } + } else { + if is_esm { + PREVIEW_IS_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_CODEBASE_HASH + } + } +} + + +pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { + match hash { + PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), + PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), + PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), + PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), + _ => None, + } +} + + +pub struct CodebaseInfo { + pub is_tar: bool, + pub is_esm: bool, +} + +pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { + let is_tar = id.ends_with(".tar"); + let is_esm = id.contains(".esm"); + CodebaseInfo { is_tar, is_esm } +} #[derive(Serialize, sqlx::FromRow)] pub struct Script { pub workspace_id: String, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1254bcb04d..5c1381ecd6 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -258,7 +258,7 @@ lazy_static::lazy_static! { // Features flags: pub static ref DISABLE_FLOW_SCRIPT: bool = std::env::var("DISABLE_FLOW_SCRIPT").ok().is_some_and(|x| x == "1" || x == "true"); - pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle/", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); + pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); } pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); @@ -470,9 +470,8 @@ pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { Ok(file) } -pub fn write_file_bytes(dir: &str, path: &str, content: &Bytes) -> error::Result { - let path = format!("{}/{}", dir, path); - let mut file = File::create(&path)?; +pub fn write_file_bytes(path: &str, content: &Bytes) -> error::Result { + let mut file = File::create(path)?; file.write_all(content)?; file.flush()?; Ok(file) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 87f15c3294..215d360879 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -25,7 +25,7 @@ use crate::{ DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; -use windmill_common::client::AuthedClient; +use windmill_common::{client::AuthedClient, s3_helpers::BundleFormat, scripts::{id_to_codebase_info, CodebaseInfo}}; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -610,15 +610,18 @@ pub async fn generate_bun_bundle( Ok(()) } -pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { +struct PulledCodebase { + is_esm: bool, +} +async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result { let path = windmill_common::s3_helpers::bundle(&w_id, &id); let bun_cache_path = format!( "{}/{}", windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR, path ); - let is_tar = id.ends_with(".tar"); + let CodebaseInfo { is_tar, is_esm } = id_to_codebase_info(id); let dst = format!( "{job_dir}/{}", if is_tar { "codebase.tar" } else { "main.js" } @@ -639,9 +642,9 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { && object_store.is_none() { let bun_cache_path = format!( - "{}{}", + "{}/{}", *windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - id + path ); if std::fs::metadata(&bun_cache_path).is_ok() { tracing::info!("loading {bun_cache_path} from standalone bundle cache"); @@ -671,7 +674,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { } } - Ok(()) + Ok(PulledCodebase { is_esm }) } fn extract_saved_codebase( @@ -907,13 +910,12 @@ pub async fn handle_bun_job( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - if codebase.is_some() { - annotation.nodejs = true - } + let main_override = job.script_entrypoint_override.as_deref(); let apply_preprocessor = job.flow_step_id.as_deref() != Some("preprocessor") && job.preprocessed == Some(false); + let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; let symlink; @@ -935,7 +937,10 @@ pub async fn handle_bun_job( )) })?; } else if let Some(codebase) = codebase.as_ref() { - pull_codebase(&job.workspace_id, codebase, job_dir).await?; + let pulled_codebase = pull_codebase(&job.workspace_id, codebase, job_dir).await?; + if pulled_codebase.is_esm { + format = BundleFormat::Esm; + } } else if let Some(reqs) = requirements_o.as_ref() { let (pkg, lock, empty, is_binary) = split_lockfile(reqs); @@ -991,6 +996,10 @@ pub async fn handle_bun_job( // } } + if codebase.is_some() && format == BundleFormat::Cjs { + annotation.nodejs = true + } + let mut init_logs = if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if has_bundle_cache { @@ -1000,7 +1009,11 @@ pub async fn handle_bun_job( "\n\n--- BUN BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } } else if codebase.is_some() { - "\n\n--- NODE CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + if format == BundleFormat::Esm { + "\n\n--- ESM CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + } else { + "\n\n--- CJS CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + } } else if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if annotation.nodejs { @@ -1612,8 +1625,13 @@ pub async fn start_worker( .await; let context_envs = build_envs_map(context.to_vec()).await; + + let mut format = BundleFormat::Cjs; if let Some(codebase) = codebase.as_ref() { - pull_codebase(w_id, codebase, job_dir).await?; + let pulled_codebase = pull_codebase(w_id, codebase, job_dir).await?; + if pulled_codebase.is_esm { + format = BundleFormat::Esm; + } } else if let Some(reqs) = requirements_o { let (pkg, lock, empty, is_binary) = split_lockfile(&reqs); if lock.is_none() { @@ -1740,7 +1758,11 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { write_file(job_dir, "wrapper.mjs", &wrapper_content)?; } - if !codebase.is_some() { + if format == BundleFormat::Esm { + annotation.nodejs = false; + } + + if !codebase.is_some() || format == BundleFormat::Esm { build_loader( job_dir, base_internal_url, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index cbac637464..5e407f3857 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -13,6 +13,8 @@ use anyhow::anyhow; use futures::TryFutureExt; use tokio::time::timeout; use windmill_common::client::AuthedClient; +use windmill_common::scripts::hash_to_codebase_id; +use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; use windmill_common::{ @@ -20,7 +22,6 @@ use windmill_common::{ apps::AppScriptId, cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, schema::{should_validate_schema, SchemaValidator}, - scripts::PREVIEW_IS_TAR_CODEBASE_HASH, utils::{create_directory_async, WarnAfterExt}, worker::{ make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, @@ -64,7 +65,7 @@ use windmill_common::{ error::{self, to_anyhow, Error}, flows::FlowNodeId, jobs::JobKind, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang, PREVIEW_IS_CODEBASE_HASH}, + scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::StripPath, worker::{CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, DB, IS_READY, @@ -2369,12 +2370,13 @@ pub async fn handle_queued_job( | JobKind::Flow | JobKind::FlowDependencies, x, - ) => match x.map(|x| x.0) { - None | Some(PREVIEW_IS_CODEBASE_HASH) | Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some( + ) => if x.map(|x| x.0).is_none_or(|x| is_special_codebase_hash(x)) { + Some( cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow.clone()) .await?, - ), - _ => None, + ) + } else { + None }, _ => None, }; @@ -2867,12 +2869,7 @@ async fn handle_code_execution_job( ScriptMetadata { language, envs, codebase, schema_validator, schema }, ) = match job.kind { JobKind::Preview => { - let codebase = match job.runnable_id.map(|x| x.0) { - Some(PREVIEW_IS_CODEBASE_HASH) => Some(job.id.to_string()), - Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some(format!("{}.tar", job.id)), - _ => None, - }; - + let codebase = job.runnable_id.and_then(|x| hash_to_codebase_id(&job.id.to_string(), x.0)); if codebase.is_none() && job.runnable_id.is_some() { (arc_data, arc_metadata) = cache::script::fetch(conn, job.runnable_id.unwrap()).await?; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index f719a008a3..b0e80b764b 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -222,9 +222,10 @@ export async function handleFile( log.info(`Started bundling ${path} ...`); const startTime = performance.now(); + const format = codebase.format ?? "cjs"; const out = await esbuild.build({ entryPoints: [path], - format: "cjs", + format: format, bundle: true, write: false, external: codebase.external, @@ -232,7 +233,7 @@ export async function handleFile( define: codebase.define, platform: "node", packages: "bundle", - target: "node20.15.1", + target: format == "cjs" ? "node20.15.1" : "esnext", }); const endTime = performance.now(); bundleContent = out.outputFiles[0].text; diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index ebe871959f..273891b42b 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -1,5 +1,9 @@ import { log, yamlParseFile, Confirm, yamlStringify } from "../../deps.ts"; -import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, isGitRepository } from "../utils/git.ts"; +import { + getCurrentGitBranch, + getOriginalBranchForWorkspaceForks, + isGitRepository, +} from "../utils/git.ts"; import { join, dirname, resolve, relative } from "node:path"; import { existsSync } from "node:fs"; import { execSync } from "node:child_process"; @@ -94,13 +98,14 @@ export interface Codebase { external?: string[]; define?: { [key: string]: string }; inject?: string[]; + format?: "cjs" | "esm"; } function getGitRepoRoot(): string | null { try { const result = execSync("git rev-parse --show-toplevel", { encoding: "utf8", - stdio: "pipe" + stdio: "pipe", }); return result.trim(); } catch (error) { @@ -182,34 +187,45 @@ export async function readConfigFile(): Promise { const migrationMessages: string[] = []; // Handle obsolete overrides format - if (conf && 'overrides' in conf) { + if (conf && "overrides" in conf) { const overrides = conf.overrides as any; - const hasSettings = overrides && typeof overrides === 'object' && Object.keys(overrides).length > 0; + const hasSettings = + overrides && + typeof overrides === "object" && + Object.keys(overrides).length > 0; if (hasSettings) { throw new Error( "❌ The 'overrides' field is no longer supported.\n" + - " The configuration system now uses Git branch-based configuration only.\n" + - " Please delete your wmill.yaml and run 'wmill init' to recreate it with the new format." + " The configuration system now uses Git branch-based configuration only.\n" + + " Please delete your wmill.yaml and run 'wmill init' to recreate it with the new format." ); } else { // Remove empty overrides delete conf.overrides; needsConfigWrite = true; - migrationMessages.push("ℹ️ Removing empty 'overrides: {}' from wmill.yaml (migrated to gitBranches format)"); + migrationMessages.push( + "ℹ️ Removing empty 'overrides: {}' from wmill.yaml (migrated to gitBranches format)" + ); } } // Handle git_branches to gitBranches migration - if (conf && 'git_branches' in conf) { + if (conf && "git_branches" in conf) { if (!conf.gitBranches) { // Deep copy git_branches to gitBranches (even if empty) conf.gitBranches = JSON.parse(JSON.stringify(conf.git_branches)); needsConfigWrite = true; - migrationMessages.push("⚠️ Migrating 'git_branches' to 'gitBranches' (camelCase). The snake_case format is deprecated."); - migrationMessages.push("✅ Successfully migrated 'git_branches' to 'gitBranches' in wmill.yaml"); + migrationMessages.push( + "⚠️ Migrating 'git_branches' to 'gitBranches' (camelCase). The snake_case format is deprecated." + ); + migrationMessages.push( + "✅ Successfully migrated 'git_branches' to 'gitBranches' in wmill.yaml" + ); } else { - migrationMessages.push("⚠️ Both 'git_branches' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'git_branches'."); + migrationMessages.push( + "⚠️ Both 'git_branches' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'git_branches'." + ); } // Always remove the old field from config object (both file and memory) delete conf.git_branches; @@ -220,20 +236,24 @@ export async function readConfigFile(): Promise { try { await Deno.writeTextFile(wmillYamlPath, yamlStringify(conf)); // Log all migration messages after successful write - migrationMessages.forEach(msg => { - if (msg.startsWith('⚠️')) { + migrationMessages.forEach((msg) => { + if (msg.startsWith("⚠️")) { log.warn(msg); } else { log.info(msg); } }); } catch (error) { - log.warn(`Could not update wmill.yaml to apply migrations: ${error instanceof Error ? error.message : error}`); + log.warn( + `Could not update wmill.yaml to apply migrations: ${ + error instanceof Error ? error.message : error + }` + ); } } else if (migrationMessages.length > 0) { // Log messages for non-write cases (like "both found") - migrationMessages.forEach(msg => { - if (msg.startsWith('⚠️')) { + migrationMessages.forEach((msg) => { + if (msg.startsWith("⚠️")) { log.warn(msg); } else { log.info(msg); @@ -248,38 +268,66 @@ export async function readConfigFile(): Promise { } return typeof conf == "object" ? conf : ({} as SyncOptions); } catch (e) { - if (e instanceof Error && (e.message.includes("overrides") || e.message.includes("Obsolete configuration format"))) { + if ( + e instanceof Error && + (e.message.includes("overrides") || + e.message.includes("Obsolete configuration format")) + ) { throw e; // Re-throw the specific obsolete format error } // Since we already found the file path, this is likely a parsing or access error if (e instanceof Error && e.message.includes("Error parsing yaml")) { - const yamlError = e.cause instanceof Error ? e.cause.message : String(e.cause); + const yamlError = + e.cause instanceof Error ? e.cause.message : String(e.cause); throw new Error( "❌ YAML syntax error in wmill.yaml:\n" + - " " + yamlError + "\n" + - " Please fix the YAML syntax in wmill.yaml or delete the file to start fresh." + " " + + yamlError + + "\n" + + " Please fix the YAML syntax in wmill.yaml or delete the file to start fresh." ); } else { // File exists but has other issues (permissions, etc.) throw new Error( "❌ Failed to read wmill.yaml:\n" + - " " + (e instanceof Error ? e.message : String(e)) + "\n" + - " Please check file permissions or fix the syntax." + " " + + (e instanceof Error ? e.message : String(e)) + + "\n" + + " Please check file permissions or fix the syntax." ); } } } // Default sync options - shared across the codebase to prevent duplication -export const DEFAULT_SYNC_OPTIONS: Readonly>> = { - defaultTs: 'bun', - includes: ['f/**'], +export const DEFAULT_SYNC_OPTIONS: Readonly< + Required< + Pick< + SyncOptions, + | "defaultTs" + | "includes" + | "excludes" + | "codebases" + | "skipVariables" + | "skipResources" + | "skipResourceTypes" + | "skipSecrets" + | "includeSchedules" + | "includeTriggers" + | "skipScripts" + | "skipFlows" + | "skipApps" + | "skipFolders" + | "includeUsers" + | "includeGroups" + | "includeSettings" + | "includeKey" + > + > +> = { + defaultTs: "bun", + includes: ["f/**"], excludes: [], codebases: [], skipVariables: false, @@ -295,7 +343,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly( @@ -306,7 +354,10 @@ export async function mergeConfigWithConfigFile( } // Validate branch configuration early in the process -export async function validateBranchConfiguration(skipValidation?: boolean, autoAccept?: boolean): Promise { +export async function validateBranchConfiguration( + skipValidation?: boolean, + autoAccept?: boolean +): Promise { if (skipValidation || !isGitRepository()) { return; } @@ -320,7 +371,9 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto let currentBranch: string | null; if (originalBranchIfForked) { - log.info(`Workspace fork detected from branch name \`${rawBranch}\`. Validating branch configuration using original branch \`${originalBranchIfForked}\``); + log.info( + `Workspace fork detected from branch name \`${rawBranch}\`. Validating branch configuration using original branch \`${originalBranchIfForked}\`` + ); currentBranch = originalBranchIfForked; } else { currentBranch = rawBranch; @@ -330,8 +383,8 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto if (!gitBranches || Object.keys(gitBranches).length === 0) { log.warn( "⚠️ WARNING: In a Git repository, the 'gitBranches' section is recommended in wmill.yaml.\n" + - " Consider adding a gitBranches section with configuration for your Git branches.\n" + - " Run 'wmill init' to recreate the configuration file with proper branch setup." + " Consider adding a gitBranches section with configuration for your Git branches.\n" + + " Run 'wmill init' to recreate the configuration file with proper branch setup." ); return; } @@ -340,24 +393,35 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto if (currentBranch && !gitBranches[currentBranch]) { // In interactive mode, offer to create the branch if (Deno.stdin.isTerminal()) { - const availableBranches = Object.keys(gitBranches).join(', '); + const availableBranches = Object.keys(gitBranches).join(", "); log.info( `Current Git branch '${currentBranch}' is not defined in the gitBranches configuration.\n` + - `Available branches: ${availableBranches}` + `Available branches: ${availableBranches}` ); - const shouldCreate = autoAccept || await Confirm.prompt({ - message: `Create empty branch configuration for '${currentBranch}'?`, - default: true, - }); + const shouldCreate = + autoAccept || + (await Confirm.prompt({ + message: `Create empty branch configuration for '${currentBranch}'?`, + default: true, + })); if (shouldCreate) { // Warn if branch name contains filesystem-unsafe characters if (/[\/\\:*?"<>|.]/.test(currentBranch)) { - const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_'); - log.warn(`⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).`); - log.warn(` Branch-specific files will be saved with sanitized name: "${sanitizedBranchName}"`); - log.warn(` Example: "file.variable.yaml" → "file.${sanitizedBranchName}.variable.yaml"`); + const sanitizedBranchName = currentBranch.replace( + /[\/\\:*?"<>|.]/g, + "_" + ); + log.warn( + `⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).` + ); + log.warn( + ` Branch-specific files will be saved with sanitized name: "${sanitizedBranchName}"` + ); + log.warn( + ` Example: "file.variable.yaml" → "file.${sanitizedBranchName}.variable.yaml"` + ); } // Read current config, add branch, and write it back @@ -370,23 +434,34 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto await Deno.writeTextFile("wmill.yaml", yamlStringify(currentConfig)); - log.info(`✅ Created empty branch configuration for '${currentBranch}'`); + log.info( + `✅ Created empty branch configuration for '${currentBranch}'` + ); } else { - log.warn("⚠️ WARNING: Branch creation cancelled. You can manually add the branch to wmill.yaml or use 'wmill gitsync-settings pull' to pull configuration from an existing windmill workspace git-sync configuration."); + log.warn( + "⚠️ WARNING: Branch creation cancelled. You can manually add the branch to wmill.yaml or use 'wmill gitsync-settings pull' to pull configuration from an existing windmill workspace git-sync configuration." + ); return; } } else { // Warn about filesystem-unsafe characters in branch name if (/[\/\\:*?"<>|.]/.test(currentBranch)) { - const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_'); - log.warn(`⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).`); - log.warn(` Branch-specific files will use sanitized name: "${sanitizedBranchName}"`); + const sanitizedBranchName = currentBranch.replace( + /[\/\\:*?"<>|.]/g, + "_" + ); + log.warn( + `⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).` + ); + log.warn( + ` Branch-specific files will use sanitized name: "${sanitizedBranchName}"` + ); } - + log.warn( `⚠️ WARNING: Current Git branch '${currentBranch}' is not defined in the gitBranches configuration.\n` + - ` Consider adding configuration for branch '${currentBranch}' in the gitBranches section of wmill.yaml.\n` + - ` Available branches: ${Object.keys(gitBranches).join(', ')}` + ` Consider adding configuration for branch '${currentBranch}' in the gitBranches section of wmill.yaml.\n` + + ` Available branches: ${Object.keys(gitBranches).join(", ")}` ); return; } @@ -394,7 +469,12 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto } // Get effective settings by merging top-level settings with branch-specific overrides -export async function getEffectiveSettings(config: SyncOptions, promotion?: string, skipBranchValidation?: boolean, suppressLogs?: boolean): Promise { +export async function getEffectiveSettings( + config: SyncOptions, + promotion?: string, + skipBranchValidation?: boolean, + suppressLogs?: boolean +): Promise { // Start with top-level settings from config const { gitBranches, ...topLevelSettings } = config; const effective = { ...topLevelSettings }; @@ -406,10 +486,12 @@ export async function getEffectiveSettings(config: SyncOptions, promotion?: stri let currentBranch: string | null; if (originalBranchIfForked) { - log.info(`Using overrides from original branch \`${originalBranchIfForked}\``); + log.info( + `Using overrides from original branch \`${originalBranchIfForked}\`` + ); currentBranch = originalBranchIfForked; } else { - currentBranch = branch + currentBranch = branch; } // If promotion is specified, use that branch's promotionOverrides or overrides @@ -425,21 +507,36 @@ export async function getEffectiveSettings(config: SyncOptions, promotion?: stri } else if (targetBranch.overrides) { Object.assign(effective, targetBranch.overrides); if (!suppressLogs) { - log.info(`Applied settings from branch: ${promotion} (no promotionOverrides found)`); + log.info( + `Applied settings from branch: ${promotion} (no promotionOverrides found)` + ); } } else { - log.debug(`No promotion or regular overrides found for branch '${promotion}', using top-level settings`); + log.debug( + `No promotion or regular overrides found for branch '${promotion}', using top-level settings` + ); } } // Otherwise use current branch overrides (existing behavior) - else if (currentBranch && gitBranches && gitBranches[currentBranch] && gitBranches[currentBranch].overrides) { + else if ( + currentBranch && + gitBranches && + gitBranches[currentBranch] && + gitBranches[currentBranch].overrides + ) { Object.assign(effective, gitBranches[currentBranch].overrides); if (!suppressLogs) { - const extraLog = originalBranchIfForked ? ` (because it is the origin of the workspace fork branch \`${branch}\`)` : ""; - log.info(`Applied settings for Git branch: ${currentBranch}${extraLog}`); + const extraLog = originalBranchIfForked + ? ` (because it is the origin of the workspace fork branch \`${branch}\`)` + : ""; + log.info( + `Applied settings for Git branch: ${currentBranch}${extraLog}` + ); } } else if (currentBranch) { - log.debug(`No branch-specific overrides found for '${currentBranch}', using top-level settings`); + log.debug( + `No branch-specific overrides found for '${currentBranch}', using top-level settings` + ); } } else { log.debug("Not in a Git repository, using top-level settings"); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0655b11329..3955a57d39 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12277,21 +12277,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 4a0f3af5dc..8deef0334a 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -180,7 +180,7 @@ replaceScript(event.data) } else if (event.data.type == 'testBundle') { if (event.data.id == lastCommandId) { - testBundle(event.data.file, event.data.isTar) + testBundle(event.data.file, event.data.isTar, event.data.format) } else { sendUserToast(`Bundle received ${lastCommandId} was obsolete, ignoring`, true) } @@ -252,7 +252,7 @@ window.parent?.postMessage({ type: 'refresh' }, '*') }) - async function testBundle(file: string, isTar: boolean) { + async function testBundle(file: string, isTar: boolean, format: 'cjs' | 'esm' | undefined) { jobLoader?.abstractRun( async () => { try { @@ -265,7 +265,8 @@ path: currentScript?.path, args, language: currentScript?.language, - tag: currentScript?.tag + tag: currentScript?.tag, + format }) ) // sendUserToast(JSON.stringify(file)) diff --git a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte index 43e45e5f12..2724b500da 100644 --- a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte +++ b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte @@ -290,12 +290,20 @@ on:click={handleClick} size={resolvedConfig.size} color={resolvedConfig.color} - title={resolvedConfig.tooltip && String(resolvedConfig.tooltip).length > 0 ? String(resolvedConfig.tooltip) : undefined} + title={resolvedConfig.tooltip && String(resolvedConfig.tooltip).length > 0 + ? String(resolvedConfig.tooltip) + : undefined} loading={resolvedConfig.runInBackground ? backgroundClickFeedback : loading} > {#if resolvedConfig.beforeIcon} {#key resolvedConfig.beforeIcon} -
0 ? "min-w-4" : ""} bind:this={beforeIconComponent}>
+
0 + ? 'min-w-4' + : ''} + bind:this={beforeIconComponent} + >
{/key} {/if} {#if resolvedConfig.label?.toString() && resolvedConfig.label?.toString()?.length > 0} @@ -303,7 +311,13 @@ {/if} {#if resolvedConfig.afterIcon} {#key resolvedConfig.afterIcon} -
0 ? "min-w-4" : ""} bind:this={afterIconComponent}>
+
0 + ? 'min-w-4' + : ''} + bind:this={afterIconComponent} + >
{/key} {/if} diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte index c4e82a4662..a68745f82c 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte @@ -44,7 +44,6 @@ function getSchema(runnable: RunnableWithFields) { if (runnable?.type == 'runnableByPath') { - console.log('runnable.schema', runnable.schema) return runnable.schema } else if (runnable?.type == 'runnableByName' && runnable.inlineScript) { return runnable.inlineScript.schema