diff --git a/CHANGELOG.md b/CHANGELOG.md index aa57cf766b..4d9a24e861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [1.589.1](https://github.com/windmill-labs/windmill/compare/v1.589.0...v1.589.1) (2025-12-03) + + +### Bug Fixes + +* **backend:** fix broken cancel selection/all ([752e43e](https://github.com/windmill-labs/windmill/commit/752e43e653c3836cde07541b7509dc5081744b75)) +* wmill app generate-locks ([#7288](https://github.com/windmill-labs/windmill/issues/7288)) ([99b1ae2](https://github.com/windmill-labs/windmill/commit/99b1ae2e034f23efe04f5ad4b7fc401910351707)) + +## [1.589.0](https://github.com/windmill-labs/windmill/compare/v1.588.0...v1.589.0) (2025-12-03) + + +### Features + +* agent workers min version support ([#7284](https://github.com/windmill-labs/windmill/issues/7284)) ([e15af78](https://github.com/windmill-labs/windmill/commit/e15af78d9330617d8c7261f47433efc2f5f95203)) + + +### Bug Fixes + +* get flow by id doesn't require path ([#7278](https://github.com/windmill-labs/windmill/issues/7278)) ([60d5ea8](https://github.com/windmill-labs/windmill/commit/60d5ea857991b370c0049f7cff8cb4611a87620e)) +* Prevent running git sync on promotion mode repos for forks ([#7276](https://github.com/windmill-labs/windmill/issues/7276)) ([2a841bb](https://github.com/windmill-labs/windmill/commit/2a841bb0e319883174a1de49d47b1fb97423191c)) + ## [1.588.0](https://github.com/windmill-labs/windmill/compare/v1.587.1...v1.588.0) (2025-12-02) diff --git a/backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json b/backend/.sqlx/query-77f13b7c1e7e488c6268a9ff6def647e47394bd0a61d44fddb1b2040d05b7f17.json similarity index 83% rename from backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json rename to backend/.sqlx/query-77f13b7c1e7e488c6268a9ff6def647e47394bd0a61d44fddb1b2040d05b7f17.json index c50b3531b0..b5b675108d 100644 --- a/backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json +++ b/backend/.sqlx/query-77f13b7c1e7e488c6268a9ff6def647e47394bd0a61d44fddb1b2040d05b7f17.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , duration_ms\n , result\n , canceled_by\n , canceled_reason\n , flow_status\n , status\n , worker\n )\n SELECT q.workspace_id\n , q.id\n , 0\n , $4\n , $1\n , 'cancel all'\n , (SELECT flow_status FROM v2_job_status WHERE id = q.id)\n , 'canceled'::job_status\n , worker\n FROM v2_job_queue q\n JOIN v2_job USING (id)\n WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL\n FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", + "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , duration_ms\n , result\n , canceled_by\n , canceled_reason\n , flow_status\n , status\n , worker\n )\n SELECT q.workspace_id\n , q.id\n , 0\n , $4\n , $1\n , 'cancel all'\n , (SELECT flow_status FROM v2_job_status WHERE id = q.id)\n , 'canceled'::job_status\n , worker\n FROM v2_job_queue q\n JOIN v2_job USING (id)\n WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule'\n FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", "describe": { "columns": [ { @@ -21,5 +21,5 @@ false ] }, - "hash": "51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9" + "hash": "77f13b7c1e7e488c6268a9ff6def647e47394bd0a61d44fddb1b2040d05b7f17" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 220a7162c9..44196532c5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -895,9 +895,9 @@ dependencies = [ [[package]] name = "aws-sdk-rds" -version = "1.118.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ddd2f95d05c73d977308579961917c3b00bbacd36a592470111058a72582ac" +checksum = "70fbf24cf9f84d18f6296857982dd970dd1c23a54a3bac208c7e9abc13a35c1b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -4627,21 +4627,22 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" dependencies = [ "proc-macro2", "quote", + "rustc_version 0.4.1", "syn 2.0.111", ] @@ -6801,9 +6802,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64 0.22.1", "bytes", @@ -7498,7 +7499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c56ff45deb0031f2a476017eed60c06872251f271b8387ad8020b8fef60960" dependencies = [ "chrono", - "derive_more 2.0.1", + "derive_more 2.1.0", "form_urlencoded", "http 1.4.0", "json-patch", @@ -7660,9 +7661,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libffi" @@ -7765,9 +7766,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222" dependencies = [ "zlib-rs", ] @@ -7825,9 +7826,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "loki-api" @@ -15162,7 +15163,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15225,7 +15226,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "argon2", @@ -15346,7 +15347,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.588.0" +version = "1.589.1" dependencies = [ "base64 0.22.1", "chrono", @@ -15361,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.588.0" +version = "1.589.1" dependencies = [ "chrono", "lazy_static", @@ -15375,7 +15376,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "axum", @@ -15394,7 +15395,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "async-recursion", @@ -15487,7 +15488,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.588.0" +version = "1.589.1" dependencies = [ "regex", "serde", @@ -15502,7 +15503,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "bytes", @@ -15526,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.588.0" +version = "1.589.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15542,7 +15543,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.588.0" +version = "1.589.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15551,7 +15552,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "lazy_static", @@ -15563,7 +15564,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "serde_json", @@ -15575,7 +15576,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "gosyn", @@ -15587,7 +15588,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "lazy_static", @@ -15599,7 +15600,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "serde_json", @@ -15611,7 +15612,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "nu-parser", @@ -15622,7 +15623,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15633,7 +15634,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15645,7 +15646,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "async-recursion", @@ -15669,7 +15670,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "lazy_static", @@ -15683,7 +15684,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15700,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "lazy_static", @@ -15714,7 +15715,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "lazy_static", @@ -15732,7 +15733,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "serde", @@ -15743,7 +15744,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "async-recursion", @@ -15780,7 +15781,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.588.0" +version = "1.589.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15790,7 +15791,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.588.0" +version = "1.589.1" dependencies = [ "anyhow", "async-once-cell", @@ -16696,9 +16697,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7" [[package]] name = "zstd" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index bd6dfee048..72fa82685a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.588.0" +version = "1.589.1" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.588.0" +version = "1.589.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/src/main.rs b/backend/src/main.rs index b8d5dfe6ea..3002ac5ae5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1254,23 +1254,36 @@ Windmill Community Edition {GIT_VERSION} tracing::error!("Error waiting for monitor handle: {e:#}") } } - Connection::Http(_) => loop { - tokio::select! { - _ = monitor_killpill_rx.recv() => { - tracing::info!("Received killpill, exiting"); - break; - }, - _ = tokio::time::sleep(Duration::from_secs(12 * 60 * 60)) => { - tracing::info!("Reloading config after 12 hours"); - initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; - if let Err(e) = reload_license_key(&conn).await { - tracing::error!("Failed to reload license key on agent: {e:#}"); + ref conn @ Connection::Http(_) => { + pub const RELOAD_FREQUENCY: Duration = Duration::from_secs(12 * 60 * 60); + let mut last_time_config_reload: Instant = Instant::now(); + + loop { + tokio::select! { + _ = monitor_killpill_rx.recv() => { + tracing::info!("Received killpill, exiting"); + break; + }, + _ = tokio::time::sleep(Duration::from_secs(30)) => { + // Reload config every 12h + if last_time_config_reload.elapsed() > RELOAD_FREQUENCY { + last_time_config_reload = Instant::now(); + tracing::info!("Reloading config after 12 hours"); + initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; + if let Err(e) = reload_license_key(&conn).await { + tracing::error!("Failed to reload license key on agent: {e:#}"); + } + #[cfg(feature = "enterprise")] + ee_oss::verify_license_key().await; + } + + // update min version explicitly. + // for sql connection it is the part of monitor_db. + windmill_common::worker::update_min_version(conn).await; } - #[cfg(feature = "enterprise")] - ee_oss::verify_license_key().await; - } + }; } - }, + } }; tracing::info!("Monitor exited"); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9be687c613..ff11d3d6ac 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.588.0 + version: 1.589.1 title: Windmill API contact: @@ -13475,6 +13475,21 @@ paths: - blacklisted_at - blacklisted_by + /agent_workers/get_min_version: + get: + summary: get minimum worker version across all workers + operationId: getMinVersion + tags: + - agent_workers + responses: + "200": + description: minimum worker version + content: + application/json: + schema: + type: string + description: Minimum semantic version across all workers (e.g. "1.583.0") + /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 8c8a3eacbe..7e4fa4ada2 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2108,6 +2108,7 @@ async fn cancel_jobs( force_cancel: bool, ) -> error::JsonResult> { let mut uuids = vec![]; + tracing::info!("Cancelling jobs: {:?}", jobs); let mut tx = db.begin().await?; let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj ( workspace_id @@ -2131,7 +2132,7 @@ async fn cancel_jobs( , worker FROM v2_job_queue q JOIN v2_job USING (id) - WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL + WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule' FOR UPDATE SKIP LOCKED ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) .fetch_all(&mut *tx) diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 1e051f064e..fa1ac93f01 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -202,6 +202,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: semver::Error) -> Self { + Self::ArgumentErr(format!("Cannot parse provided semver: {e}")) + } +} + impl Error { /// https://docs.rs/anyhow/1/anyhow/struct.Error.html#display-representations pub fn alt(&self) -> String { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 76c04cbe42..4801b93a78 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -34,6 +34,7 @@ use crate::{ global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, + utils::GIT_SEM_VERSION, KillpillSender, BASE_INTERNAL_URL, DB, }; @@ -1238,19 +1239,16 @@ pub fn get_windmill_memory_usage() -> Option { } } -pub async fn update_min_version(conn: &Connection) -> bool { - tracing::debug!("Updating min version"); - use crate::utils::{GIT_SEM_VERSION, GIT_VERSION}; +pub async fn get_min_version(conn: &Connection) -> error::Result { + use crate::utils::GIT_VERSION; - let cur_version = GIT_SEM_VERSION.clone(); - - let min_version = match conn { + let fetched = match conn { Connection::Sql(pool) => { // fetch all pings with a different version than self from the last 5 minutes. let pings = sqlx::query_scalar!( "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'", GIT_VERSION - ).fetch_all(pool).await.unwrap_or_default(); + ).fetch_all(pool).await?; pings .iter() @@ -1259,10 +1257,34 @@ pub async fn update_min_version(conn: &Connection) -> bool { semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok() }) .min() - .unwrap_or_else(|| cur_version.clone()) } - Connection::Http(_) => { - // TODO: get min version from server, for now we use the current version. Min version should be of no interest for http mode workers + Connection::Http(client) => { + // Fetch min version from server + Some( + client + .get::("/api/agent_workers/get_min_version") + .await + .map(|v| Version::parse(&v))??, + ) + } + }; + + Ok(fetched.unwrap_or_else(|| GIT_SEM_VERSION.clone())) +} + +pub async fn update_min_version(conn: &Connection) -> bool { + tracing::debug!("Updating min version"); + use crate::utils::GIT_SEM_VERSION; + + let cur_version = GIT_SEM_VERSION.clone(); + + let min_version = match get_min_version(conn).await { + Ok(v) => v, + Err(e) => { + tracing::error!( + "Failed to fetch min version: {:#?}, using current version", + e + ); cur_version.clone() } }; diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index c58b73f551..d9d34682ba 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -442,6 +442,7 @@ pub async fn do_bigquery( token.as_str(), timeout_ms, if i == queries.len() - 1 + && s3.is_none() && collection_strategy.collect_last_statement_only(queries.len()) && !collection_strategy.collect_scalar() { diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 2163ac9890..feeb1bd108 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -308,6 +308,7 @@ pub async fn do_mysql( &statement_values, conn_a_ref.clone(), if i == queries.len() - 1 + && s3.is_none() && collection_strategy.collect_last_statement_only(queries.len()) && !collection_strategy.collect_scalar() { diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index d83edf0104..95537a658c 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -431,6 +431,7 @@ pub async fn do_oracledb( vals, conn_a.clone(), if i == queries.len() - 1 + && s3.is_none() && collection_strategy.collect_last_statement_only(queries.len()) && !collection_strategy.collect_scalar() { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index beb66b39a5..08ff7a08e9 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -377,6 +377,7 @@ pub async fn do_postgresql( ¶m_idx_to_arg_and_value, client, if i == queries.len() - 1 + && s3.is_none() && collection_strategy.collect_last_statement_only(queries.len()) && !collection_strategy.collect_scalar() { diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 38eff99ae1..dc0b46e094 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -430,6 +430,7 @@ pub async fn do_snowflake( &token, token_is_keypair, if i == queries.len() - 1 + && s3.is_none() && collection_strategy.collect_last_statement_only(queries.len()) && !collection_strategy.collect_scalar() { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index a31dfbabb8..4434ab9b47 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.588.0"; +export const VERSION = "v1.589.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index b4d37c0cba..25c3fccbd2 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -16,22 +16,30 @@ import { inferSchema, getRawWorkspaceDependencies, } from "../../utils/metadata.ts"; -import { ScriptLanguage, workspaceDependenciesLanguages } from "../../utils/script_common.ts"; import { - inferContentTypeFromFilePath, + ScriptLanguage, + workspaceDependenciesLanguages, } from "../../utils/script_common.ts"; +import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; -import { AppFile } from "./raw_apps.ts"; -import { replaceInlineScripts } from "./apps.ts"; +import { AppFile as RawAppFile } from "./raw_apps.ts"; +import { replaceInlineScripts, AppFile as NormalAppFile } from "./apps.ts"; import { newPathAssigner, SupportedLanguage, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; +import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { requireLogin } from "../../core/auth.ts"; const TOP_HASH = "__app_hash"; +export const APP_BACKEND_FOLDER = "backend"; + +// Union type for app files that can be either raw or normal apps +type AppFile = RawAppFile | NormalAppFile; /** * Generates a hash for all inline scripts in an app directory @@ -39,27 +47,25 @@ const TOP_HASH = "__app_hash"; async function generateAppHash( rawReqs: Record | undefined, folder: string, + rawApp: boolean, defaultTs: "bun" | "deno" | undefined ): Promise> { - const runnablesFolder = path.join(folder, "runnables"); + const runnablesFolder = rawApp + ? path.join(folder, APP_BACKEND_FOLDER) + : folder; const hashes: Record = {}; try { const elems = await FSFSElement(runnablesFolder, [], true); for await (const f of elems.getChildren()) { + if (!rawApp && !f.path.includes(".inline_script.")) { + continue; + } if (exts.some((e) => f.path.endsWith(e))) { - let reqs: string | undefined; - if (rawReqs) { - // Get language name from path - const lang = inferContentTypeFromFilePath(f.path, defaultTs); - // Get lock for that language - [, reqs] = - Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; - } // Embed lock into hash const relativePath = f.path.replace(runnablesFolder + SEP, ""); hashes[relativePath] = await generateHash( - (await f.getContentText()) + (reqs ?? "") + (await f.getContentText()) + JSON.stringify(rawReqs) ); } } @@ -78,13 +84,14 @@ async function generateAppHash( */ export async function generateAppLocksInternal( appFolder: string, + rawApp: boolean, dryRun: boolean, workspace: Workspace, opts: GlobalOptions & { defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean, + noStaleMessage?: boolean ): Promise { if (appFolder.endsWith(SEP)) { appFolder = appFolder.substring(0, appFolder.length - 1); @@ -96,10 +103,15 @@ export async function generateAppLocksInternal( log.info(`Generating locks for app ${appFolder} at ${remote_path}`); } - const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies: Record = + await getRawWorkspaceDependencies(); - - let hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs); + let hashes = await generateAppHash( + rawWorkspaceDependencies, + appFolder, + rawApp, + opts.defaultTs + ); const conf = await import("../../utils/metadata.ts").then((m) => m.readLockfile() @@ -128,7 +140,10 @@ export async function generateAppLocksInternal( } // Read the app file - const appFilePath = path.join(appFolder, "raw_app.yaml"); + const appFilePath = path.join( + appFolder, + rawApp ? "raw_app.yaml" : "app.yaml" + ); const appFile = (await yamlParseFile(appFilePath)) as AppFile; if (!justUpdateMetadataLock) { @@ -148,20 +163,38 @@ export async function generateAppLocksInternal( `Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}` ); - const runnablesPath = path.join(appFolder, "runnables") + SEP; + if (rawApp) { + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER) + SEP; + const rawAppFile = appFile as RawAppFile; - // Replace inline scripts for changed runnables - await replaceInlineScripts(appFile.runnables, runnablesPath); + // Replace inline scripts for changed runnables + replaceInlineScripts(rawAppFile.runnables, runnablesPath, false); - // Update the app runnables with new locks - appFile.runnables = await updateAppRunnables( - workspace, - appFile.runnables, - remote_path, - appFolder, - rawWorkspaceDependencies, - opts.defaultTs - ); + // Update the app runnables with new locks + rawAppFile.runnables = await updateRawAppRunnables( + workspace, + rawAppFile.runnables, + remote_path, + appFolder, + rawWorkspaceDependencies, + opts.defaultTs + ); + } else { + const normalAppFile = appFile as NormalAppFile; + + // Replace inline scripts for normal apps + replaceInlineScripts(normalAppFile.value, appFolder + SEP, false); + + // Update the app value with new locks + normalAppFile.value = await updateAppInlineScripts( + workspace, + normalAppFile.value, + remote_path, + appFolder, + rawWorkspaceDependencies, + opts.defaultTs + ); + } // Write the updated app file writeIfChanged( @@ -174,7 +207,12 @@ export async function generateAppLocksInternal( } // Regenerate hashes after updates - hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs); + hashes = await generateAppHash( + rawWorkspaceDependencies, + appFolder, + rawApp, + opts.defaultTs + ); await clearGlobalLock(appFolder); for (const [scriptPath, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(appFolder, hash, scriptPath); @@ -183,10 +221,68 @@ export async function generateAppLocksInternal( } /** - * Updates locks for all runnables in an app, generating locks inline script by inline script + * Callback type for processing inline scripts found during traversal + */ +type InlineScriptProcessor = ( + inlineScript: any, + context: { + path: string[]; + parentKey: string; + parentObject: any; + } +) => Promise; + +/** + * Traverses an app structure (either app.value for normal apps or app.runnables for raw apps) + * and processes all inline scripts found, returning the updated structure + */ +async function traverseAndProcessInlineScripts( + obj: any, + processor: InlineScriptProcessor, + currentPath: string[] = [] +): Promise { + if (!obj || typeof obj !== "object") { + return obj; + } + + if (Array.isArray(obj)) { + return await Promise.all( + obj.map((item, index) => + traverseAndProcessInlineScripts(item, processor, [ + ...currentPath, + `[${index}]`, + ]) + ) + ); + } + + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + if (key === "inlineScript" && typeof value === "object") { + // Found an inline script - process it + result[key] = await processor(value, { + path: currentPath, + parentKey: key, + parentObject: obj, + }); + } else { + // Recursively process nested objects + result[key] = await traverseAndProcessInlineScripts(value, processor, [ + ...currentPath, + key, + ]); + } + } + + return result; +} + +/** + * Updates locks for all runnables in a raw app, generating locks inline script by inline script * Also writes content and locks back to the runnables folder */ -async function updateAppRunnables( +async function updateRawAppRunnables( workspace: Workspace, runnables: Record, remotePath: string, @@ -194,8 +290,7 @@ async function updateAppRunnables( rawDeps?: Record, defaultTs: "bun" | "deno" = "bun" ): Promise> { - const updatedRunnables = { ...runnables }; - const runnablesFolder = path.join(appFolder, "runnables"); + const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); // Ensure runnables folder exists try { @@ -205,9 +300,14 @@ async function updateAppRunnables( } const pathAssigner = newPathAssigner(defaultTs); + + // Process each runnable + const updatedRunnables: Record = {}; + for (const [runnableId, runnable] of Object.entries(runnables)) { // Only process inline scripts (runnableByName with inlineScript) if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) { + updatedRunnables[runnableId] = runnable; continue; } @@ -216,6 +316,7 @@ async function updateAppRunnables( const content = inlineScript.content; if (!content || !language) { + updatedRunnables[runnableId] = runnable; continue; } @@ -226,21 +327,19 @@ async function updateAppRunnables( `Runnable ${runnableId} content is still an !inline reference, skipping` ) ); + updatedRunnables[runnableId] = runnable; continue; } // Skip frontend scripts - they don't need locks if (language === "frontend") { + updatedRunnables[runnableId] = runnable; continue; } - // Find raw deps for this language if available - const langRawDeps = rawDeps?.[language]; - log.info( colors.gray( - `Generating lock for runnable ${runnableId} (${language})${ - langRawDeps ? " with raw deps" : "" + `Generating lock for runnable ${runnableId} (${language}) }` ) ); @@ -251,7 +350,7 @@ async function updateAppRunnables( content, language, `${remotePath}/${runnableId}`, - langRawDeps + rawDeps ); // Determine file extension for this language @@ -294,12 +393,120 @@ async function updateAppRunnables( ) ); // Continue with other runnables even if one fails + updatedRunnables[runnableId] = runnable; } } return updatedRunnables; } +/** + * Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables + * but for the app.value structure instead of app.runnables + */ +async function updateAppInlineScripts( + workspace: Workspace, + appValue: any, + remotePath: string, + appFolder: string, + rawDeps?: Record, + defaultTs: "bun" | "deno" = "bun" +): Promise { + const pathAssigner = newPathAssigner(defaultTs); + + const processor: InlineScriptProcessor = async (inlineScript, context) => { + const language = inlineScript.language as SupportedLanguage; + const content = inlineScript.content; + + if (!content || !language) { + return inlineScript; + } + + // Skip if content is still an !inline reference (should have been replaced by replaceInlineScripts) + if (typeof content === "string" && content.startsWith("!inline ")) { + log.warn( + colors.yellow( + `Inline script at ${context.path.join( + "." + )} is still an !inline reference, skipping` + ) + ); + return inlineScript; + } + + // Skip frontend scripts - they don't need locks + if (language === "frontend") { + return inlineScript; + } + + // Get the name from the parent object (following extractInlineScriptsForApps pattern) + // For normal apps, the name is stored in the component's "name" property + const scriptName = context.parentObject?.["name"] || "unnamed"; + const scriptPath = `${remotePath}/${context.path.join("/")}`; + + log.info( + colors.gray( + `Generating lock for inline script "${scriptName}" at ${context.path.join( + "." + )} (${language})` + ) + ); + + try { + const lock = await generateInlineScriptLock( + workspace, + content, + language, + scriptPath, + rawDeps + ); + + // Determine file extension for this language (following extractInlineScriptsForApps pattern) + const [basePathO, ext] = pathAssigner.assignPath(scriptName, language); + const basePath = basePathO.replaceAll(SEP, "/"); + const contentPath = path.join(appFolder, `${basePath}${ext}`); + const lockPath = path.join(appFolder, `${basePath}lock`); + + // Write content to file + writeIfChanged(contentPath, content); + + // Write lock to file if it exists + if (lock && lock !== "") { + writeIfChanged(lockPath, lock); + } + + // Update the inline script with !inline references + const inlineContentRef = `!inline ${basePath}${ext}`; + const inlineLockRef = + lock && lock !== "" ? `!inline ${basePath}lock` : ""; + + log.info( + colors.gray( + ` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` + ) + ); + + return { + ...inlineScript, + content: inlineContentRef, + lock: inlineLockRef, + }; + } catch (error: any) { + log.error( + colors.red( + `Failed to generate lock for inline script at ${context.path.join( + "." + )}: ${error.message}` + ) + ); + // Return original on error + return inlineScript; + } + }; + + return await traverseAndProcessInlineScripts(appValue, processor); +} + /** * Generates a lock for a single inline script using the dependencies endpoint */ @@ -308,7 +515,7 @@ async function generateInlineScriptLock( content: string, language: string, scriptPath: string, - rawDeps?: string + rawWorkspaceDependencies: Record | undefined ): Promise { const extraHeaders = getHeaders(); @@ -329,7 +536,11 @@ async function generateInlineScriptLock( script_path: scriptPath, }, ], - raw_deps: rawDeps, + raw_workspace_dependencies: + rawWorkspaceDependencies && + Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies + : null, entrypoint: scriptPath, }), } @@ -386,7 +597,7 @@ export interface InferredSchemaResult { */ export async function inferRunnableSchemaFromFile( appFolder: string, - runnableFilePath: string, + runnableFilePath: string ): Promise { // Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable") const fileName = path.basename(runnableFilePath); @@ -406,7 +617,7 @@ export async function inferRunnableSchemaFromFile( // Read the app file to get the language const appFilePath = path.join(appFolder, "raw_app.yaml"); - const appFile = (await yamlParseFile(appFilePath)) as AppFile; + const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; if (!appFile.runnables?.[runnableId]) { log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`)); @@ -423,10 +634,12 @@ export async function inferRunnableSchemaFromFile( const inlineScript = runnable.inlineScript; const language = inlineScript.language as SupportedLanguage; - - // Read the actual content from the file - const fullFilePath = path.join(appFolder, "runnables", runnableFilePath); + const fullFilePath = path.join( + appFolder, + APP_BACKEND_FOLDER, + runnableFilePath + ); let content: string; try { content = await Deno.readTextFile(fullFilePath); @@ -461,3 +674,144 @@ export async function inferRunnableSchemaFromFile( return undefined; } } + +function getAppFolders(elems: Record, extension: string) { + return Object.keys(elems) + .filter((p) => p.endsWith(SEP + extension)) + .map((p) => p.substring(0, p.length - (SEP + extension).length)); +} + +export async function generateLocksCommand( + opts: GlobalOptions & { + yes?: boolean; + dryRun?: boolean; + defaultTs?: "bun" | "deno"; + } & SyncOptions, + appPath: string | undefined +) { + const { generateAppLocksInternal } = await import("./app_metadata.ts"); + const { elementsToMap, FSFSElement } = await import("../sync/sync.ts"); + const { ignoreF } = await import("../sync/sync.ts"); + const { Confirm } = await import("../../../deps.ts"); + + if (appPath == "") { + appPath = undefined; + } + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + opts = await mergeConfigWithConfigFile(opts); + + if (appPath) { + //TODO: Generate metadata for a specific raw app but handle normal apps to + throw new Error("Not implemented"); + // Generate metadata for a specific app + // await generateAppLocksInternal( + // appPath, + // true, + // false, + // workspace, + // opts, + // false, + // false + // ); + } else { + // Generate metadata for all apps + const ignore = await ignoreF(opts); + const elems = await elementsToMap( + await FSFSElement(Deno.cwd(), [], true), + (p, isD) => { + return ( + ignore(p, isD) || + (!isD && + !p.endsWith(SEP + "raw_app.yaml") && + !p.endsWith(SEP + "app.yaml")) + ); + }, + false, + {} + ); + + const rawAppFolders = getAppFolders(elems, "raw_app.yaml"); + const appFolders = getAppFolders(elems, "app.yaml"); + + let hasAny = false; + log.info( + `Checking metadata for all apps (${appFolders.length}) and raw apps (${rawAppFolders.length})` + ); + for (const appFolder of rawAppFolders) { + const candidate = await generateAppLocksInternal( + appFolder, + true, + true, + workspace, + opts, + false, + true + ); + if (candidate) { + hasAny = true; + log.info(colors.green(`+ ${candidate}`)); + } + } + + for (const appFolder of appFolders) { + const candidate = await generateAppLocksInternal( + appFolder, + false, + true, + workspace, + opts, + false, + true + ); + if (candidate) { + hasAny = true; + log.info(colors.green(`+ ${candidate}`)); + } + } + + if (hasAny) { + if (opts.dryRun) { + log.info(colors.gray(`Dry run complete.`)); + return; + } + if ( + !opts.yes && + !(await Confirm.prompt({ + message: "Update the metadata of the above apps?", + default: true, + })) + ) { + return; + } + } else { + log.info(colors.green.bold("No metadata to update")); + return; + } + + for (const appFolder of rawAppFolders) { + await generateAppLocksInternal( + appFolder, + true, + false, + workspace, + opts, + false, + true + ); + } + + for (const appFolder of appFolders) { + await generateAppLocksInternal( + appFolder, + false, + false, + workspace, + opts, + false, + true + ); + } + } +} diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index 91933c0f02..e2b715b199 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -16,7 +16,7 @@ import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; -import { isVersionsGeq15851 } from "../sync/global.ts"; +import { isVersionsGeq1585 } from "../sync/global.ts"; export interface AppFile { value: any; @@ -52,19 +52,31 @@ export function repopulateFields(runnables: Record) { } }); } -export function replaceInlineScripts(rec: any, localPath: string) { +export function replaceInlineScripts( + rec: any, + localPath: string, + addType: boolean +) { if (!rec) { return; } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { if (k == "runType") { - if (isVersionsGeq15851()) { - rec["type"] = "path"; + if (addType) { + if (isVersionsGeq1585()) { + rec["type"] = "path"; + } else { + rec["type"] = "runnableByPath"; + } } } else if (k == "inlineScript" && typeof v == "object") { - if (isVersionsGeq15851()) { - rec["type"] = "inline"; + if (addType) { + if (isVersionsGeq1585()) { + rec["type"] = "inline"; + } else { + rec["type"] = "runnableByName"; + } } const o: Record = v as any; @@ -77,7 +89,7 @@ export function replaceInlineScripts(rec: any, localPath: string) { o["lock"] = readInlinePathSync(basePath); } } else { - replaceInlineScripts(v, localPath); + replaceInlineScripts(v, localPath, addType); } }); } @@ -121,7 +133,7 @@ export async function pushApp( const path = localPath + "app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts(localApp.value, localPath); + replaceInlineScripts(localApp.value, localPath, true); await generatingPolicy( localApp, remotePath, @@ -230,7 +242,7 @@ const command = new Command() "Default TypeScript runtime (bun or deno)" ) .action(async (opts: any, appFolder: string | undefined) => { - const { generateLocksCommand } = await import("./raw_apps.ts"); + const { generateLocksCommand } = await import("./app_metadata.ts"); await generateLocksCommand(opts, appFolder); }); diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index ed8b565e10..327f06864e 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -7,6 +7,7 @@ import { open, windmillUtils, yamlParseFile, + SEP, } from "../../../deps.ts"; import { GlobalOptions } from "../../types.ts"; import * as http from "node:http"; @@ -16,7 +17,12 @@ import process from "node:process"; import { Buffer } from "node:buffer"; import { writeFileSync } from "node:fs"; import { WebSocketServer, WebSocket } from "npm:ws"; -import { getDevBuildOptions, ensureNodeModules, createFrameworkPlugins, detectFrameworks } from "./bundle.ts"; +import { + getDevBuildOptions, + ensureNodeModules, + createFrameworkPlugins, + detectFrameworks, +} from "./bundle.ts"; import { wmillTsDev as wmillTs } from "./wmillTsDev.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { resolveWorkspace } from "../../core/context.ts"; @@ -24,7 +30,10 @@ import { requireLogin } from "../../core/auth.ts"; import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts"; import { replaceInlineScripts } from "./apps.ts"; import { Runnable } from "./metadata.ts"; -import { inferRunnableSchemaFromFile } from "./app_metadata.ts"; +import { + APP_BACKEND_FOLDER, + inferRunnableSchemaFromFile, +} from "./app_metadata.ts"; const DEFAULT_PORT = 4000; const DEFAULT_HOST = "localhost"; @@ -92,8 +101,8 @@ async function dev(opts: DevOptions) { log.error( colors.red( `Error: The dev command must be run inside a .raw_app folder.\n` + - `Current directory: ${currentDirName}\n` + - `Please navigate to a folder ending with '.raw_app' before running this command.` + `Current directory: ${currentDirName}\n` + + `Please navigate to a folder ending with '.raw_app' before running this command.` ) ); Deno.exit(1); @@ -105,7 +114,7 @@ async function dev(opts: DevOptions) { log.error( colors.red( `Error: raw_app.yaml not found in current directory.\n` + - `The dev command must be run in a .raw_app folder containing a raw_app.yaml file.` + `The dev command must be run in a .raw_app folder containing a raw_app.yaml file.` ) ); Deno.exit(1); @@ -133,7 +142,8 @@ async function dev(opts: DevOptions) { // Detect frameworks to determine default entry point const frameworks = detectFrameworks(process.cwd()); - const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const defaultEntry = + frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; const entryPoint = opts.entry ?? defaultEntry; // Verify entry point exists @@ -206,7 +216,6 @@ async function dev(opts: DevOptions) { }, }; - // Create esbuild context const ctx = await esbuild.context({ ...buildOptions, @@ -242,7 +251,7 @@ async function dev(opts: DevOptions) { await ctx.rebuild(); // Watch runnables folder for changes - const runnablesPath = path.join(process.cwd(), "runnables"); + const runnablesPath = path.join(process.cwd(), APP_BACKEND_FOLDER); let runnablesWatcher: Deno.FsWatcher | undefined; if (fs.existsSync(runnablesPath)) { @@ -262,7 +271,10 @@ async function dev(opts: DevOptions) { // Process each changed path with individual debouncing for (const changedPath of event.paths) { const relativePath = path.relative(process.cwd(), changedPath); - const relativeToRunnables = path.relative(runnablesPath, changedPath); + const relativeToRunnables = path.relative( + runnablesPath, + changedPath + ); // Skip non-modify events for schema inference if (event.kind !== "modify" && event.kind !== "create") { @@ -276,7 +288,9 @@ async function dev(opts: DevOptions) { // Log the change event log.info( - colors.cyan(`📝 Runnable changed [${event.kind}]: ${relativePath}`) + colors.cyan( + `📝 Runnable changed [${event.kind}]: ${relativePath}` + ) ); // Debounce schema inference per file (wait for typing to finish) @@ -288,7 +302,9 @@ async function dev(opts: DevOptions) { delete schemaInferenceTimeouts[changedPath]; try { - log.info(colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`)); + log.info( + colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`) + ); // Infer schema for this runnable (returns schema in memory, doesn't write to file) const result = await inferRunnableSchemaFromFile( process.cwd(), @@ -299,7 +315,15 @@ async function dev(opts: DevOptions) { // log.info(colors.green(` Runnable ID: ${result.runnableId}`)); // Store inferred schema in memory inferredSchemas[result.runnableId] = result.schema; - log.info(colors.green(` Inferred Schemas: ${JSON.stringify(inferredSchemas, null, 2)}`)); + log.info( + colors.green( + ` Inferred Schemas: ${JSON.stringify( + inferredSchemas, + null, + 2 + )}` + ) + ); // Regenerate wmill.d.ts with updated schema from memory await genRunnablesTs(inferredSchemas); } @@ -468,7 +492,9 @@ async function dev(opts: DevOptions) { case "backendAsync": { // Run a runnable asynchronously and return job ID immediately log.info( - colors.blue(`[backendAsync] Running runnable async: ${runnable_id}`) + colors.blue( + `[backendAsync] Running runnable async: ${runnable_id}` + ) ); try { const runnables = await loadRunnables(); @@ -628,7 +654,10 @@ const command = new Command() .option("--host ", "Host to bind the dev server to", { default: DEFAULT_HOST, }) - .option("--entry ", "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)") + .option( + "--entry ", + "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)" + ) .option("--no-open", "Don't automatically open the browser") .action(dev as any); @@ -671,7 +700,11 @@ async function loadRunnables(): Promise> { const rawApp = (await yamlParseFile( path.join(localPath, "raw_app.yaml") )) as any; - replaceInlineScripts(rawApp.runnables, path.join(localPath, "runnables/")); + replaceInlineScripts( + rawApp.runnables, + path.join(localPath, APP_BACKEND_FOLDER) + SEP, + true + ); return rawApp?.runnables ?? {}; } catch (error: any) { @@ -707,7 +740,10 @@ async function executeRunnable( } } - if ((runnable.type === "inline" || runnable.type === "runnableByName") && runnable.inlineScript) { + if ( + (runnable.type === "inline" || runnable.type === "runnableByName") && + runnable.inlineScript + ) { const inlineScript = runnable.inlineScript; if (inlineScript.id !== undefined) { requestBody.id = inlineScript.id; @@ -719,7 +755,10 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; - } else if ((runnable.type === "path" || runnable.type === "runnableByPath") && runnable.path) { + } else if ( + (runnable.type === "path" || runnable.type === "runnableByPath") && + runnable.path + ) { const runType = runnable.runType ?? "script"; requestBody.path = runType !== "hubscript" diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index cb2d42f05b..200b697024 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -15,7 +15,7 @@ import { GlobalOptions, isSuperset } from "../../types.ts"; import { replaceInlineScripts, repopulateFields } from "./apps.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; -import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; +import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; export interface AppFile { runnables: any; @@ -39,7 +39,12 @@ async function collectAppFiles( if (entry.isDirectory) { // Skip the runnables and node_modules subfolders - if (entry.name === "runnables" || entry.name === "node_modules" || entry.name === "dist" || entry.name === ".claude") { + if ( + entry.name === APP_BACKEND_FOLDER || + entry.name === "node_modules" || + entry.name === "dist" || + entry.name === ".claude" + ) { continue; } await readDirRecursive(fullPath + SEP, relativePath + SEP); @@ -96,15 +101,20 @@ export async function pushRawApp( } const path = localPath + "raw_app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts(localApp.runnables, localPath + SEP + "runnables/"); - repopulateFields(localApp.runnables) + replaceInlineScripts( + localApp.runnables, + localPath + SEP + APP_BACKEND_FOLDER + SEP, + true + ); + repopulateFields(localApp.runnables); await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false); const files = await collectAppFiles(localPath); async function createBundleRaw() { log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); // Detect frameworks to determine entry point const frameworks = detectFrameworks(localPath); - const entryFile = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const entryFile = + frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; const entryPoint = localPath + entryFile; return await createBundle({ entryPoint: entryPoint, @@ -183,102 +193,6 @@ export async function generatingPolicy( } } -export async function generateLocksCommand( - opts: GlobalOptions & { - yes?: boolean; - dryRun?: boolean; - defaultTs?: "bun" | "deno"; - } & SyncOptions, - appPath: string | undefined -) { - const { generateAppLocksInternal } = await import("./app_metadata.ts"); - const { elementsToMap, FSFSElement } = await import("../sync/sync.ts"); - const { ignoreF } = await import("../sync/sync.ts"); - const { Confirm } = await import("../../../deps.ts"); - - if (appPath == "") { - appPath = undefined; - } - - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); - opts = await mergeConfigWithConfigFile(opts); - - if (appPath) { - // Generate metadata for a specific app - await generateAppLocksInternal( - appPath, - false, - workspace, - opts, - false, - false, - ); - } else { - // Generate metadata for all apps - const ignore = await ignoreF(opts); - const elems = await elementsToMap( - await FSFSElement(Deno.cwd(), [], true), - (p, isD) => { - return ignore(p, isD) || (!isD && !p.endsWith(SEP + "raw_app.yaml")); - }, - false, - {} - ); - - const appFolders = Object.keys(elems) - .filter((p) => p.endsWith(SEP + "raw_app.yaml")) - .map((p) => p.substring(0, p.length - (SEP + "raw_app.yaml").length)); - - let hasAny = false; - log.info("Checking metadata for all apps:"); - for (const appFolder of appFolders) { - const candidate = await generateAppLocksInternal( - appFolder, - true, - workspace, - opts, - false, - true, - ); - if (candidate) { - hasAny = true; - log.info(colors.green(`+ ${candidate}`)); - } - } - - if (hasAny) { - if (opts.dryRun) { - log.info(colors.gray(`Dry run complete.`)); - return; - } - if ( - !opts.yes && - !(await Confirm.prompt({ - message: "Update the metadata of the above apps?", - default: true, - })) - ) { - return; - } - } else { - log.info(colors.green.bold("No metadata to update")); - return; - } - - for (const appFolder of appFolders) { - await generateAppLocksInternal( - appFolder, - false, - workspace, - opts, - false, - true, - ); - } - } -} - async function pushRawAppCommand( opts: GlobalOptions, filePath: string, diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index fb5c5539da..2eadf1b705 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -18,9 +18,8 @@ import { } from "../../utils/metadata.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; - import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; -import { exts, } from "../script/script.ts"; +import { exts } from "../script/script.ts"; import { FSFSElement } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; import { FlowFile } from "./flow.ts"; @@ -67,8 +66,13 @@ export async function generateFlowLockInternal( } // Always get out-of-sync workspace dependencies - const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); - let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); + const rawWorkspaceDependencies: Record = + await getRawWorkspaceDependencies(); + let hashes = await generateFlowHash( + rawWorkspaceDependencies, + folder, + opts.defaultTs + ); const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { @@ -115,7 +119,7 @@ export async function generateFlowLockInternal( log, folder + SEP!, SEP, - changedScripts, + changedScripts // (path: string, newPath: string) => Deno.renameSync(path, newPath), // (path: string) => Deno.removeSync(path) ); @@ -145,7 +149,11 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); + hashes = await generateFlowHash( + rawWorkspaceDependencies, + folder, + opts.defaultTs + ); await clearGlobalLock(folder); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); @@ -153,8 +161,6 @@ export async function generateFlowLockInternal( log.info(colors.green(`Flow ${remote_path} lockfiles updated`)); } - - export async function updateFlow( workspace: Workspace, flow_value: FlowValue, @@ -164,7 +170,9 @@ export async function updateFlow( let rawResponse; if (Object.keys(rawWorkspaceDependencies).length > 0) { - log.info(colors.blue("Using raw workspace dependencies for flow dependencies")); + log.info( + colors.blue("Using raw workspace dependencies for flow dependencies") + ); // generate the script lock running a dependency job in Windmill and update it inplace const extraHeaders = getHeaders(); @@ -181,9 +189,10 @@ export async function updateFlow( flow_value, path: remotePath, use_local_lockfiles: true, - raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 - ? rawWorkspaceDependencies - : null, + raw_workspace_dependencies: + Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies + : null, }), } ); diff --git a/cli/src/commands/sync/global.ts b/cli/src/commands/sync/global.ts index bd64e1452e..1b5e975aa6 100644 --- a/cli/src/commands/sync/global.ts +++ b/cli/src/commands/sync/global.ts @@ -21,11 +21,11 @@ export function updateGlobalVersions(version: string) { } } -export function isVersionsGeq15851(): boolean { +export function isVersionsGeq1585(): boolean { return ( GLOBAL_VERSIONS.remoteMajor !== undefined && GLOBAL_VERSIONS.remoteMajor >= 1 && GLOBAL_VERSIONS.remoteMinor !== undefined && - GLOBAL_VERSIONS.remoteMinor >= 5851 + GLOBAL_VERSIONS.remoteMinor >= 585 ); } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 8fffa5ed32..57222f482b 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -36,6 +36,7 @@ import { import { handleFile } from "../script/script.ts"; import { deepEqual, + fetchRemoteVersion, isFileResource, isRawAppFile, isWorkspaceDependencies, @@ -74,8 +75,10 @@ import { import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; import { isExecutionModeAnonymous } from "../app/apps.ts"; -import { generateAppLocksInternal } from "../app/app_metadata.ts"; -import { updateGlobalVersions } from "./global.ts"; +import { + APP_BACKEND_FOLDER, + generateAppLocksInternal, +} from "../app/app_metadata.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< @@ -549,7 +552,7 @@ function ZipFSElement( for (const s of inlineScripts) { yield { isDirectory: false, - path: path.join(finalPath, "runnables", s.path), + path: path.join(finalPath, APP_BACKEND_FOLDER, s.path), async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { @@ -1737,6 +1740,19 @@ export async function pull( await generateAppLocksInternal( change, false, + true, + workspace, + opts, + true, + true + ); + } + for (const change of tracker.apps) { + log.info(`Updating lock metadata for app ${change}`); + await generateAppLocksInternal( + change, + false, + false, workspace, opts, true, @@ -1983,6 +1999,7 @@ export async function push( const staleScripts: string[] = []; const staleFlows: string[] = []; + const staleApps: string[] = []; for (const change of tracker.scripts) { const stale = await generateScriptMetadataInternal( @@ -2036,16 +2053,51 @@ export async function push( log.info(""); } - const version = await fetchVersion(workspace.remote); - if (version) { - updateGlobalVersions(version); + for (const change of tracker.apps) { + const stale = await generateAppLocksInternal( + change, + false, + true, + workspace, + opts, + true, + true + ); + if (stale) { + staleApps.push(stale); + } } - log.info(colors.gray("Remote version: " + version)); + + for (const change of tracker.rawApps) { + const stale = await generateAppLocksInternal( + change, + true, + true, + workspace, + opts, + true, + true + ); + if (stale) { + staleApps.push(stale); + } + } + + if (staleApps.length > 0) { + log.warn( + "Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:" + ); + for (const stale of staleApps) { + log.warn(stale); + } + log.info(""); + } + + await fetchRemoteVersion(workspace); log.info( `remote (${workspace.name}) <- local: ${changes.length} changes to apply` ); - // Handle JSON output for dry-run if (opts.dryRun && opts.jsonOutput) { const result = { diff --git a/cli/src/main.ts b/cli/src/main.ts index 125429b81f..227ba4e003 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -25,7 +25,6 @@ import instance from "./commands/instance/instance.ts"; import workerGroups from "./commands/worker-groups/worker_groups.ts"; import dev from "./commands/dev/dev.ts"; -import { fetchVersion } from "./core/context.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; import { getHeaders, getIsWin } from "./utils/utils.ts"; @@ -39,6 +38,7 @@ import queues from "./commands/queues/queues.ts"; import dependencies from "./commands/dependencies/dependencies.ts"; import init from "./commands/init/init.ts"; import jobs from "./commands/jobs/jobs.ts"; +import { fetchVersion } from "./core/context.ts"; export { flow, @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.588.0"; +export const VERSION = "1.589.1"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index bda8dea78a..ceee3d9d7a 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -4,6 +4,8 @@ import { colors, encodeHex, log, SEP } from "../../deps.ts"; import crypto from "node:crypto"; +import { fetchVersion } from "../core/context.ts"; +import { updateGlobalVersions } from "../commands/sync/global.ts"; export function deepEqual(a: T, b: T): boolean { if (a === b) return true; @@ -151,11 +153,11 @@ export function isFileResource(path: string): boolean { } export function isRawAppFile(path: string): boolean { - return path.includes(".raw_app" + SEP) ; + return path.includes(".raw_app" + SEP); } export function isWorkspaceDependencies(path: string): boolean { - return path.startsWith("dependencies/") + return path.startsWith("dependencies/"); } export function printSync(input: string | Uint8Array, to = Deno.stdout) { @@ -263,3 +265,13 @@ export function writeIfChanged(path: string, content: string): boolean { Deno.writeTextFileSync(path, content); return true; // File was written } + +export async function fetchRemoteVersion( + workspace: Workspace +): Promise { + const version = await fetchVersion(workspace.remote); + if (version) { + updateGlobalVersions(version); + } + log.info(colors.gray("Remote version: " + version)); +} diff --git a/flake.nix b/flake.nix index b818e0245a..8801302dee 100644 --- a/flake.nix +++ b/flake.nix @@ -143,14 +143,12 @@ ] ++ (with pkgs; [ # Essentials rust - cargo-watch - cargo-sweep git - xcaddy sqlx-cli - sccache - nsjail + + # Build/helper scripts jq + gnused # other implementations are inconsistent on osx # Python flock @@ -171,6 +169,7 @@ oracle-instantclient ansible ruby_3_4 + cargo-sweep # We use it for rust # LSP/Local dev svelte-language-server @@ -182,6 +181,12 @@ kubernetes-helm conntrack-tools # To run minikube without driver (--driver=none) cri-tools + + # Extra + xcaddy + cargo-watch + nsjail + sccache ]); packages = [ (pkgs.writeScriptBin "wm-caddy" '' diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f2cdecbef7..29c8aa120a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.588.0", + "version": "1.589.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.588.0", + "version": "1.589.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index ccfbc59b6a..4380aa4359 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.588.0", + "version": "1.589.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index c46f85410d..326f08b686 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -782,11 +782,19 @@ {:then schema} {#if !schema || !schema.properties} - {#await import('$lib/components/JsonEditor.svelte')} - - {:then Module} - - {/await} +
+
+ {#await import('$lib/components/JsonEditor.svelte')} + + {:then Module} + + {/await} +
+
+ Error loading json schema resource {format.substring('jsonschema-'.length)}, please + check if the resource exists and is a valid json schema. +
+
{:else}
+ console.log( + 'suspendStatusVal', + Object.entries(suspendStatus.val || {}).map(([k, v]) => [k, v.job.id]) + ) + ) diff --git a/frontend/src/lib/components/schema/jsonSchemaResource.svelte.ts b/frontend/src/lib/components/schema/jsonSchemaResource.svelte.ts index 0d4fee8f2a..b51c802838 100644 --- a/frontend/src/lib/components/schema/jsonSchemaResource.svelte.ts +++ b/frontend/src/lib/components/schema/jsonSchemaResource.svelte.ts @@ -18,6 +18,10 @@ export async function getJsonSchemaFromResource(path: string, workspace: string) path, workspace }) + if (!resourceValue || !resourceValue == null) { + console.warn(`JSON schema resource ${path} not found`) + return null + } const parsedResource = jsonSchemaResourceSchema.safeParse(resourceValue) if (parsedResource.success) { @@ -29,7 +33,11 @@ export async function getJsonSchemaFromResource(path: string, workspace: string) return parsedResource.data.schema } else { console.error('Invalid JSON schema resource:', parsedResource.error) - sendUserToast('Invalid JSON schema resource: ' + parsedResource.error, true) + sendUserToast( + `Error loading json schema resource ${path}, Invalid JSON schema resource: ` + + parsedResource.error, + true + ) } } catch (err) { console.error(err) diff --git a/lsp/Pipfile b/lsp/Pipfile index bc4412c77d..c1bca421e6 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.588.0" -wmill_pg = ">=1.588.0" +wmill = ">=1.589.1" +wmill_pg = ">=1.589.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index bb797b7ff1..d0d23c5875 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.588.0 + version: 1.589.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f05ba1ad19..5a28e62798 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.588.0' + ModuleVersion = '1.589.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index df1a0f99ce..a5724e1e2f 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.588.0" +version = "1.589.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 64de8115dc..e54fcce2c5 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.588.0" +version = "1.589.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 763b5bcd87..71db9fc91a 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.588.0", + "version": "1.589.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index e271fd0707..aa3cf1a53f 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.588.0", + "version": "1.589.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 31f17667ee..77d77680b5 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.588.0 +1.589.1