From 02596e888f6e6d7e19fac0ac97de866e6df409ec Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:49:31 +0100 Subject: [PATCH] fix: mixed version error (#7686) * fix mixed version error Signed-off-by: pyranota * fix comp error Signed-off-by: pyranota * fix Signed-off-by: pyranota --------- Signed-off-by: pyranota --- backend/src/main.rs | 2 +- backend/src/monitor.rs | 1 + backend/windmill-api/openapi.yaml | 18 ++- backend/windmill-api/src/lib.rs | 12 +- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/min_version.rs | 111 +++++++++++++----- .../(root)/(logged)/workers/+page.svelte | 30 +++-- 7 files changed, 126 insertions(+), 49 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 9450e8aa14..6dafd54d74 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1516,7 +1516,7 @@ Windmill Community Edition {GIT_VERSION} // update min version explicitly. // for sql connection it is the part of monitor_db. // TODO: pass worker names for min keep-alive alerts (for HTTP connection) - windmill_common::min_version::update_min_version(conn, false, vec![], false).await; + windmill_common::min_version::update_min_version(conn, true, vec![], false).await; } }; } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ba892394e4..0d65daaa89 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -219,6 +219,7 @@ pub async fn initial_load( e ) } + windmill_common::min_version::store_min_keep_alive_version(db).await; } } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index dd5e0d98d9..a28e820316 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1413,17 +1413,27 @@ paths: /min_keep_alive_version: get: - summary: get minimum worker version required to stay alive + summary: get minimum worker versions required to stay alive operationId: getMinKeepAliveVersion tags: - setting responses: "200": - description: minimum keep-alive version + description: minimum keep-alive versions for workers and agents content: - text/plain: + application/json: schema: - type: string + type: object + required: + - worker + - agent + properties: + worker: + type: string + description: minimum version for normal workers + agent: + type: string + description: minimum version for agent workers /.well-known/jwks.json: get: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index db561b7ea6..c18c077880 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -38,7 +38,7 @@ use axum::body::Body; use axum::extract::DefaultBodyLimit; use axum::http::HeaderValue; use axum::response::Response; -use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router}; +use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router}; use db::DB; use reqwest::Client; #[cfg(feature = "oauth2")] @@ -906,9 +906,13 @@ async fn git_v() -> String { format!("CE {GIT_VERSION}") } -async fn min_keep_alive_version() -> String { - let v = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION; - format!("{}.{}.{}", v.0, v.1, v.2) +async fn min_keep_alive_version() -> Json { + let worker = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION; + let agent = windmill_common::min_version::AGENT_MIN_KEEP_ALIVE_VERSION; + Json(serde_json::json!({ + "worker": format!("{}.{}.{}", worker.0, worker.1, worker.2), + "agent": format!("{}.{}.{}", agent.0, agent.1, agent.2) + })) } #[cfg(not(feature = "enterprise"))] diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 890fba7821..59a941701b 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -49,6 +49,7 @@ pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; +pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version"; pub const ENV_SETTINGS: &[&str] = &[ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/min_version.rs b/backend/windmill-common/src/min_version.rs index 85e0d48950..0af6e5f64e 100644 --- a/backend/windmill-common/src/min_version.rs +++ b/backend/windmill-common/src/min_version.rs @@ -19,17 +19,23 @@ pub const MIN_VERSION_IS_AT_LEAST_1_427: VC = vc(1, 427, 0, "Flow version lite t // workers below this version should be terminated automatically. /// Minimum version workers must have to stay connected. -/// Served via: GET /api/settings/min_keep_alive_version +/// Served via: GET /api/min_keep_alive_version (returns { worker, agent }) /// Also used by vc() for compile-time checks. pub const MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 400, 0); -// Compile-time check: must lag at least 50 minor versions behind current. -// NOTE: The 50 version lag is a constant and should NEVER be changed. If this check +/// Minimum version agent workers must have to stay connected. +/// Served via: GET /api/min_keep_alive_version (returns { worker, agent }) +pub const AGENT_MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 0, 0); + +// Compile-time check: MIN_KEEP_ALIVE_VERSION must lag at least 50 minor versions behind current, +// AGENT_MIN_KEEP_ALIVE_VERSION must lag at least 100 minor versions behind current. +// NOTE: These version lags are constants and should NEVER be changed. If this check // fails, wait until enough versions have passed rather than reducing the lag requirement. // Skip check if GIT_VERSION is "unknown-version" (no git tags available during build) const _: () = assert!( !const_str::contains!(crate::utils::GIT_VERSION, ".") || - const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50 + (const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50 + && const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - AGENT_MIN_KEEP_ALIVE_VERSION.1 >= 100) ); // ============ Implementation ============ @@ -124,8 +130,8 @@ pub async fn get_min_version(conn: &Connection) -> error::Result { } /// Updates MIN_VERSION and optionally checks min keep-alive version for workers. -/// If `_worker_mode` is true, fetches min keep-alive version from server and sends alerts for each worker. -/// If `initial_load` is true, skips the HTTP fetch to min_keep_alive_version endpoint (server may not be ready). +/// If `_worker_mode` is true, checks min keep-alive version and sends critical alerts. +/// If `initial_load` is true, skips the min keep-alive check (server may not be ready). pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_names: Vec, _initial_load: bool) { // Update MIN_VERSION match get_min_version(conn).await { @@ -141,34 +147,79 @@ pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_n Err(e) => tracing::error!("Failed to fetch min version: {:#?}", e), } - // Workers fetch min keep-alive version from server and send alerts + // Workers check min keep-alive version and send critical alerts // Skip on initial_load since the server may not be ready yet #[cfg(all(feature = "enterprise", feature = "private"))] if _worker_mode && !_initial_load { - if let Connection::Sql(db) = conn { - let url = format!("{}/api/min_keep_alive_version", *crate::BASE_INTERNAL_URL); - match crate::utils::HTTP_CLIENT.get(&url).send().await { - Ok(resp) => match resp.text().await { - Ok(v) => match Version::parse(&v) { - Ok(min_keep_alive) => { - let current = GIT_SEM_VERSION.clone(); - for worker_name in &_worker_names { - crate::ee::simple_alert_helper( - format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade recommended."), - format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."), - &format!("worker-below-min-keep-alive-{worker_name}"), - || current < min_keep_alive, - Some("admins"), - db, - ).await; - } - } - Err(e) => tracing::error!("Failed to parse min keep-alive version: {:#?}", e), - }, - Err(e) => tracing::error!("Failed to read min keep-alive version response: {:#?}", e), - }, - Err(e) => tracing::error!("Failed to fetch min keep-alive version: {:#?}", e), + let min_keep_alive_version: Option = match conn { + Connection::Sql(db) => { + match crate::global_settings::load_value_from_global_settings( + db, + crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING, + ).await { + Ok(Some(v)) => v.as_str().and_then(|s| Version::parse(s).ok()), + Ok(None) => None, + Err(e) => { + tracing::error!("Failed to load min keep-alive version from global_settings: {:#?}", e); + None + } + } + } + Connection::Http(client) => { + match client.get::("/api/min_keep_alive_version").await { + Ok(resp) => resp + .get("agent") + .and_then(|v| v.as_str()) + .and_then(|v| Version::parse(v).ok()), + Err(e) => { + tracing::error!("Failed to fetch min keep-alive version: {:#?}", e); + None + } + } + } + }; + + if let Some(min_keep_alive) = min_keep_alive_version { + let current = GIT_SEM_VERSION.clone(); + match conn { + Connection::Sql(db) => { + for worker_name in &_worker_names { + crate::ee::simple_alert_helper( + format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."), + format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."), + &format!("worker-below-min-keep-alive-{worker_name}"), + || current < min_keep_alive, + Some("admins"), + db, + ).await; + } + } + Connection::Http(_) => { + if current < min_keep_alive { + tracing::warn!( + "Agent worker version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately." + ); + } + } } } } } + +/// Stores the min keep-alive version in global_settings. +/// Called by server on startup, NOT by workers. +pub async fn store_min_keep_alive_version(db: &sqlx::Pool) { + let version = format!( + "{}.{}.{}", + MIN_KEEP_ALIVE_VERSION.0, + MIN_KEEP_ALIVE_VERSION.1, + MIN_KEEP_ALIVE_VERSION.2 + ); + if let Err(e) = crate::global_settings::set_value_in_global_settings( + db, + crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING, + serde_json::json!(version), + ).await { + tracing::error!("Failed to store min keep-alive version in global_settings: {:#?}", e); + } +} diff --git a/frontend/src/routes/(root)/(logged)/workers/+page.svelte b/frontend/src/routes/(root)/(logged)/workers/+page.svelte index b9d7a93399..bf959dfb58 100644 --- a/frontend/src/routes/(root)/(logged)/workers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workers/+page.svelte @@ -56,6 +56,7 @@ let customTags: string[] | undefined = $state(undefined) let serverVersion: string | undefined = $state(undefined) let minKeepAliveVersion: string | undefined = $state(undefined) + let agentMinKeepAliveVersion: string | undefined = $state(undefined) type VersionWarning = 'none' | 'note' | 'warning' | 'critical' | 'newer' @@ -68,7 +69,7 @@ return [major, minor || 0, patch || 0] } - function getVersionWarning(workerVersion: string): VersionWarning { + function getVersionWarning(workerVersion: string, isAgent: boolean = false): VersionWarning { if (!serverVersion) return 'none' const server = parseVersion(serverVersion) const worker = parseVersion(workerVersion) @@ -82,15 +83,19 @@ const minorLag = server[1] - worker[1] if (minorLag <= 0) return 'none' - // Check against min keep alive - if (minKeepAliveVersion) { - const minKeepAlive = parseVersion(minKeepAliveVersion) + // Check against min keep alive (different for agents vs normal workers) + const minVersion = isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion + if (minVersion) { + const minKeepAlive = parseVersion(minVersion) if (minKeepAlive && (worker[0] < minKeepAlive[0] || (worker[0] === minKeepAlive[0] && worker[1] < minKeepAlive[1]))) { return 'critical' } } + // Agent workers: no warning for version lag, only critical if below min keep alive + if (isAgent) return 'none' + if (minorLag > 50) return 'warning' if (minorLag > 0) return 'note' return 'none' @@ -104,7 +109,8 @@ for (const w of workers) { // Only check alive workers (pinged within last 60 seconds, accounting for time since refresh) if (w.last_ping == null || w.last_ping + timeSinceLastPing >= 60) continue - const warning = getVersionWarning(w.wm_version) + const isAgent = w.worker.startsWith('ag-') + const warning = getVersionWarning(w.wm_version, isAgent) if (priority[warning] > priority[worst]) worst = warning } return worst @@ -263,7 +269,10 @@ loadWorkerGroups() loadCustomTags() SettingsService.backendVersion().then((v) => (serverVersion = v)).catch((e) => console.error('Failed to fetch server version:', e)) - SettingService.getMinKeepAliveVersion().then((v) => (minKeepAliveVersion = v)).catch((e) => console.error('Failed to fetch min keep-alive version:', e)) + SettingService.getMinKeepAliveVersion().then((v) => { + minKeepAliveVersion = v.worker + agentMinKeepAliveVersion = v.agent + }).catch((e) => console.error('Failed to fetch min keep-alive version:', e)) onDestroy(() => { if (intervalId) { @@ -686,7 +695,7 @@ {#if worstVersionWarning === 'critical'} - One or more workers are running below the minimum supported version ({minKeepAliveVersion}). + One or more workers are running below the minimum supported version. This may cause undefined behavior and cluster instability. Upgrade these workers immediately—running workers this old is untested and strongly discouraged. @@ -956,7 +965,7 @@ - {@const versionWarning = getVersionWarning(wm_version)} + {@const versionWarning = getVersionWarning(wm_version, worker.startsWith('ag-'))}
{wm_version.split('-')[0]} @@ -967,11 +976,12 @@ {#snippet text()} + {@const isAgent = worker.startsWith('ag-')}
{#if versionWarning === 'critical'} - Critical: This worker is running below the minimum supported version ({minKeepAliveVersion}). + Critical: This {isAgent ? 'agent worker' : 'worker'} is running below the minimum supported version ({isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion}). This may cause undefined behavior and cluster instability. - Upgrade this worker immediately—running workers this old is untested and strongly discouraged. + Upgrade this {isAgent ? 'agent worker' : 'worker'} immediately—running {isAgent ? 'agent workers' : 'workers'} this old is untested and strongly discouraged. {:else if versionWarning === 'warning'} Warning: This worker is significantly behind the server ({serverVersion}) by more than 50 minor versions. While it should still function, the risk of issues is elevated.