fix: mixed version error (#7686)

* fix mixed version error

Signed-off-by: pyranota <pyra@duck.com>

* fix comp error

Signed-off-by: pyranota <pyra@duck.com>

* fix

Signed-off-by: pyranota <pyra@duck.com>

---------

Signed-off-by: pyranota <pyra@duck.com>
This commit is contained in:
Pyra
2026-01-26 16:49:31 +01:00
committed by GitHub
parent 89c6d86857
commit 02596e888f
7 changed files with 126 additions and 49 deletions
+1 -1
View File
@@ -1516,7 +1516,7 @@ Windmill Community Edition {GIT_VERSION}
// update min version explicitly.
// for sql connection it is the part of monitor_db.
// TODO: pass worker names for min keep-alive alerts (for HTTP connection)
windmill_common::min_version::update_min_version(conn, false, vec![], false).await;
windmill_common::min_version::update_min_version(conn, true, vec![], false).await;
}
};
}
+1
View File
@@ -219,6 +219,7 @@ pub async fn initial_load(
e
)
}
windmill_common::min_version::store_min_keep_alive_version(db).await;
}
}
+14 -4
View File
@@ -1413,17 +1413,27 @@ paths:
/min_keep_alive_version:
get:
summary: get minimum worker version required to stay alive
summary: get minimum worker versions required to stay alive
operationId: getMinKeepAliveVersion
tags:
- setting
responses:
"200":
description: minimum keep-alive version
description: minimum keep-alive versions for workers and agents
content:
text/plain:
application/json:
schema:
type: string
type: object
required:
- worker
- agent
properties:
worker:
type: string
description: minimum version for normal workers
agent:
type: string
description: minimum version for agent workers
/.well-known/jwks.json:
get:
+8 -4
View File
@@ -38,7 +38,7 @@ use axum::body::Body;
use axum::extract::DefaultBodyLimit;
use axum::http::HeaderValue;
use axum::response::Response;
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router};
use db::DB;
use reqwest::Client;
#[cfg(feature = "oauth2")]
@@ -906,9 +906,13 @@ async fn git_v() -> String {
format!("CE {GIT_VERSION}")
}
async fn min_keep_alive_version() -> String {
let v = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
format!("{}.{}.{}", v.0, v.1, v.2)
async fn min_keep_alive_version() -> Json<serde_json::Value> {
let worker = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
let agent = windmill_common::min_version::AGENT_MIN_KEEP_ALIVE_VERSION;
Json(serde_json::json!({
"worker": format!("{}.{}.{}", worker.0, worker.1, worker.2),
"agent": format!("{}.{}.{}", agent.0, agent.1, agent.2)
}))
}
#[cfg(not(feature = "enterprise"))]
@@ -49,6 +49,7 @@ pub const OTEL_SETTING: &str = "otel";
pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy";
pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
pub const SECRET_BACKEND_SETTING: &str = "secret_backend";
pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version";
pub const ENV_SETTINGS: &[&str] = &[
"DISABLE_NSJAIL",
+81 -30
View File
@@ -19,17 +19,23 @@ pub const MIN_VERSION_IS_AT_LEAST_1_427: VC = vc(1, 427, 0, "Flow version lite t
// workers below this version should be terminated automatically.
/// Minimum version workers must have to stay connected.
/// Served via: GET /api/settings/min_keep_alive_version
/// Served via: GET /api/min_keep_alive_version (returns { worker, agent })
/// Also used by vc() for compile-time checks.
pub const MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 400, 0);
// Compile-time check: must lag at least 50 minor versions behind current.
// NOTE: The 50 version lag is a constant and should NEVER be changed. If this check
/// Minimum version agent workers must have to stay connected.
/// Served via: GET /api/min_keep_alive_version (returns { worker, agent })
pub const AGENT_MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 0, 0);
// Compile-time check: MIN_KEEP_ALIVE_VERSION must lag at least 50 minor versions behind current,
// AGENT_MIN_KEEP_ALIVE_VERSION must lag at least 100 minor versions behind current.
// NOTE: These version lags are constants and should NEVER be changed. If this check
// fails, wait until enough versions have passed rather than reducing the lag requirement.
// Skip check if GIT_VERSION is "unknown-version" (no git tags available during build)
const _: () = assert!(
!const_str::contains!(crate::utils::GIT_VERSION, ".") ||
const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50
(const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50
&& const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - AGENT_MIN_KEEP_ALIVE_VERSION.1 >= 100)
);
// ============ Implementation ============
@@ -124,8 +130,8 @@ pub async fn get_min_version(conn: &Connection) -> error::Result<Version> {
}
/// Updates MIN_VERSION and optionally checks min keep-alive version for workers.
/// If `_worker_mode` is true, fetches min keep-alive version from server and sends alerts for each worker.
/// If `initial_load` is true, skips the HTTP fetch to min_keep_alive_version endpoint (server may not be ready).
/// If `_worker_mode` is true, checks min keep-alive version and sends critical alerts.
/// If `initial_load` is true, skips the min keep-alive check (server may not be ready).
pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_names: Vec<String>, _initial_load: bool) {
// Update MIN_VERSION
match get_min_version(conn).await {
@@ -141,34 +147,79 @@ pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_n
Err(e) => tracing::error!("Failed to fetch min version: {:#?}", e),
}
// Workers fetch min keep-alive version from server and send alerts
// Workers check min keep-alive version and send critical alerts
// Skip on initial_load since the server may not be ready yet
#[cfg(all(feature = "enterprise", feature = "private"))]
if _worker_mode && !_initial_load {
if let Connection::Sql(db) = conn {
let url = format!("{}/api/min_keep_alive_version", *crate::BASE_INTERNAL_URL);
match crate::utils::HTTP_CLIENT.get(&url).send().await {
Ok(resp) => match resp.text().await {
Ok(v) => match Version::parse(&v) {
Ok(min_keep_alive) => {
let current = GIT_SEM_VERSION.clone();
for worker_name in &_worker_names {
crate::ee::simple_alert_helper(
format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade recommended."),
format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."),
&format!("worker-below-min-keep-alive-{worker_name}"),
|| current < min_keep_alive,
Some("admins"),
db,
).await;
}
}
Err(e) => tracing::error!("Failed to parse min keep-alive version: {:#?}", e),
},
Err(e) => tracing::error!("Failed to read min keep-alive version response: {:#?}", e),
},
Err(e) => tracing::error!("Failed to fetch min keep-alive version: {:#?}", e),
let min_keep_alive_version: Option<Version> = match conn {
Connection::Sql(db) => {
match crate::global_settings::load_value_from_global_settings(
db,
crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING,
).await {
Ok(Some(v)) => v.as_str().and_then(|s| Version::parse(s).ok()),
Ok(None) => None,
Err(e) => {
tracing::error!("Failed to load min keep-alive version from global_settings: {:#?}", e);
None
}
}
}
Connection::Http(client) => {
match client.get::<serde_json::Value>("/api/min_keep_alive_version").await {
Ok(resp) => resp
.get("agent")
.and_then(|v| v.as_str())
.and_then(|v| Version::parse(v).ok()),
Err(e) => {
tracing::error!("Failed to fetch min keep-alive version: {:#?}", e);
None
}
}
}
};
if let Some(min_keep_alive) = min_keep_alive_version {
let current = GIT_SEM_VERSION.clone();
match conn {
Connection::Sql(db) => {
for worker_name in &_worker_names {
crate::ee::simple_alert_helper(
format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."),
format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."),
&format!("worker-below-min-keep-alive-{worker_name}"),
|| current < min_keep_alive,
Some("admins"),
db,
).await;
}
}
Connection::Http(_) => {
if current < min_keep_alive {
tracing::warn!(
"Agent worker version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."
);
}
}
}
}
}
}
/// Stores the min keep-alive version in global_settings.
/// Called by server on startup, NOT by workers.
pub async fn store_min_keep_alive_version(db: &sqlx::Pool<sqlx::Postgres>) {
let version = format!(
"{}.{}.{}",
MIN_KEEP_ALIVE_VERSION.0,
MIN_KEEP_ALIVE_VERSION.1,
MIN_KEEP_ALIVE_VERSION.2
);
if let Err(e) = crate::global_settings::set_value_in_global_settings(
db,
crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING,
serde_json::json!(version),
).await {
tracing::error!("Failed to store min keep-alive version in global_settings: {:#?}", e);
}
}
@@ -56,6 +56,7 @@
let customTags: string[] | undefined = $state(undefined)
let serverVersion: string | undefined = $state(undefined)
let minKeepAliveVersion: string | undefined = $state(undefined)
let agentMinKeepAliveVersion: string | undefined = $state(undefined)
type VersionWarning = 'none' | 'note' | 'warning' | 'critical' | 'newer'
@@ -68,7 +69,7 @@
return [major, minor || 0, patch || 0]
}
function getVersionWarning(workerVersion: string): VersionWarning {
function getVersionWarning(workerVersion: string, isAgent: boolean = false): VersionWarning {
if (!serverVersion) return 'none'
const server = parseVersion(serverVersion)
const worker = parseVersion(workerVersion)
@@ -82,15 +83,19 @@
const minorLag = server[1] - worker[1]
if (minorLag <= 0) return 'none'
// Check against min keep alive
if (minKeepAliveVersion) {
const minKeepAlive = parseVersion(minKeepAliveVersion)
// Check against min keep alive (different for agents vs normal workers)
const minVersion = isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion
if (minVersion) {
const minKeepAlive = parseVersion(minVersion)
if (minKeepAlive && (worker[0] < minKeepAlive[0] ||
(worker[0] === minKeepAlive[0] && worker[1] < minKeepAlive[1]))) {
return 'critical'
}
}
// Agent workers: no warning for version lag, only critical if below min keep alive
if (isAgent) return 'none'
if (minorLag > 50) return 'warning'
if (minorLag > 0) return 'note'
return 'none'
@@ -104,7 +109,8 @@
for (const w of workers) {
// Only check alive workers (pinged within last 60 seconds, accounting for time since refresh)
if (w.last_ping == null || w.last_ping + timeSinceLastPing >= 60) continue
const warning = getVersionWarning(w.wm_version)
const isAgent = w.worker.startsWith('ag-')
const warning = getVersionWarning(w.wm_version, isAgent)
if (priority[warning] > priority[worst]) worst = warning
}
return worst
@@ -263,7 +269,10 @@
loadWorkerGroups()
loadCustomTags()
SettingsService.backendVersion().then((v) => (serverVersion = v)).catch((e) => console.error('Failed to fetch server version:', e))
SettingService.getMinKeepAliveVersion().then((v) => (minKeepAliveVersion = v)).catch((e) => console.error('Failed to fetch min keep-alive version:', e))
SettingService.getMinKeepAliveVersion().then((v) => {
minKeepAliveVersion = v.worker
agentMinKeepAliveVersion = v.agent
}).catch((e) => console.error('Failed to fetch min keep-alive version:', e))
onDestroy(() => {
if (intervalId) {
@@ -686,7 +695,7 @@
{#if worstVersionWarning === 'critical'}
<Alert type="error" title="Critical: Workers below minimum version" class="my-4">
One or more workers are running below the minimum supported version ({minKeepAliveVersion}).
One or more workers are running below the minimum supported version.
This may cause undefined behavior and cluster instability.
Upgrade these workers immediately—running workers this old is untested and strongly discouraged.
</Alert>
@@ -956,7 +965,7 @@
</div>
</Cell>
<Cell class="text-secondary">
{@const versionWarning = getVersionWarning(wm_version)}
{@const versionWarning = getVersionWarning(wm_version, worker.startsWith('ag-'))}
<div class="flex items-center gap-1">
<div class="!text-2xs" title={wm_version}>
{wm_version.split('-')[0]}
@@ -967,11 +976,12 @@
<TriangleAlert size={12} />
</Badge>
{#snippet text()}
{@const isAgent = worker.startsWith('ag-')}
<div class="max-w-xs text-xs">
{#if versionWarning === 'critical'}
<strong>Critical:</strong> This worker is running below the minimum supported version ({minKeepAliveVersion}).
<strong>Critical:</strong> This {isAgent ? 'agent worker' : 'worker'} is running below the minimum supported version ({isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion}).
This may cause undefined behavior and cluster instability.
Upgrade this worker immediately—running workers this old is untested and strongly discouraged.
Upgrade this {isAgent ? 'agent worker' : 'worker'} immediately—running {isAgent ? 'agent workers' : 'workers'} this old is untested and strongly discouraged.
{:else if versionWarning === 'warning'}
<strong>Warning:</strong> This worker is significantly behind the server ({serverVersion}) by more than 50 minor versions.
While it should still function, the risk of issues is elevated.