mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-22 16:02:24 +00:00
fix: mixed version error (#7686)
* fix mixed version error Signed-off-by: pyranota <pyra@duck.com> * fix comp error Signed-off-by: pyranota <pyra@duck.com> * fix Signed-off-by: pyranota <pyra@duck.com> --------- Signed-off-by: pyranota <pyra@duck.com>
This commit is contained in:
+1
-1
@@ -1516,7 +1516,7 @@ Windmill Community Edition {GIT_VERSION}
|
||||
// update min version explicitly.
|
||||
// for sql connection it is the part of monitor_db.
|
||||
// TODO: pass worker names for min keep-alive alerts (for HTTP connection)
|
||||
windmill_common::min_version::update_min_version(conn, false, vec![], false).await;
|
||||
windmill_common::min_version::update_min_version(conn, true, vec![], false).await;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -219,6 +219,7 @@ pub async fn initial_load(
|
||||
e
|
||||
)
|
||||
}
|
||||
windmill_common::min_version::store_min_keep_alive_version(db).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1413,17 +1413,27 @@ paths:
|
||||
|
||||
/min_keep_alive_version:
|
||||
get:
|
||||
summary: get minimum worker version required to stay alive
|
||||
summary: get minimum worker versions required to stay alive
|
||||
operationId: getMinKeepAliveVersion
|
||||
tags:
|
||||
- setting
|
||||
responses:
|
||||
"200":
|
||||
description: minimum keep-alive version
|
||||
description: minimum keep-alive versions for workers and agents
|
||||
content:
|
||||
text/plain:
|
||||
application/json:
|
||||
schema:
|
||||
type: string
|
||||
type: object
|
||||
required:
|
||||
- worker
|
||||
- agent
|
||||
properties:
|
||||
worker:
|
||||
type: string
|
||||
description: minimum version for normal workers
|
||||
agent:
|
||||
type: string
|
||||
description: minimum version for agent workers
|
||||
|
||||
/.well-known/jwks.json:
|
||||
get:
|
||||
|
||||
@@ -38,7 +38,7 @@ use axum::body::Body;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::http::HeaderValue;
|
||||
use axum::response::Response;
|
||||
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
|
||||
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router};
|
||||
use db::DB;
|
||||
use reqwest::Client;
|
||||
#[cfg(feature = "oauth2")]
|
||||
@@ -906,9 +906,13 @@ async fn git_v() -> String {
|
||||
format!("CE {GIT_VERSION}")
|
||||
}
|
||||
|
||||
async fn min_keep_alive_version() -> String {
|
||||
let v = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
|
||||
format!("{}.{}.{}", v.0, v.1, v.2)
|
||||
async fn min_keep_alive_version() -> Json<serde_json::Value> {
|
||||
let worker = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
|
||||
let agent = windmill_common::min_version::AGENT_MIN_KEEP_ALIVE_VERSION;
|
||||
Json(serde_json::json!({
|
||||
"worker": format!("{}.{}.{}", worker.0, worker.1, worker.2),
|
||||
"agent": format!("{}.{}.{}", agent.0, agent.1, agent.2)
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
|
||||
@@ -49,6 +49,7 @@ pub const OTEL_SETTING: &str = "otel";
|
||||
pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy";
|
||||
pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
|
||||
pub const SECRET_BACKEND_SETTING: &str = "secret_backend";
|
||||
pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version";
|
||||
|
||||
pub const ENV_SETTINGS: &[&str] = &[
|
||||
"DISABLE_NSJAIL",
|
||||
|
||||
@@ -19,17 +19,23 @@ pub const MIN_VERSION_IS_AT_LEAST_1_427: VC = vc(1, 427, 0, "Flow version lite t
|
||||
// workers below this version should be terminated automatically.
|
||||
|
||||
/// Minimum version workers must have to stay connected.
|
||||
/// Served via: GET /api/settings/min_keep_alive_version
|
||||
/// Served via: GET /api/min_keep_alive_version (returns { worker, agent })
|
||||
/// Also used by vc() for compile-time checks.
|
||||
pub const MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 400, 0);
|
||||
|
||||
// Compile-time check: must lag at least 50 minor versions behind current.
|
||||
// NOTE: The 50 version lag is a constant and should NEVER be changed. If this check
|
||||
/// Minimum version agent workers must have to stay connected.
|
||||
/// Served via: GET /api/min_keep_alive_version (returns { worker, agent })
|
||||
pub const AGENT_MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 0, 0);
|
||||
|
||||
// Compile-time check: MIN_KEEP_ALIVE_VERSION must lag at least 50 minor versions behind current,
|
||||
// AGENT_MIN_KEEP_ALIVE_VERSION must lag at least 100 minor versions behind current.
|
||||
// NOTE: These version lags are constants and should NEVER be changed. If this check
|
||||
// fails, wait until enough versions have passed rather than reducing the lag requirement.
|
||||
// Skip check if GIT_VERSION is "unknown-version" (no git tags available during build)
|
||||
const _: () = assert!(
|
||||
!const_str::contains!(crate::utils::GIT_VERSION, ".") ||
|
||||
const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50
|
||||
(const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50
|
||||
&& const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - AGENT_MIN_KEEP_ALIVE_VERSION.1 >= 100)
|
||||
);
|
||||
|
||||
// ============ Implementation ============
|
||||
@@ -124,8 +130,8 @@ pub async fn get_min_version(conn: &Connection) -> error::Result<Version> {
|
||||
}
|
||||
|
||||
/// Updates MIN_VERSION and optionally checks min keep-alive version for workers.
|
||||
/// If `_worker_mode` is true, fetches min keep-alive version from server and sends alerts for each worker.
|
||||
/// If `initial_load` is true, skips the HTTP fetch to min_keep_alive_version endpoint (server may not be ready).
|
||||
/// If `_worker_mode` is true, checks min keep-alive version and sends critical alerts.
|
||||
/// If `initial_load` is true, skips the min keep-alive check (server may not be ready).
|
||||
pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_names: Vec<String>, _initial_load: bool) {
|
||||
// Update MIN_VERSION
|
||||
match get_min_version(conn).await {
|
||||
@@ -141,34 +147,79 @@ pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_n
|
||||
Err(e) => tracing::error!("Failed to fetch min version: {:#?}", e),
|
||||
}
|
||||
|
||||
// Workers fetch min keep-alive version from server and send alerts
|
||||
// Workers check min keep-alive version and send critical alerts
|
||||
// Skip on initial_load since the server may not be ready yet
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
if _worker_mode && !_initial_load {
|
||||
if let Connection::Sql(db) = conn {
|
||||
let url = format!("{}/api/min_keep_alive_version", *crate::BASE_INTERNAL_URL);
|
||||
match crate::utils::HTTP_CLIENT.get(&url).send().await {
|
||||
Ok(resp) => match resp.text().await {
|
||||
Ok(v) => match Version::parse(&v) {
|
||||
Ok(min_keep_alive) => {
|
||||
let current = GIT_SEM_VERSION.clone();
|
||||
for worker_name in &_worker_names {
|
||||
crate::ee::simple_alert_helper(
|
||||
format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade recommended."),
|
||||
format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."),
|
||||
&format!("worker-below-min-keep-alive-{worker_name}"),
|
||||
|| current < min_keep_alive,
|
||||
Some("admins"),
|
||||
db,
|
||||
).await;
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Failed to parse min keep-alive version: {:#?}", e),
|
||||
},
|
||||
Err(e) => tracing::error!("Failed to read min keep-alive version response: {:#?}", e),
|
||||
},
|
||||
Err(e) => tracing::error!("Failed to fetch min keep-alive version: {:#?}", e),
|
||||
let min_keep_alive_version: Option<Version> = match conn {
|
||||
Connection::Sql(db) => {
|
||||
match crate::global_settings::load_value_from_global_settings(
|
||||
db,
|
||||
crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING,
|
||||
).await {
|
||||
Ok(Some(v)) => v.as_str().and_then(|s| Version::parse(s).ok()),
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to load min keep-alive version from global_settings: {:#?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Connection::Http(client) => {
|
||||
match client.get::<serde_json::Value>("/api/min_keep_alive_version").await {
|
||||
Ok(resp) => resp
|
||||
.get("agent")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|v| Version::parse(v).ok()),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to fetch min keep-alive version: {:#?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(min_keep_alive) = min_keep_alive_version {
|
||||
let current = GIT_SEM_VERSION.clone();
|
||||
match conn {
|
||||
Connection::Sql(db) => {
|
||||
for worker_name in &_worker_names {
|
||||
crate::ee::simple_alert_helper(
|
||||
format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."),
|
||||
format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."),
|
||||
&format!("worker-below-min-keep-alive-{worker_name}"),
|
||||
|| current < min_keep_alive,
|
||||
Some("admins"),
|
||||
db,
|
||||
).await;
|
||||
}
|
||||
}
|
||||
Connection::Http(_) => {
|
||||
if current < min_keep_alive {
|
||||
tracing::warn!(
|
||||
"Agent worker version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores the min keep-alive version in global_settings.
|
||||
/// Called by server on startup, NOT by workers.
|
||||
pub async fn store_min_keep_alive_version(db: &sqlx::Pool<sqlx::Postgres>) {
|
||||
let version = format!(
|
||||
"{}.{}.{}",
|
||||
MIN_KEEP_ALIVE_VERSION.0,
|
||||
MIN_KEEP_ALIVE_VERSION.1,
|
||||
MIN_KEEP_ALIVE_VERSION.2
|
||||
);
|
||||
if let Err(e) = crate::global_settings::set_value_in_global_settings(
|
||||
db,
|
||||
crate::global_settings::MIN_KEEP_ALIVE_VERSION_SETTING,
|
||||
serde_json::json!(version),
|
||||
).await {
|
||||
tracing::error!("Failed to store min keep-alive version in global_settings: {:#?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@
|
||||
let customTags: string[] | undefined = $state(undefined)
|
||||
let serverVersion: string | undefined = $state(undefined)
|
||||
let minKeepAliveVersion: string | undefined = $state(undefined)
|
||||
let agentMinKeepAliveVersion: string | undefined = $state(undefined)
|
||||
|
||||
type VersionWarning = 'none' | 'note' | 'warning' | 'critical' | 'newer'
|
||||
|
||||
@@ -68,7 +69,7 @@
|
||||
return [major, minor || 0, patch || 0]
|
||||
}
|
||||
|
||||
function getVersionWarning(workerVersion: string): VersionWarning {
|
||||
function getVersionWarning(workerVersion: string, isAgent: boolean = false): VersionWarning {
|
||||
if (!serverVersion) return 'none'
|
||||
const server = parseVersion(serverVersion)
|
||||
const worker = parseVersion(workerVersion)
|
||||
@@ -82,15 +83,19 @@
|
||||
const minorLag = server[1] - worker[1]
|
||||
if (minorLag <= 0) return 'none'
|
||||
|
||||
// Check against min keep alive
|
||||
if (minKeepAliveVersion) {
|
||||
const minKeepAlive = parseVersion(minKeepAliveVersion)
|
||||
// Check against min keep alive (different for agents vs normal workers)
|
||||
const minVersion = isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion
|
||||
if (minVersion) {
|
||||
const minKeepAlive = parseVersion(minVersion)
|
||||
if (minKeepAlive && (worker[0] < minKeepAlive[0] ||
|
||||
(worker[0] === minKeepAlive[0] && worker[1] < minKeepAlive[1]))) {
|
||||
return 'critical'
|
||||
}
|
||||
}
|
||||
|
||||
// Agent workers: no warning for version lag, only critical if below min keep alive
|
||||
if (isAgent) return 'none'
|
||||
|
||||
if (minorLag > 50) return 'warning'
|
||||
if (minorLag > 0) return 'note'
|
||||
return 'none'
|
||||
@@ -104,7 +109,8 @@
|
||||
for (const w of workers) {
|
||||
// Only check alive workers (pinged within last 60 seconds, accounting for time since refresh)
|
||||
if (w.last_ping == null || w.last_ping + timeSinceLastPing >= 60) continue
|
||||
const warning = getVersionWarning(w.wm_version)
|
||||
const isAgent = w.worker.startsWith('ag-')
|
||||
const warning = getVersionWarning(w.wm_version, isAgent)
|
||||
if (priority[warning] > priority[worst]) worst = warning
|
||||
}
|
||||
return worst
|
||||
@@ -263,7 +269,10 @@
|
||||
loadWorkerGroups()
|
||||
loadCustomTags()
|
||||
SettingsService.backendVersion().then((v) => (serverVersion = v)).catch((e) => console.error('Failed to fetch server version:', e))
|
||||
SettingService.getMinKeepAliveVersion().then((v) => (minKeepAliveVersion = v)).catch((e) => console.error('Failed to fetch min keep-alive version:', e))
|
||||
SettingService.getMinKeepAliveVersion().then((v) => {
|
||||
minKeepAliveVersion = v.worker
|
||||
agentMinKeepAliveVersion = v.agent
|
||||
}).catch((e) => console.error('Failed to fetch min keep-alive version:', e))
|
||||
|
||||
onDestroy(() => {
|
||||
if (intervalId) {
|
||||
@@ -686,7 +695,7 @@
|
||||
|
||||
{#if worstVersionWarning === 'critical'}
|
||||
<Alert type="error" title="Critical: Workers below minimum version" class="my-4">
|
||||
One or more workers are running below the minimum supported version ({minKeepAliveVersion}).
|
||||
One or more workers are running below the minimum supported version.
|
||||
This may cause undefined behavior and cluster instability.
|
||||
Upgrade these workers immediately—running workers this old is untested and strongly discouraged.
|
||||
</Alert>
|
||||
@@ -956,7 +965,7 @@
|
||||
</div>
|
||||
</Cell>
|
||||
<Cell class="text-secondary">
|
||||
{@const versionWarning = getVersionWarning(wm_version)}
|
||||
{@const versionWarning = getVersionWarning(wm_version, worker.startsWith('ag-'))}
|
||||
<div class="flex items-center gap-1">
|
||||
<div class="!text-2xs" title={wm_version}>
|
||||
{wm_version.split('-')[0]}
|
||||
@@ -967,11 +976,12 @@
|
||||
<TriangleAlert size={12} />
|
||||
</Badge>
|
||||
{#snippet text()}
|
||||
{@const isAgent = worker.startsWith('ag-')}
|
||||
<div class="max-w-xs text-xs">
|
||||
{#if versionWarning === 'critical'}
|
||||
<strong>Critical:</strong> This worker is running below the minimum supported version ({minKeepAliveVersion}).
|
||||
<strong>Critical:</strong> This {isAgent ? 'agent worker' : 'worker'} is running below the minimum supported version ({isAgent ? agentMinKeepAliveVersion : minKeepAliveVersion}).
|
||||
This may cause undefined behavior and cluster instability.
|
||||
Upgrade this worker immediately—running workers this old is untested and strongly discouraged.
|
||||
Upgrade this {isAgent ? 'agent worker' : 'worker'} immediately—running {isAgent ? 'agent workers' : 'workers'} this old is untested and strongly discouraged.
|
||||
{:else if versionWarning === 'warning'}
|
||||
<strong>Warning:</strong> This worker is significantly behind the server ({serverVersion}) by more than 50 minor versions.
|
||||
While it should still function, the risk of issues is elevated.
|
||||
|
||||
Reference in New Issue
Block a user