From 7558ce27a9736bef1a9b9db9b0799150dccb11a5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 13 Oct 2026 17:22:07 +0200 Subject: [PATCH] feat: record each instance's python version and start new ones on 3.13 (#11656) * feat: pin the default python version of an instance at initialization Co-Authored-By: Claude Opus 5.5 * fix: detect a new instance from the schema it created, not from its age Co-Authored-By: Claude Opus 5.5 * fix: record the python baseline on the migration connection Co-Authored-By: Claude Opus 5.5 * refactor: record the instance python version in the setting itself Co-Authored-By: Claude Opus 5.5 * feat: start new instances on python 3.15 Co-Authored-By: Claude Opus 5.5 * fix: keep the forced python version ahead of the stored one when a worker reads it late Co-Authored-By: Claude Opus 5.5 * feat: start new instances on python 3.14 rather than 3.15 Co-Authored-By: Claude Opus 5.5 * feat: start new instances on python 3.13 Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to 06610ecc950ed64a2709d7f1aa8c7b49a95fdbae This commit updates the EE repository reference after PR #869 was merged in windmill-ee-private. Previous ee-repo-ref: 599eae0034b34c5e0d3d42df37cf18bbe01f9b49 New ee-repo-ref: 06610ecc950ed64a2709d7f1aa8c7b49a95fdbae Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] --- Dockerfile | 10 ++-- backend/windmill-api/src/db.rs | 51 +++++++++++++++++++ .../windmill-common/src/instance_config.rs | 1 + backend/windmill-common/src/worker.rs | 13 +++++ .../windmill-worker/src/python_versions.rs | 29 +++++++++-- docker/DockerfileSlim | 2 + docker/DockerfileSlimEe | 2 + .../src/lib/components/instanceSettings.ts | 9 ---- 8 files changed, 99 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6a4ccc55f..4e022d4b53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -152,11 +152,12 @@ ARG WITH_HELM=true ARG WITH_GIT=true ARG features="" -# To change latest stable version: -# 1. Change placeholder in instanceSettings.ts -# 2. Change LATEST_STABLE_PY in dockerfile -# 3. Change #[default] annotation for PyVersion in backend +# Instances keep the Python version they were initialized with, so both are preinstalled: +# LATEST_STABLE_PY is what instances initialized before the version was raised are on +# (PyVAlias::default() in the backend), NEW_INSTANCE_PY the one new instances start on +# (PyVAlias::NEW_INSTANCE). When raising the latter, keep installing every earlier value. ARG LATEST_STABLE_PY=3.12 +ARG NEW_INSTANCE_PY=3.13 ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime ENV UV_PYTHON_PREFERENCE=only-managed @@ -249,6 +250,7 @@ RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releas # timestamps or Python's mtime-based .pyc invalidation discards these compiled files. RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 --compile-bytecode RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode +RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $NEW_INSTANCE_PY --compile-bytecode RUN curl -sL https://deb.nodesource.com/setup_24.x | bash - diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 9619e96ebc..21b738891e 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -17,7 +17,9 @@ use tokio::task::JoinHandle; pub use windmill_common::db::DB; use windmill_common::{ error::Error, + global_settings::INSTANCE_PYTHON_VERSION_SETTING, utils::{generate_lock_id, GIT_VERSION}, + worker::PyVAlias, }; #[allow(unused_imports)] @@ -298,6 +300,12 @@ pub async fn migrate( let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; + let new_instance = + sqlx::query_scalar::<_, bool>("SELECT to_regclass('_sqlx_migrations') IS NULL") + .fetch_one(custom_migrator.connection()) + .await + .unwrap_or(false); + if let Err(err) = sqlx::query!( "DELETE FROM _sqlx_migrations WHERE version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR @@ -372,12 +380,55 @@ pub async fn migrate( } crate::live_migrations::custom_migrations(&mut custom_migrator).await?; + if let Err(e) = seed_instance_python_version(custom_migrator.connection(), new_instance).await { + tracing::error!("Could not record the instance python version: {e:#}"); + } Ok(Some(crate::live_migrations::spawn_background_migrations( db.clone(), killpill_rx, ))) } +/// Gives `instance_python_version` a concrete value when the instance has none, so the version +/// scripts resolve around is recorded per instance instead of following whatever a release +/// compiles in. Only the process that created the schema knows the instance is new, so its value +/// replaces what a server that started alongside it wrote; every other start fills in the frozen +/// default, and only where no version was chosen (`default` is what the settings page stored +/// for that). +/// +/// Runs on the migration connection: a `DATABASE_CONNECTIONS=1` pool has no second one to give. +async fn seed_instance_python_version( + conn: &mut PgConnection, + new_instance: bool, +) -> Result<(), Error> { + let (version, on_conflict) = if new_instance { + ( + PyVAlias::NEW_INSTANCE, + "DO UPDATE SET value = EXCLUDED.value", + ) + } else { + ( + PyVAlias::default(), + "DO UPDATE SET value = EXCLUDED.value WHERE global_settings.value = '\"default\"'::jsonb", + ) + }; + // The stored setting outranks the `INSTANCE_PYTHON_VERSION` environment variable, so an + // instance configured through it would otherwise be moved off its version by this write. + let version = std::env::var("INSTANCE_PYTHON_VERSION") + .ok() + .filter(|v| !v.is_empty() && v.split('.').all(|part| part.parse::().is_ok())) + .unwrap_or_else(|| version.version_string()); + sqlx::query(&format!( + "INSERT INTO global_settings (name, value) VALUES ($1, to_jsonb($2::text)) + ON CONFLICT (name) {on_conflict}" + )) + .bind(INSTANCE_PYTHON_VERSION_SETTING) + .bind(version) + .execute(conn) + .await?; + Ok(()) +} + pub async fn wait_for_migrations( db: &DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 7c76fab2ee..d19de41dcc 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1063,6 +1063,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "rsa_keys", "jwt_secret", "min_keep_alive_version", + "instance_python_version", // Deleting it would let a worker sign OIDC tokens with a local key again. "oidc_external_signer", ]; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 58223c5d4a..30eb2403b1 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2891,6 +2891,9 @@ pub fn lockfile_line_has_continuation(line: &str) -> bool { pub enum PyVAlias { Py310 = 10, Py311, + // Frozen: what a lock with no `# py:` line runs on, and what the server writes into + // `instance_python_version` for an instance that never chose one. Raise `NEW_INSTANCE` + // instead, which existing instances never pick up. #[default] Py312, Py313, @@ -2911,6 +2914,11 @@ impl Into for PyVAlias { } impl PyVAlias { + /// What the server writes into `instance_python_version` when it initializes an instance. + /// Raising it also means the images must keep preinstalling every earlier value, or + /// instances that started on one download their runtime on first use. + pub const NEW_INSTANCE: PyVAlias = PyVAlias::Py313; + pub fn all>() -> Vec { use PyVAlias::*; vec![ @@ -2922,6 +2930,11 @@ impl PyVAlias { Py315.into(), ] } + /// `MAJOR.MINOR`, the form the instance settings store. + pub fn version_string(self) -> String { + format!("{}.{}", self.major(), self as u32) + } + // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) fn major(&self) -> u32 { use PyVAlias::*; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index b52352915a..3a15de17b5 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -14,6 +14,7 @@ use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; use uuid::Uuid; use windmill_common::{ error::{self, Error}, + global_settings::{load_value_from_global_settings, INSTANCE_PYTHON_VERSION_SETTING}, worker::{try_parse_locked_python_version_from_requirements, Connection, PyVAlias}, }; @@ -49,10 +50,6 @@ impl From for PyVAlias { } } -// To change latest stable version: -// 1. Change placeholder in instanceSettings.ts -// 2. Change LATEST_STABLE_PY in dockerfile -// 3. Change #[default] annotation for PyVersion in backend #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PyV(pub pep440_rs::Version); @@ -260,7 +257,29 @@ impl PyV { conn: Option, ) -> Self { let mut err = None; - let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { + let mut setting = INSTANCE_PYTHON_VERSION.read().await.clone(); + // A worker that loaded its settings while the server was still initializing the instance + // read no version, and may have started listening for setting changes only after it was + // written. Jobs only exist once it is, so reading it here settles that worker for good. + if setting.is_none() { + if let Some(Connection::Sql(db)) = conn.as_ref() { + // Same precedence as the settings load: forced value, stored value, environment. + let env = |name: &str| std::env::var(name).ok().filter(|v| !v.is_empty()); + setting = match env("FORCE_INSTANCE_PYTHON_VERSION") { + Some(forced) => Some(forced), + None => load_value_from_global_settings(db, INSTANCE_PYTHON_VERSION_SETTING) + .await + .ok() + .flatten() + .and_then(|v| serde_json::from_value::(v).ok()) + .or_else(|| env("INSTANCE_PYTHON_VERSION")), + }; + if setting.is_some() { + *INSTANCE_PYTHON_VERSION.write().await = setting.clone(); + } + } + } + let pyv = match setting { Some(v) if &v == "default" => PyVAlias::default().into(), Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| { let v = PyVAlias::default().into(); diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index a37ced0eca..ef976f83d3 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -30,6 +30,7 @@ FROM ${DEBIAN_IMAGE} ARG APP=/usr/src/app ARG LATEST_STABLE_PY=3.12.13 +ARG NEW_INSTANCE_PY=3.13.16 # UV configuration ENV UV_CACHE_DIR=/tmp/windmill/cache/uv @@ -65,6 +66,7 @@ RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releas # under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve # timestamps or Python's mtime-based .pyc invalidation discards these compiled files. RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $NEW_INSTANCE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 687b1506ce..67de36eac7 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -30,6 +30,7 @@ FROM ${DEBIAN_IMAGE} ARG APP=/usr/src/app ARG LATEST_STABLE_PY=3.12.13 +ARG NEW_INSTANCE_PY=3.13.16 # UV configuration ENV UV_CACHE_DIR=/tmp/windmill/cache/uv @@ -65,6 +66,7 @@ RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releas # under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve # timestamps or Python's mtime-based .pyc invalidation discards these compiled files. RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $NEW_INSTANCE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 27906dddee..9681e821e6 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -838,17 +838,8 @@ export const settings: Record = { description: 'Default python version for newly deployed scripts', key: 'instance_python_version', fieldType: 'select_python', - // To change latest stable version: - // 1. Change placeholder in instanceSettings.ts - // 2. Change LATEST_STABLE_PY in dockerfile - // 3. Change #[default] annotation for PyVersion in backend placeholder: '3.10,3.11,3.12,3.13,3.14,3.15', select_items: [ - { - label: 'Latest Stable', - value: 'default', - tooltip: 'python-3.12' - }, { label: '3.10' },