From a07aef30d5b11110fdded45f70812a8cabaaded2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 6 Nov 2026 03:51:39 +0000 Subject: [PATCH] fix: refill every free concurrency slot on completion, at constant cost (#12261) * perf: top up deferred concurrency jobs with a bounded index walk Co-Authored-By: Claude Opus 5.5 * fix: compute the wake bounds in bigint Co-Authored-By: Claude Opus 5.5 * fix: filter wake candidates before bounding the walk Co-Authored-By: Claude Opus 5.5 * fix: skip the wake when the completed job is limited per time window Co-Authored-By: Claude Opus 5.5 * revert: wake on every completion that frees a slot, whatever its time window The completed job's window does not decide whether a waiter without one can take its slot. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ...9e3d59bd4df85375c7fd35b0dfe16283fa8e7.json | 15 ++++ ...19fb76d423193411bbd8ac80a1eac8eb1448c.json | 14 ---- ...203_concurrency_key_waiting_index.down.sql | 1 + ...03203_concurrency_key_waiting_index.up.sql | 11 +++ .../tests/concurrency_wake_on_completion.rs | 37 ++++++---- backend/windmill-api/src/db.rs | 3 + backend/windmill-queue/src/jobs.rs | 69 ++++++++++++------- 7 files changed, 96 insertions(+), 54 deletions(-) create mode 100644 backend/.sqlx/query-254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7.json delete mode 100644 backend/.sqlx/query-f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c.json create mode 100644 backend/migrations/20261105203203_concurrency_key_waiting_index.down.sql create mode 100644 backend/migrations/20261105203203_concurrency_key_waiting_index.up.sql diff --git a/backend/.sqlx/query-254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7.json b/backend/.sqlx/query-254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7.json new file mode 100644 index 0000000000..2e9fe70fea --- /dev/null +++ b/backend/.sqlx/query-254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH head AS (\n SELECT q.running, q.scheduled_for <= now() AS due\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND q.canceled_by IS NULL\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 2 * $2::bigint + 1\n ), taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*) FROM head WHERE due AND NOT running) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0)\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now()\n FROM next\n WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running\n AND v2_job_queue.scheduled_for > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7" +} diff --git a/backend/.sqlx/query-f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c.json b/backend/.sqlx/query-f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c.json deleted file mode 100644 index 287c158d58..0000000000 --- a/backend/.sqlx/query-f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*)\n FROM concurrency_key due_ck\n JOIN v2_job_queue due ON due.id = due_ck.job_id\n WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL\n AND NOT due.running AND due.canceled_by IS NULL\n AND due.scheduled_for <= now()) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 1\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c" -} diff --git a/backend/migrations/20261105203203_concurrency_key_waiting_index.down.sql b/backend/migrations/20261105203203_concurrency_key_waiting_index.down.sql new file mode 100644 index 0000000000..b0f657ca2e --- /dev/null +++ b/backend/migrations/20261105203203_concurrency_key_waiting_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS concurrency_key_waiting_idx; diff --git a/backend/migrations/20261105203203_concurrency_key_waiting_index.up.sql b/backend/migrations/20261105203203_concurrency_key_waiting_index.up.sql new file mode 100644 index 0000000000..9f4faf2a6d --- /dev/null +++ b/backend/migrations/20261105203203_concurrency_key_waiting_index.up.sql @@ -0,0 +1,11 @@ +-- Serves wake_deferred_concurrency_job in windmill-queue/src/jobs.rs, which walks the jobs a +-- concurrency limit deferred on one key in the order they were first due and stops after a +-- few. Without it, every completion of a concurrency-limited job sorts the key's whole backlog. +-- Dropped before it is built because the OVERRIDDEN_MIGRATIONS rewrite in +-- windmill-api/src/db.rs runs both CONCURRENTLY: an interrupted concurrent build leaves an +-- invalid index that IF NOT EXISTS would skip rebuilding. +DROP INDEX IF EXISTS concurrency_key_waiting_idx; + +CREATE INDEX IF NOT EXISTS concurrency_key_waiting_idx + ON concurrency_key (key, original_scheduled_for, job_id) + WHERE ended_at IS NULL AND original_scheduled_for IS NOT NULL; diff --git a/backend/tests/concurrency_wake_on_completion.rs b/backend/tests/concurrency_wake_on_completion.rs index 3961039b68..be10a63817 100644 --- a/backend/tests/concurrency_wake_on_completion.rs +++ b/backend/tests/concurrency_wake_on_completion.rs @@ -53,36 +53,43 @@ async fn due(db: &Pool, id: Uuid) -> anyhow::Result { ) } -/// Completions wake the jobs the limit deferred first, as many as the free slots, never -/// a job scheduled for later or one limited per time window. +/// One completion tops the key up: it wakes the jobs the limit deferred first, as many as +/// the free slots, and a later completion wakes none while they are due. Never a job +/// scheduled for later or one limited per time window, and older time-window jobs sharing +/// the key, more of them than the head the wake counts in, do not hide the eligible ones. #[sqlx::test(fixtures("base"))] -async fn completions_wake_the_first_deferred_jobs(db: Pool) -> anyhow::Result<()> { - let windowed = waiting_job(&db, Some(40), Some(10)).await?; +async fn completion_fills_the_free_slots(db: Pool) -> anyhow::Result<()> { + let mut windowed = vec![]; + for i in 0..(2 * LIMIT + 2) as i64 { + windowed.push(waiting_job(&db, Some(100 - i), Some(10)).await?); + } let first = waiting_job(&db, Some(30), None).await?; let second = waiting_job(&db, Some(20), None).await?; let third = waiting_job(&db, Some(10), None).await?; let scheduled_later = waiting_job(&db, None, None).await?; - for _ in 0..3 { - wake_deferred_concurrency_job(&db, KEY).await?; - } + wake_deferred_concurrency_job(&db, KEY, LIMIT).await?; + let second_woken_at_once = due(&db, second).await?; + wake_deferred_concurrency_job(&db, KEY, LIMIT).await?; + assert!( + second_woken_at_once, + "a free slot was left to the deferral estimate" + ); assert!( due(&db, first).await?, "the first deferred job was not woken" ); - assert!( - due(&db, second).await?, - "a free slot was left to the deferral estimate" - ); assert!( !due(&db, third).await?, "more jobs were woken than slots are free" ); - assert!( - !due(&db, windowed).await?, - "a job limited per time window was woken" - ); + for w in windowed { + assert!( + !due(&db, w).await?, + "a job limited per time window was woken" + ); + } assert!( !due(&db, scheduled_later).await?, "a job scheduled for later was woken" diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index b3853a5fb8..ffcde90155 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -120,6 +120,9 @@ lazy_static::lazy_static! { (20261015114712, include_str!( "../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20261105203203, include_str!( + "../../migrations/20261105203203_concurrency_key_waiting_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), // The widening fails every query the previous release's still-running // servers prepared on the column; `variables::widen_variable_value` does it // once they are gone. diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index ba5d7409eb..8dc2b38cfd 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1601,16 +1601,16 @@ async fn commit_completed_job( // Resolve the concurrency-limit settings on the pool *before* opening the // completion transaction: doing it inside the tx would hold a second // simultaneous connection from the small per-worker pool. + let settings_concurrent_limit = + windmill_common::runnable_settings::prefetch_cached_from_handle( + completed_job.runnable_settings_handle, + db, + ) + .await? + .1 + .concurrent_limit; let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit) - || has_active_concurrency_limit( - windmill_common::runnable_settings::prefetch_cached_from_handle( - completed_job.runnable_settings_handle, - db, - ) - .await? - .1 - .concurrent_limit, - ); + || has_active_concurrency_limit(settings_concurrent_limit); // A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that // the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape @@ -1868,7 +1868,11 @@ async fn commit_completed_job( // After the commit, so the woken job finds the slot this one freed. #[cfg(all(feature = "enterprise", feature = "private"))] if let Some(key) = released_concurrency_key { - if let Err(e) = wake_deferred_concurrency_job(db, &key).await { + let limit = completed_job + .concurrent_limit + .max(settings_concurrent_limit) + .unwrap_or(0); + if let Err(e) = wake_deferred_concurrency_job(db, &key, limit).await { tracing::error!("Could not wake a deferred job for concurrency key {key}: {e:#}"); } } @@ -5309,25 +5313,36 @@ pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result< Ok(true) } -/// Make due now the job of `key` that a concurrency limit deferred first, so the slot a -/// completed job freed is taken rather than left empty until the deferral's estimate. -/// No-op while the jobs of the key already due fill the free slots, and for a job limited -/// per time window, whose slot a completion does not free. Queue bookkeeping, not an -/// authorization boundary: only for the completion path, never with a key taken from a request. +/// Make due now the earliest-deferred jobs of `key`, as many as the slots no admitted or +/// already-due job holds. It tops up to the limit: waking one job per completion never +/// refills a slot whose woken job was lost, and the key stays below its limit. +/// Already-due jobs are counted in the first `2 * limit + 1` deferred jobs not limited per +/// time window: wakes go to the earliest of those, so they come first, and missing one can +/// only over-wake, which admission re-defers. Both walks filter before they stop, so older +/// time-window jobs or ones at a full lower limit never hide an eligible job on a shared key. +/// `limit` is the completed job's. Runs whatever the completed job's own time window: its slot +/// can go to a waiter that has none. Queue bookkeeping, not an authorization boundary. #[cfg(all(feature = "enterprise", feature = "private"))] -pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result<()> { +pub async fn wake_deferred_concurrency_job(db: &DB, key: &str, limit: i32) -> error::Result<()> { sqlx::query!( - "WITH taken AS ( + "WITH head AS ( + SELECT q.running, q.scheduled_for <= now() AS due + FROM concurrency_key ck + JOIN v2_job_queue q ON q.id = ck.job_id + JOIN v2_job j ON j.id = ck.job_id + LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle + LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings + WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL + AND q.canceled_by IS NULL + AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0 + ORDER BY ck.original_scheduled_for, ck.job_id + LIMIT 2 * $2::bigint + 1 + ), taken AS ( SELECT (SELECT count(*) FROM concurrency_counter, jsonb_object_keys(job_uuids) WHERE concurrency_id = $1) - + (SELECT count(*) - FROM concurrency_key due_ck - JOIN v2_job_queue due ON due.id = due_ck.job_id - WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL - AND NOT due.running AND due.canceled_by IS NULL - AND due.scheduled_for <= now()) AS n + + (SELECT count(*) FROM head WHERE due AND NOT running) AS n ), next AS ( SELECT q.id FROM concurrency_key ck @@ -5340,11 +5355,15 @@ pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result< AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0 AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0) ORDER BY ck.original_scheduled_for, ck.job_id - LIMIT 1 + LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0) FOR UPDATE OF q SKIP LOCKED ) - UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id", + UPDATE v2_job_queue SET scheduled_for = now() + FROM next + WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running + AND v2_job_queue.scheduled_for > now()", key, + i64::from(limit), ) .execute(db) .await?;