From 1f92c4fa44edabdaa4b7885d07e788c3e92f05fa Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 9 Nov 2026 17:34:05 +0100 Subject: [PATCH] feat: let dedicated_workers take folder patterns like ws:f/dw/* (#12268) * feat: reload dedicated workers in place and restart them one at a time Co-Authored-By: Claude Opus 5.5 * fix: reload dedicated importers on deploy, refresh python imports, count native readiness Co-Authored-By: Claude Opus 5.5 * fix: count a native dedicated worker as started once its isolate is ready Co-Authored-By: Claude Opus 5.5 * feat: let dedicated_workers take folder patterns like ws:f/dw/* Co-Authored-By: Claude Opus 5.5 * fix: re-save pattern groups on redeploy over an archived script, reject workspace in pattern input Co-Authored-By: Claude Opus 5.5 * fix: autoscale pattern groups on matched script tags, keep legacy dedicated_worker literal Co-Authored-By: Claude Opus 5.5 * docs: state who may supply entries to expand_dedicated_workers Co-Authored-By: Claude Opus 5.5 * feat: show which listed scripts a folder pattern covers and how many it matches Co-Authored-By: Claude Opus 5.5 * feat: explain staggered restarts in the dedicated worker group config Co-Authored-By: Claude Opus 5.5 * fix: state when dedicated deploys restart workers in the group config copy Co-Authored-By: Claude Opus 5.5 * fix: shorten the dedicated worker group restart copy Co-Authored-By: Claude Opus 5.5 * feat: warn about staggered restarts when applying a dedicated group config Co-Authored-By: Claude Opus 5.5 * fix: show the staggered restart note only for groups with several workers Co-Authored-By: Claude Opus 5.5 * fix: say when a dedicated deploy restarts workers in the group config note Co-Authored-By: Claude Opus 5.5 * fix: clarify when dedicated deploys restart workers and tighten the dedicated tab spacing Co-Authored-By: Claude Opus 5.5 * fix: keep only the supervisor note in the dedicated tab Co-Authored-By: Claude Opus 5.5 * fix: word the staggered restart alert in terms of workers Co-Authored-By: Claude Opus 5.5 * refactor: use checked sqlx macros for the dedicated restart turn queries Co-Authored-By: Claude Opus 5.5 * feat: list what restarts dedicated workers in a tooltip on the group config Co-Authored-By: Claude Opus 5.5 * fix: simplify the dedicated deploy restart bullet Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to 51a806ae14dcca95ca2221a9f5cc6434ed755bca This commit updates the EE repository reference after PR #950 was merged in windmill-ee-private. Previous ee-repo-ref: ce1eba465e5891bab71173c06723a1a6cbd2d869 New ee-repo-ref: 51a806ae14dcca95ca2221a9f5cc6434ed755bca Automated by sync-ee-ref workflow. * chore: update ee-repo-ref to 276f5f5cacc259bad1dc138430ae03c53d3db43d This commit updates the EE repository reference after PR #964 was merged in windmill-ee-private. Previous ee-repo-ref: 3511c351a2fd81268d2fb1f7ca1c2ba74c3c1467 New ee-repo-ref: 276f5f5cacc259bad1dc138430ae03c53d3db43d Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] Co-authored-by: Ruben Fiszel --- backend/ee-repo-ref.txt | 2 +- backend/src/monitor.rs | 14 ++- backend/windmill-api-scripts/src/scripts.rs | 26 ++++ backend/windmill-common/src/worker.rs | 59 ++++++++- backend/windmill-queue/src/tags.rs | 42 ++++--- .../DedicatedWorkersSelector.svelte | 116 ++++++++++++++++-- .../src/lib/components/dedicated_worker.ts | 14 ++- 7 files changed, 236 insertions(+), 37 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9e5923a17e..73a5c27013 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -51a806ae14dcca95ca2221a9f5cc6434ed755bca +276f5f5cacc259bad1dc138430ae03c53d3db43d diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f462ec7c04..57b96db902 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -5915,11 +5915,19 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b // Pausing or resuming must not restart the workers, dedicated ones included: a reload // whose only difference is the flag skips every restart below. - let only_pause_changed = wc.paused != config.paused - && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc; + // Tags only decide what the workers pull, and the pull query is swapped below: a change + // to them alone needs no restart either. A script toggled under a `dedicated_workers` + // pattern comes in this way, its dependency job tagged for the group before it has a lock. + let only_live_fields_changed = **wc != config + && WorkerConfig { + paused: wc.paused, + worker_tags: wc.worker_tags.clone(), + priority_tags_sorted: wc.priority_tags_sorted.clone(), + ..config.clone() + } == **wc; if **wc != config || has_dedicated { - if kill_if_change && !only_pause_changed { + if kill_if_change && !only_live_fields_changed { // Every worker of a dedicated group sees the same change at once: they restart // one at a time instead, so the group keeps warm workers to serve its jobs. // A cache clear still restarts at once: the cache goes away under the workers. diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 61d83e76fe..35682c60ca 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -3044,6 +3044,32 @@ async fn create_script_internal<'c>( } else { tracing::info!("creating script {hash:?} at path {script_path} on workspace {w_id}",); } + if ns.dedicated_worker == Some(true) && resolved_job_token_scopes.is_none() { + // Its dependency job is tagged for a dedicated worker. One taking it through a + // `dedicated_workers` pattern only listens to the scripts it found at its last config + // load: re-saving the group's config reloads it, unless the live version it replaces + // was served already (a version adopted from a deleted path was not). + sqlx::query( + "UPDATE config SET config = config + WHERE name LIKE 'worker\\_\\_%' + AND jsonb_typeof(config->'dedicated_workers') = 'array' + AND EXISTS ( + SELECT 1 FROM jsonb_array_elements_text(config->'dedicated_workers') e + WHERE right(e, 2) = '/*' AND starts_with($1 || ':' || $2, left(e, -1))) + AND NOT EXISTS ( + SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $3 AND path = $2 + AND dedicated_worker = true AND job_token_scopes IS NULL)", + ) + .bind(&w_id) + .bind(&ns.path) + .bind( + p_hashes + .filter(|_| !parent_adopted_from_retired_path) + .and_then(|h| h.first().copied()), + ) + .execute(&mut *tx) + .await?; + } if needs_lock_gen { let tag = windmill_common::scripts::dependency_job_tag(ns.tag, &lang); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 81d815f7a6..c1c1e9f582 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2604,6 +2604,49 @@ pub fn is_hashed_dedicated_worker_tag(tag: &str) -> bool { tag.len() == MAX_TAG_LEN && tag.as_bytes().get(MAX_TAG_LEN - 1 - HASH_SUFFIX_LEN) == Some(&b'#') } +/// The path prefix a `dedicated_workers` entry ending in `/*` (`f/dw/*`) stands for: every +/// script under it, nested folders included. Flows are listed one by one. +pub fn dedicated_worker_pattern_prefix(path: &str) -> Option<&str> { + path.strip_suffix('*') + .filter(|p| p.ends_with('/') && p.len() > 1 && !p.starts_with("flow/")) +} + +/// `entries` with each pattern replaced by the scripts under it that a dedicated worker serves: +/// those enabling `dedicated_worker` with an unrestricted job token, the only ones whose jobs +/// get the dedicated tag on push. Reads every workspace's scripts unchecked: `entries` must come +/// from a worker group config, which only an instance admin or devops user can write, never from a +/// request. +pub async fn expand_dedicated_workers( + db: &DB, + entries: &[WorkspacedPath], +) -> error::Result> { + let mut out: Vec = vec![]; + for wp in entries { + let Some(prefix) = dedicated_worker_pattern_prefix(&wp.path) else { + out.push(wp.clone()); + continue; + }; + let paths = sqlx::query_scalar::<_, String>( + "SELECT DISTINCT path FROM script + WHERE workspace_id = $1 AND starts_with(path, $2) AND archived = false + AND dedicated_worker = true AND job_token_scopes IS NULL + ORDER BY path", + ) + .bind(&wp.workspace_id) + .bind(prefix) + .fetch_all(db) + .await?; + out.extend( + paths + .into_iter() + .map(|path| WorkspacedPath { workspace_id: wp.workspace_id.clone(), path }), + ); + } + let mut seen = HashSet::new(); + out.retain(|wp| seen.insert((wp.workspace_id.clone(), wp.path.clone()))); + Ok(out) +} + pub fn dedicated_worker_tag(workspace_id: &str, path: &str) -> String { let full_tag = format!("{}:{}", workspace_id, path); if full_tag.len() <= MAX_TAG_LEN { @@ -2703,6 +2746,10 @@ pub async fn load_worker_config( .collect::, _>>() }) .transpose()?; + let dedicated_scripts = match dedicated_workers.as_deref() { + Some(dws) => Some(expand_dedicated_workers(db, dws).await?), + None => None, + }; if *WORKER_GROUP == "default" && dedicated_worker.is_none() { let mut all_tags = config .worker_tags @@ -2737,7 +2784,7 @@ pub async fn load_worker_config( .worker_tags .or_else(|| { // Check for multiple dedicated workers - if let Some(ref dws) = dedicated_workers.as_ref() { + if let Some(ref dws) = dedicated_scripts.as_ref() { let mut dedi_tags: Vec = dws .iter() .map(|dw| dedicated_worker_tag(&dw.workspace_id, &dw.path)) @@ -3891,6 +3938,16 @@ mod tests { assert!(marked.applies_to_workspace(&chain(&["other"]))); } + #[test] + fn dedicated_worker_patterns_are_script_folders() { + assert_eq!(dedicated_worker_pattern_prefix("f/dw/*"), Some("f/dw/")); + assert_eq!(dedicated_worker_pattern_prefix("u/alice/*"), Some("u/alice/")); + assert_eq!(dedicated_worker_pattern_prefix("f/dw/script"), None); + assert_eq!(dedicated_worker_pattern_prefix("f/dw*"), None); + assert_eq!(dedicated_worker_pattern_prefix("/*"), None); + assert_eq!(dedicated_worker_pattern_prefix("flow/f/dw/*"), None); + } + #[test] fn test_dedicated_worker_tag_short() { let tag = dedicated_worker_tag("demo", "u/alice/script"); diff --git a/backend/windmill-queue/src/tags.rs b/backend/windmill-queue/src/tags.rs index b31e3d99c3..6a9cfc1200 100644 --- a/backend/windmill-queue/src/tags.rs +++ b/backend/windmill-queue/src/tags.rs @@ -1,10 +1,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::error::Error; use windmill_common::worker::{ - dedicated_worker_tag, dev_workspace_tag_suffix, is_hashed_dedicated_worker_tag, - is_isolated_workspace, with_dev_workspace_suffix, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, DEV_WORKSPACE_TAG_SUFFIX, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, - MAX_TAG_LEN, WORKER_SERVES_DEV_WORKSPACES, + dedicated_worker_tag, dev_workspace_tag_suffix, expand_dedicated_workers, + is_hashed_dedicated_worker_tag, is_isolated_workspace, with_dev_workspace_suffix, + WorkspacedPath, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, DEV_WORKSPACE_TAG_SUFFIX, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, MAX_TAG_LEN, WORKER_SERVES_DEV_WORKSPACES, }; use windmill_common::workspaces::WM_FORK_PREFIX; @@ -207,21 +207,25 @@ async fn is_dedicated_worker_tag_of( let configs = sqlx::query_scalar!("SELECT config FROM config WHERE name LIKE 'worker\\_\\_%'") .fetch_all(db) .await?; - Ok(configs.into_iter().flatten().any(|config| { - let single = config.get("dedicated_worker").cloned().into_iter(); - let many = config - .get("dedicated_workers") - .and_then(|v| v.as_array().cloned()) - .unwrap_or_default(); - single.chain(many).any(|entry| { - entry - .as_str() - .and_then(|e| e.split_once(':')) - .is_some_and(|(ws, path)| { - ws == workspace_id && dedicated_worker_tag(ws, path) == tag - }) - }) - })) + let of_workspace = |entry: &serde_json::Value| { + let (ws, path) = entry.as_str()?.split_once(':')?; + (ws == workspace_id) + .then(|| WorkspacedPath { workspace_id: ws.to_string(), path: path.to_string() }) + }; + let mut single = vec![]; + let mut many = vec![]; + for config in configs.into_iter().flatten() { + single.extend(config.get("dedicated_worker").and_then(of_workspace)); + if let Some(entries) = config.get("dedicated_workers").and_then(|v| v.as_array()) { + many.extend(entries.iter().filter_map(of_workspace)); + } + } + // Only the list takes patterns, as in `load_worker_config`. + many = expand_dedicated_workers(db, &many).await?; + Ok(single + .iter() + .chain(many.iter()) + .any(|wp| dedicated_worker_tag(&wp.workspace_id, &wp.path) == tag)) } /// Gives `new_id` the lineage of `old_id` before a rename commits: the rename re-pushes the diff --git a/frontend/src/lib/components/DedicatedWorkersSelector.svelte b/frontend/src/lib/components/DedicatedWorkersSelector.svelte index d318faf9e5..c355f1cf50 100644 --- a/frontend/src/lib/components/DedicatedWorkersSelector.svelte +++ b/frontend/src/lib/components/DedicatedWorkersSelector.svelte @@ -15,16 +15,18 @@ CodeXml, ExternalLink, TriangleAlert, - Layers + Layers, + Folder } from 'lucide-svelte' import { Button } from './common' + import TextInput from './text_input/TextInput.svelte' import Select from './select/Select.svelte' import { sendUserToast } from '$lib/toast' import Badge from './common/badge/Badge.svelte' import { SvelteMap } from 'svelte/reactivity' import { untrack } from 'svelte' import BarsStaggered from './icons/BarsStaggered.svelte' - import { parseTag } from './dedicated_worker' + import { isDedicatedPattern, parseTag } from './dedicated_worker' import Tooltip from './Tooltip.svelte' // A "Runnable" is a script or flow with dedicated_worker=true @@ -73,13 +75,15 @@ interface SelectedTagInfo { tag: string workspace: string - type: 'script' | 'flow' + type: 'script' | 'flow' | 'pattern' path: string language?: string runners?: FlowRunner[] expanded?: boolean loading?: boolean workspaceDeps?: string[] + // Patterns: how many dedicated scripts the pattern covers today + matches?: number } let selectedTagsInfo: SvelteMap = $state(new SvelteMap()) @@ -183,7 +187,10 @@ tags.map(async (tag) => { // Check if we already have this info cached const existing = currentInfo.get(tag) - if (existing && (existing.type === 'script' || existing.runners !== undefined)) { + if ( + existing && + (existing.type !== 'flow' || existing.runners !== undefined) + ) { // Backfill workspaceDeps and language if missing if (existing.type === 'script' && !existing.workspaceDeps) { const depInfo = depsPerWorkspace.get(existing.workspace)?.get(existing.path) @@ -218,6 +225,21 @@ const parsed = parseTag(tag) if (!parsed) return + if (parsed.type === 'pattern') { + const matches = disabled + ? undefined + : await ScriptService.listScripts({ + workspace: parsed.workspace, + dedicatedWorker: true, + pathStart: parsed.path.slice(0, -1), + perPage: 1000 + }) + .then((s) => s.length) + .catch(() => undefined) + newInfo.set(tag, { tag, ...parsed, matches }) + return + } + if (disabled) { // Read-only: just show path info, skip API calls newInfo.set(tag, { @@ -520,11 +542,39 @@ // Keep tags from other workspaces, update only the currently visible ones const visibleTags = new Set(runnables.map((r) => r.tag)) const otherTags = selectedTags.filter((t) => !visibleTags.has(t)) + for (const r of runnables) if (coveringPattern(r)) r.selected = false const newVisibleTags = runnables.filter((r) => r.selected).map((r) => r.tag) selectedTags = [...otherTags, ...newVisibleTags] onchange?.(selectedTags) } + let patternInput = $state('') + let patternTag = $derived( + selectedWorkspace && isDedicatedPattern(patternInput.trim()) + ? `${selectedWorkspace}:${patternInput.trim()}` + : undefined + ) + + // The selected pattern that already serves this script, which makes its own entry redundant + function coveringPattern(r: Runnable): string | undefined { + if (r.type !== 'script') return undefined + return selectedTags.find((t) => { + const p = parseTag(t) + return ( + p?.type === 'pattern' && + t.startsWith(`${selectedWorkspace}:`) && + r.path.startsWith(p.path.slice(0, -1)) + ) + }) + } + + function addPattern() { + if (!patternTag || selectedTags.includes(patternTag)) return + selectedTags = [...selectedTags, patternTag] + patternInput = '' + updateSelectedTags() + } + function removeTag(tag: string) { selectedTags = selectedTags.filter((t) => t !== tag) // Also update runnable state if visible @@ -541,7 +591,7 @@ } }) - let selectedCount = $derived(runnables.filter((r) => r.selected).length) + let selectedCount = $derived(runnables.filter((r) => r.selected || coveringPattern(r)).length) // Compute shared runner groups: scripts sharing a (workspace, dep_name, language) tuple interface RunnerGroup { @@ -651,12 +701,29 @@ {#if info} {#if info.type === 'flow'} + {:else if info.type === 'pattern'} + {:else} {/if} {info.path} ({info.workspace}) + {#if info.type === 'pattern'} + + Serves every script under this folder that has the dedicated worker setting enabled. + A script enabling it later is picked up once deployed. + + {#if info.matches === 0} + No matching scripts yet + {:else if info.matches !== undefined} + + {info.matches} script{info.matches !== 1 ? 's' : ''} + + {:else} + All dedicated scripts + {/if} + {/if} {#if !tagRunnerGroup.has(tag)} {#if info.workspaceDeps} {#each info.workspaceDeps as dep} @@ -803,6 +870,8 @@ {:else}
{#each runnables as runnable (runnable.tag)} + {@const via = coveringPattern(runnable)} + {@const checked = runnable.selected || !!via}
{#if runnable.type === 'flow' && runnable.runners && runnable.runners.length > 0} @@ -822,24 +891,30 @@ {/if}
{/if} + + {#if selectedWorkspace} +
+ Folder pattern +
+ + +
+ Every script under the folder that enables the dedicated worker setting, including + ones deployed later. +
+ {/if}
{/if}
diff --git a/frontend/src/lib/components/dedicated_worker.ts b/frontend/src/lib/components/dedicated_worker.ts index f3b90fde57..21ec971990 100644 --- a/frontend/src/lib/components/dedicated_worker.ts +++ b/frontend/src/lib/components/dedicated_worker.ts @@ -17,9 +17,17 @@ export async function dedicatedWorkerTag(workspaceId: string, path: string): Pro return `${fullTag.substring(0, prefixLen)}#${hexHash.substring(0, HASH_SUFFIX_LEN)}` } +// Same rule as `dedicated_worker_pattern_prefix()` in backend/windmill-common/src/worker.rs: +// `f/dw/*` stands for every dedicated script under `f/dw/`. Flows are listed one by one. +export function isDedicatedPattern(path: string): boolean { + return ( + path.endsWith('/*') && path.length > 2 && !path.startsWith('flow/') && !path.includes(':') + ) +} + export function parseTag( tag: string -): { workspace: string; type: 'script' | 'flow'; path: string } | null { +): { workspace: string; type: 'script' | 'flow' | 'pattern'; path: string } | null { const colonIndex = tag.indexOf(':') if (colonIndex === -1) return null @@ -28,6 +36,8 @@ export function parseTag( if (rest.startsWith('flow/')) { return { workspace, type: 'flow', path: rest.substring(5) } + } else if (isDedicatedPattern(rest)) { + return { workspace, type: 'pattern', path: rest } } else { return { workspace, type: 'script', path: rest } } @@ -35,7 +45,7 @@ export function parseTag( export async function computeHashedTag(rawTag: string): Promise { const parsed = parseTag(rawTag) - if (!parsed) return rawTag + if (!parsed || parsed.type === 'pattern') return rawTag const fullPath = parsed.type === 'flow' ? `flow/${parsed.path}` : parsed.path return dedicatedWorkerTag(parsed.workspace, fullPath) }