fix: report a missing worker tag instead of spinning in data table UIs (#10456)

* fix: report a missing worker tag instead of spinning in db/datatable UIs Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: cancel the unpickable job, confirm the tag lookup, back off the poll Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: confirm an unserved tag over ~90s and never hide a failed refresh Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: leave the job queued so the autoscaler still sees the backlog Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: cancel writes before reporting them, leave reads queued Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: read the write's terminal state instead of trusting the cancel request Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: never abandon a write, explain the wait instead of cancelling Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-18 08:01:26 +00:00 · 2026-08-02 13:29:07 +02:00
parent 57b59a8ec8
commit eca24bdfb5
19 changed files with 558 additions and 117 deletions
@@ -1,9 +1,15 @@
 <script lang="ts">
 	import { dbSchemas, workspaceStore, type DBSchema } from '$lib/stores'
-	import { sendUserToast, sortArray } from '$lib/utils'
-	import { Loader2 } from 'lucide-svelte'
-	import { dbSupportsSchemas } from './apps/components/display/dbtable/utils'
+	import { sortArray } from '$lib/utils'
+	import { Loader2, RefreshCcw } from 'lucide-svelte'
+	import Alert from './common/alert/Alert.svelte'
+	import Button from './common/button/Button.svelte'
+	import {
+		dbSupportsSchemas,
+		getLanguageByResourceType
+	} from './apps/components/display/dbtable/utils'
 	import DbManager from './DBManager.svelte'
+	import MissingWorkerTagAlert from './jobs/MissingWorkerTagAlert.svelte'
 	import {
 		dbSchemaOpsWithPreviewScripts,
 		dbTableOpsWithPreviewScripts,
@@ -82,30 +88,61 @@
 		return `${ws}:${getDbSchemasPath(input)}`
 	}

+	// Reported in place of the loading spinner: both queries run as jobs, so
+	// anything from a bad connection to a tag no worker serves surfaces here
+	// instead of leaving the manager spinning with no explanation. Each query
+	// owns its slot so neither can clear the other's error on a refetch.
+	let schemaError = $state<string | undefined>(undefined)
+	let colDefsError = $state<string | undefined>(undefined)
+	let loadError = $derived(
+		schemaError
+			? { title: 'Could not load the database schema', message: schemaError }
+			: colDefsError
+				? { title: 'Could not load the tables of this database', message: colDefsError }
+				: undefined
+	)
+
 	let colDefs = resource(
 		() => [input, ws],
 		async () => {
+			colDefsError = undefined
 			if (!input) return
-			return await loadAllTablesMetaData(ws, input)
+			try {
+				return await loadAllTablesMetaData(ws, input)
+			} catch (e) {
+				colDefsError = 'Error loading tables metadata: ' + ((e as Error)?.message || e)
+				return
+			}
 		}
 	)
+
 	let dbSchemasPromise = resource(
 		() => [input, ws],
 		async () => {
+			schemaError = undefined
 			if (!input) return
 			const dbSchemasPath = schemaCacheKey(input)
 			if (input.type == 'database') {
-				$dbSchemas[dbSchemasPath] = await getDbSchemas(
+				const schema = await getDbSchemas(
 					input.resourceType,
 					input.resourcePath,
 					ws,
-					(message: string) => sendUserToast(message, true)
+					(message: string) => (schemaError = message)
 				)
+				if (!schema) {
+					schemaError ??= 'The schema query returned no schema'
+					return
+				}
+				$dbSchemas[dbSchemasPath] = schema
 			} else if (input.type == 'ducklake') {
-				$dbSchemas[dbSchemasPath] = await getDucklakeSchema({
-					workspace: ws!,
-					ducklake: input.ducklake
-				})
+				try {
+					$dbSchemas[dbSchemasPath] = await getDucklakeSchema({
+						workspace: ws!,
+						ducklake: input.ducklake
+					})
+				} catch (e) {
+					schemaError = 'Error fetching schema: ' + ((e as Error)?.message || e)
+				}
 			}
 		}
 	)
@@ -117,6 +154,24 @@
 		return colDefs.loading || dbSchemasPromise.loading
 	}

+	// Tag the schema/metadata jobs run on: for every DB the manager supports, the
+	// language name is the tag, which is what makes the missing-worker hint below
+	// possible without waiting for the poller to give up.
+	let jobTag = $derived(input ? getLanguageByResourceType(getDbType(input)) : undefined)
+
+	// A job queued behind busy workers still loads eventually, so this is a hint
+	// rather than an error. The no-worker-at-all case fails outright instead.
+	const SLOW_LOAD_MS = 10_000
+	let slowLoad = $state(false)
+	$effect(() => {
+		if (!isLoading()) {
+			slowLoad = false
+			return
+		}
+		const t = setTimeout(() => (slowLoad = true), SLOW_LOAD_MS)
+		return () => clearTimeout(t)
+	})
+
 	let replPanelSize = $state(36)
 	const REPL_MIN_SIZE = 1.5

@@ -145,7 +200,23 @@
 	}}
 />

-{#if dbSchema && ws && input}
+<!-- The error branch comes first on purpose: `dbSchema` is read from a cache that
+	survives a failed refetch, so ordering it first would hide the failure behind
+	stale content. -->
+{#if loadError}
+	<div class="h-full w-full flex flex-col items-center justify-center gap-3 p-8">
+		<div class="max-w-2xl w-full flex flex-col gap-3">
+			<Alert type="error" title={loadError.title} size="xs">
+				{loadError.message}
+			</Alert>
+			<div class="self-start">
+				<Button size="xs" color="light" startIcon={{ icon: RefreshCcw }} on:click={() => refresh()}>
+					Retry
+				</Button>
+			</div>
+		</div>
+	</div>
+{:else if dbSchema && ws && input}
 	{@const _input = input}
 	{@const dbType = getDbType(_input)}
 	<Splitpanes horizontal>
@@ -238,8 +309,22 @@
 	</Splitpanes>
 {:else}
 	<Splitpanes>
-		<Pane class="relative flex justify-center items-center">
+		<Pane class="relative flex flex-col justify-center items-center gap-3 p-8">
 			<Loader2 class="animate-spin" size={32} />
+			{#if slowLoad}
+				<span class="text-xs text-tertiary max-w-md text-center">
+					The schema query is taking a while. It runs as a job, so it waits for a worker serving its
+					tag to be free.
+				</span>
+				{#if jobTag}
+					<MissingWorkerTagAlert
+						tag={jobTag}
+						subject="Database queries"
+						workspace={ws}
+						class="max-w-2xl w-full"
+					/>
+				{/if}
+			{/if}
 		</Pane>
 	</Splitpanes>
 {/if}
@@ -70,8 +70,8 @@
 								.then(() => {
 									sendUserToast('Value updated')
 								})
-								.catch(() => {
-									sendUserToast('Error updating value', true)
+								.catch((e) => {
+									sendUserToast('Error updating value: ' + ((e as Error)?.message || e), true)
 									refresh?.()
 								})
 						}
@@ -14,6 +14,7 @@
 	import { sendUserToast } from '$lib/toast'
 	import { userWorkspaces } from '$lib/stores'
 	import { runScriptAndPollResult } from '$lib/components/jobs/utils'
+	import { writingJobOptions } from '$lib/components/jobs/writingJob'
 	import YAML from 'yaml'
 	import DrawerContent from './common/drawer/DrawerContent.svelte'
 	import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte'
@@ -228,14 +229,17 @@
 					throw runErr
 				}
 			} else {
-				await runScriptAndPollResult({
-					workspace: targetWorkspace,
-					requestBody: {
-						args: { database: `datatable://${dtName}` },
-						language: 'postgresql',
-						content: migrationSql
-					}
-				})
+				await runScriptAndPollResult(
+					{
+						workspace: targetWorkspace,
+						requestBody: {
+							args: { database: `datatable://${dtName}` },
+							language: 'postgresql',
+							content: migrationSql
+						}
+					},
+					writingJobOptions
+				)
 			}
 		} catch (e: any) {
 			sendUserToast(e?.body ?? e?.message ?? String(e), true)
@@ -2,6 +2,7 @@
 	import { CornerDownLeft, Loader2 } from 'lucide-svelte'
 	import Button from './common/button/Button.svelte'
 	import { runScriptAndPollResult } from './jobs/utils'
+	import { writingJobOptions } from './jobs/writingJob'
 	import { workspaceStore } from '$lib/stores'
 	import { sendUserToast } from '$lib/toast'
 	import { untrack } from 'svelte'
@@ -123,7 +124,8 @@
 						args: dbArg
 					}
 				},
-				{ withJobData: true }
+				// The user types arbitrary SQL here, so treat every run as a write.
+				{ withJobData: true, ...writingJobOptions }
 			)) as any
 			if (statements.length > 1) {
 				result = result[result.length - 1]
@@ -105,7 +105,12 @@
 				}
 			})

-			let result: any = await pollJobResult(jobId, $workspaceStore!)
+			// The shell tag is the worker's name prefix, which its shell loop pulls
+			// directly instead of advertising it in `worker_ping`, so the missing-worker
+			// check would read it as unserved.
+			let result: any = await pollJobResult(jobId, $workspaceStore!, {
+				failIfNoWorkerForTag: false
+			})

 			if (isOnlyCdCommand) {
 				working_directory = (result as string).replace(/(\r\n|\n|\r)/g, '')
@@ -365,9 +370,10 @@
 					>
 						Full path

-						<Tooltip
-							class="absolute top-0.5"
-						>Commands run in the default directory. Run a standalone ‘cd’ to change it. Chained or invalid ‘cd’ commands won’t apply.</Tooltip>
+						<Tooltip class="absolute top-0.5"
+							>Commands run in the default directory. Run a standalone ‘cd’ to change it. Chained or
+							invalid ‘cd’ commands won’t apply.</Tooltip
+						>
 					</Badge>
 				</div>
 				<input type="text" disabled bind:value={working_directory} />
@@ -399,7 +399,13 @@
 			resolvedConfig.type.configuration[selected].table
 		)

-		if (!tableMetadata) return
+		if (!tableMetadata) {
+			//@ts-ignore
+			gridItem.data.configuration.columnDefs.loading = false
+			gridItem.data = gridItem.data
+			$app = $app
+			return
+		}

 		let old: TableMetadata = (columnDefs?.value as TableMetadata) ?? []
 		if (!Array.isArray(old)) {
@@ -1,5 +1,3 @@
-import { JobService } from '$lib/gen'
-
 import { runScriptAndPollResult } from '$lib/components/jobs/utils'
 import type { DbInput } from '$lib/components/dbTypes'
 import {
@@ -43,50 +41,33 @@ export async function loadTableMetaData(
 	// back to `DATABASE()`), so we don't read the resource value client-side for it.
 	const content = makeMetadataMarker('LOAD_TABLE_METADATA', { table }, ducklake)

-	const job = await JobService.runScriptPreview({
-		workspace,
-		requestBody: { language, content, args: dbArg }
-	})
+	try {
+		const rows = (await runScriptAndPollResult({
+			workspace,
+			requestBody: { language, content, args: dbArg }
+		})) as Record<string, any>[]
+		const result = rows.map(lowercaseKeys)

-	const maxRetries = 8
-	let attempts = 0
-	while (attempts < maxRetries) {
-		try {
-			await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6)))
-
-			const testResult = (await JobService.getCompletedJob({
-				workspace,
-				id: job
-			})) as any
-
-			if (testResult.success) {
-				attempts = maxRetries
-
-				const result = testResult.result.map(lowercaseKeys)
-
-				// For Snowflake, fetch primary keys separately
-				if (
-					input.type === 'database' &&
-					(input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth')
-				) {
-					const map: Record<string, TableMetadata> = { [table]: result }
-					await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table)
-					return map[table]
-				}
-
-				return result
-			} else {
-				attempts++
-			}
-		} catch (error) {
-			attempts++
+		// For Snowflake, fetch primary keys separately
+		if (
+			input.type === 'database' &&
+			(input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth')
+		) {
+			const map: Record<string, TableMetadata> = { [table]: result }
+			await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table)
+			return map[table]
 		}
-	}

-	console.error('Failed to load table metadata after maximum retries.')
-	return undefined
+		return result
+	} catch (e) {
+		console.error('Failed to load table metadata', e)
+		sendUserToast('Error loading table metadata: ' + ((e as Error)?.message || e), true)
+		return undefined
+	}
 }

+/** Throws on failure without reporting it: every caller renders the error in its
+ * own pane, so toasting here would double-report it. */
 export async function loadAllTablesMetaData(
 	workspace: string | undefined,
 	input: DbInput
@@ -120,7 +101,7 @@ export async function loadAllTablesMetaData(

 		return map
 	} catch (e) {
-		sendUserToast('Error loading tables metadata: ' + e, 'error')
+		console.error('Failed to load tables metadata', e)
 		throw e
 	}
 }
@@ -171,7 +152,7 @@ async function fetchSnowflakePrimaryKeys(
 	const payload: Record<string, unknown> = {}
 	if (tableKey) payload.table = tableKey
 	const content = makeMetadataMarker('SNOWFLAKE_PRIMARY_KEYS', payload, undefined)
-	return (await JobService.runScriptPreviewAndWaitResult({
+	return (await runScriptAndPollResult({
 		workspace,
 		requestBody: {
 			language: 'snowflake',
@@ -206,7 +187,7 @@ export async function getDbSchemas(

 	let result: unknown
 	try {
-		result = await JobService.runScriptPreviewAndWaitResult({
+		result = await runScriptAndPollResult({
 			workspace,
 			requestBody: {
 				language: sqlScript.lang as Preview['language'],
@@ -102,6 +102,11 @@
 			: undefined
 	)

+	// Kept apart from "no columns for this table": a failed metadata read says
+	// nothing about whether the table exists, so reporting it as missing would
+	// point the user at the wrong problem.
+	let colDefsError = $state<string | undefined>(undefined)
+
 	// Metadata for every table in the datatable. We query the lot rather
 	// than just the one we care about because `loadAllTablesMetaData` is
 	// what `dbTableOpsWithPreviewScripts` and the rest of the DB manager
@@ -111,14 +116,12 @@
 	let colDefs = resource(
 		() => [input, refreshKey],
 		async ([_input]) => {
+			colDefsError = undefined
 			if (!_input || !$workspaceStore) return undefined
 			try {
 				return await loadAllTablesMetaData($workspaceStore, _input)
-			} catch {
-				// Connection/permission errors look identical to "table
-				// missing" from the user's POV inside the preview pane;
-				// the full error is surfaced via the existing sendUserToast
-				// path inside loadAllTablesMetaData.
+			} catch (e) {
+				colDefsError = (e as Error)?.message || String(e)
 				return undefined
 			}
 		}
@@ -187,6 +190,12 @@
 		<div class="absolute inset-0 flex items-center justify-center text-tertiary">
 			<Loader2 class="animate-spin" size={20} />
 		</div>
+	{:else if colDefsError}
+		<div class="absolute inset-0 flex flex-col items-center justify-center gap-2 p-6 text-center">
+			<AlertTriangle size={20} class="text-red-500" />
+			<span class="text-sm font-medium text-emphasis">Could not read the datatable</span>
+			<span class="text-2xs text-tertiary max-w-md">{colDefsError}</span>
+		</div>
 	{:else if !tableColDefs}
 		<!-- Datatable exists but the specific table doesn't — most often
 		     because no upstream pipeline script has run yet to create it. -->
@@ -53,15 +53,19 @@
 		scoped && partition ? `_wm_partition = '${partition.replaceAll("'", "''")}'` : undefined
 	)

+	// Distinct from "no columns for this table": a failed metadata read says
+	// nothing about whether the table exists.
+	let colDefsError = $state<string | undefined>(undefined)
+
 	let colDefs = resource(
 		() => [input, refreshKey] as const,
 		async ([_input]) => {
+			colDefsError = undefined
 			if (!_input || !$workspaceStore) return undefined
 			try {
 				return await loadAllTablesMetaData($workspaceStore, _input)
-			} catch {
-				// A load failure reads the same as "table missing" from the preview's
-				// POV; the underlying error is surfaced by loadAllTablesMetaData.
+			} catch (e) {
+				colDefsError = (e as Error)?.message || String(e)
 				return undefined
 			}
 		}
@@ -119,6 +123,11 @@
 		<div class="flex items-center justify-center p-4 text-tertiary">
 			<Loader2 class="animate-spin" size={18} />
 		</div>
+	{:else if colDefsError}
+		<div class="flex items-start gap-2 p-3 text-2xs text-tertiary">
+			<AlertTriangle size={14} class="text-red-500 shrink-0 mt-0.5" />
+			{colDefsError}
+		</div>
 	{:else if !tableColDefs}
 		<div class="flex items-center gap-2 p-3 text-2xs text-tertiary">
 			<AlertTriangle size={14} class="text-amber-500" />
@@ -5,6 +5,7 @@ import {
 	type TableMetadata
 } from './apps/components/display/dbtable/utils'
 import { runScriptAndPollResult } from './jobs/utils'
+import { writingJobOptions } from './jobs/writingJob'
 import type { DBSchema, SQLSchema } from '$lib/stores'
 import { stringifySchema } from './copilot/lib'
 import type { DbInput, DbType } from './dbTypes'
@@ -114,28 +115,31 @@ export function dbTableOpsWithPreviewScripts({
 				column: colDef,
 				columns: colDefs
 			})
-			await runScriptAndPollResult({
-				workspace,
-				requestBody: {
-					args: { ...dbArg, value_to_update: newValue, ...values },
-					language,
-					content
-				}
-			})
+			await runScriptAndPollResult(
+				{
+					workspace,
+					requestBody: {
+						args: { ...dbArg, value_to_update: newValue, ...values },
+						language,
+						content
+					}
+				},
+				writingJobOptions
+			)
 		},
 		onDelete: async ({ values }) => {
 			const content = makeMarker('DELETE', { table: tableKey, columns: colDefs })
-			await runScriptAndPollResult({
-				workspace,
-				requestBody: { args: { ...dbArg, ...values }, language, content }
-			})
+			await runScriptAndPollResult(
+				{ workspace, requestBody: { args: { ...dbArg, ...values }, language, content } },
+				writingJobOptions
+			)
 		},
 		onInsert: async ({ values }) => {
 			const content = makeMarker('INSERT', { table: tableKey, columns: colDefs })
-			await runScriptAndPollResult({
-				workspace,
-				requestBody: { args: { ...dbArg, ...values }, language, content }
-			})
+			await runScriptAndPollResult(
+				{ workspace, requestBody: { args: { ...dbArg, ...values }, language, content } },
+				writingJobOptions
+			)
 		}
 	}
 }
@@ -340,7 +344,10 @@ export function dbSchemaOpsWithPreviewScripts({
 			? await WorkspaceService.getDatatableMigrationsStatus({ workspace, datatableName })
 			: undefined
 		if (!datatableName || !status?.enabled) {
-			await runScriptAndPollResult({ workspace, requestBody: { args: dbArg, content, language } })
+			await runScriptAndPollResult(
+				{ workspace, requestBody: { args: dbArg, content, language } },
+				writingJobOptions
+			)
 			return
 		}
 		// The new migration gets the highest timestamp, so any still-pending
@@ -0,0 +1,47 @@
+<script lang="ts">
+	import { base } from '$app/paths'
+	import Alert from '$lib/components/common/alert/Alert.svelte'
+	import { WorkerService } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { resource } from 'runed'
+	import { hasWorkerForTag } from './missingWorker'
+
+	interface Props {
+		/** Worker tag the feature's jobs run on. */
+		tag: string
+		/** What breaks without it, used as the sentence subject, e.g. "Data table queries". */
+		subject: string
+		workspace?: string
+		class?: string
+	}
+
+	let { tag, subject, workspace = undefined, class: className = '' }: Props = $props()
+
+	let ws = $derived(workspace ?? $workspaceStore)
+
+	// With per-workspace default tags, jobs run on a workspace-suffixed variant of
+	// `tag`, so probing the bare tag would warn about a tag nothing uses. The
+	// in-flight check in `pollJobResult` reads the job's real tag and still covers
+	// those instances.
+	const served = resource(
+		() => [ws, tag] as const,
+		async ([ws, tag]) => {
+			if (!ws || (await WorkerService.isDefaultTagsPerWorkspace())) return true
+			return await hasWorkerForTag(ws, tag)
+		}
+	)
+</script>
+
+{#if served.current === false}
+	<div class={className}>
+		<!-- Deliberately not phrased as "this tag is not configured": a correctly
+			tagged group sitting at zero replicas is indistinguishable from one in
+			`worker_ping`, and queueing a job is what scales it back up. -->
+		<Alert type="warning" title="No worker is running for the &quot;{tag}&quot; tag" size="xs">
+			{subject} run as Windmill jobs tagged <b>{tag}</b>, and no worker is currently listening to
+			that tag, so they stay queued until one is. If no worker group is meant to serve it, add
+			<b>{tag}</b>
+			to a group's worker tags on the <a href="{base}/workers">workers page</a>.
+		</Alert>
+	</div>
+{/if}
@@ -0,0 +1,121 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
+
+const getCompletedJobResultMaybe = vi.fn()
+const getJob = vi.fn()
+const cancelQueuedJob = vi.fn()
+const existsWorkersWithTags = vi.fn()
+
+vi.mock('$lib/gen', () => ({
+	JobService: {
+		getCompletedJobResultMaybe: (...a: unknown[]) => getCompletedJobResultMaybe(...(a as [])),
+		getJob: (...a: unknown[]) => getJob(...(a as [])),
+		cancelQueuedJob: (...a: unknown[]) => cancelQueuedJob(...(a as []))
+	},
+	WorkerService: {
+		existsWorkersWithTags: (...a: unknown[]) => existsWorkersWithTags(...(a as []))
+	}
+}))
+
+import { pollJobResult } from './utils'
+import { hasWorkerForTag, NoWorkerForTagError } from './missingWorker'
+
+function settlementTracker(promise: Promise<unknown>) {
+	const state = { settled: false }
+	promise.then(
+		() => (state.settled = true),
+		() => (state.settled = true)
+	)
+	return state
+}
+
+beforeEach(() => {
+	getCompletedJobResultMaybe.mockReset()
+	getJob.mockReset()
+	cancelQueuedJob.mockReset()
+	existsWorkersWithTags.mockReset()
+	getCompletedJobResultMaybe.mockResolvedValue({ completed: false })
+	getJob.mockResolvedValue({ type: 'QueuedJob', running: false, tag: 'postgresql' })
+	cancelQueuedJob.mockResolvedValue(undefined)
+	vi.useFakeTimers()
+})
+
+afterEach(() => {
+	vi.useRealTimers()
+})
+
+// Long enough for the whole confirmation window (first probe + 2 intervals).
+const PAST_CONFIRMATION_WINDOW_MS = 120_000
+
+describe('pollJobResult', () => {
+	it('reports a queued read whose tag stays unserved without cancelling it', async () => {
+		existsWorkersWithTags.mockResolvedValue({ postgresql: false })
+
+		const promise = pollJobResult('job-1', 'ws')
+		const rejects = expect(promise).rejects.toBeInstanceOf(NoWorkerForTagError)
+		await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS)
+		await rejects
+		// The backlog is what the autoscaler scales up on: cancelling would stop a
+		// group coming back from zero from ever recovering.
+		expect(cancelQueuedJob).not.toHaveBeenCalled()
+	})
+
+	it('never abandons a write, and reports once why it is waiting', async () => {
+		existsWorkersWithTags.mockResolvedValue({ postgresql: false })
+		const onNoWorkerForTag = vi.fn()
+
+		const promise = pollJobResult('job-1', 'ws', { sideEffecting: true, onNoWorkerForTag })
+		const tracker = settlementTracker(promise)
+
+		await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS * 3)
+		// Reporting failure while the write stays executable would let it apply after
+		// the caller gave up and duplicate on retry; cancelling it first cannot be
+		// done atomically from the client.
+		expect(tracker.settled).toBe(false)
+		expect(cancelQueuedJob).not.toHaveBeenCalled()
+		expect(onNoWorkerForTag).toHaveBeenCalledTimes(1)
+		expect(onNoWorkerForTag).toHaveBeenCalledWith('postgresql')
+
+		getCompletedJobResultMaybe.mockResolvedValue({ completed: true, success: true, result: 7 })
+		await vi.advanceTimersByTimeAsync(3_000)
+		await expect(promise).resolves.toBe(7)
+	})
+
+	it('does not give up while a worker group could still be coming up', async () => {
+		// A worker group booting is absent from worker_ping exactly like an unserved
+		// tag; only a run of empty lookups distinguishes them.
+		existsWorkersWithTags.mockResolvedValueOnce({ postgresql: false })
+		existsWorkersWithTags.mockResolvedValueOnce({ postgresql: false })
+		existsWorkersWithTags.mockResolvedValue({ postgresql: true })
+
+		const promise = pollJobResult('job-1', 'ws')
+		const tracker = settlementTracker(promise)
+
+		await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS)
+		expect(tracker.settled).toBe(false)
+		expect(cancelQueuedJob).not.toHaveBeenCalled()
+	})
+
+	it('keeps waiting on a job queued behind a busy worker that serves its tag', async () => {
+		existsWorkersWithTags.mockResolvedValue({ postgresql: true })
+
+		const promise = pollJobResult('job-1', 'ws')
+		const tracker = settlementTracker(promise)
+
+		await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS)
+		expect(tracker.settled).toBe(false)
+
+		getCompletedJobResultMaybe.mockResolvedValue({ completed: true, success: true, result: 42 })
+		await vi.advanceTimersByTimeAsync(3_000)
+		await expect(promise).resolves.toBe(42)
+	})
+})
+
+describe('hasWorkerForTag', () => {
+	it('treats an answer it did not get as a worker being there', async () => {
+		// `existsWorkersWithTags` returns an empty map when TAGS_ARE_SENSITIVE hides
+		// the tag from the caller. Reading that as "unserved" would diagnose a
+		// perfectly healthy instance.
+		existsWorkersWithTags.mockResolvedValue({})
+		await expect(hasWorkerForTag('ws', 'postgresql')).resolves.toBe(true)
+	})
+})
@@ -0,0 +1,76 @@
+import { JobService, WorkerService } from '$lib/gen'
+
+/**
+ * A queued job whose tag no running worker serves is never picked up: without
+ * this the UI polls until the server-side `run_wait_result` timeout (10min by
+ * default) or, for the client-side pollers, forever.
+ *
+ * The most common cause is a language that defaults to a native tag
+ * (`postgresql`, `mysql`, `bigquery`, …) on an instance whose worker groups
+ * only declare the default tags.
+ */
+export class NoWorkerForTagError extends Error {
+	tag: string
+
+	constructor(tag: string) {
+		super(
+			`No worker has been listening to the tag "${tag}" while this job waited, so it was never ` +
+				`picked up. It stays queued and will run once a worker with that tag comes online. ` +
+				`Add "${tag}" to the worker tags of one of your worker groups (Workers page), or run a ` +
+				`worker that serves it.`
+		)
+		this.name = 'NoWorkerForTagError'
+		this.tag = tag
+	}
+}
+
+/** Shown while a write waits, which is never abandoned (see `sideEffecting`). */
+export function queuedWithoutWorkerMessage(tag: string): string {
+	return (
+		`No worker is listening to the tag "${tag}", so this operation is queued and will only run ` +
+		`once one is. Add "${tag}" to the worker tags of one of your worker groups (Workers page), ` +
+		`or run a worker that serves it.`
+	)
+}
+
+/** How long a job may sit un-started before the first lookup for a worker serving its tag. */
+export const NO_WORKER_FIRST_PROBE_MS = 10_000
+/** How long to wait between lookups while the job stays queued. */
+export const NO_WORKER_PROBE_INTERVAL_MS = 40_000
+/**
+ * How many consecutive lookups must come back empty before the caller stops
+ * waiting. A worker group scaling from zero, or every worker down for a rollout,
+ * is indistinguishable from an unserved tag in any single lookup, so no single
+ * empty reading is acted on.
+ */
+export const NO_WORKER_CONFIRMATIONS = 3
+
+/**
+ * Whether any worker pinged in the last minute declares `tag`. Unknown answers
+ * (the endpoint returns an empty map when tags are sensitive and the caller may
+ * not see them) count as "yes", so an opaque instance never gets a wrong
+ * diagnosis.
+ */
+export async function hasWorkerForTag(workspace: string, tag: string): Promise<boolean> {
+	const existing = await WorkerService.existsWorkersWithTags({ workspace, tags: tag })
+	return existing[tag] !== false
+}
+
+/**
+ * The tag of `jobId` when it is still queued and no worker serves it, else
+ * undefined. Never throws: a failed lookup means "can't tell", and the caller
+ * keeps waiting rather than reporting a cause it did not establish.
+ */
+export async function missingWorkerTagOfQueuedJob(
+	workspace: string,
+	jobId: string
+): Promise<string | undefined> {
+	try {
+		const job = await JobService.getJob({ workspace, id: jobId, noCode: true, noLogs: true })
+		if (job.type !== 'QueuedJob' || job.running || !job.tag) return undefined
+		return (await hasWorkerForTag(workspace, job.tag)) ? undefined : job.tag
+	} catch (err) {
+		console.warn('Could not determine whether a worker serves the job tag', err)
+		return undefined
+	}
+}
@@ -1,4 +1,11 @@
 import { JobService, type RunScriptByPathData, type RunScriptPreviewData } from '$lib/gen'
+import {
+	missingWorkerTagOfQueuedJob,
+	NoWorkerForTagError,
+	NO_WORKER_CONFIRMATIONS,
+	NO_WORKER_FIRST_PROBE_MS,
+	NO_WORKER_PROBE_INTERVAL_MS
+} from './missingWorker'

 function isRunScriptByPathData(
 	arg: RunScriptPreviewData | RunScriptByPathData
@@ -9,13 +16,26 @@ function isRunScriptByPathData(
 type RunScriptOptions = {
 	maxRetries?: number
 	withJobData?: boolean
+	/** Set to false to keep polling a job that no worker can pick up. */
+	failIfNoWorkerForTag?: boolean
+	/**
+	 * The job writes (insert/update/delete, DDL, arbitrary SQL). Such a job is
+	 * never given up on: reporting it as failed while it stays executable invites
+	 * a duplicate retry, and cancelling it first is not something the client can
+	 * do atomically: a worker can claim it between the tag probe and the cancel,
+	 * and a soft-cancelled statement may already have committed. `onNoWorkerForTag`
+	 * is what tells the user why it is waiting.
+	 */
+	sideEffecting?: boolean
+	/** Called once when the job has stayed queued on a tag no worker serves. */
+	onNoWorkerForTag?: (tag: string) => void
 }

 /**
 * @function runScript
 * @param {RunScriptPreviewData | RunScriptByPathData} data - Data for running the script.
 * @returns {Promise<string>} A UUID representing the running script.
- * 
+ *
 * @example
 * const uuid = await runScript(data)
 */
@@ -29,6 +49,15 @@ export async function runScript(data: RunScriptPreviewData | RunScriptByPathData
 	return uuid
 }

+/** Tight at first so a quick job feels instant, then slower: a schema
+ * introspection or a DDL migration can run for minutes, and a fixed sub-second
+ * tick would cost hundreds of round-trips for it. */
+function pollDelayMs(poll: number): number {
+	if (poll < 4) return 375
+	if (poll < 12) return 750
+	return 2000
+}
+
 /**
 * @function pollJobResult
 * @description Polls a job result by UUID until success, failure, or max retries reached.
@@ -36,19 +65,35 @@ export async function runScript(data: RunScriptPreviewData | RunScriptByPathData
 * @param {string} workspace - Workspace identifier.
 * @param {RunScriptOptions} [options] - Optional settings like retries and job data inclusion.
 * @returns {Promise<unknown>} Final job result or throws error if it fails.
- * 
+ *
 * @example
 * const result = await pollJobResult(uuid, 'my-workspace', { maxRetries: 5, withJobData: true });
 */
 export async function pollJobResult(
 	uuid: string,
 	workspace: string,
-	{ maxRetries = 7, withJobData }: RunScriptOptions = {}
+	{
+		maxRetries = 7,
+		withJobData,
+		failIfNoWorkerForTag = true,
+		sideEffecting = false,
+		onNoWorkerForTag
+	}: RunScriptOptions = {}
 ): Promise<unknown> {
 	let attempts = 0
+	let polls = 0
+	// `attempts` only advances on errors, so a queued job would poll forever. The
+	// one case that never resolves on its own is a tag no worker serves, which
+	// takes NO_WORKER_CONFIRMATIONS consecutive empty lookups to establish, since
+	// a worker group booting reads like an unserved tag in any single one.
+	let noWorkerProbeAt = Date.now() + NO_WORKER_FIRST_PROBE_MS
+	let unservedProbes = 0
+	let reportedNoWorker = false
 	while (attempts < maxRetries) {
 		try {
-			await new Promise((resolve) => setTimeout(resolve, 500 * (attempts || 0.75)))
+			await new Promise((resolve) =>
+				setTimeout(resolve, attempts ? 500 * attempts : pollDelayMs(polls++))
+			)
 			const job = await JobService.getCompletedJobResultMaybe({
 				id: uuid,
 				workspace
@@ -65,8 +110,26 @@ export async function pollJobResult(
 				if (typeof errorMsg !== 'string') errorMsg = undefined
 				console.error('JOB FAILED', job.result)
 				throw new Error(errorMsg ?? 'Job failed')
+			} else if (failIfNoWorkerForTag && Date.now() >= noWorkerProbeAt) {
+				const tag = await missingWorkerTagOfQueuedJob(workspace, uuid)
+				noWorkerProbeAt = Date.now() + NO_WORKER_PROBE_INTERVAL_MS
+				unservedProbes = tag ? unservedProbes + 1 : 0
+				if (tag && unservedProbes >= NO_WORKER_CONFIRMATIONS) {
+					if (!reportedNoWorker) {
+						reportedNoWorker = true
+						onNoWorkerForTag?.(tag)
+					}
+					// Reads give up the wait but leave the job queued: cancelling one would
+					// remove the very backlog the autoscaler scales up on, so a group coming
+					// back from zero (300s cooldown) would never recover. Writes keep
+					// waiting instead (see `sideEffecting`).
+					if (!sideEffecting) throw new NoWorkerForTagError(tag)
+				}
 			}
 		} catch (e) {
+			if (e instanceof NoWorkerForTagError) {
+				throw e
+			}
 			if (attempts == maxRetries) {
 				throw e
 			}
@@ -0,0 +1,12 @@
+import { sendUserToast } from '$lib/toast'
+import { queuedWithoutWorkerMessage } from './missingWorker'
+
+/**
+ * Poll options for a job that writes (row edits, DDL, arbitrary SQL). Such a job
+ * is never abandoned (see `sideEffecting` in `pollJobResult`), so this is what
+ * explains the wait when it sits on a tag no worker serves.
+ */
+export const writingJobOptions = {
+	sideEffecting: true,
+	onNoWorkerForTag: (tag: string) => sendUserToast(queuedWithoutWorkerMessage(tag), true)
+}
@@ -54,6 +54,7 @@
 	} from 'lucide-svelte'
 	import DraggableTabs, { type TabItem } from '$lib/components/common/tabs/DraggableTabs.svelte'
 	import { runScriptAndPollResult } from '../jobs/utils'
+	import { writingJobOptions } from '../jobs/writingJob'
 	import { RawAppHistoryManager } from './RawAppHistoryManager.svelte'
 	import { sendUserToast } from '$lib/utils'
 	import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
@@ -1020,14 +1021,17 @@
 				}

 				try {
-					const result = await runScriptAndPollResult({
-						workspace: opWorkspace,
-						requestBody: {
-							language: 'postgresql',
-							content: sql,
-							args: { database: `datatable://${datatableName}` }
-						}
-					})
+					const result = await runScriptAndPollResult(
+						{
+							workspace: opWorkspace,
+							requestBody: {
+								language: 'postgresql',
+								content: sql,
+								args: { database: `datatable://${datatableName}` }
+							}
+						},
+						writingJobOptions
+					)

 					// If newTable was specified and the query succeeded, add it to data.tables
 					if (newTable) {
@@ -25,6 +25,7 @@
 	import Tooltip from '../Tooltip.svelte'
 	import Portal from '$lib/components/Portal.svelte'
 	import DropdownV2 from '../DropdownV2.svelte'
+	import MissingWorkerTagAlert from '../jobs/MissingWorkerTagAlert.svelte'
 	import { superadmin, userStore } from '$lib/stores'

 	let {
@@ -422,6 +423,7 @@
 				{/if}
 			</div>
 		{:else}
+			<MissingWorkerTagAlert tag="postgresql" subject="Migrations" {workspace} class="mb-2" />
 			{#if loadError}
 				<div class="text-xs text-red-500">
 					Could not read applied status from the data table: {loadError}
@@ -87,6 +87,7 @@
 	import { clone } from '$lib/utils'
 	import SettingsFooter from './SettingsFooter.svelte'
 	import Alert from '../common/alert/Alert.svelte'
+	import MissingWorkerTagAlert from '../jobs/MissingWorkerTagAlert.svelte'
 	import { isCloudHosted } from '$lib/cloud'

 	type Props = {
@@ -252,6 +253,8 @@
 	</Alert>
 {/if}

+<MissingWorkerTagAlert tag="postgresql" subject="Browsing and querying data tables" class="mb-4" />
+
 <DataTable>
 	<Head>
 		<tr>
@@ -23,6 +23,7 @@ import { updateRawAppPolicy } from '$lib/sharedUtils'
 import { apiErrorMessage as errorMessage } from '$lib/utils'
 import type { App } from '$lib/components/apps/types'
 import { runScriptAndPollResult } from '$lib/components/jobs/utils'
+import { writingJobOptions } from '$lib/components/jobs/writingJob'
 import {
 	classifyPath,
 	collectExportVarPaths,
@@ -233,14 +234,17 @@ async function applyOneMigration(
 			only: created.timestamp
 		})
 	} else {
-		await runScriptAndPollResult({
-			workspace,
-			requestBody: {
-				language: 'postgresql',
-				content: m.sql,
-				args: { database: `datatable://${m.datatable_name}` }
-			}
-		})
+		await runScriptAndPollResult(
+			{
+				workspace,
+				requestBody: {
+					language: 'postgresql',
+					content: m.sql,
+					args: { database: `datatable://${m.datatable_name}` }
+				}
+			},
+			writingJobOptions
+		)
 	}
 }