feat: resolve canceled jobs and re-run them in limited batches (#11797)

* feat: resolve canceled jobs and batch re-run them in limited batches Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: offer the re-run batch limit only for failed or canceled filters Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: label an unresolved canceled run as canceled in its status badge Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: tell the user that sources a batch could not re-run block the next batch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat: warn when a re-run batch includes resolved runs and offer to hide them Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat: say in the re-run confirmation that failed or canceled sources get resolved Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 08:03:06 +00:00 · 2026-10-15 18:19:23 +02:00
parent 7da2344813
commit f827ba0566
16 changed files with 330 additions and 85 deletions
@@ -1,28 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n            SELECT c.id, c.workspace_id, $4, COALESCE($5, $7), false\n                FROM v2_job_completed c\n                JOIN v2_job j ON j.id = c.id\n                WHERE c.id = ANY($1)\n                    AND c.workspace_id = $2\n                    AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))\n                    AND c.status = 'failure'\n                    -- Resolution is a top-level triage state: a step resolved on its own\n                    -- would render orange inside a flow whose status is still red.\n                    AND j.flow_step_id IS NULL\n                    -- A supersession claim has to be proven, not trusted: a later success of the\n                    -- same identified runnable, itself visible to the caller. An unproven claim\n                    -- resolves nothing, so the caller learns it was rejected instead of having\n                    -- the fiction recorded as provenance.\n                    AND ($6::UUID IS NULL OR EXISTS (\n                        SELECT 1 FROM v2_job_completed sc\n                            JOIN v2_job sj ON sj.id = sc.id\n                            WHERE sc.id = $6\n                                AND sc.workspace_id = $2\n                                -- Tag scope is a read restriction enforced outside RLS, so it has\n                                -- to bind the evidence as well: otherwise the result reveals\n                                -- whether an out-of-scope run succeeded.\n                                AND ($3::TEXT[] IS NULL OR sj.tag = ANY($3))\n                                AND sc.status = 'success'\n                                AND sc.completed_at >= c.completed_at\n                                AND (j.runnable_id IS NOT NULL OR j.runnable_path IS NOT NULL)\n                                AND sj.runnable_id IS NOT DISTINCT FROM j.runnable_id\n                                AND sj.runnable_path IS NOT DISTINCT FROM j.runnable_path\n                    ))\n            ON CONFLICT (job_id) DO UPDATE SET\n                resolved_at = now(),\n                -- Both COALESCEd: `resolution_attribution` returns NULLs outside EE and once the\n                -- licence lapses, and bulk selections routinely include already-resolved rows,\n                -- so overwriting would erase metadata recorded while it was valid. Clear either\n                -- by unresolving first.\n                resolved_by = COALESCE($4, job_resolution.resolved_by),\n                -- A person's explanation replaces what was there; machine provenance only fills\n                -- a blank, so re-running an already-explained failure never erases their words.\n                note = COALESCE($5, job_resolution.note, $7),\n                -- A human taking over an automatic resolution makes it no longer automatic.\n                automatic = false\n            RETURNING job_id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "job_id",
-        "type_info": "Uuid"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "UuidArray",
-        "Text",
-        "TextArray",
-        "Varchar",
-        "Text",
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "13d8e65ffd3ef6bcfb963525fc3ec0859463b1c153950f894a830a4b5ec33fcc"
-}
@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n                SELECT c.id, c.workspace_id, $4, $5, false\n                    FROM v2_job_completed c\n                    JOIN v2_job j ON j.id = c.id\n                    WHERE c.id = $1\n                        AND c.workspace_id = $2\n                        AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))\n                        AND c.status IN ('failure', 'canceled')\n                        AND j.flow_step_id IS NULL\n                ON CONFLICT (job_id) DO NOTHING\n                RETURNING job_id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "TextArray",
+        "Varchar",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "dbd821d6d58aff9ed34cf3f14982ffe92d3cd13915be8a07f9364a14a01e549f"
+}
@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n            SELECT c.id, c.workspace_id, $4, COALESCE($5, $7), false\n                FROM v2_job_completed c\n                JOIN v2_job j ON j.id = c.id\n                WHERE c.id = ANY($1)\n                    AND c.workspace_id = $2\n                    AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))\n                    AND c.status IN ('failure', 'canceled')\n                    -- Resolution is a top-level triage state: a step resolved on its own\n                    -- would render orange inside a flow whose status is still red.\n                    AND j.flow_step_id IS NULL\n                    -- A supersession claim has to be proven, not trusted: a later success of the\n                    -- same identified runnable, itself visible to the caller. An unproven claim\n                    -- resolves nothing, so the caller learns it was rejected instead of having\n                    -- the fiction recorded as provenance.\n                    AND ($6::UUID IS NULL OR EXISTS (\n                        SELECT 1 FROM v2_job_completed sc\n                            JOIN v2_job sj ON sj.id = sc.id\n                            WHERE sc.id = $6\n                                AND sc.workspace_id = $2\n                                -- Tag scope is a read restriction enforced outside RLS, so it has\n                                -- to bind the evidence as well: otherwise the result reveals\n                                -- whether an out-of-scope run succeeded.\n                                AND ($3::TEXT[] IS NULL OR sj.tag = ANY($3))\n                                AND sc.status = 'success'\n                                AND sc.completed_at >= c.completed_at\n                                AND (j.runnable_id IS NOT NULL OR j.runnable_path IS NOT NULL)\n                                AND sj.runnable_id IS NOT DISTINCT FROM j.runnable_id\n                                AND sj.runnable_path IS NOT DISTINCT FROM j.runnable_path\n                    ))\n            ON CONFLICT (job_id) DO UPDATE SET\n                resolved_at = now(),\n                -- Both COALESCEd: `resolution_attribution` returns NULLs outside EE and once the\n                -- licence lapses, and bulk selections routinely include already-resolved rows,\n                -- so overwriting would erase metadata recorded while it was valid. Clear either\n                -- by unresolving first.\n                resolved_by = COALESCE($4, job_resolution.resolved_by),\n                -- A person's explanation replaces what was there; machine provenance only fills\n                -- a blank, so re-running an already-explained failure never erases their words.\n                note = COALESCE($5, job_resolution.note, $7),\n                -- A human taking over an automatic resolution makes it no longer automatic.\n                automatic = false\n            RETURNING job_id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "UuidArray",
+        "Text",
+        "TextArray",
+        "Varchar",
+        "Text",
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e4bb7af19003baca7f4d88e954b603843edce422c522c035b81b88cb169b88b1"
+}
@@ -29,12 +29,21 @@ const FLOW_VERSION: i64 = 2222222222;
 /// Mark a queued job as completed with success — needed so it's eligible for
 /// `list_selected_job_groups` / `batch_rerun_jobs` (both join `v2_job_completed`).
 async fn complete_job(db: &Pool<Postgres>, job_id: Uuid) -> anyhow::Result<()> {
+    complete_job_with_status(db, job_id, "success").await
+}
+
+async fn complete_job_with_status(
+    db: &Pool<Postgres>,
+    job_id: Uuid,
+    status: &str,
+) -> anyhow::Result<()> {
    sqlx::query(
        "INSERT INTO v2_job_completed (workspace_id, id, result, status, duration_ms, started_at, completed_at)
-         VALUES ($1, $2, '{}'::jsonb, 'success', 0, now(), now())",
+         VALUES ($1, $2, '{}'::jsonb, $3::job_status, 0, now(), now())",
    )
    .bind(WORKSPACE)
    .bind(job_id)
+    .bind(status)
    .execute(db)
    .await?;
    Ok(())
@@ -660,3 +669,56 @@ async fn batch_rerun_denies_invisible_source_job(db: Pool<Postgres>) -> anyhow::
    assert_eq!(pushed, 0);
    Ok(())
 }
+
+/// Re-running a failed or canceled job resolves it, pointing at the re-run, which is what lets
+/// "hide resolved" exclude it from the next batch. A success has nothing to resolve.
+#[sqlx::test(fixtures("base", "batch_rerun"))]
+async fn batch_rerun_resolves_failed_and_canceled_sources(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{}", server.addr.port()),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    let mut sources = vec![];
+    for status in ["success", "failure", "canceled"] {
+        let id = RunJob::from(script_payload())
+            .arg("name".to_string(), json!(status))
+            .push(&db)
+            .await;
+        complete_job_with_status(&db, id, status).await?;
+        sources.push(id);
+    }
+
+    let results = batch_rerun(
+        &client,
+        json!({
+            "job_ids": sources,
+            "script_options_by_path": {},
+            "flow_options_by_path": {},
+        }),
+    )
+    .await?;
+    assert!(results.iter().all(|r| r.is_ok()), "{results:?}");
+
+    let resolved: Vec<(String, String)> = sqlx::query_as(
+        "SELECT o.args->>'name', n.args->>'name'
+            FROM job_resolution r
+            JOIN v2_job o ON o.id = r.job_id
+            JOIN v2_job n ON r.note = 'Re-run as ' || n.id
+            ORDER BY 1",
+    )
+    .fetch_all(&db)
+    .await?;
+    assert_eq!(
+        resolved,
+        vec![
+            ("canceled".to_string(), "canceled".to_string()),
+            ("failure".to_string(), "failure".to_string()),
+        ]
+    );
+    Ok(())
+}
@@ -16117,6 +16117,9 @@ paths:
  /w/{workspace}/jobs/run/batch_rerun_jobs:
    post:
      summary: re-run multiple jobs
+      description: >-
+        Re-runs each job and marks every failed or canceled original that was re-run as
+        resolved, with the new job's id in its note. Operators re-run without resolving.
      operationId: batchReRunJobs
      tags:
        - job
@@ -16878,6 +16881,7 @@ paths:
        - job
      parameters:
        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/OrderDesc"
        - $ref: "#/components/parameters/CreatedBy"
        - $ref: "#/components/parameters/Label"
        - $ref: "#/components/parameters/Worker"
@@ -16910,8 +16914,8 @@ paths:
            type: boolean
        - name: resolved
          description: >-
-            filter on whether a failure has been marked as handled. true keeps only
-            resolved failures, false hides them
+            filter on whether a failed or canceled run has been marked as handled. true
+            keeps only resolved runs, false hides them
          in: query
          schema:
            type: boolean
@@ -17222,8 +17226,8 @@ paths:
            type: boolean
        - name: resolved
          description: >-
-            filter on whether a failure has been marked as handled. true keeps only
-            resolved failures, false hides them
+            filter on whether a failed or canceled run has been marked as handled. true
+            keeps only resolved runs, false hides them
          in: query
          schema:
            type: boolean
@@ -17409,8 +17413,8 @@ paths:
            type: boolean
        - name: resolved
          description: >-
-            filter on whether a failure has been marked as handled. true keeps only
-            resolved failures, false hides them
+            filter on whether a failed or canceled run has been marked as handled. true
+            keeps only resolved runs, false hides them
          in: query
          schema:
            type: boolean
@@ -18268,11 +18272,12 @@ paths:

  /w/{workspace}/jobs/completed/resolve:
    post:
-      summary: mark failed jobs as resolved
+      summary: mark failed or canceled jobs as resolved
      description: >-
-        Marks failed jobs as handled so triage surfaces stop showing them as failures.
-        The job status itself is unchanged. Returns the ids actually resolved: ids that
-        are not visible to the caller or did not fail are silently skipped.
+        Marks failed or canceled jobs as handled so triage surfaces stop showing them as
+        unhandled. The job status itself is unchanged. Returns the ids actually resolved: ids
+        that are not visible to the caller or neither failed nor were canceled are silently
+        skipped.
      operationId: resolveCompletedJobs
      tags:
        - job
@@ -26130,8 +26135,8 @@ paths:
            type: boolean
        - name: resolved
          description: >-
-            filter on whether a failure has been marked as handled. true keeps only
-            resolved failures, false hides them
+            filter on whether a failed or canceled run has been marked as handled. true
+            keeps only resolved runs, false hides them
          in: query
          schema:
            type: boolean
@@ -31052,7 +31057,7 @@ components:
          type: boolean
        resolved:
          type: boolean
-          description: whether this failure has been marked as handled
+          description: whether this failed or canceled run has been marked as handled
        resolved_by:
          type: string
          description: >-
@@ -7058,6 +7058,7 @@ async fn batch_rerun_handle_job(
            )
            .await;
            if let Ok((uuid, _, _, _)) = result {
+                resolve_batch_rerun_source(authed, user_db, w_id, job.id, uuid).await;
                return Ok(uuid.to_string());
            }
        }
@@ -7091,6 +7092,7 @@ async fn batch_rerun_handle_job(
                .map(|r| r.0)
            };
            if let Ok(uuid) = result {
+                resolve_batch_rerun_source(authed, user_db, w_id, job.id, uuid).await;
                return Ok(uuid.to_string());
            }
        }
@@ -7101,6 +7103,67 @@ async fn batch_rerun_handle_job(
    ))
 }

+/// Marks a failed or canceled source of a batch re-run as handled, with the re-run as its note.
+/// This is the only moment the link exists (the re-run is pushed with no back-pointer), and it
+/// is what lets "hide resolved" exclude already re-run jobs from the next batch. Never fails the
+/// re-run, which is already queued: a source that cannot be resolved is logged and left as is.
+async fn resolve_batch_rerun_source(
+    authed: &ApiAuthed,
+    user_db: &UserDB,
+    w_id: &str,
+    source: Uuid,
+    rerun: Uuid,
+) {
+    if authed.is_operator {
+        return;
+    }
+    let result: error::Result<()> = async {
+        let mut tx = user_db.clone().begin(authed).await?;
+        let tags = get_scope_tags(authed);
+        let (resolved_by, _) = resolution_attribution(authed, None);
+        // Same authorization and eligibility as `resolve_completed_jobs`. DO NOTHING keeps a
+        // resolution someone already explained.
+        let resolved = sqlx::query_scalar!(
+            "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)
+                SELECT c.id, c.workspace_id, $4, $5, false
+                    FROM v2_job_completed c
+                    JOIN v2_job j ON j.id = c.id
+                    WHERE c.id = $1
+                        AND c.workspace_id = $2
+                        AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))
+                        AND c.status IN ('failure', 'canceled')
+                        AND j.flow_step_id IS NULL
+                ON CONFLICT (job_id) DO NOTHING
+                RETURNING job_id",
+            source,
+            w_id,
+            tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>,
+            resolved_by,
+            format!("Re-run as {rerun}"),
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+        if resolved.is_some() {
+            audit_log(
+                &mut *tx,
+                authed,
+                "jobs.resolve",
+                ActionKind::Update,
+                w_id,
+                Some(&source.to_string()),
+                None,
+            )
+            .await?;
+        }
+        tx.commit().await?;
+        Ok(())
+    }
+    .await;
+    if let Err(e) = result {
+        tracing::error!("Could not resolve job {source} after re-running it as {rerun}: {e}");
+    }
+}
+
 pub async fn run_flow_by_path(
    authed: ApiAuthed,
    Extension(db): Extension<DB>,
@@ -12531,8 +12594,8 @@ fn check_resolution_request(
    Ok(())
 }

-/// Marks failed jobs as handled. Returns the ids actually affected: an id that is not
-/// visible to the caller, carries an out-of-scope tag, or did not fail is silently
+/// Marks failed or canceled jobs as handled. Returns the ids actually affected: an id that is
+/// not visible to the caller, carries an out-of-scope tag, or did not fail nor get canceled is silently
 /// absent rather than an error, so a bulk selection never fails as a whole.
 async fn resolve_completed_jobs(
    authed: ApiAuthed,
@@ -12547,7 +12610,7 @@ async fn resolve_completed_jobs(
    let system_note = req.superseded_by.map(|_| SUPERSEDED_NOTE);
    // The join on v2_job is what authorizes this write: v2_job_completed has RLS
    // disabled, so v2_job's policies are the only thing scoping rows to the caller.
-    // `status = 'failure'` keeps the invariant that only a failure can be resolved.
+    // Only a failed or canceled run can be resolved: a success has nothing to handle.
    let resolved = sqlx::query_scalar!(
        "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)
            SELECT c.id, c.workspace_id, $4, COALESCE($5, $7), false
@@ -12556,7 +12619,7 @@ async fn resolve_completed_jobs(
                WHERE c.id = ANY($1)
                    AND c.workspace_id = $2
                    AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))
-                    AND c.status = 'failure'
+                    AND c.status IN ('failure', 'canceled')
                    -- Resolution is a top-level triage state: a step resolved on its own
                    -- would render orange inside a flow whose status is still red.
                    AND j.flow_step_id IS NULL
@@ -1846,7 +1846,7 @@ is, a different one moves it there and archives the old path"),
                },
                "resolved": {
                        "type": "boolean",
-                        "description": "filter on whether a failure has been marked as handled. true keeps only resolved failures, false hides them"
+                        "description": "filter on whether a failed or canceled run has been marked as handled. true keeps only resolved runs, false hides them"
                },
                "is_flow_step": {
                        "type": "boolean",
@@ -34,7 +34,7 @@
 	<Badge {large} color="orange" title={job.resolution_note}>
 		<!-- `resolved_by` is absent both for an automatic resolution and for a manual one
 		     outside EE, so the automatic wording comes from `resolved_automatically`. -->
-		Failed after {msToReadableTime(job.duration_ms)}, resolved{job.resolved_automatically
+		{job.canceled ? 'Canceled' : 'Failed'} after {msToReadableTime(job.duration_ms)}, resolved{job.resolved_automatically
 			? ' automatically'
 			: job.resolved_by
 				? ` by ${job.resolved_by}`
@@ -49,7 +49,7 @@
 	</Badge>
 {:else if job && 'success' in job}
 	<Badge {large} color="red">
-		Failed after {msToReadableTime(job.duration_ms)}
+		{job.canceled ? 'Canceled' : 'Failed'} after {msToReadableTime(job.duration_ms)}
 		{#if job.self_wait_time_ms || job.aggregate_wait_time_ms}
 			<WaitTimeWarning
 				self_wait_time_ms={job.self_wait_time_ms}
@@ -152,6 +152,7 @@
 				title: string
 				confirmBtnText: string
 				loading?: boolean
+				description?: string
 				preContent?: string
 				onConfirm?: (forceCancel: boolean) => void
 				type?: ConfirmationModal['$$prop_def']['type']
@@ -437,6 +438,7 @@

 	async function reRunJobs(jobIdsToReRun: string[], batchReRunOptions: BatchReRunOptions) {
 		if (!$operatingWorkspace) return
+		const batching = filterMatchedIds !== undefined

 		if (askingForConfirmation) {
 			askingForConfirmation.loading = true
@@ -486,7 +488,11 @@
 						}
 						if (reRanUuids.length !== jobIdsToReRun.length) {
 							sendUserToast(
-								`Failed to re-run ${jobIdsToReRun.length - reRanUuids.length} jobs. Check console for details`,
+								`Failed to re-run ${jobIdsToReRun.length - reRanUuids.length} jobs. Check console for details` +
+									// Only a re-run resolves its source, so these stay first in line.
+									(batching
+										? '. They stay unresolved and the next batch will try them again: resolve them to skip them'
+										: ''),
 								true
 							)
 							// We do not get explicit error from backend if the job script don't exist
@@ -509,13 +515,31 @@
 		jobsLoader?.loadJobs(true, true)
 	}

+	// Every failed or canceled job "Re-run all jobs matching filters" found, oldest first, so a
+	// batch limit takes the oldest ones. Re-running resolves them, so under "Hide resolved runs"
+	// the next click picks up where the previous batch stopped.
+	let filterMatchedIds: string[] | undefined = $state()
+	let rerunLimit: number | undefined = $state()
+	// The pane closes from several places, the table included; a stale match list would then
+	// offer its limit over a selection it no longer describes.
+	$effect(() => {
+		if (!batchRerunOptionsIsOpen) untrack(() => (filterMatchedIds = undefined))
+	})
+
+	function setRerunLimit(limit: number | undefined) {
+		if (!filterMatchedIds) return
+		rerunLimit = limit && limit > 0 ? limit : undefined
+		selectedIds = filterMatchedIds.slice(0, rerunLimit ?? filterMatchedIds.length)
+	}
+
 	async function onRerunAllJobsMatchingFilters() {
 		const selectedFilters = getSelectedFilters()
 		selectedIds = []

 		const loadingToast = sendUserToast('Loading job ids', 'info')

-		if (filters.val.job_kinds !== 'runs') {
+		// Selecting "runs" deletes the key, since it is the default.
+		if ((filters.val.job_kinds ?? 'runs') !== 'runs') {
 			sendUserToast('Batch re-run is only supported for scripts and flows', true)
 			loadingToast.destroy()
 			return
@@ -524,11 +548,21 @@
 		// workspace. started_at >= minTs implies completed_at >= minTs, so completedAfter adds a
 		// lossless indexed lower bound ((workspace_id, completed_at DESC)); started_at stays the
 		// exact recheck. (completedBefore is omitted: it would drop jobs that finish after maxTs.)
-		selectedIds = await JobService.listFilteredJobsUuids({
+		const ids = await JobService.listFilteredJobsUuids({
 			...selectedFilters,
 			completedAfter: selectedFilters.startedAfter,
-			jobKinds: 'script,flow'
+			orderDesc: false
 		})
+		// Batching needs both halves of its promise: only failed or canceled sources are resolved
+		// by a re-run, and only a status filter keeps the queue out of the lookup, whose union
+		// drops the ordering. Anything else re-runs every match, as one batch.
+		if (selectedFilters.status === 'failure' || selectedFilters.status === 'canceled') {
+			filterMatchedIds = ids
+			setRerunLimit(rerunLimit)
+		} else {
+			filterMatchedIds = undefined
+			selectedIds = ids
+		}
 		loadingToast.destroy()
 		batchRerunOptionsIsOpen = true
 	}
@@ -538,6 +572,10 @@
 		askingForConfirmation = {
 			title: `Confirm re-running the selected jobs`,
 			confirmBtnText: `Re-run ${jobIdsToReRun.length} jobs`,
+			// The server skips resolution for operators, who cannot resolve.
+			description: operatingUser.current?.operator
+				? undefined
+				: "Failed or canceled jobs among them will be marked resolved, with the new run's id as the note.",
 			type: 'reload',
 			onConfirm: async () => {
 				await reRunJobs(jobIdsToReRun, batchReRunOptions)
@@ -867,6 +905,9 @@
 		askingForConfirmation = undefined
 	}}
 >
+	{#if askingForConfirmation?.description}
+		<p class="text-xs text-secondary">{askingForConfirmation.description}</p>
+	{/if}
 	{#if askingForConfirmation?.preContent}
 		<pre>{askingForConfirmation.preContent}</pre>
 		<Toggle
@@ -1121,6 +1162,7 @@
 								action: () => {
 									manualSelectionMode = 'rerun'
 									selectedIds = []
+									filterMatchedIds = undefined
 									batchRerunOptionsIsOpen = true
 								}
 							},
@@ -1128,7 +1170,7 @@
 								? [
 										{
 											// Operators are rejected by the endpoint, so offering it would only 403.
-											displayName: 'Resolve failed jobs',
+											displayName: 'Resolve failed or canceled jobs',
 											action: () => (
 												(manualSelectionMode = 'resolve'),
 												(selectedIds = []),
@@ -1210,7 +1252,7 @@
 				class="rounded-md bg-surface-tertiary border absolute inset-0 mb-4 flex flex-col items-center justify-center gap-3 p-4"
 			>
 				<p class="text-xs text-secondary text-center max-w-xs">
-					Resolving keeps the run a failure but stops it showing as one in the runs list.
+					Resolving keeps the run's status but marks it as handled in the runs list.
 				</p>
 				<TextInput
 					bind:value={resolutionNote}
@@ -1303,15 +1345,59 @@
 {/snippet}

 {#snippet batchRerunOptions()}
-	{#await import('$lib/components/runs/BatchReRunOptionsPane.svelte') then BatchReRunOptionsPane}
-		<BatchReRunOptionsPane.default
-			{selectedIds}
-			onCancel={() => ((batchRerunOptionsIsOpen = false), (manualSelectionMode = undefined))}
-			onConfirm={async (options) => {
-				await onReRunSelectedJobs(options)
-			}}
-		/>
-	{/await}
+	<div class="flex flex-col h-full">
+		{#if filterMatchedIds}
+			<div class="flex flex-col gap-1 pb-2 text-xs text-secondary">
+				<div class="flex flex-row items-center gap-2">
+					<span>{filterMatchedIds.length} jobs match the filters, re-run at most</span>
+					<TextInput
+						inputProps={{ type: 'number', min: 1, placeholder: 'All' }}
+						size="sm"
+						class="w-24"
+						bind:value={
+							() => rerunLimit?.toString() ?? '',
+							(v) => setRerunLimit(v ? parseInt(v.toString()) : undefined)
+						}
+					/>
+				</div>
+				{#if filters.val.resolved === 'unresolved'}
+					<span class="text-hint">
+						Oldest first. Re-run jobs are marked resolved, so the next batch starts where this one
+						stops.
+					</span>
+				{:else}
+					<div class="flex flex-row items-center gap-2 text-orange-600 dark:text-orange-400">
+						<TriangleAlertIcon size={14} class="shrink-0" />
+						<span>
+							Resolved runs are included, so the next batch will pick these same jobs again.
+						</span>
+						<Button
+							variant="default"
+							unifiedSize="xs"
+							btnClasses="shrink-0"
+							onClick={() => {
+								filters.val.resolved = 'unresolved'
+								onRerunAllJobsMatchingFilters()
+							}}
+						>
+							Hide resolved runs
+						</Button>
+					</div>
+				{/if}
+			</div>
+		{/if}
+		<div class="flex-1 min-h-0 flex flex-col">
+			{#await import('$lib/components/runs/BatchReRunOptionsPane.svelte') then BatchReRunOptionsPane}
+				<BatchReRunOptionsPane.default
+					{selectedIds}
+					onCancel={() => ((batchRerunOptionsIsOpen = false), (manualSelectionMode = undefined))}
+					onConfirm={async (options) => {
+						await onReRunSelectedJobs(options)
+					}}
+				/>
+			{/await}
+		</div>
+	</div>
 {/snippet}

 {#snippet jobPreview(pending?: Snippet)}
@@ -514,7 +514,7 @@
 										variant="subtle"
 										disabled={togglingResolution}
 										onClick={() => setResolution(false)}
-										title="Show this run as a failure again"
+										title="Show this run as unhandled again"
 									>
 										Unresolve
 									</Button>
@@ -525,7 +525,7 @@
 												size="xs2"
 												variant="subtle"
 												nonCaptureEvent
-												title="Mark this failure as handled"
+												title="Mark this run as handled"
 											>
 												Mark resolved
 											</Button>
@@ -533,8 +533,7 @@
 										{#snippet content({ close })}
 											<div class="p-3 flex flex-col gap-2 w-80">
 												<p class="text-xs text-secondary">
-													Resolving keeps the run a failure but stops it showing as one in the runs
-													list.
+													Resolving keeps the run's status but marks it as handled in the runs list.
 												</p>
 												<TextInput
 													bind:value={resolutionNote}
@@ -29,7 +29,12 @@
 			<ShieldQuestion size={14} />
 		</Badge>
 	{:else if job.canceled && 'success' in job}
-		<Badge color="gray" {roundedFull} baseClass={roundedFull ? '' : '!px-1.5'} title="Canceled">
+		<Badge
+			color="gray"
+			{roundedFull}
+			baseClass={roundedFull ? '' : '!px-1.5'}
+			title={job.resolved ? 'Canceled, marked as resolved' : 'Canceled'}
+		>
 			<Ban size={14} />
 		</Badge>
 	{:else if 'success' in job && job.success}
@@ -37,7 +37,7 @@ export function offerToResolveOriginal(origin: PendingRerun, onResolved?: () =>
 		'success',
 		[
 			{
-				label: 'Resolve the original failure',
+				label: 'Resolve the original run',
 				callback: async () => {
 					const affected = await JobService.resolveCompletedJobs({
 						workspace: origin.workspace,
@@ -50,10 +50,10 @@ export function offerToResolveOriginal(origin: PendingRerun, onResolved?: () =>
 						}
 					})
 					if (affected.length === 0) {
-						sendUserToast('Could not resolve the original failure', true)
+						sendUserToast('Could not resolve the original run', true)
 						return
 					}
-					sendUserToast('Original failure resolved')
+					sendUserToast('Original run resolved')
 					onResolved?.()
 				}
 			}
@@ -186,7 +186,7 @@ export function buildRunsFilterSearchbarSchema({
 			],
 			label: 'Resolution',
 			icon: Wrench,
-			description: 'Filter by whether a failure has been marked as handled'
+			description: 'Filter by whether a failed or canceled run has been marked as handled'
 		},
 		job_trigger_kind: {
 			type: 'oneof' as const,
@@ -254,7 +254,7 @@ export const buildRunsFilterPresets = ({
 	{ name: 'Hide schedules', value: 'job_trigger_kind:\\ !schedule' },
 	{ name: 'Hide future jobs', value: 'show_future_jobs:\\ false' },
 	{ name: 'Show skipped', value: 'show_skipped:\\ true' },
-	{ name: 'Hide resolved failures', value: 'resolved:\\ unresolved' },
+	{ name: 'Hide resolved runs', value: 'resolved:\\ unresolved' },
 	...(isSuperAdminOrDevops && isAdminsWorkspace
 		? [{ name: 'All workspaces', value: 'all_workspaces:\\ true' }]
 		: [])
@@ -1853,7 +1853,7 @@ export const mcpEndpointTools: EndpointTool[] = [
                },
                "resolved": {
                        "type": "boolean",
-                        "description": "filter on whether a failure has been marked as handled. true keeps only resolved failures, false hides them"
+                        "description": "filter on whether a failed or canceled run has been marked as handled. true keeps only resolved runs, false hides them"
                },
                "is_flow_step": {
                        "type": "boolean",
@@ -13,17 +13,16 @@ import {
 	apiErrorMessage
 } from './utils'

-// Mirrors the backend invariant that only `status = 'failure'` rows can carry a
-// resolution; loosening this to `!success` would offer the action on queued and
-// canceled runs the API then silently skips.
+// Mirrors the backend invariant that only failed or canceled rows can carry a resolution;
+// offering the action anywhere else would send ids the API then silently skips.
 describe('isJobResolvable', () => {
 	const completed = (o: object) => ({ type: 'CompletedJob', ...o }) as any
-	it('accepts a plain failure', () => {
+	it('accepts a plain failure and a cancellation', () => {
 		expect(isJobResolvable(completed({ success: false, canceled: false }))).toBe(true)
+		expect(isJobResolvable(completed({ success: false, canceled: true }))).toBe(true)
 	})
-	it('rejects success, cancellation, flow steps and jobs still queued', () => {
+	it('rejects success, flow steps and jobs still queued', () => {
 		expect(isJobResolvable(completed({ success: true, canceled: false }))).toBe(false)
-		expect(isJobResolvable(completed({ success: false, canceled: true }))).toBe(false)
 		// a step resolved alone would show orange inside a flow that is still red
 		expect(
 			isJobResolvable(completed({ success: false, canceled: false, is_flow_step: true }))
@@ -83,13 +83,13 @@ export function isJobReRunnable(j: Job): boolean {
 	return (j.job_kind === 'script' || j.job_kind === 'flow') && j.parent_job === undefined
 }

-// Only a top-level failure can carry a resolution: cancellations and skipped steps are not
-// failures, and a flow step resolved on its own would render orange inside a flow whose status
-// is still red. The resolve endpoint enforces the same three conditions.
+// Only a top-level failed or canceled run can carry a resolution: successes and skipped steps
+// have nothing to handle, and a flow step resolved on its own would render orange inside a flow
+// whose status is still red. The resolve endpoint enforces the same conditions.
 // Runs obscured from another workspace stand in with `id: '-'`; they look like failures but
 // address nothing, and one reaching the endpoint fails the whole batch on UUID parsing.
 export function isJobResolvable(j: Job): j is CompletedJob {
-	return j.type === 'CompletedJob' && !j.success && !j.canceled && !j.is_flow_step && j.id !== '-'
+	return j.type === 'CompletedJob' && !j.success && !j.is_flow_step && j.id !== '-'
 }

 // Mirror the caps enforced by /jobs/completed/resolve, so the UI can chunk and validate