From 354aa2581dcbccfe67781a269f92fc7693bf4456 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 12 Nov 2026 12:41:10 +0100 Subject: [PATCH] fix: report a failed mcp script or flow run as a tool error (#12365) * fix: report a failed mcp script or flow run as a tool error Co-Authored-By: Claude Opus 5.5 * fix: take mcp run success from the job and cap failure text Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/tests/mcp_runnable_tool_failure.rs | 107 +++++++++++++++++++++ backend/windmill-api-jobs/src/execution.rs | 4 +- backend/windmill-api/src/jobs.rs | 34 ++++--- backend/windmill-api/src/mcp/core.rs | 41 ++++++-- backend/windmill-mcp/src/server/backend.rs | 15 ++- backend/windmill-mcp/src/server/mod.rs | 4 +- backend/windmill-mcp/src/server/runner.rs | 90 +++++++++++++++-- 7 files changed, 265 insertions(+), 30 deletions(-) create mode 100644 backend/tests/mcp_runnable_tool_failure.rs diff --git a/backend/tests/mcp_runnable_tool_failure.rs b/backend/tests/mcp_runnable_tool_failure.rs new file mode 100644 index 0000000000..5f27c82c29 --- /dev/null +++ b/backend/tests/mcp_runnable_tool_failure.rs @@ -0,0 +1,107 @@ +//! A script tool on a selection-scoped MCP token reports a run that failed as a tool execution +//! error (`isError`) naming the run, and a run that succeeded as its result. +#![cfg(feature = "mcp")] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +async fn seed(db: &Pool) -> anyhow::Result<()> { + for (hash, path, content) in [ + ( + 7000001_i64, + "u/test-user/fails", + "echo about to fail\nexit 3", + ), + (7000002_i64, "u/test-user/works", "echo all good"), + ] { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock) + VALUES ('test-workspace', $1, $2, $3, 'bash', 'script', 'test-user', + '{\"type\": \"object\", \"properties\": {}}', '', '', '')", + ) + .bind(hash) + .bind(path) + .bind(content) + .execute(db) + .await?; + } + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, workspace_id, scopes) + VALUES (encode(sha256('MCP_SELECTION'::bytea), 'hex'), 'MCP_SELECT', 'MCP_SELECTION', + 'test@windmill.dev', 'mcp', 'test-workspace', + ARRAY['mcp:scripts:u/test-user/fails,u/test-user/works'])", + ) + .execute(db) + .await?; + Ok(()) +} + +async fn rpc(base: &str, method: &str, params: Value) -> anyhow::Result { + let body = reqwest::Client::new() + .post(format!("{base}/api/mcp/w/test-workspace/mcp")) + .bearer_auth("MCP_SELECTION") + .header("Accept", "application/json, text/event-stream") + .json(&json!({ "jsonrpc": "2.0", "id": 1, "method": method, "params": params })) + .send() + .await? + .text() + .await?; + let data = body + .lines() + .filter_map(|l| l.strip_prefix("data:")) + .find(|l| !l.trim().is_empty()) + .unwrap_or(&body); + Ok(serde_json::from_str(data.trim())?) +} + +async fn call(base: &str, tools: &[Value], path_suffix: &str) -> anyhow::Result { + let name = tools + .iter() + .filter_map(|t| t["name"].as_str()) + .find(|n| n.ends_with(path_suffix)) + .unwrap_or_else(|| panic!("no tool for {path_suffix} in {tools:?}")); + let res = rpc(base, "tools/call", json!({ "name": name, "arguments": {} })).await?; + Ok(res["result"].clone()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_failed_run_is_a_tool_error(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}"); + + in_test_worker( + &db, + async { + let tools = rpc(&base, "tools/list", json!({})).await?; + let tools = tools["result"]["tools"] + .as_array() + .cloned() + .unwrap_or_default(); + + let failed = call(&base, &tools, "fails").await?; + assert_eq!(failed["isError"], json!(true), "{failed}"); + let text = failed["content"][0]["text"].as_str().unwrap_or_default(); + assert!(text.contains("exit code"), "{text}"); + let job_id: String = sqlx::query_scalar( + "SELECT id::text FROM v2_job WHERE runnable_path = 'u/test-user/fails'", + ) + .fetch_one(&db) + .await?; + assert!(text.contains(&job_id), "{text}"); + + let ok = call(&base, &tools, "works").await?; + assert_ne!(ok["isError"], json!(true), "{ok}"); + let text = ok["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!(text, "\"all good\""); + anyhow::Ok(()) + }, + port, + ) + .await?; + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index ffa49379a0..731f803406 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -1167,7 +1167,7 @@ pub async fn run_flow_and_wait_result( run_query: RunJobQuery, args: PushArgsOwned, trigger: Option, -) -> error::Result { +) -> error::Result<(Box, bool)> { let (uuid, early_return, has_failure_module, _) = run_flow( authed, db, @@ -1182,7 +1182,7 @@ pub async fn run_flow_and_wait_result( ) .await?; - run_wait_result( + run_wait_result_internal( &db, uuid, w_id, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 7c5270587b..e939fd4f22 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -8383,7 +8383,10 @@ pub async fn run_wait_result_flow_by_path_get( ) .await?; - run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id).await + let (result, success) = + run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id) + .await?; + result_to_response(result, success) } pub async fn run_wait_result_script_by_path( @@ -8411,8 +8414,11 @@ pub async fn run_wait_result_script_by_path( ) .await?; - run_wait_result_script_by_path_internal(db, run_query, &script, authed, user_db, w_id, args) - .await + let (result, success) = run_wait_result_script_by_path_internal( + db, run_query, &script, authed, user_db, w_id, args, + ) + .await?; + result_to_response(result, success) } pub async fn run_wait_result_script_by_path_internal( @@ -8423,7 +8429,7 @@ pub async fn run_wait_result_script_by_path_internal( user_db: UserDB, w_id: String, args: PushArgsOwned, -) -> error::Result { +) -> error::Result<(Box, bool)> { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; @@ -8498,7 +8504,8 @@ pub async fn run_wait_result_script_by_path_internal( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, &w_id, None, false, &authed.username).await; + let wait_result = + run_wait_result_internal(&db, uuid, &w_id, None, false, &authed.username).await; handle_delete_after_completion(&db, uuid, &w_id, delete_after_use, delete_after_secs).await?; return wait_result; } @@ -8661,7 +8668,10 @@ pub async fn run_wait_result_flow_by_path( ) .await?; - run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id).await + let (result, success) = + run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id) + .await?; + result_to_response(result, success) } pub async fn stream_flow_by_path( @@ -8905,7 +8915,7 @@ pub async fn run_wait_result_flow_by_path_internal( user_db: UserDB, args: PushArgsOwned, w_id: String, -) -> error::Result { +) -> error::Result<(Box, bool)> { check_queue_too_long(&db, run_query.queue_limit).await?; let flow_path = flow.path(); @@ -8968,7 +8978,7 @@ pub async fn run_wait_result_flow_by_version_get( let flow_version_info = get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?; - run_flow_and_wait_result( + let (result, success) = run_flow_and_wait_result( &authed, &db, user_db, @@ -8979,7 +8989,8 @@ pub async fn run_wait_result_flow_by_version_get( args, None, ) - .await + .await?; + result_to_response(result, success) } pub async fn run_wait_result_flow_by_version( @@ -9011,7 +9022,7 @@ pub async fn run_wait_result_flow_by_version( let flow_version_info = get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?; - run_flow_and_wait_result( + let (result, success) = run_flow_and_wait_result( &authed, &db, user_db, @@ -9022,7 +9033,8 @@ pub async fn run_wait_result_flow_by_version( args, None, ) - .await + .await?; + result_to_response(result, success) } /// Whether request-supplied SQL from an operator may run. Operators can only run deployed diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 0bfa6cd717..a4498855a7 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -15,13 +15,14 @@ use windmill_mcp::common::types::{ }; use windmill_mcp::server::{ BackendResult, CallToolResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter, - SessionAccess, Tool, + RunOutcome, SessionAccess, Tool, }; use crate::auth::AuthCache; use crate::db::ApiAuthed; use crate::jobs::{ - run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, + result_to_response, run_wait_result_flow_by_path_internal, + run_wait_result_script_by_path_internal, RunJobQuery, }; use super::auto_generated_endpoints::all_tools; @@ -110,6 +111,28 @@ impl WindmillBackend { } } +/// A successful result goes through the same response shaping as the wait-result endpoint, so a +/// `windmill_status_code`/`result` wrapper is unwrapped exactly as it is there. +async fn run_outcome( + workspace_id: &str, + job_id: uuid::Uuid, + (result, success): (Box, bool), +) -> BackendResult { + if success { + let response = result_to_response(result, true) + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + return Ok(RunOutcome::Success(parse_response_body(response).await?)); + } + Ok(RunOutcome::Failed { + job_id: job_id.to_string(), + run_url: format!( + "{}/run/{job_id}?workspace={workspace_id}", + windmill_common::BASE_URL.load() + ), + result: serde_json::from_str(result.get()).unwrap_or(Value::Null), + }) +} + #[async_trait] impl McpBackend for WindmillBackend { type Auth = ApiAuthed; @@ -287,7 +310,7 @@ impl McpBackend for WindmillBackend { path: &str, args: Value, request: &McpRequest<'_>, - ) -> BackendResult { + ) -> BackendResult { windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "script"); // Called directly rather than through the jobs router, so its run gate does not apply. windmill_common::workspaces::check_role_can_run(&self.db, workspace_id, auth) @@ -299,9 +322,10 @@ impl McpBackend for WindmillBackend { .await .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + let job_id: uuid::Uuid = ulid::Ulid::new().into(); let result = run_wait_result_script_by_path_internal( self.db.clone(), - RunJobQuery::default(), + RunJobQuery { job_id: Some(job_id), ..Default::default() }, &script, auth.clone(), self.user_db.clone(), @@ -311,7 +335,7 @@ impl McpBackend for WindmillBackend { .await .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; - parse_response_body(result).await + run_outcome(workspace_id, job_id, result).await } async fn run_flow( @@ -321,7 +345,7 @@ impl McpBackend for WindmillBackend { path: &str, args: Value, request: &McpRequest<'_>, - ) -> BackendResult { + ) -> BackendResult { windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "flow"); // Called directly rather than through the jobs router, so its run gate does not apply. windmill_common::workspaces::check_role_can_run(&self.db, workspace_id, auth) @@ -333,9 +357,10 @@ impl McpBackend for WindmillBackend { .await .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + let job_id: uuid::Uuid = ulid::Ulid::new().into(); let result = run_wait_result_flow_by_path_internal( self.db.clone(), - RunJobQuery::default(), + RunJobQuery { job_id: Some(job_id), ..Default::default() }, &flow, auth.clone(), self.user_db.clone(), @@ -345,7 +370,7 @@ impl McpBackend for WindmillBackend { .await .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; - parse_response_body(result).await + run_outcome(workspace_id, job_id, result).await } async fn call_endpoint( diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index 98831279d5..83332174cc 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -17,6 +17,17 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// How a script or flow run by a tool ended. +pub enum RunOutcome { + Success(Value), + /// The job ran and failed. `result` is the job's result, its error object included. + Failed { + job_id: String, + run_url: String, + result: Value, + }, +} + /// What a v2 token may do: read, read and run deployed scripts, flows and agents /// (`mcp:read_run`), or everything its owner can. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -202,7 +213,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { path: &str, args: Value, request: &McpRequest<'_>, - ) -> BackendResult; + ) -> BackendResult; /// Run a flow and wait for result async fn run_flow( @@ -212,7 +223,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { path: &str, args: Value, request: &McpRequest<'_>, - ) -> BackendResult; + ) -> BackendResult; /// Call an endpoint tool (generated API endpoint) async fn call_endpoint( diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index 545812a411..69ce01be4f 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -14,7 +14,9 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, NoEndpointTools, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter, SessionAccess}; +pub use backend::{ + BackendResult, McpAuth, McpBackend, McpRequest, PathFilter, RunOutcome, SessionAccess, +}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, non_empty_body_fields, with_workspace_id_arg, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index 6e18f36c7b..9d636924de 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -14,7 +14,7 @@ use crate::common::types::{ McpToken, MultiWorkspaceMcp, NoEndpointTools, ResourceInfo, SchemaType, ToolableItem, WorkspaceId, }; -use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; +use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter, RunOutcome}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; @@ -87,6 +87,55 @@ pub(super) fn truncate_tool_result(text: String) -> String { truncated } +const MAX_PARTIAL_RESULT_BYTES: usize = 2_000; + +/// A failed run as a tool execution error: the error's name and message, the run to open, and a +/// short excerpt of whatever else the result holds. The stack trace is left to the run's page. +fn failed_run_result( + tool_type: &str, + job_id: &str, + run_url: &str, + mut result: Value, +) -> CallToolResult { + let mut headline = None; + if let Some(obj) = result.as_object_mut() { + if let Some(Value::Object(error)) = obj.get_mut("error") { + let name = error.remove("name"); + let message = error.remove("message"); + error.remove("stack"); + if error.get("job_id").and_then(Value::as_str) == Some(job_id) { + error.remove("job_id"); + } + let [name, message] = [&name, &message].map(|v| v.as_ref().and_then(Value::as_str)); + headline = match (name, message) { + (Some(n), Some(m)) => Some(format!("{n}: {m}")), + (n, m) => n.or(m).map(str::to_string), + }; + if error.is_empty() { + obj.remove("error"); + } + } + } + // The run id comes first so that truncating a huge message never cuts it off. + let mut text = format!("The {tool_type} failed. Run id: {job_id} ({run_url})"); + if let Some(h) = headline { + text.push_str(&format!("\n{h}")); + } + if !matches!(&result, Value::Object(o) if o.is_empty()) && !result.is_null() { + let mut rest = result.to_string(); + if rest.len() > MAX_PARTIAL_RESULT_BYTES { + let mut end = MAX_PARTIAL_RESULT_BYTES; + while !rest.is_char_boundary(end) { + end -= 1; + } + rest.truncate(end); + rest.push_str("… [truncated, the full result is on the run]"); + } + text.push_str(&format!("\nPartial result: {rest}")); + } + CallToolResult::error(vec![ContentBlock::text(truncate_tool_result(text))]) +} + /// MCP Server Runner - generic over the backend implementation /// /// This struct implements the MCP ServerHandler trait and uses a McpBackend @@ -1098,11 +1147,16 @@ impl Runner { }; match result { - Ok(value) => Ok(CallToolResult::success(vec![ContentBlock::text( - truncate_tool_result( - serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()), - ), - )])), + Ok(RunOutcome::Success(value)) => { + Ok(CallToolResult::success(vec![ContentBlock::text( + truncate_tool_result( + serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()), + ), + )])) + } + Ok(RunOutcome::Failed { job_id, run_url, result }) => { + Ok(failed_run_result(tool_type, &job_id, &run_url, result)) + } Err(e) => Err(ErrorData::internal_error( format!("Failed to run {}: {}", tool_type, e.message), None, @@ -1606,4 +1660,28 @@ mod tests { ) .is_ok()); } + + #[test] + fn failed_run_is_a_tool_error_without_the_stack() { + let result = json!({ "error": { + "name": "Error", + "message": "boom", + "stack": "Error: boom\n at main (/tmp/main.ts:2:9)", + "step_id": "a", + "job_id": "0192-id", + }}); + let res = failed_run_result("script", "0192-id", "http://wm/run/0192-id", result); + assert_eq!(res.is_error, Some(true)); + let text = res.content[0].as_text().unwrap().text.clone(); + assert!(text.contains("Error: boom"), "{text}"); + assert!(text.contains("0192-id"), "{text}"); + assert!(text.contains(r#"{"error":{"step_id":"a"}}"#), "{text}"); + assert!(!text.contains("main.ts"), "{text}"); + + let huge = json!({ "error": { "name": "Error", "message": "x".repeat(200_000) } }); + let res = failed_run_result("script", "0192-id", "http://wm/run/0192-id", huge); + let text = &res.content[0].as_text().unwrap().text; + assert!(text.len() < MAX_TOOL_RESULT_BYTES + 100, "{}", text.len()); + assert!(text.contains("0192-id")); + } }