fix: report a failed mcp script or flow run as a tool error (#12365)

* fix: report a failed mcp script or flow run as a tool error

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: take mcp run success from the job and cap failure text

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-11-12 12:41:10 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent d4f9975121
commit 354aa2581d
7 changed files with 265 additions and 30 deletions
+107
View File
@@ -0,0 +1,107 @@
//! A script tool on a selection-scoped MCP token reports a run that failed as a tool execution
//! error (`isError`) naming the run, and a run that succeeded as its result.
#![cfg(feature = "mcp")]
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
async fn seed(db: &Pool<Postgres>) -> anyhow::Result<()> {
for (hash, path, content) in [
(
7000001_i64,
"u/test-user/fails",
"echo about to fail\nexit 3",
),
(7000002_i64, "u/test-user/works", "echo all good"),
] {
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by,
schema, summary, description, lock)
VALUES ('test-workspace', $1, $2, $3, 'bash', 'script', 'test-user',
'{\"type\": \"object\", \"properties\": {}}', '', '', '')",
)
.bind(hash)
.bind(path)
.bind(content)
.execute(db)
.await?;
}
sqlx::query(
"INSERT INTO token (token_hash, token_prefix, token, email, label, workspace_id, scopes)
VALUES (encode(sha256('MCP_SELECTION'::bytea), 'hex'), 'MCP_SELECT', 'MCP_SELECTION',
'test@windmill.dev', 'mcp', 'test-workspace',
ARRAY['mcp:scripts:u/test-user/fails,u/test-user/works'])",
)
.execute(db)
.await?;
Ok(())
}
async fn rpc(base: &str, method: &str, params: Value) -> anyhow::Result<Value> {
let body = reqwest::Client::new()
.post(format!("{base}/api/mcp/w/test-workspace/mcp"))
.bearer_auth("MCP_SELECTION")
.header("Accept", "application/json, text/event-stream")
.json(&json!({ "jsonrpc": "2.0", "id": 1, "method": method, "params": params }))
.send()
.await?
.text()
.await?;
let data = body
.lines()
.filter_map(|l| l.strip_prefix("data:"))
.find(|l| !l.trim().is_empty())
.unwrap_or(&body);
Ok(serde_json::from_str(data.trim())?)
}
async fn call(base: &str, tools: &[Value], path_suffix: &str) -> anyhow::Result<Value> {
let name = tools
.iter()
.filter_map(|t| t["name"].as_str())
.find(|n| n.ends_with(path_suffix))
.unwrap_or_else(|| panic!("no tool for {path_suffix} in {tools:?}"));
let res = rpc(base, "tools/call", json!({ "name": name, "arguments": {} })).await?;
Ok(res["result"].clone())
}
#[sqlx::test(fixtures("base"))]
async fn test_failed_run_is_a_tool_error(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
seed(&db).await?;
let server = ApiServer::start_mcp(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}");
in_test_worker(
&db,
async {
let tools = rpc(&base, "tools/list", json!({})).await?;
let tools = tools["result"]["tools"]
.as_array()
.cloned()
.unwrap_or_default();
let failed = call(&base, &tools, "fails").await?;
assert_eq!(failed["isError"], json!(true), "{failed}");
let text = failed["content"][0]["text"].as_str().unwrap_or_default();
assert!(text.contains("exit code"), "{text}");
let job_id: String = sqlx::query_scalar(
"SELECT id::text FROM v2_job WHERE runnable_path = 'u/test-user/fails'",
)
.fetch_one(&db)
.await?;
assert!(text.contains(&job_id), "{text}");
let ok = call(&base, &tools, "works").await?;
assert_ne!(ok["isError"], json!(true), "{ok}");
let text = ok["content"][0]["text"].as_str().unwrap_or_default();
assert_eq!(text, "\"all good\"");
anyhow::Ok(())
},
port,
)
.await?;
Ok(())
}
+2 -2
View File
@@ -1167,7 +1167,7 @@ pub async fn run_flow_and_wait_result(
run_query: RunJobQuery,
args: PushArgsOwned,
trigger: Option<TriggerMetadata>,
) -> error::Result<Response> {
) -> error::Result<(Box<RawValue>, bool)> {
let (uuid, early_return, has_failure_module, _) = run_flow(
authed,
db,
@@ -1182,7 +1182,7 @@ pub async fn run_flow_and_wait_result(
)
.await?;
run_wait_result(
run_wait_result_internal(
&db,
uuid,
w_id,
+23 -11
View File
@@ -8383,7 +8383,10 @@ pub async fn run_wait_result_flow_by_path_get(
)
.await?;
run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id).await
let (result, success) =
run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id)
.await?;
result_to_response(result, success)
}
pub async fn run_wait_result_script_by_path(
@@ -8411,8 +8414,11 @@ pub async fn run_wait_result_script_by_path(
)
.await?;
run_wait_result_script_by_path_internal(db, run_query, &script, authed, user_db, w_id, args)
.await
let (result, success) = run_wait_result_script_by_path_internal(
db, run_query, &script, authed, user_db, w_id, args,
)
.await?;
result_to_response(result, success)
}
pub async fn run_wait_result_script_by_path_internal(
@@ -8423,7 +8429,7 @@ pub async fn run_wait_result_script_by_path_internal(
user_db: UserDB,
w_id: String,
args: PushArgsOwned,
) -> error::Result<Response> {
) -> error::Result<(Box<RawValue>, bool)> {
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() };
@@ -8498,7 +8504,8 @@ pub async fn run_wait_result_script_by_path_internal(
.await?;
tx.commit().await?;
let wait_result = run_wait_result(&db, uuid, &w_id, None, false, &authed.username).await;
let wait_result =
run_wait_result_internal(&db, uuid, &w_id, None, false, &authed.username).await;
handle_delete_after_completion(&db, uuid, &w_id, delete_after_use, delete_after_secs).await?;
return wait_result;
}
@@ -8661,7 +8668,10 @@ pub async fn run_wait_result_flow_by_path(
)
.await?;
run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id).await
let (result, success) =
run_wait_result_flow_by_path_internal(db, run_query, &flow, authed, user_db, args, w_id)
.await?;
result_to_response(result, success)
}
pub async fn stream_flow_by_path(
@@ -8905,7 +8915,7 @@ pub async fn run_wait_result_flow_by_path_internal(
user_db: UserDB,
args: PushArgsOwned,
w_id: String,
) -> error::Result<Response> {
) -> error::Result<(Box<RawValue>, bool)> {
check_queue_too_long(&db, run_query.queue_limit).await?;
let flow_path = flow.path();
@@ -8968,7 +8978,7 @@ pub async fn run_wait_result_flow_by_version_get(
let flow_version_info =
get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?;
run_flow_and_wait_result(
let (result, success) = run_flow_and_wait_result(
&authed,
&db,
user_db,
@@ -8979,7 +8989,8 @@ pub async fn run_wait_result_flow_by_version_get(
args,
None,
)
.await
.await?;
result_to_response(result, success)
}
pub async fn run_wait_result_flow_by_version(
@@ -9011,7 +9022,7 @@ pub async fn run_wait_result_flow_by_version(
let flow_version_info =
get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?;
run_flow_and_wait_result(
let (result, success) = run_flow_and_wait_result(
&authed,
&db,
user_db,
@@ -9022,7 +9033,8 @@ pub async fn run_wait_result_flow_by_version(
args,
None,
)
.await
.await?;
result_to_response(result, success)
}
/// Whether request-supplied SQL from an operator may run. Operators can only run deployed
+33 -8
View File
@@ -15,13 +15,14 @@ use windmill_mcp::common::types::{
};
use windmill_mcp::server::{
BackendResult, CallToolResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter,
SessionAccess, Tool,
RunOutcome, SessionAccess, Tool,
};
use crate::auth::AuthCache;
use crate::db::ApiAuthed;
use crate::jobs::{
run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery,
result_to_response, run_wait_result_flow_by_path_internal,
run_wait_result_script_by_path_internal, RunJobQuery,
};
use super::auto_generated_endpoints::all_tools;
@@ -110,6 +111,28 @@ impl WindmillBackend {
}
}
/// A successful result goes through the same response shaping as the wait-result endpoint, so a
/// `windmill_status_code`/`result` wrapper is unwrapped exactly as it is there.
async fn run_outcome(
workspace_id: &str,
job_id: uuid::Uuid,
(result, success): (Box<serde_json::value::RawValue>, bool),
) -> BackendResult<RunOutcome> {
if success {
let response = result_to_response(result, true)
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
return Ok(RunOutcome::Success(parse_response_body(response).await?));
}
Ok(RunOutcome::Failed {
job_id: job_id.to_string(),
run_url: format!(
"{}/run/{job_id}?workspace={workspace_id}",
windmill_common::BASE_URL.load()
),
result: serde_json::from_str(result.get()).unwrap_or(Value::Null),
})
}
#[async_trait]
impl McpBackend for WindmillBackend {
type Auth = ApiAuthed;
@@ -287,7 +310,7 @@ impl McpBackend for WindmillBackend {
path: &str,
args: Value,
request: &McpRequest<'_>,
) -> BackendResult<Value> {
) -> BackendResult<RunOutcome> {
windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "script");
// Called directly rather than through the jobs router, so its run gate does not apply.
windmill_common::workspaces::check_role_can_run(&self.db, workspace_id, auth)
@@ -299,9 +322,10 @@ impl McpBackend for WindmillBackend {
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
let job_id: uuid::Uuid = ulid::Ulid::new().into();
let result = run_wait_result_script_by_path_internal(
self.db.clone(),
RunJobQuery::default(),
RunJobQuery { job_id: Some(job_id), ..Default::default() },
&script,
auth.clone(),
self.user_db.clone(),
@@ -311,7 +335,7 @@ impl McpBackend for WindmillBackend {
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
parse_response_body(result).await
run_outcome(workspace_id, job_id, result).await
}
async fn run_flow(
@@ -321,7 +345,7 @@ impl McpBackend for WindmillBackend {
path: &str,
args: Value,
request: &McpRequest<'_>,
) -> BackendResult<Value> {
) -> BackendResult<RunOutcome> {
windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "flow");
// Called directly rather than through the jobs router, so its run gate does not apply.
windmill_common::workspaces::check_role_can_run(&self.db, workspace_id, auth)
@@ -333,9 +357,10 @@ impl McpBackend for WindmillBackend {
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
let job_id: uuid::Uuid = ulid::Ulid::new().into();
let result = run_wait_result_flow_by_path_internal(
self.db.clone(),
RunJobQuery::default(),
RunJobQuery { job_id: Some(job_id), ..Default::default() },
&flow,
auth.clone(),
self.user_db.clone(),
@@ -345,7 +370,7 @@ impl McpBackend for WindmillBackend {
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
parse_response_body(result).await
run_outcome(workspace_id, job_id, result).await
}
async fn call_endpoint(
+13 -2
View File
@@ -17,6 +17,17 @@ use crate::server::endpoints::EndpointTool;
/// Result type for backend operations using rmcp's ErrorData directly
pub type BackendResult<T> = Result<T, ErrorData>;
/// How a script or flow run by a tool ended.
pub enum RunOutcome {
Success(Value),
/// The job ran and failed. `result` is the job's result, its error object included.
Failed {
job_id: String,
run_url: String,
result: Value,
},
}
/// What a v2 token may do: read, read and run deployed scripts, flows and agents
/// (`mcp:read_run`), or everything its owner can.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -202,7 +213,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static {
path: &str,
args: Value,
request: &McpRequest<'_>,
) -> BackendResult<Value>;
) -> BackendResult<RunOutcome>;
/// Run a flow and wait for result
async fn run_flow(
@@ -212,7 +223,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static {
path: &str,
args: Value,
request: &McpRequest<'_>,
) -> BackendResult<Value>;
) -> BackendResult<RunOutcome>;
/// Call an endpoint tool (generated API endpoint)
async fn call_endpoint(
+3 -1
View File
@@ -14,7 +14,9 @@ pub mod tools;
// Re-export main types
pub use crate::common::types::{McpToken, MultiWorkspaceMcp, NoEndpointTools, WorkspaceInfo};
pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter, SessionAccess};
pub use backend::{
BackendResult, McpAuth, McpBackend, McpRequest, PathFilter, RunOutcome, SessionAccess,
};
pub use endpoints::{
endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only,
list_workspaces_tool, non_empty_body_fields, with_workspace_id_arg, EndpointTool,
+84 -6
View File
@@ -14,7 +14,7 @@ use crate::common::types::{
McpToken, MultiWorkspaceMcp, NoEndpointTools, ResourceInfo, SchemaType, ToolableItem,
WorkspaceId,
};
use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter};
use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter, RunOutcome};
use crate::server::endpoints::{
endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool,
};
@@ -87,6 +87,55 @@ pub(super) fn truncate_tool_result(text: String) -> String {
truncated
}
const MAX_PARTIAL_RESULT_BYTES: usize = 2_000;
/// A failed run as a tool execution error: the error's name and message, the run to open, and a
/// short excerpt of whatever else the result holds. The stack trace is left to the run's page.
fn failed_run_result(
tool_type: &str,
job_id: &str,
run_url: &str,
mut result: Value,
) -> CallToolResult {
let mut headline = None;
if let Some(obj) = result.as_object_mut() {
if let Some(Value::Object(error)) = obj.get_mut("error") {
let name = error.remove("name");
let message = error.remove("message");
error.remove("stack");
if error.get("job_id").and_then(Value::as_str) == Some(job_id) {
error.remove("job_id");
}
let [name, message] = [&name, &message].map(|v| v.as_ref().and_then(Value::as_str));
headline = match (name, message) {
(Some(n), Some(m)) => Some(format!("{n}: {m}")),
(n, m) => n.or(m).map(str::to_string),
};
if error.is_empty() {
obj.remove("error");
}
}
}
// The run id comes first so that truncating a huge message never cuts it off.
let mut text = format!("The {tool_type} failed. Run id: {job_id} ({run_url})");
if let Some(h) = headline {
text.push_str(&format!("\n{h}"));
}
if !matches!(&result, Value::Object(o) if o.is_empty()) && !result.is_null() {
let mut rest = result.to_string();
if rest.len() > MAX_PARTIAL_RESULT_BYTES {
let mut end = MAX_PARTIAL_RESULT_BYTES;
while !rest.is_char_boundary(end) {
end -= 1;
}
rest.truncate(end);
rest.push_str("… [truncated, the full result is on the run]");
}
text.push_str(&format!("\nPartial result: {rest}"));
}
CallToolResult::error(vec![ContentBlock::text(truncate_tool_result(text))])
}
/// MCP Server Runner - generic over the backend implementation
///
/// This struct implements the MCP ServerHandler trait and uses a McpBackend
@@ -1098,11 +1147,16 @@ impl<B: McpBackend> Runner<B> {
};
match result {
Ok(value) => Ok(CallToolResult::success(vec![ContentBlock::text(
truncate_tool_result(
serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()),
),
)])),
Ok(RunOutcome::Success(value)) => {
Ok(CallToolResult::success(vec![ContentBlock::text(
truncate_tool_result(
serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()),
),
)]))
}
Ok(RunOutcome::Failed { job_id, run_url, result }) => {
Ok(failed_run_result(tool_type, &job_id, &run_url, result))
}
Err(e) => Err(ErrorData::internal_error(
format!("Failed to run {}: {}", tool_type, e.message),
None,
@@ -1606,4 +1660,28 @@ mod tests {
)
.is_ok());
}
#[test]
fn failed_run_is_a_tool_error_without_the_stack() {
let result = json!({ "error": {
"name": "Error",
"message": "boom",
"stack": "Error: boom\n at main (/tmp/main.ts:2:9)",
"step_id": "a",
"job_id": "0192-id",
}});
let res = failed_run_result("script", "0192-id", "http://wm/run/0192-id", result);
assert_eq!(res.is_error, Some(true));
let text = res.content[0].as_text().unwrap().text.clone();
assert!(text.contains("Error: boom"), "{text}");
assert!(text.contains("0192-id"), "{text}");
assert!(text.contains(r#"{"error":{"step_id":"a"}}"#), "{text}");
assert!(!text.contains("main.ts"), "{text}");
let huge = json!({ "error": { "name": "Error", "message": "x".repeat(200_000) } });
let res = failed_run_result("script", "0192-id", "http://wm/run/0192-id", huge);
let text = &res.content[0].as_text().unwrap().text;
assert!(text.len() < MAX_TOOL_RESULT_BYTES + 100, "{}", text.len());
assert!(text.contains("0192-id"));
}
}