From 039f49ef124eacefc0ee34dcb011ea5773383845 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 29 Sep 2026 22:30:50 +0200 Subject: [PATCH 1/9] feat: dispatch ai agent tools through the worker queue (#11227) * feat: dispatch ai agent tools through the worker queue * fix: respect worker tags when executing ai agent tools locally * fix: cancel only direct-child tool jobs on agent failure * fix: cancel queued descendant tool jobs on agent failure * fix: route ai agent tools by the flow's tag rule --- ...9e7801129570106b5acd49c0f708398606771.json | 23 + ...ca7cb85bdfc66110b6534260f8642f91f006a.json | 22 + ...239a136ee5132bc97f4b06cc1ee5beaa621e5.json | 41 - ...5a1754ccb8ddeb08c23795dfe14e9f332e9dc.json | 35 + ...6f59430530b8b7428e42d1777927e63395ff8.json | 16 + backend/tests/ai_tool_queue.rs | 220 +++++ backend/windmill-common/src/worker.rs | 18 + backend/windmill-queue/src/jobs.rs | 39 + backend/windmill-worker/src/ai/tools.rs | 764 ++++++++---------- backend/windmill-worker/src/ai_executor.rs | 90 ++- backend/windmill-worker/src/worker.rs | 1 + backend/windmill-worker/src/worker_flow.rs | 2 +- 12 files changed, 774 insertions(+), 497 deletions(-) create mode 100644 backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json create mode 100644 backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json delete mode 100644 backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json create mode 100644 backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json create mode 100644 backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json create mode 100644 backend/tests/ai_tool_queue.rs diff --git a/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json new file mode 100644 index 0000000000..a7b61d7282 --- /dev/null +++ b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE descendants AS (\n SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id\n WHERE j.workspace_id = $2\n ) SELECT d.id AS \"id!\" FROM descendants d JOIN v2_job_queue q ON q.id = d.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771" +} diff --git a/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json new file mode 100644 index 0000000000..8209f67c75 --- /dev/null +++ b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a" +} diff --git a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json b/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json deleted file mode 100644 index cc6a57c005..0000000000 --- a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT c.id IS NOT NULL AS \"completed!\",\n c.result AS \"result: sqlx::types::Json>\",\n c.status = 'success' AS \"success\",\n EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\"\n FROM (SELECT 1) one\n LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "success", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "queued!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - true, - null, - null - ] - }, - "hash": "52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5" -} diff --git a/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json new file mode 100644 index 0000000000..9702cbf61d --- /dev/null +++ b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json>\"\n FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + true + ] + }, + "hash": "789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc" +} diff --git a/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json new file mode 100644 index 0000000000..197ecaf3d9 --- /dev/null +++ b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now()\n WHERE id = $2 AND tag = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8" +} diff --git a/backend/tests/ai_tool_queue.rs b/backend/tests/ai_tool_queue.rs new file mode 100644 index 0000000000..4d39176cc9 --- /dev/null +++ b/backend/tests/ai_tool_queue.rs @@ -0,0 +1,220 @@ +use axum::{ + extract::State, + routing::{get, post}, + Json, Router, +}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Notify; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"done"}) + } else { + json!({"role":"assistant", "tool_calls": (0..3).map(|i| json!({ + "index":i, "id":format!("call_{i}"), "type":"function", + "function":{"name":format!("tool_{i}"), "arguments":"{}"} + })).collect::>()}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +async fn run_batch( + db: Pool, + parallel: bool, + limited: bool, + unsupported: bool, +) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let server = ApiServer::start(db.clone()).await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let gate = Arc::new(Notify::new()); + let router = Router::new() + .route("/v1/chat/completions", post(model)) + .route( + "/first", + get(|State(gate): State>| async move { + gate.notified().await; + "ok" + }), + ) + .route( + "/second", + get(|State(gate): State>| async move { + gate.notify_one(); + "ok" + }), + ) + .with_state(gate); + let stub = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let tools: Vec = (0..3).map(|i| { + let wait = if parallel && !limited && i < 2 { + format!("await fetch('{base}/{}');", if i == 0 {"first"} else {"second"}) + } else { String::new() }; + let finish = if i == 2 { "throw new Error('expected tool failure');".to_string() } + else { format!("return {i};") }; + json!({"id":format!("t{i}"),"summary":format!("tool_{i}"),"value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "tag": if unsupported && i != 1 { "unserved-tool-tag" } else { "bun" }, + "concurrent_limit": if limited { Some(1) } else { None }, + "custom_concurrency_key": if limited { Some("agent-tool-test") } else { None }, + "content":format!("export async function main() {{ {wait} await Bun.sleep({}); {finish} }}", if i == 0 {200} else {0}) + }}) + }).collect(); + let flow: FlowValue = serde_json::from_value( + json!({"modules":[{"id":"agent","timeout":if unsupported {Some(3)} else {None},"value":{ + "type":"aiagent", "tools":tools, "input_transforms":{ + "provider":{"type":"static","value":{"kind":"customai","model":"queue-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"run the tools"}, + "max_iterations":{"type":"static","value":3} + } + }}]}), + )?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/agent_queue".into()), + restarted_from: None, + }) + .push(&db) + .await; + let notifications = listen_for_completed_jobs(&db).await; + let wait = async { + if parallel { + in_test_worker(&db, notifications.find(&id), server.addr.port()).await; + } else { + notifications.find(&id).await; + } + }; + tokio::time::timeout( + Duration::from_secs(45), + in_test_worker(&db, wait, server.addr.port()), + ) + .await?; + let result = completed_job(id, &db).await; + if unsupported { + assert!( + !result.success, + "agent must time out waiting for unsupported tools" + ); + let children: Vec<(String, bool, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.status = 'success', c.started_at IS NULL + FROM v2_job j JOIN v2_job_completed c USING(id) + WHERE j.parent_job IN (SELECT id FROM v2_job WHERE parent_job = $1) + ORDER BY j.runnable_path", + ) + .bind(id) + .fetch_all(&db) + .await?; + assert_eq!(children.len(), 3); + assert_eq!( + children.iter().map(|c| (c.1, c.2)).collect::>(), + vec![(false, true), (true, false), (false, true)], + "neither the first-tool reservation nor fallback may execute unsupported tags" + ); + stub.abort(); + server.close().await?; + return Ok(()); + } + assert!(result.success, "{:?}", result.result); + let result = result.json_result().expect("agent result"); + let messages: Vec<_> = result["messages"] + .as_array() + .unwrap() + .iter() + .filter(|m| m["role"] == "tool") + .collect(); + assert_eq!(messages.len(), 3); + for (index, message) in messages.iter().enumerate() { + assert_eq!(message["tool_call_id"], format!("call_{index}")); + } + assert_eq!(messages[0]["content"], "0"); + assert_eq!(messages[1]["content"], "1"); + assert!(messages[2]["content"] + .as_str() + .unwrap() + .contains("expected tool failure")); + let (parent_id, parent_worker): (uuid::Uuid, String) = sqlx::query_as( + "SELECT j.id, c.worker FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1" + ).bind(id).fetch_one(&db).await?; + let children: Vec<(String, String, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.worker, c.status = 'success' FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1 ORDER BY j.runnable_path" + ).bind(parent_id).fetch_all(&db).await?; + assert_eq!(children.len(), 3); + assert_eq!( + children[0].1, parent_worker, + "first tool must stay on the parent worker" + ); + assert_eq!( + children.iter().map(|c| c.2).collect::>(), + vec![true, true, false] + ); + if limited { + let overlapping: i64 = sqlx::query_scalar( + "SELECT count(*) FROM v2_job j1 JOIN v2_job_completed c1 ON c1.id = j1.id + JOIN v2_job j2 ON j2.parent_job = j1.parent_job AND j2.id > j1.id + JOIN v2_job_completed c2 ON c2.id = j2.id + WHERE j1.parent_job = $1 AND c1.started_at < c2.completed_at AND c2.started_at < c1.completed_at" + ).bind(parent_id).fetch_one(&db).await?; + assert_eq!( + overlapping, 0, + "the reserved first job must count toward the shared limit" + ); + } else if parallel { + assert_ne!( + children[1].1, parent_worker, + "second tool must unblock the first from another worker" + ); + } else { + assert!(children.iter().all(|child| child.1 == parent_worker)); + } + stub.abort(); + server.close().await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_drains_tools_without_another_worker(db: Pool) -> anyhow::Result<()> { + run_batch(db, false, false, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_keeps_first_tool_while_another_worker_runs_siblings( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, true, false, false).await +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn reserved_tool_obeys_shared_concurrency_limit(db: Pool) -> anyhow::Result<()> { + run_batch(db, true, true, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_leaves_unsupported_first_and_remaining_tools_queued( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, false, false, true).await +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index d480a89851..d661483182 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -864,6 +864,24 @@ pub fn make_pull_query(tags: &[String]) -> String { query } +/// Claim a parent's tool jobs matching its worker tags. The caller must supply only child IDs +/// it owns; this query is an internal scheduling primitive and does not authorize job access. +pub fn make_tool_job_pull_query(job_ids: &[uuid::Uuid], tags: &[String]) -> String { + // pull() binds only the worker name. These literals come from typed UUIDs, never input SQL. + let ids = job_ids.iter().map(|id| format!("'{id}'::uuid")).join(", "); + let tags = tags + .iter() + .map(|tag| format!("'{}'", tag.replace('\'', "''"))) + .join(", "); + format_pull_query(format!( + "SELECT id FROM v2_job_queue + WHERE running = false AND id = ANY(ARRAY[{ids}]::uuid[]) AND scheduled_for <= now() + AND tag = ANY(ARRAY[{tags}]::text[]) + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED LIMIT 1" + )) +} + // Variant of `make_pull_query` that additionally excludes jobs whose workspace_id is in the // overloaded-list bind parameter ($2::text[]). Built as a separate string (rather than reusing // `make_pull_query` with an always-bound array) so the planner can keep using the same indexes diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index b555e1ea92..c2c42c649c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4909,6 +4909,45 @@ pub fn has_active_concurrency_limit(concurrent_limit: Option) -> bool { concurrent_limit.is_some_and(|n| n > 0) } +/// Admit a job already owned by a worker without releasing its queue reservation. +/// The caller must maintain its heartbeat while waiting and complete it on failure. +pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result { + #[cfg(all(feature = "private", feature = "enterprise"))] + { + let settings = windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, + ) + .await? + .1 + .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); + if has_active_concurrency_limit(settings.concurrent_limit) + && !*DISABLE_CONCURRENCY_LIMIT + && job.canceled_by.is_none() + { + let key = concurrency_key(db, &job.id).await?.ok_or_else(|| { + Error::internal_err(format!("No concurrency key found for job {}", job.id)) + })?; + if !key.is_empty() { + return Ok(crate::jobs_ee::update_concurrency_counter( + db, + &job.id, + key, + serde_json::json!({ job.id.to_string(): {} }), + job.id.to_string(), + settings.concurrency_time_window_s.unwrap_or(0), + settings.concurrent_limit.unwrap_or_default(), + ) + .await? + .0); + } + } + } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + let _ = (db, job); + Ok(true) +} + pub async fn custom_concurrency_key( db: &Pool, job_id: &Uuid, diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 682714292e..930a329be4 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -7,16 +7,15 @@ use crate::ai::utils::{ use crate::common::OccupancyMetrics; use crate::result_processor::handle_non_flow_job_error; use crate::worker_flow::{ - evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag, -}; -use crate::{ - create_job_dir, handle_queued_job, JobCompletedReceiver, JobCompletedSender, SendResult, - SendResultPayload, + evaluate_input_transform, raw_script_to_payload, resolve_flow_step_tag, script_to_payload, + JobPayloadWithTag, }; +use crate::{create_job_dir, handle_queued_job, JobCompletedSender}; use anyhow::Context; use mappable_rc::Marc; use serde_json::value::RawValue; -use std::{collections::HashMap, sync::Arc}; +use sqlx::types::Json; +use std::{collections::HashMap, sync::Arc, time::Duration}; use uuid::Uuid; use windmill_ai::{ai_types::OpenAIToolCall, query_builder::StreamEventSink, types::*}; use windmill_common::jobs::JobPayload; @@ -36,11 +35,12 @@ use windmill_common::{ flow_conversations::{MessageExtras, MessageType}, flow_status::AgentAction, flows::FlowModuleValue, - worker::{to_raw_value, Connection}, + worker::{make_tool_job_pull_query, to_raw_value, Connection, WORKER_CONFIG}, }; use windmill_queue::{ - add_completed_job, add_completed_job_error, check_tag_available_for_push, get_mini_pulled_job, - push, resolve_push_tag, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, + append_logs, check_tag_available_for_push, get_mini_pulled_job, pull, push, resolve_push_tag, + tag_reads_flow_expr, try_admit_owned_job, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, }; /// Shared collection of abort handles for spawned tool tasks. @@ -74,6 +74,8 @@ pub struct ToolExecutionContext<'a> { pub stream_event_processor: Option<&'a StreamEventProcessor>, pub flow_context: &'a mut FlowContext, pub omit_output_from_conversation: bool, + /// The agent's flow `preserve_step_tags`: its tools pick their tag as that flow's steps do. + pub preserve_step_tags: bool, /// The thinking that led to this round's calls, stored on the first tool row written. /// None when the round wrote text, whose row carries it. pub reasoning: Option, @@ -82,6 +84,7 @@ pub struct ToolExecutionContext<'a> { // Abort handles for spawned tool tasks (used for force-cancel cleanup) pub tool_abort_handles: ToolAbortHandles, + pub job_completed_tx: JobCompletedSender, } /// Execute all tool calls from an AI response @@ -98,7 +101,8 @@ pub async fn execute_tool_calls( let mut used_structured_output_tool = false; let mut final_content = None; - for tool_call in tool_calls.iter() { + let mut calls = tool_calls.iter().peekable(); + while let Some(tool_call) = calls.next() { // Stream tool call progress if let Some(stream_event_processor) = ctx.stream_event_processor { let event = StreamingEvent::ToolExecution { @@ -150,15 +154,24 @@ pub async fn execute_tool_calls( ) .await?; } else if tool.module.is_some() { - execute_windmill_tool( - &mut ctx, - tool_call, - tool, - actions, - &mut messages, - final_events_str, - ) - .await?; + let mut batch = vec![(tool_call, tool)]; + while let Some(next_call) = calls.peek() { + if structured_output_tool_name.as_deref() + == Some(next_call.function.name.as_str()) + { + break; + } + let Some(next_tool) = tools.iter().find(|t| { + t.def.function.name == next_call.function.name + && t.mcp_source.is_none() + && t.module.is_some() + }) else { + break; + }; + batch.push((calls.next().unwrap(), next_tool)); + } + execute_windmill_tools(&mut ctx, &batch, actions, &mut messages, final_events_str) + .await?; } else { return Err(Error::internal_err(format!( "Tool type not supported: {}", @@ -313,14 +326,13 @@ async fn execute_mcp_tool_call( } /// Execute a Windmill tool (script or flow) -async fn execute_windmill_tool( - ctx: &mut ToolExecutionContext<'_>, +async fn enqueue_windmill_tool( + ctx: &ToolExecutionContext<'_>, tool_call: &OpenAIToolCall, tool: &Tool, actions: &mut Vec, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { + reserved: bool, +) -> Result<(Uuid, bool), Error> { // Regular Windmill tools must have a module let tool_module = tool.module.as_ref().ok_or_else(|| { Error::internal_err(format!("Tool {} has no module", tool_call.function.name)) @@ -402,8 +414,6 @@ async fn execute_windmill_tool( tool_call_args.insert(key.clone(), result); } - let is_ai_agent_tool = matches!(tool_value, FlowModuleValue::AIAgent { .. }); - let job_payload = match tool_value { FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => { script_to_payload( @@ -473,7 +483,6 @@ async fn execute_windmill_tool( )); } let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id); - // a nested agent always runs inline on this worker, under this agent's tag JobPayloadWithTag { payload: JobPayload::AIAgent { path }, tag: None, @@ -502,18 +511,34 @@ async fn execute_windmill_tool( let push_args = PushArgs { args: &tool_call_args, extra: None }; - // A tool with a tag of its own naming another queue than this agent's must run on a worker - // that tag selects, and like any tag the caller chose it must pass the workspace's custom tag - // restrictions. It runs inline when this worker serves that queue: the agent holds this - // worker while it waits, so dispatching to a queue it serves could wait on itself. - let mut resolved_tool_tag = None; - if let Some(tag) = job_payload.tag.as_deref() { - resolved_tool_tag = resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db) - .await - .filter(|resolved| *resolved != ctx.job.tag); + // The agent's tag stands in for the flow's, as a sub-flow's does for its own steps. + let tool_tag = resolve_flow_step_tag( + false, + &ctx.job.tag, + &ctx.job.workspace_id, + ctx.preserve_step_tags, + job_payload.tag.as_deref(), + ); + if tool_tag.as_deref().is_some_and(tag_reads_flow_expr) { + append_logs( + &ctx.job.id, + &ctx.job.workspace_id, + format!( + "Tool '{}' is tagged with `$flow_expr[...]`, which AI agent tools do not resolve, so it runs on its default tag.\n", + tool_call.function.name + ), + ctx.conn, + ) + .await; } - let tool_tag = match (job_payload.tag.as_deref(), resolved_tool_tag.as_ref()) { - (Some(tag), Some(_)) => { + + // A tool's own tag routes its job, so like any tag the caller chose it must pass the + // workspace's custom tag restrictions. The agent's tag was already checked when it was pushed. + if let Some(tag) = tool_tag.as_deref() { + if resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db) + .await + .is_some_and(|resolved| resolved != ctx.job.tag) + { let is_super_admin = windmill_common::auth::is_super_admin_email(ctx.db, email).await?; check_tag_available_for_push( ctx.db, @@ -531,16 +556,8 @@ async fn execute_windmill_tool( )), e => e, })?; - Some(tag.to_string()) } - _ => None, - }; - let run_inline = resolved_tool_tag.is_none_or(|resolved| { - windmill_common::worker::WORKER_CONFIG - .load() - .worker_tags - .contains(&resolved) - }); + } let mut tx = ctx.db.begin().await?; @@ -573,77 +590,71 @@ async fn execute_windmill_tool( false, None, ctx.job.visible_to_owner, - Some(tool_tag.unwrap_or_else(|| ctx.job.tag.clone())), + tool_tag, job_payload.timeout, None, job_priority, job_perms.as_ref(), - run_inline, + false, None, None, None, ) .await?; - tx.commit().await?; - - if !run_inline { - // Like an inline tool's failure, a lost wait reaches the model rather than failing the - // agent. - let (result, success) = - match wait_for_dispatched_tool_job(ctx.db, &uuid, &ctx.job.workspace_id).await { - Ok(outcome) => outcome, - Err(e) => ( - to_raw_value(&serde_json::json!({ "error": { "message": e.to_string() } })), - false, - ), - }; - return report_tool_result( - ctx, - tool_call, - tool_module, - job_id, - &result, - success, - is_ai_agent_tool && success, - messages, - final_events_str, + let mut tx = tx; + let reserved = if reserved { + let tags = local_tool_tags(); + // Reserve a supported first tool before commit so other workers cannot race it. + let claimed = sqlx::query!( + "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now() + WHERE id = $2 AND tag = ANY($3)", + ctx.worker_name, + uuid, + &tags, ) - .await; - } - - let tool_job = get_mini_pulled_job(ctx.db, &uuid).await?; - - let Some(tool_job) = tool_job else { - return Err(Error::internal_err("Tool job not found".to_string())); + .execute(&mut *tx) + .await? + .rows_affected() + == 1; + if claimed { + sqlx::query!("UPDATE v2_job_runtime SET ping = now() WHERE id = $1", uuid) + .execute(&mut *tx) + .await?; + } + claimed + } else { + false }; + tx.commit().await?; + Ok((uuid, reserved)) +} - // The tool gets a token of its own, as it would on any worker: the agent's token would - // identify the tool as the agent job (its OIDC path and job id). - let tool_client = AuthedClient::new( - ctx.base_internal_url.to_string(), - tool_job.workspace_id.clone(), - windmill_queue::create_token(ctx.db, &tool_job, None).await, - None, - ); - let tool_job = Arc::new(tool_job); +fn local_tool_tags() -> Vec { + WORKER_CONFIG + .load() + .priority_tags_sorted + .iter() + .flat_map(|group| group.tags.iter().cloned()) + .collect() +} - let (inner_job_completed_tx, inner_job_completed_rx) = JobCompletedSender::new(ctx.conn, 1); - - let inner_job_completed_rx = inner_job_completed_rx.expect( - "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", - ); +fn spawn_local_tool( + ctx: &ToolExecutionContext<'_>, + tool_job: MiniPulledJob, + reserved: bool, +) -> tokio::task::JoinHandle> { + let mut tool_job = Arc::new(tool_job); // Spawn handle_queued_job on separate task to prevent tokio stack overflow // Clone everything needed for the spawned task - let tool_job_spawn = tool_job.clone(); + let db = ctx.db.clone(); let conn_spawn = ctx.conn.clone(); - let client_spawn = tool_client; let hostname_spawn = ctx.hostname.to_string(); let worker_name_spawn = ctx.worker_name.to_string(); let worker_dir_spawn = ctx.worker_dir.to_string(); let base_internal_url_spawn = ctx.base_internal_url.to_string(); - let inner_job_completed_tx_spawn = inner_job_completed_tx.clone(); + let job_completed_tx = ctx.job_completed_tx.clone(); let mut occupancy_metrics_spawn = ctx.occupancy_metrics.clone(); let mut killpill_rx_spawn = ctx.killpill_rx.resubscribe(); @@ -652,147 +663,270 @@ async fn execute_windmill_tool( #[cfg(feature = "benchmark")] let mut bench_spawn = windmill_common::bench::BenchmarkIter::new(); - let job_dir = create_job_dir(&worker_dir_spawn, tool_job_spawn.id).await; + let result = async { + if reserved { + loop { + let queued = get_mini_pulled_job(&db, &tool_job.id) + .await? + .ok_or_else(|| Error::AlreadyCompleted("Tool job already completed".to_string()))?; + tool_job = Arc::new(queued); + sqlx::query!( + "UPDATE v2_job_runtime SET ping = now() WHERE id = $1", + tool_job.id, + ) + .execute(&db) + .await?; + if try_admit_owned_job(&db, &tool_job).await? { + let started_at = sqlx::query_scalar!( + "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at", + tool_job.id, + ) + .fetch_optional(&db) + .await? + .flatten(); + Arc::make_mut(&mut tool_job).started_at = started_at; + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + let perms = + windmill_common::auth::get_job_perms(&db, &tool_job.id, &tool_job.workspace_id).await?; + let token = windmill_queue::create_token(&db, &tool_job, perms).await; + let client_spawn = AuthedClient::new( + base_internal_url_spawn.clone(), + tool_job.workspace_id.clone(), + token, + None, + ); + let job_dir = create_job_dir(&worker_dir_spawn, tool_job.id).await; - let result = handle_queued_job( - tool_job_spawn, - None, - None, - None, - None, - &conn_spawn, - &client_spawn, - &hostname_spawn, - &worker_name_spawn, - &worker_dir_spawn, - &job_dir, - None, - &base_internal_url_spawn, - inner_job_completed_tx_spawn, - &mut occupancy_metrics_spawn, - &mut killpill_rx_spawn, - None, - None, - #[cfg(feature = "benchmark")] - &mut bench_spawn, - ) + handle_queued_job( + tool_job.clone(), + None, + None, + None, + None, + &conn_spawn, + &client_spawn, + &hostname_spawn, + &worker_name_spawn, + &worker_dir_spawn, + &job_dir, + None, + &base_internal_url_spawn, + job_completed_tx, + &mut occupancy_metrics_spawn, + &mut killpill_rx_spawn, + None, + None, + #[cfg(feature = "benchmark")] + &mut bench_spawn, + ) + .await + } .await; - // Return both result and updated metrics - (result, occupancy_metrics_spawn) + match result { + Err(err) => { + let err_string = format!("{}: {}", err.name(), err); + handle_non_flow_job_error( + &db, + &MiniCompletedJob::from(tool_job), + 0, + None, + err_string, + windmill_common::worker::error_to_value(&err), + &worker_name_spawn, + ) + .await?; + } + Ok(_) => {} + } + Ok(occupancy_metrics_spawn) }); // Register abort handle so the task can be killed on force-cancel let abort_handle = join_handle.abort_handle(); // unwrap safe: lock is only held briefly for push/drain, no panic possible inside ctx.tool_abort_handles.lock().unwrap().push(abort_handle); - - // Await the spawned task - let (handle_result, updated_occupancy) = join_handle.await.map_err(|e| { - if e.is_cancelled() { - Error::ExecutionErr("Tool execution task was cancelled".to_string()) - } else { - Error::internal_err(format!("Tool execution task failed: {}", e)) - } - })?; - - // Merge occupancy metrics back - ctx.occupancy_metrics.total_duration_of_running_jobs = - updated_occupancy.total_duration_of_running_jobs; - - match handle_result { - Err(err) => { - handle_tool_execution_error( - ctx, - tool_call, - tool_module, - &MiniCompletedJob::from(tool_job), - job_id, - err, - messages, - final_events_str, - ) - .await?; - } - Ok(outcome) => { - handle_tool_execution_success( - ctx, - tool_call, - tool_module, - job_id, - outcome.is_success(), - is_ai_agent_tool, - inner_job_completed_rx, - messages, - final_events_str, - ) - .await?; - } - } - - Ok(()) + join_handle } -/// Handle tool execution error -async fn handle_tool_execution_error( +async fn execute_windmill_tools( ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - tool_job: &MiniCompletedJob, - job_id: Uuid, - err: Error, + batch: &[(&OpenAIToolCall, &Tool)], + actions: &mut Vec, messages: &mut Vec, final_events_str: &mut String, ) -> Result<(), Error> { - let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = windmill_common::worker::error_to_value(&err); - let _ = handle_non_flow_job_error( - ctx.db, - tool_job, - 0, - None, - err_string.clone(), - err_json, - ctx.worker_name, - ) - .await; - - let error_message = format!("Error running tool: {}", err_string); - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(error_message.clone())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool_module.id.clone(), - }), - ..Default::default() - }); - - // Stream tool result (error case) - if let Some(stream_event_processor) = ctx.stream_event_processor { - let tool_result_event = StreamingEvent::ToolResult { - call_id: tool_call.id.clone(), - function_name: tool_call.function.name.clone(), - result: error_message.clone(), - success: false, - }; - stream_event_processor - .send(tool_result_event, final_events_str) - .await?; + let mut job_ids = Vec::with_capacity(batch.len()); + let mut local = None; + for (index, (call, tool)) in batch.iter().enumerate() { + if index > 0 { + if let Some(processor) = ctx.stream_event_processor { + processor + .send( + StreamingEvent::ToolExecution { + call_id: call.id.clone(), + function_name: call.function.name.clone(), + }, + final_events_str, + ) + .await?; + } + } + let (id, reserved) = enqueue_windmill_tool(ctx, call, tool, actions, index == 0).await?; + job_ids.push(id); + if reserved { + let job = get_mini_pulled_job(ctx.db, &id) + .await? + .ok_or_else(|| Error::internal_err("Reserved tool job not found".to_string()))?; + local = Some(spawn_local_tool(ctx, job, true)); + } } - if let Some(parent_job) = ctx.parent_job { - update_flow_status_module_with_actions_success(ctx.db, parent_job, false).await?; + let mut results: HashMap = HashMap::new(); + let mut next_result = 0; + let mut poll = tokio::time::interval(Duration::from_millis(100)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + while next_result < batch.len() || local.is_some() { + tokio::select! { + result = async { local.as_mut().unwrap().await }, if local.is_some() => { + local = None; + let metrics = result.map_err(|e| Error::internal_err(format!("Tool task failed: {e}")))??; + ctx.occupancy_metrics.total_duration_of_running_jobs = metrics.total_duration_of_running_jobs; + ctx.tool_abort_handles.lock().unwrap().retain(|handle| !handle.is_finished()); + } + _ = poll.tick() => {} + } + + let pending: Vec = job_ids[next_result..] + .iter() + .filter(|id| !results.contains_key(id)) + .copied() + .collect(); + if !pending.is_empty() { + let completed = sqlx::query!( + "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json>\" + FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + ctx.job.workspace_id, + &pending, + ).fetch_all(ctx.db).await?; + for completed in completed { + let index = job_ids + .iter() + .position(|id| *id == completed.id) + .ok_or_else(|| Error::internal_err("Unexpected tool completion".to_string()))?; + let (call, tool) = batch[index]; + let result = completed + .result + .map(|value| value.0) + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let is_agent = tool.module.as_ref().is_some_and(|module| { + matches!(module.get_value(), Ok(FlowModuleValue::AIAgent { .. })) + }); + let content = if is_agent && completed.success { + extract_ai_agent_output(&result).unwrap_or_else(|| result.get().to_string()) + } else { + result.get().to_string() + }; + if let Some(processor) = ctx.stream_event_processor { + processor + .send( + StreamingEvent::ToolResult { + call_id: call.id.clone(), + function_name: call.function.name.clone(), + result: content.clone(), + success: completed.success, + }, + final_events_str, + ) + .await?; + } + results.insert(completed.id, (completed.success, content)); + } + } + + // Transcript rows, model messages, and positional action statuses share call order. + while next_result < batch.len() { + let job_id = job_ids[next_result]; + let Some((success, content)) = results.remove(&job_id) else { + break; + }; + let (call, tool) = batch[next_result]; + let module = tool + .module + .as_ref() + .ok_or_else(|| Error::internal_err("Windmill tool has no module".to_string()))?; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(content.clone())), + tool_call_id: Some(call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: call.function.name.clone(), + module_id: module.id.clone(), + }), + ..Default::default() + }); + if let Some(parent) = ctx.parent_job { + update_flow_status_module_with_actions_success(ctx.db, parent, success).await?; + } + let (content, extras) = windmill_tool_row(call, success, &content); + add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await; + next_result += 1; + } + + if local.is_none() && next_result < batch.len() { + let pending: Vec = job_ids[next_result..] + .iter() + .filter(|id| !results.contains_key(id)) + .copied() + .collect(); + if !pending.is_empty() { + local = claim_local_tool(ctx, &pending).await?; + } + } } - - let (content, extras) = windmill_tool_row(tool_call, false, &error_message); - add_tool_message_to_chat(ctx, Some(job_id), &content, false, Some(extras)).await; - Ok(()) } +async fn claim_local_tool( + ctx: &ToolExecutionContext<'_>, + pending: &[Uuid], +) -> Result>>, Error> { + let query = ( + String::new(), + make_tool_job_pull_query(pending, &local_tool_tags()), + ); + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + let mut pulled = pull( + ctx.db, + false, + ctx.worker_name, + Some(&query), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await?; + if let Err(err) = pulled.maybe_apply_debouncing(ctx.db).await { + pulled.error_while_preprocessing = Some(err.to_string()); + } + match pulled.to_pulled_job() { + Ok(job) => Ok(job.map(|job| spawn_local_tool(ctx, job.job, false))), + Err( + windmill_queue::PulledJobResultToJobErr::MissingConcurrencyKey(job) + | windmill_queue::PulledJobResultToJobErr::ErrorWhilePreprocessing(job), + ) => { + ctx.job_completed_tx.send_job(job, true).await?; + Ok(None) + } + } +} + /// Extract the `output` field of an `AIAgentResult` envelope, serialized back to JSON. /// Returns `None` if the result is not a JSON object carrying an `output` field. fn extract_ai_agent_output(result: &RawValue) -> Option { @@ -802,200 +936,6 @@ fn extract_ai_agent_output(result: &RawValue) -> Option { .map(|output| output.get().to_string()) } -/// Handle tool execution success -async fn handle_tool_execution_success( - ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - job_id: Uuid, - success: bool, - is_ai_agent_tool: bool, - inner_job_completed_rx: JobCompletedReceiver, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { - let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); - - let (result, job_success) = if let Some(SendResult { - result: SendResultPayload::JobCompleted(ref jc), - .. - }) = send_result - { - let result = jc.result.clone(); - // Write tool completion to the DB inline instead of forwarding through - // the parent channel. Forwarding would deadlock for nested agents: the - // sub-tool result would fill the parent's bounded(1) channel, leaving - // no room for the agent's own completion from process_result. - if jc.success { - add_completed_job( - ctx.db, - &jc.job, - true, - false, - sqlx::types::Json(&*jc.result), - jc.result_columns.clone(), - jc.mem_peak, - jc.canceled_by.clone(), - false, - jc.duration, - jc.from_cache.unwrap_or(false), - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to add completed job: {e}")))?; - } else { - let error_value: serde_json::Value = - serde_json::from_str(jc.result.get()).unwrap_or_else(|_| { - serde_json::json!({ "message": format!("Non serializable error: {}", jc.result.get()) }) - }); - add_completed_job_error( - ctx.db, - &jc.job, - jc.mem_peak, - jc.canceled_by.clone(), - error_value, - ctx.worker_name, - false, - jc.duration, - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to add completed job error: {e}")))?; - } - (result, jc.success) - } else { - return Err(Error::internal_err( - "Tool job completed but no result".to_string(), - )); - }; - - report_tool_result( - ctx, - tool_call, - tool_module, - job_id, - &result, - success, - is_ai_agent_tool && job_success, - messages, - final_events_str, - ) - .await -} - -/// Wait for a tool job another worker runs; returns its result and whether it succeeded. -/// Canceling the agent cancels this job along with it, as a child of the agent's job. -async fn wait_for_dispatched_tool_job( - db: &DB, - id: &Uuid, - w_id: &str, -) -> Result<(Box, bool), Error> { - const MAX_CONSECUTIVE_POLL_ERRORS: u32 = 10; - let mut interval = std::time::Duration::from_millis(50); - let mut poll_errors = 0; - loop { - // One statement reads both tables from one snapshot, so a job completing between two - // reads cannot look like one that vanished. - let state = match sqlx::query!( - "SELECT c.id IS NOT NULL AS \"completed!\", - c.result AS \"result: sqlx::types::Json>\", - c.status = 'success' AS \"success\", - EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\" - FROM (SELECT 1) one - LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2", - id, - w_id - ) - .fetch_one(db) - .await - { - Ok(state) => { - poll_errors = 0; - state - } - Err(e) if poll_errors < MAX_CONSECUTIVE_POLL_ERRORS => { - poll_errors += 1; - tracing::warn!("polling tool job {id} failed, retrying: {e}"); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - Err(e) => return Err(e.into()), - }; - if state.completed { - let result = state - .result - .map(|r| r.0) - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - return Ok((result, state.success.unwrap_or(false))); - } - if !state.queued { - return Err(Error::internal_err(format!( - "tool job {id} is neither queued nor completed" - ))); - } - tokio::time::sleep(interval).await; - interval = std::cmp::min(interval * 2, std::time::Duration::from_secs(1)); - } -} - -/// Hand a finished tool job's result to the model, the stream, the flow status and the chat. -async fn report_tool_result( - ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - job_id: Uuid, - result: &RawValue, - success: bool, - is_ai_agent_output: bool, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { - // A nested agent returns the whole `AIAgentResult` envelope: on top of `output` it carries - // the child's entire message history, stream log and token usage. Feeding that back would - // grow the caller's context by the child's full transcript on every call, so the caller only - // sees `output`. The envelope stays intact in the tool job's completed row. - let tool_result = if is_ai_agent_output { - extract_ai_agent_output(result).unwrap_or_else(|| result.get().to_string()) - } else { - result.get().to_string() - }; - - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(tool_result.clone())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool_module.id.clone(), - }), - ..Default::default() - }); - - let (content, extras) = windmill_tool_row(tool_call, success, &tool_result); - - // The job ran; whether it ran successfully is `success`, and the row stored below is - // worded from it. The stream has to carry the same value, or the card the reader watches - // and the row that replaces it describe the same call differently. - if let Some(stream_event_processor) = ctx.stream_event_processor { - let tool_result_event = StreamingEvent::ToolResult { - call_id: tool_call.id.clone(), - function_name: tool_call.function.name.clone(), - result: tool_result, - success, - }; - stream_event_processor - .send(tool_result_event, final_events_str) - .await?; - } - - if let Some(parent_job) = ctx.parent_job { - update_flow_status_module_with_actions_success(ctx.db, parent_job, success).await?; - } - - add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await; - - Ok(()) -} - /// A Windmill tool's conversation row: worded from the tool, carrying the model's call and /// the exact text the model got back, the same text agent memory keeps for that tool /// message, so a card needs no job fetch. The call is the model's arguments, not the job's @@ -1060,9 +1000,7 @@ async fn add_tool_message_to_chat( .or(ctx.job.flow_step_id.as_deref()); let step_name = get_step_name_from_flow(ctx.summary.as_deref(), effective_step_id); - // Awaited, not spawned: `created_seq` is the transcript's order, so a round's rows - // must commit in the order of its calls. Calls run one after another; running them - // in parallel would need their rows written in call order all the same. + // created_seq defines transcript order, so these writes must stay sequential. if let Err(e) = add_message_to_conversation( ctx.db, &memory_id, diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index d7ded3652f..958caeaefc 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -607,6 +607,7 @@ pub async fn handle_ai_agent_job( hostname: &str, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, has_stream: &mut bool, + job_completed_tx: crate::JobCompletedSender, ) -> Result, Error> { // build_args_map returns None if no $res:/$var: transforms needed, in which case use original args let local_args = match build_args_map(job, client, conn).await? { @@ -686,6 +687,7 @@ pub async fn handle_ai_agent_job( }; let value = flow_data.value(); + let preserve_step_tags = value.preserve_step_tags; let module = if direct_parent_job_kind == JobKind::AIAgent { let parent_agent_step_id = direct_parent_job_flow_step_id.as_deref().ok_or_else(|| { @@ -1111,8 +1113,10 @@ pub async fn handle_ai_agent_job( has_stream, has_websearch, omit_output_from_conversation, + preserve_step_tags, cancel_rx, tool_abort_handles.clone(), + job_completed_tx, ); let mut occupancy_opt = Some(occupancy_metrics); @@ -1131,13 +1135,10 @@ pub async fn handle_ai_agent_job( cancel_tx, CANCEL_GRACE_PERIOD, ) - .await? + .await }; // agent_fut and update_job are now dropped — borrows on mcp_clients and canceled_by released - // Cleanup MCP clients - cleanup_mcp_clients(mcp_clients).await; - let format_cancel_info = |cb: &Option| { cb.as_ref() .map_or(("unknown".to_string(), "unknown".to_string()), |x| { @@ -1148,43 +1149,42 @@ pub async fn handle_ai_agent_job( }) }; - match outcome { - GracefulPollOutcome::Ok(result) => Ok(result), - GracefulPollOutcome::Timeout(ms) => { - tracing::error!("AI agent timeout after {}s", ms / 1000); - // A tool dispatched to another worker group would otherwise still run later, after - // the agent that wanted its result is gone. - let reason = format!("parent AI agent {} timed out", job.id); - let cb = CanceledBy { username: None, reason: Some(reason) }; - cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, Some(cb)).await; - Err(Error::ExecutionErr(format!( - "AI agent timeout after (>{}s)", - ms / 1000 - ))) - } - GracefulPollOutcome::Cancelled { canceled_by: cb } => { - let (by, reason) = format_cancel_info(&cb); - Err(Error::ExecutionErr(format!( - "Job cancelled by {by} (reason: {reason})" - ))) - } - GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => { - let (by, reason) = format_cancel_info(&cb); - // Abort any still-running spawned tool tasks - // unwrap safe: lock is only held briefly for push/drain, no panic possible inside - for handle in tool_abort_handles.lock().unwrap().drain(..) { - handle.abort(); + let result = match outcome { + Err(error) => Err(error), + Ok(outcome) => match outcome { + GracefulPollOutcome::Ok(result) => Ok(result), + GracefulPollOutcome::Timeout(ms) => { + tracing::error!("AI agent timeout after {}s", ms / 1000); + Err(Error::ExecutionErr(format!( + "AI agent timeout after (>{}s)", + ms / 1000 + ))) } - // Hard timeout: clean up orphaned jobs still stuck in v2_job_queue - cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, cb).await; - Err(Error::ExecutionErr(format!( - "Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)" - ))) - } - GracefulPollOutcome::AlreadyCompleted => { - Err(Error::AlreadyCompleted("Job already completed".to_string())) + GracefulPollOutcome::Cancelled { canceled_by: cb } => { + let (by, reason) = format_cancel_info(&cb); + Err(Error::ExecutionErr(format!( + "Job cancelled by {by} (reason: {reason})" + ))) + } + GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => { + let (by, reason) = format_cancel_info(&cb); + Err(Error::ExecutionErr(format!( + "Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)" + ))) + } + GracefulPollOutcome::AlreadyCompleted => { + Err(Error::AlreadyCompleted("Job already completed".to_string())) + } + }, + }; + if result.is_err() { + for handle in tool_abort_handles.lock().unwrap().drain(..) { + handle.abort(); } + cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, canceled_by.clone()).await; } + cleanup_mcp_clients(mcp_clients).await; + result } /// OpenAI rejects a `prompt_cache_key` over 64 characters @@ -1231,12 +1231,14 @@ pub async fn run_agent( has_stream: &mut bool, has_websearch: bool, omit_output_from_conversation: bool, + preserve_step_tags: bool, // cancellation signal from parent cancel_rx: tokio::sync::watch::Receiver, // abort handles for spawned tool tasks tool_abort_handles: ToolAbortHandles, + job_completed_tx: crate::JobCompletedSender, ) -> error::Result> { let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); let credentials = args.provider.to_provider_credentials(db).await?; @@ -2105,6 +2107,7 @@ pub async fn run_agent( stream_event_processor: stream_event_processor.as_ref(), flow_context: &mut flow_context, omit_output_from_conversation, + preserve_step_tags, reasoning: if structured_output_first { None } else { @@ -2113,6 +2116,7 @@ pub async fn run_agent( previous_result: &previous_result, id_context: &id_context, tool_abort_handles: tool_abort_handles.clone(), + job_completed_tx: job_completed_tx.clone(), }; let (tool_messages, tool_content, tool_used_structured_output) = @@ -3093,11 +3097,13 @@ async fn cleanup_orphaned_tool_jobs( ) }); - // Find direct child jobs still in v2_job_queue (agent tool jobs are always direct children) let orphaned_ids: Vec = match sqlx::query_scalar!( - r#"SELECT j.id FROM v2_job j - JOIN v2_job_queue q ON q.id = j.id - WHERE j.parent_job = $1 AND j.workspace_id = $2"#, + r#"WITH RECURSIVE descendants AS ( + SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id + WHERE j.workspace_id = $2 + ) SELECT d.id AS "id!" FROM descendants d JOIN v2_job_queue q ON q.id = d.id"#, parent_job_id, w_id, ) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f45c295b7a..ce3fe26545 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4990,6 +4990,7 @@ pub async fn handle_queued_job( hostname, killpill_rx, &mut has_stream, + job_completed_tx.clone(), )) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index a3257ace36..7c1ea23de4 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -3239,7 +3239,7 @@ struct PushNextFlowJobRec { /// - when the flow opts into `preserve_step_tags` and the child declares its own non-empty tag, /// that tag is honored instead of being overridden by the flow tag; /// - otherwise the child inherits the parent flow job's tag. -fn resolve_flow_step_tag( +pub(crate) fn resolve_flow_step_tag( is_preprocessor_step: bool, flow_tag: &str, workspace_id: &str, From cec8eaeadb125c6c0273eb7b6f567aaf0e63b757 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Sep 2026 22:36:32 +0200 Subject: [PATCH 2/9] test: fix CLI and Windows cargo test failures on 1.820.0 (#11429) * test(cli): stop CLI tests resolving a just-published windmill-client Co-Authored-By: Claude Opus 5.5 (1M context) * test: quote the Windows cert path in the pg attach assertion Co-Authored-By: Claude Opus 5.5 (1M context) * test(cli): note when the windmill-client pin must move Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-worker/src/duckdb_executor.rs | 2 +- cli/test/preview.test.ts | 10 ++++++++-- cli/test/raw_app_sync.test.ts | 3 +-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index d3bd976f21..b8c850a2f4 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2969,7 +2969,7 @@ mod tests { assert!( attach.starts_with(&format!( "ATTACH 'sslmode=verify-full sslrootcert=''{}''", - root + root.replace('\\', "\\\\") )), "{attach}" ); diff --git a/cli/test/preview.test.ts b/cli/test/preview.test.ts index ffa900c8f7..b0b00a2c7e 100644 --- a/cli/test/preview.test.ts +++ b/cli/test/preview.test.ts @@ -455,6 +455,12 @@ schema: // SCRIPT WITH MODULES PREVIEW TESTS // ============================================================================= +// Pinned: a release commit runs these tests while the same release publishes a new +// windmill-client, and the worker resolving that version before npm serves it fails the job. +// These are the only CLI tests running task/taskScript/workflow on a real client: bump the pin +// to a published release whenever the WAC client surface changes. +const WINDMILL_CLIENT = "windmill-client@1.819.0"; + test("script preview: script with modules (taskScript pattern)", async () => { await withTestBackend(async (backend, tempDir) => { await createWmillConfig(tempDir, { defaultTs: "bun" }); @@ -463,7 +469,7 @@ test("script preview: script with modules (taskScript pattern)", async () => { await createScript( tempDir, "f/test/wac_script.ts", - `import { task, taskScript, workflow } from "windmill-client"; + `import { task, taskScript, workflow } from "${WINDMILL_CLIENT}"; const helper = taskScript("./helper.ts"); @@ -512,7 +518,7 @@ test("script preview: script with modules (folder layout)", async () => { // Entry point script await writeFile( `${modDir}/script.ts`, - `import { task, taskScript, workflow } from "windmill-client"; + `import { task, taskScript, workflow } from "${WINDMILL_CLIENT}"; const helper = taskScript("./helper.ts"); diff --git a/cli/test/raw_app_sync.test.ts b/cli/test/raw_app_sync.test.ts index 315b274d80..33f1d3f6a6 100644 --- a/cli/test/raw_app_sync.test.ts +++ b/cli/test/raw_app_sync.test.ts @@ -61,8 +61,7 @@ root.render(); const PACKAGE_JSON = `{ "dependencies": { "react": "19.0.0", - "react-dom": "19.0.0", - "windmill-client": "^1" + "react-dom": "19.0.0" }, "devDependencies": { "@types/react-dom": "^19.0.0", From c728346a151e4a0a2d1ecaae397068871186323d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Sep 2026 09:10:12 +0200 Subject: [PATCH 3/9] fix(apps): hide custom component host while it is not rendered (#11434) Semi-lazy mode mounts unvisited subgrids' components with render=false so they initialize outputs. The custom component always kept its host div visible and the user's renderer draws into it regardless of render, so its content landed in the parent's layout flow and pushed the visible tab's components down until the owning tab was first opened. Co-authored-by: Claude Opus 5.5 (1M context) --- .../apps/components/display/AppCustomComponent.svelte | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte b/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte index 1e094b97ce..380501f7b7 100644 --- a/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte +++ b/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte @@ -123,7 +123,13 @@ -
+ +
{#if !loaded && render} From 9028432e8a71b9601ff65160a29e683a9cc24989 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Sep 2026 09:20:58 +0200 Subject: [PATCH 4/9] fix: embed a VERSIONINFO resource in the Windows binary (#11433) Co-authored-by: Claude Opus 5.5 (1M context) --- backend/Cargo.lock | 10 ++++++++++ backend/Cargo.toml | 3 +++ backend/build.rs | 15 +++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c1cdb2326c..114fb76b67 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14876,6 +14876,7 @@ dependencies = [ "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", + "winresource", ] [[package]] @@ -17382,6 +17383,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winresource" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" +dependencies = [ + "version_check", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5970ada6d8..9c09265dbd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -329,6 +329,9 @@ strum.workspace = true windows-service = "0.7" windows-sys = { version = "0.52", features = ["Win32_System_Services", "Win32_System_Console", "Win32_Foundation"] } +[target.'cfg(windows)'.build-dependencies] +winresource = { version = "0.1.31", default-features = false } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } diff --git a/backend/build.rs b/backend/build.rs index d5068697c7..88917b0492 100644 --- a/backend/build.rs +++ b/backend/build.rs @@ -2,4 +2,19 @@ fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); + + // Embed a VERSIONINFO resource so the exe's file properties carry the release + // version (from Cargo.toml), which Windows worker update scripts read. + // `cfg(windows)` is the host, matching the Windows-only build-dependency, so + // cross builds to Windows get no version resource. + #[cfg(windows)] + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") { + winresource::WindowsResource::new() + .set("ProductName", "Windmill") + .set("FileDescription", "Windmill") + .set("CompanyName", "Windmill Labs, Inc.") + .set("LegalCopyright", "Copyright Windmill Labs, Inc.") + .compile() + .expect("failed to embed the Windows version resource"); + } } From ab2a973357524c2c3aa81cc4aeb716d8cfd108d1 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 30 Sep 2026 10:33:20 +0200 Subject: [PATCH 5/9] fix: collapse run card in ai chat when the run is declined (#11435) * fix: collapse run card in ai chat when the run is declined Co-Authored-By: Claude Opus 5.5 (1M context) * refactor: simplify run card collapse condition and comment Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/lib/components/copilot/chat/RunScriptCard.svelte | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte index 4aff51d002..ca5e717891 100644 --- a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -208,9 +208,12 @@ // Keyed by call id: a bare flag would carry one card's collapse onto the next message // reusing this instance. Open by default, since the run is what was asked for — // except for an inspection, which is usually a step in the reasoning rather than - // the answer, and which pays a fetch for being opened. + // the answer, and which pays a fetch for being opened. A run cancelled before it started + // is closed too: the row already says Cancelled, and there is no run to show. let toggled = $state<{ id: string; open: boolean } | undefined>(undefined) - const expanded = $derived(toggled?.id === message.tool_call_id ? toggled.open : !inspected) + const expanded = $derived( + toggled?.id === message.tool_call_id ? toggled.open : !inspected && (ran || !canceled) + ) $effect(() => { const target = inspected From 1d8d4a16b4396acb70484c858058353eb1ba9d6f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Sep 2026 10:41:59 +0200 Subject: [PATCH 6/9] fix(cli): resolve fork specific items from the parent workspace entry (#11436) * test(cli): stop CLI tests resolving a just-published windmill-client Co-Authored-By: Claude Opus 5.5 (1M context) * test: quote the Windows cert path in the pg attach assertion Co-Authored-By: Claude Opus 5.5 (1M context) * test(cli): note when the windmill-client pin must move Co-Authored-By: Claude Opus 5.5 (1M context) * fix(cli): resolve fork specific items from the parent workspace entry Co-Authored-By: Claude Opus 5.5 (1M context) * fix(cli): keep fork resolution to the id-guarded inference Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- cli/src/commands/sync/sync.ts | 15 ++++++--- cli/src/core/conf.ts | 25 ++++++++++++++ cli/test/fork_specific_items_unit.test.ts | 40 +++++++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 cli/test/fork_specific_items_unit.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 9dfc26ca85..54887028f0 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -77,6 +77,7 @@ import { import { getEffectiveSettings, inferWsNameFromProfile, + inferWsNameFromForkBranch, resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, @@ -3752,9 +3753,12 @@ export async function pull( } } - // If wsNameForConfig wasn't set from flags, infer from the resolved profile + // If wsNameForConfig wasn't set from flags, infer from the resolved profile, + // then (for a fork) from the parent's entry if (!wsNameForConfig) { - wsNameForConfig = inferWsNameFromProfile(opts, workspace); + wsNameForConfig = + inferWsNameFromProfile(opts, workspace) ?? + inferWsNameFromForkBranch(opts, workspace.workspaceId); } // Resolve effective sync options with workspace awareness @@ -4802,9 +4806,12 @@ export async function push( const workspace = await resolveWorkspace(opts, wsNameForConfig); await requireLogin(opts); - // If wsNameForConfig wasn't set from flags, infer from the resolved profile + // If wsNameForConfig wasn't set from flags, infer from the resolved profile, + // then (for a fork) from the parent's entry if (!wsNameForConfig) { - wsNameForConfig = inferWsNameFromProfile(opts, workspace); + wsNameForConfig = + inferWsNameFromProfile(opts, workspace) ?? + inferWsNameFromForkBranch(opts, workspace.workspaceId); } // Resolve effective sync options with workspace awareness diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 7c6eb13274..455e876292 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -5,6 +5,7 @@ import { stringify as yamlStringify } from "yaml"; import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, + getWorkspaceIdForWorkspaceForkFromBranchName, isGitRepository, } from "../utils/git.ts"; import { join, dirname, resolve, relative } from "node:path"; @@ -725,6 +726,30 @@ export function inferWsNameFromProfile( return undefined; } +/** + * The workspace config entry a fork inherits from its parent: the one bound to + * the base branch of the checked-out `wm-fork//` branch, when + * `workspaceId` is the fork that branch names. A fork id never has an entry of + * its own, so without this its specific items and file suffix fall back to + * the common/base files and clobber what the fork cloned from its parent. + */ +export function inferWsNameFromForkBranch( + opts: SyncOptions, + workspaceId: string +): string | undefined { + if (!opts.workspaces || !isGitRepository()) return undefined; + const branch = getCurrentGitBranch(); + const baseBranch = getOriginalBranchForWorkspaceForks(branch); + if ( + !branch || + !baseBranch || + getWorkspaceIdForWorkspaceForkFromBranchName(branch) !== workspaceId + ) { + return undefined; + } + return findWorkspaceByGitBranch(opts.workspaces, baseBranch)?.[0]; +} + /** * `syncBehavior` as the workspace being pushed to sees it. The top level alone * misses a `workspaces..overrides.syncBehavior`, which is where a repo diff --git a/cli/test/fork_specific_items_unit.test.ts b/cli/test/fork_specific_items_unit.test.ts new file mode 100644 index 0000000000..0201958a8e --- /dev/null +++ b/cli/test/fork_specific_items_unit.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { execSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; + +import { inferWsNameFromForkBranch, type SyncOptions } from "../src/core/conf.ts"; + +// A fork id never has a wmill.yaml entry of its own. Unless it resolves to its +// parent's entry, a sync on the fork branch pushes the base files over the +// workspace-specific values the fork cloned from its parent. +test("a fork branch resolves to its parent's workspace entry", async () => { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_fork_specific_")); + const originalCwd = process.cwd(); + try { + execSync( + `git init -q -b wm-fork/dev-branch/t1 && git -c user.email=t@t -c user.name=t commit -q --allow-empty -m init`, + { cwd: repoDir }, + ); + process.chdir(repoDir); + + const config: SyncOptions = { + workspaces: { + dev: { + baseUrl: "http://localhost:8000/", + workspaceId: "dev-ws", + gitBranch: "dev-branch", + specificItems: { folders: ["f/team"] }, + }, + }, + }; + + expect(inferWsNameFromForkBranch(config, "wm-fork-t1")).toEqual("dev"); + // Only the fork the branch names inherits the parent's entry. + expect(inferWsNameFromForkBranch(config, "other-ws")).toBeUndefined(); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + } +}); From 7664001b36afc80bd8d26cb12c917c971e24e96f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Sep 2026 10:54:17 +0200 Subject: [PATCH 7/9] fix: bump git sync hub scripts to cli 1.820.1 (#11438) Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-common/src/workspaces.rs | 4 ++-- frontend/src/lib/hubPaths.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 1c72873377..448e13af85 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -184,7 +184,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -192,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28970/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28993/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 831380748e..2c46c48b2c 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28950/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28970/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28993/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", From 94ac34a4891a8572ad2493bdbbbf2f4911236aff Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 30 Sep 2026 13:02:30 +0200 Subject: [PATCH 8/9] fix: await async def main in python scripts (#11437) * fix: await async main and preprocessor in python scripts * fix: reuse one event loop per python process for async main * fix: reuse the event loop a python script sets at import * docs: clarify python await helper comment --- backend/tests/python_jobs.rs | 74 +++++++++++++++++++ .../windmill-worker/src/python_executor.rs | 45 +++++++++-- 2 files changed, 111 insertions(+), 8 deletions(-) diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index b7aa671b21..9616df267a 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -465,6 +465,37 @@ def main(x: int): ); } + #[test] + fn test_python_execd_async() { + let script = r#" +import asyncio + +_import_loop = asyncio.new_event_loop() +asyncio.set_event_loop(_import_loop) + +async def preprocessor(x: int): + return {"x": x * 2} + +async def main(x: int): + return {"x": x + 100, "same_loop": asyncio.get_running_loop() is _import_loop} +"#; + let results = run_py_raw_protocol_test( + &[("f/test/async", script)], + vec![ + ProtocolCmd::ExecdPreprocess { args: serde_json::json!({"x": 5}) }, + ProtocolCmd::Execd { args: serde_json::json!({"x": 7}) }, + ], + ); + assert_eq!( + results, + vec![ + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})), + DedicatedWorkerResult::Success(serde_json::json!({"x": 110, "same_loop": true})), + DedicatedWorkerResult::Success(serde_json::json!({"x": 107, "same_loop": true})), + ] + ); + } + // ==================== Argument Transformation Tests ==================== #[test] @@ -760,6 +791,49 @@ def main(): Ok(()) } +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base"))] +async fn test_python_async_main(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +import asyncio + +async def main(x: int): + await asyncio.sleep(0) + return x + 1 + "# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Python3, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + }); + + let result = RunJob::from(job) + .arg("x", serde_json::json!(41)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(42)); + Ok(()) +} + #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_result_preserves_infinity_in_string(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 5f5ca170b4..6b24b93e77 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -828,7 +828,7 @@ pub async fn handle_python_job( for k, v in list(pre_args.items()): if v == '': del pre_args[k] - kwargs = inner_script.preprocessor(**pre_args) + kwargs = _await(inner_script.preprocessor(**pre_args)) kwrags_json = res_to_json(kwargs, type(kwargs)) with open("args.json", 'w', encoding="utf-8") as f: f.write(kwrags_json)"# @@ -969,6 +969,8 @@ result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.j def res_to_json(res, typ): {res_to_json_body} +{PY_AWAIT_HELPER} + try: {preprocessor} {spread} @@ -977,7 +979,7 @@ try: del args[k] if inner_script.{main_override} is None or not callable(inner_script.{main_override}): raise ValueError("{main_override} function is missing") - res = inner_script.{main_override}(**args) + res = _await(inner_script.{main_override}(**args)) typ = type(res) if hasattr(res, '__iter__') and not isinstance(res, (str, dict, list, bytes, tuple, set, frozenset, range, memoryview, bytearray)) and typ.__name__ != 'DataFrame': for chunk in res: @@ -1513,6 +1515,8 @@ _fix=lambda s:s if 'Infinity' not in s and 'NaN' not in s and '\\u0000' not in s def res_to_json(res, typ): {res_to_json_body} + +{PY_AWAIT_HELPER} {functions} {registrations} @@ -1546,11 +1550,11 @@ for line in sys.stdin: continue kwargs = json.loads(args_json, strict=False) pre_args = entry['pre_transform'](kwargs) - preprocessed = mod.preprocessor(**pre_args) + preprocessed = _await(mod.preprocessor(**pre_args)) preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '') sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") main_args = entry['transform'](preprocessed if preprocessed else {{}}) - res = mod.main(**main_args) + res = _await(mod.main(**main_args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1574,11 +1578,11 @@ for line in sys.stdin: continue kwargs = json.loads(args_json, strict=False) pre_args = entry['pre_transform'](kwargs) - preprocessed = mod.preprocessor(**pre_args) + preprocessed = _await(mod.preprocessor(**pre_args)) preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '') sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") main_args = entry['transform'](preprocessed if preprocessed else {{}}) - res = mod.main(**main_args) + res = _await(mod.main(**main_args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1596,7 +1600,7 @@ for line in sys.stdin: entry = next(iter(scripts.values())) kwargs = json.loads(args_json, strict=False) args = entry['transform'](kwargs) - res = entry['mod'].main(**args) + res = _await(entry['mod'].main(**args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1624,7 +1628,7 @@ for line in sys.stdin: kwargs = json.loads(args_json, strict=False) args = entry['transform'](kwargs) - res = entry['mod'].main(**args) + res = _await(entry['mod'].main(**args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -3330,6 +3334,31 @@ This is not normal behavior, please make sure all workers have enough memory.\n }; } +/// Python helper running what an `async def` main or preprocessor returns. +/// One loop per process: a dedicated worker's module-level async clients and +/// locks are bound to the loop they were first used on, so `asyncio.run`'s +/// loop-per-call would break them from the second job on. A loop the script +/// set while importing (`asyncio.set_event_loop`) is reused. asyncio is +/// imported lazily so sync jobs don't pay its import time. +const PY_AWAIT_HELPER: &str = r#"_loop = None +def _await(r): + global _loop + if not hasattr(r, '__await__'): + return r + import asyncio + if _loop is None or _loop.is_closed(): + import warnings + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + _loop = asyncio.get_event_loop() + if _loop.is_closed(): + raise RuntimeError + except RuntimeError: + _loop = asyncio.new_event_loop() + asyncio.set_event_loop(_loop) + return _loop.run_until_complete(r)"#; + /// Python function body for `res_to_json(res, typ)`. /// Handles DataFrame, bytes, dict coercion + JSON serialization. fn python_res_to_json_body(postprocessor: &str) -> String { From e117871becc04a84baabe2aa0db6ae6e58cdd058 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 30 Sep 2026 13:10:24 +0200 Subject: [PATCH 9/9] weekly ai evals on current models, and claude 5.5/gpt-6 support (#11409) * feat: run ai evals weekly on current models and post results to a dashboard Co-Authored-By: Claude Opus 5.5 * feat: add current flagship models, a reasoning flag and claude 5.5 defaults Co-Authored-By: Claude Opus 5.5 * fix: never send a reasoning disable claude 5.5 or gpt-6-astra reject, and treat gpt-6 as a reasoning model Co-Authored-By: Claude Opus 5.5 * fix: address review on gpt-6 support, chat completions tools and model metadata Co-Authored-By: Claude Opus 5.5 * fix: leave tiered gpt-6 unpriced and drop the off sentinel on gpt-5 and o-series Co-Authored-By: Claude Opus 5.5 * docs: point the ai_evals readme at the model registry instead of copying it Co-Authored-By: Claude Opus 5.5 * refactor: encode the reasoning rules as per-family maps with a shared parity fixture Co-Authored-By: Claude Opus 5.5 * fix: keep the chat completions tools rule open-ended past gpt-5.6 and scope the parity fixture Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ai-evals-scheduled.yml | 171 ++++++ ai_evals/README.md | 17 +- .../frontend/core/script/scriptEvalRunner.ts | 10 +- .../frontend/core/shared/providerConfig.ts | 13 + ai_evals/bun.lock | 120 +---- ai_evals/cli/index.ts | 18 + ai_evals/core/judge.ts | 14 +- ai_evals/core/models.ts | 71 +++ ai_evals/package.json | 4 +- backend/windmill-ai/src/providers/mod.rs | 164 ++++++ backend/windmill-ai/src/providers/other.rs | 10 + backend/windmill-ai/src/types.rs | 5 +- frontend/src/lib/components/copilot/lib.ts | 22 +- .../components/copilot/modelConfig.test.ts | 7 + .../src/lib/components/copilot/modelConfig.ts | 4 +- .../components/copilot/modelPricing.test.ts | 10 + .../lib/components/copilot/modelPricing.ts | 5 + .../components/copilot/reasoningParity.json | 84 +++ .../copilot/reasoningRegistry.test.ts | 57 ++ .../components/copilot/reasoningRegistry.ts | 496 ++++++++---------- 20 files changed, 888 insertions(+), 414 deletions(-) create mode 100644 .github/workflows/ai-evals-scheduled.yml create mode 100644 frontend/src/lib/components/copilot/reasoningParity.json diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml new file mode 100644 index 0000000000..964e5d0cab --- /dev/null +++ b/.github/workflows/ai-evals-scheduled.yml @@ -0,0 +1,171 @@ +name: AI Evals (scheduled) + +# Full ai_evals suites on current models, posted to the AI evals dashboard in +# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked +# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M +# tokens per model: every suite runs on MODELS, and global (the mode users get) +# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by +# hand to measure a branch against main. +on: + schedule: + - cron: "0 3 * * 1" + workflow_dispatch: + inputs: + modes: + description: "Space-separated modes" + default: "global flow app script cli" + models: + description: "Space-separated model aliases (bun run cli -- models)" + default: "sonnet-5.5" + global_extra_models: + description: "Extra model aliases for global mode only" + default: "gpt-6-astra gemini-3.8-flash" + runs: + description: "Runs per case" + default: "3" + reasoning: + description: "Reasoning effort for frontend modes (empty: the product default)" + default: "" + +concurrency: + group: ai-evals-scheduled-${{ github.ref }} + +env: + MODELS: ${{ inputs.models || 'sonnet-5.5' }} + GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }} + RUNS: ${{ inputs.runs || '3' }} + REASONING: ${{ inputs.reasoning }} + INGEST_URL: https://app.windmill.dev/api/r/f/ai/ingest_ai_eval_run + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + modes: ${{ steps.modes.outputs.modes }} + steps: + - id: modes + env: + MODES: ${{ inputs.modes || 'global flow app script cli' }} + run: | + echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ") + | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT" + + evals: + needs: setup + runs-on: ubicloud-standard-16 + timeout-minutes: 330 + strategy: + fail-fast: false + matrix: + mode: ${{ fromJSON(needs.setup.outputs.modes) }} + services: + postgres: + image: postgres:16 + ports: + - 5432:5432 + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.97.0 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.0 + + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: Build Windmill + working-directory: ./backend + env: + SQLX_OFFLINE: true + CARGO_BUILD_JOBS: 12 + RUSTFLAGS: "" + run: cargo build --features quickjs + + - name: Start Windmill + working-directory: ./backend + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: info + run: | + mkdir -p ../ai_evals/logs + ./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 & + for i in $(seq 1 60); do + curl -sf http://localhost:8000/api/version > /dev/null 2>&1 && break + sleep 2 + done + curl -sf http://localhost:8000/api/version > /dev/null || { tail -50 ../ai_evals/logs/windmill.log; exit 1; } + + - name: Install frontend deps + generate client + working-directory: ./frontend + run: | + npm ci + npm run generate-backend-client + + - name: Install CLI deps + generate CLI client + working-directory: ./cli + run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh + + - name: Run ${{ matrix.mode }} evals + working-directory: ./ai_evals + env: + MODE: ${{ matrix.mode }} + WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 + WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + bun install + mkdir -p results + models="$MODELS" + [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS" + reasoning=() + [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING") + for m in $models; do + bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \ + --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored" + done + + - name: Post results to the dashboard + if: always() + working-directory: ./ai_evals + env: + MODE: ${{ matrix.mode }} + INGEST_TOKEN: ${{ secrets.AI_EVALS_INGEST_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + [ -n "$INGEST_TOKEN" ] || { echo "::warning::AI_EVALS_INGEST_TOKEN is not set"; exit 0; } + shopt -s nullglob + for f in results/"$MODE"-*.json; do + # Keep what the dashboard reads; the full traces stay in the run's artifacts. + jq -c --arg ref "$GITHUB_REF" --arg trigger "$GITHUB_EVENT_NAME" --arg url "$RUN_URL" ' + {result_json: (del(.cases[].prompt, .cases[].initialPath, .cases[].expectedPath) + | .cases[].attempts[] |= {attempt, passed, durationMs, toolCallCount, judgeScore, + judgeSummary, error, tokenUsage, checks: [.checks[]? | {name, passed}]}), + git_ref: $ref, trigger: $trigger, run_url: $url}' "$f" \ + | curl -sf --retry 3 -X POST "$INGEST_URL" \ + -H "Authorization: Bearer $INGEST_TOKEN" -H 'content-type: application/json' \ + --data-binary @- && echo " <- $f" || echo "::warning::failed to post $f" + done + + - name: Archive logs and results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ai-evals-${{ matrix.mode }} + path: | + ai_evals/logs + ai_evals/results diff --git a/ai_evals/README.md b/ai_evals/README.md index 603f28e5f9..bff4294e52 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -77,31 +77,20 @@ Public CLI surface: - `--verbose`: stream assistant output for frontend runs - `--skip-judge`: skip LLM judge scoring for the run - `--execution-only`: only require the model/proxy/frontend loop to complete; skip validators, tool expectations, backend artifact validation, and judge scoring +- `--reasoning `: reasoning effort for frontend modes (`off`, `low`, `medium`, `high`, `max`, …); without it the product's default applies (`high` on models that can reason). The effort is appended to the recorded model label (`anthropic:claude-sonnet-5-5@max`) - `--record`: append a compact tracked summary line to `ai_evals/history/.jsonl` for full-suite runs only - `--backend-validation `: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals ## Models -Use `bun run cli -- models` to see the current aliases. - -Today: - -- `haiku` -- `sonnet` -- `opus` -- `4o` -- `gpt-5.5` -- `gemini-3-flash-preview` -- `gemini-3.1-pro-preview` -- `deepseek-v4-flash` -- `deepseek-v4-pro` +Use `bun run cli -- models` to see the current aliases; `core/models.ts` is the list. Notes: - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5` - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases - `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there -- the judge model is separate and currently defaults to `claude-sonnet-4-6`; use `--skip-judge` for deterministic-only runs +- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs ## Case Format diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index b9c80f3554..0ff352f3d4 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -15,6 +15,7 @@ import { runEval } from "../shared"; import type { ModeRunContext } from "../../../../core/types"; import type { TokenUsage, ToolCallDetail } from "../shared/types"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; +import { evalReasoningEffort } from "../shared/providerConfig"; export interface ScriptEvalResult { success: boolean; @@ -41,14 +42,15 @@ export interface ScriptEvalOptions { function resolveModelProvider( model: string, provider?: AIProvider, -): AIProviderModel { +): AIProviderModel & { reasoning?: string } { + const reasoning = evalReasoningEffort(); if (provider) { - return { provider, model }; + return { provider, model, reasoning }; } if (model.startsWith("claude")) { - return { provider: "anthropic", model }; + return { provider: "anthropic", model, reasoning }; } - return { provider: "openai", model }; + return { provider: "openai", model, reasoning }; } export async function runScriptEval( diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.ts index 9d9e315217..49d5cb6ef8 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.ts @@ -12,6 +12,12 @@ export interface EvalClients { export interface ResolvedEvalModelProvider { provider: FrontendEvalProvider; model: string; + reasoning?: string; +} + +/** `run --reasoning` hands the effort to the frontend runtime through the environment. */ +export function evalReasoningEffort(): string | undefined { + return process.env.WMILL_AI_EVAL_REASONING || undefined; } export interface WindmillAiProxyClientConfig { @@ -73,6 +79,13 @@ export function createEvalClients(input: { export function resolveEvalModelProvider( model: string, provider?: FrontendEvalProvider, +): ResolvedEvalModelProvider { + return { ...resolveProvider(model, provider), reasoning: evalReasoningEffort() }; +} + +function resolveProvider( + model: string, + provider?: FrontendEvalProvider, ): ResolvedEvalModelProvider { if (provider) { return { provider, model }; diff --git a/ai_evals/bun.lock b/ai_evals/bun.lock index eaed1db99a..798022d49a 100644 --- a/ai_evals/bun.lock +++ b/ai_evals/bun.lock @@ -5,8 +5,8 @@ "": { "name": "windmill-ai-evals", "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.25", - "@anthropic-ai/sdk": "^0.39.0", + "@anthropic-ai/claude-agent-sdk": "^0.3.284", + "@anthropic-ai/sdk": "^0.129.0", "commander": "^14.0.3", "openai": "^6.9.1", "yaml": "^2.8.3", @@ -18,66 +18,44 @@ }, }, "packages": { - "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="], + "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.284", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.284" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-NSoJwEq6nFSf8dtaacYx37QdGgqApI3eHFUlxclMLYi8irb6ZJwEaUnPnLUCyAyg9W/tMkgZC0GXWLRCc01I0w=="], - "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="], + "@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.284", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gKY9MUjY83398uCiPLHsd87kyzu7agIM7ApqWpJkpSINep6hxx4rNoR8bUbNWg49Aoe/PW/DJkQByAQuNzR9rg=="], - "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], + "@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.284", "", { "os": "darwin", "cpu": "x64" }, "sha512-P+q6Z7sKeYz99uE7RJc4au1IK+4SiWe69pyHY3QXMnjZq8hSFU/5Mo8iSurLk9jRlihsMltV1rLA5naouYyCAQ=="], + + "@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-LDpuYDaz+pCdG29iy1pw4P1D2YVtpZOb4rWOPxxSpg2fFyUeQ5WrfnuszOxhDWpNh1k1ekDiKm3A9OzGcrkdtA=="], + + "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-U3/uAv1TS8sMWsgWQSxYbdND5AaRXXpP5RxHU1oYC5ZizlZE8/xAVum9dj1QX3rVZJg8EzsqZveq/AkEv5ABfQ=="], + + "@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-yGytBCCwJvWeg1FzFpgnLM9BOM2vKVExtv6pMJHMtF82J5yhKHcxodRaYlQVd/HzJJ2nruKb9EnrhRBZ//4yRw=="], + + "@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-4x0Q8CFkCNbeENE7pRhuUmkwh/sQfTq1yWi4ZE56stve9pgoKzu6ZDa1SEtvK1YIooGE/qM5rVS8xFLE1gbQ0w=="], + + "@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.284", "", { "os": "win32", "cpu": "arm64" }, "sha512-zOXkdPHhElyxyFJ6eY5R+YGUswqBDZ+ZHNaRmQGJ28RkQYgwG5imNy/nfFR+/CPGlMnH9ZzbGc2rKYApm1K4Tg=="], + + "@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.284", "", { "os": "win32", "cpu": "x64" }, "sha512-VGaFRDCOPloj5IjvJTyy5JsSl9sE2HR3W+A6f/cjgh23/7Y5vK7/ka+1JQD+AkoGO9tbHqP2O6god3IBrLvOaw=="], + + "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.129.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw=="], + + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], "@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="], - "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], - - "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], - - "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], - - "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], - - "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], - - "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], - - "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], - - "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], - - "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], - - "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], - - "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], - - "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], - - "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], - - "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], - - "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], - - "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], + "@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="], + "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], - "@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], - - "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], - - "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], + "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], - "agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="], - "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], - "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], - "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], @@ -88,8 +66,6 @@ "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], - "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], - "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], @@ -106,8 +82,6 @@ "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], - "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], @@ -122,14 +96,10 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], - "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], - "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], - "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], - "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -140,16 +110,12 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + "fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="], + "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], - - "form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="], - - "formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="], - "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], @@ -164,16 +130,12 @@ "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], - "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], - "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], "hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="], - "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], @@ -208,10 +170,6 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], - "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], - - "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], - "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -262,30 +220,24 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], - "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - "undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="], - - "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], - - "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], @@ -295,19 +247,5 @@ "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], - - "@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="], - - "@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], - - "bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], - - "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - - "@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], - - "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], - - "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], } } diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 504b8a8d13..8a5f63c8b2 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -108,6 +108,10 @@ async function main() { "--record", "append a compact summary line to ai_evals/history/.jsonl", ) + .option( + "--reasoning ", + "reasoning effort for frontend modes (e.g. off, low, medium, high, max); default: the product's", + ) .option( "--backend-validation ", `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`, @@ -126,6 +130,7 @@ async function main() { executionOnly?: boolean; record?: boolean; backendValidation?: string; + reasoning?: string; }, ) => { await handleRun({ @@ -140,6 +145,7 @@ async function main() { executionOnly: options.executionOnly ?? false, record: options.record ?? false, backendValidation: options.backendValidation, + reasoning: options.reasoning, }); }, ); @@ -190,6 +196,7 @@ async function handleRun(input: { executionOnly: boolean; record: boolean; backendValidation?: string; + reasoning?: string; }) { if (input.record && input.caseIds.length > 0) { throw new Error( @@ -217,6 +224,13 @@ async function handleRun(input: { "--backend-validation currently supports only flow and script modes", ); } + if (input.reasoning) { + if (input.mode === "cli") { + throw new Error("--reasoning only applies to frontend modes"); + } + // The frontend runtime runs in a child process, which inherits it. + process.env.WMILL_AI_EVAL_REASONING = input.reasoning; + } if (input.mode !== "cli") { await assertWindmillBackendReachable(resolveWindmillBackendSettings()); } @@ -257,6 +271,10 @@ async function handleRun(input: { backendValidation, }); + if (input.reasoning && result.runModel) { + result.runModel = `${result.runModel}@${input.reasoning}`; + } + const resolvedOutputPath = models.length === 1 ? resolveRunOutputPath(input.mode, input.outputPath) diff --git a/ai_evals/core/judge.ts b/ai_evals/core/judge.ts index 5e67510027..f1df60f286 100644 --- a/ai_evals/core/judge.ts +++ b/ai_evals/core/judge.ts @@ -1,7 +1,7 @@ import Anthropic from "@anthropic-ai/sdk"; import type { EvalMode, JudgeResult } from "./types"; -export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6"; +export const DEFAULT_JUDGE_MODEL = "claude-sonnet-5-5"; const JUDGE_TOOL_NAME = "submit_judgement"; @@ -29,6 +29,7 @@ export async function judgeOutput(input: { const system = [ "You evaluate benchmark outputs for Windmill AI generation.", + `Always answer by calling the ${JUDGE_TOOL_NAME} tool.`, "Deterministic checks already run separately. Focus on whether the final output satisfies the user request.", "If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.", "If a checklist is provided, treat it as the explicit acceptance criteria for this case.", @@ -69,8 +70,8 @@ export async function judgeOutput(input: { try { const response = await client.messages.create({ model, - max_tokens: 1024, - temperature: 0, + // The judge thinks by default, and thinking shares this budget with the verdict. + max_tokens: 16000, system, messages: [{ role: "user", content: user }], tools: [ @@ -93,11 +94,8 @@ export async function judgeOutput(input: { }, }, ], - tool_choice: { - type: "tool", - name: JUDGE_TOOL_NAME, - disable_parallel_tool_use: true, - }, + // Current models refuse a forced tool_choice ("tool"/"any"). + tool_choice: { type: "auto", disable_parallel_tool_use: true }, }); const toolUseBlock = response.content.find( diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 295cd36135..1580cf5b0b 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -78,6 +78,32 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "opus", }, }, + { + id: "sonnet-5.5", + label: "Claude Sonnet 5.5", + aliases: ["sonnet-5.5", "claude-sonnet-5.5", "claude-sonnet-5-5"], + frontend: { + provider: "anthropic", + model: "claude-sonnet-5-5", + }, + cli: { + provider: "anthropic", + model: "claude-sonnet-5-5", + }, + }, + { + id: "opus-5.5", + label: "Claude Opus 5.5", + aliases: ["opus-5.5", "claude-opus-5.5", "claude-opus-5-5"], + frontend: { + provider: "anthropic", + model: "claude-opus-5-5", + }, + cli: { + provider: "anthropic", + model: "claude-opus-5-5", + }, + }, { id: "4o", label: "GPT-4o", @@ -96,6 +122,51 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-5.5", }, }, + { + id: "gpt-5.6-sol", + label: "GPT-5.6 Sol", + aliases: ["gpt-5.6-sol"], + frontend: { + provider: "openai", + model: "gpt-5.6-sol", + }, + }, + { + id: "gpt-6-astra", + label: "GPT-6 Astra", + aliases: ["gpt-6-astra", "gpt-6"], + frontend: { + provider: "openai", + model: "gpt-6-astra", + }, + }, + { + id: "gpt-6-sol", + label: "GPT-6 Sol", + aliases: ["gpt-6-sol"], + frontend: { + provider: "openai", + model: "gpt-6-sol", + }, + }, + { + id: "gpt-6-luna", + label: "GPT-6 Luna", + aliases: ["gpt-6-luna"], + frontend: { + provider: "openai", + model: "gpt-6-luna", + }, + }, + { + id: "gemini-3.8-flash", + label: "Gemini 3.8 Flash", + aliases: ["gemini-3.8-flash"], + frontend: { + provider: "googleai", + model: "gemini-3.8-flash", + }, + }, { id: "gemini-3-flash-preview", label: "Gemini 3 Flash Preview", diff --git a/ai_evals/package.json b/ai_evals/package.json index 6720c910cb..424e49ffae 100644 --- a/ai_evals/package.json +++ b/ai_evals/package.json @@ -8,8 +8,8 @@ "test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.25", - "@anthropic-ai/sdk": "^0.39.0", + "@anthropic-ai/claude-agent-sdk": "^0.3.284", + "@anthropic-ai/sdk": "^0.129.0", "commander": "^14.0.3", "openai": "^6.9.1", "yaml": "^2.8.3" diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 908f007978..b5b6281264 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -14,6 +14,130 @@ use std::time::{Duration, Instant}; /// its `thinking` param, Gemini to a zero budget or the model's floor). pub(crate) const REASONING_OFF_SENTINEL: &str = "none"; +/// The effort to send for a model, dropping the off sentinel on a model that rejects every +/// disable: the model then reasons at its default instead of failing the request. The UI +/// never offers off on these, so this guards an agent step saved before it stopped, or an +/// effort passed in as a flow input. +pub fn effective_reasoning_effort<'a>(model: &str, effort: Option<&'a str>) -> Option<&'a str> { + match effort { + Some(effort) + if effort == REASONING_OFF_SENTINEL + && reasoning_rule(model).is_some_and(|rule| !rule.can_disable) => + { + None + } + effort => effort, + } +} + +/// Whether a request with function tools must send the off sentinel on Chat Completions: +/// the model refuses tools there while it reasons, and does accept being turned off. +pub(crate) fn completions_tools_need_reasoning_off(model: &str) -> bool { + reasoning_rule(model).is_some_and(|rule| rule.completions_tools_need_off && rule.can_disable) +} + +/// What the backend needs to know about a model's reasoning: the rows of `REASONING_RULES` +/// in the frontend's `reasoningRegistry.ts`, cut down to the two facts the wire needs. +/// `reasoningParity.json` next to that file is checked by both sides' tests. +struct ReasoningRule { + matches: fn(&str) -> bool, + /// False when the provider rejects every disable, so the off sentinel must not be sent. + can_disable: bool, + /// Chat Completions refuses function tools while the model reasons, even with the + /// effort omitted (live-verified); the Responses API has no such limit. + completions_tools_need_off: bool, +} + +/// Matched in order against the lowercased model id, first match wins. A model no row +/// matches keeps whatever effort it was given. +const REASONING_RULES: &[ReasoningRule] = &[ + // Live-verified: Fable, Mythos and the 5.x point releases reject `thinking: disabled`. + ReasoningRule { + matches: |m| m.contains("claude-fable") || m.contains("claude-mythos"), + can_disable: false, + completions_tools_need_off: false, + }, + ReasoningRule { + matches: is_claude_5_point_release, + can_disable: false, + completions_tools_need_off: false, + }, + // Live-verified: astra takes low..max only, where sol and luna also take `none`. + ReasoningRule { + matches: |m| base_id(m).starts_with("gpt-6-astra"), + can_disable: false, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| gpt_version(m).is_some_and(|(major, _)| major >= 6), + can_disable: true, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, Some(minor))) if minor >= 5), + can_disable: true, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, Some(_)))), + can_disable: true, + completions_tools_need_off: false, + }, + // gpt-5 and the o-series reject `none`. + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, None))), + can_disable: false, + completions_tools_need_off: false, + }, + ReasoningRule { + matches: |m| { + let base = base_id(m); + base.starts_with('o') && base[1..].starts_with(|c: char| c.is_ascii_digit()) + }, + can_disable: false, + completions_tools_need_off: false, + }, +]; + +fn reasoning_rule(model: &str) -> Option<&'static ReasoningRule> { + let model = model.to_lowercase(); + REASONING_RULES.iter().find(|rule| (rule.matches)(&model)) +} + +/// The id after a gateway's `vendor/` and before a `:variant`. +fn base_id(model: &str) -> &str { + let last = model.rsplit('/').next().unwrap_or(model); + last.split(':').next().unwrap_or(last) +} + +/// `(major, minor)` of a `gpt-` id. The major is one digit then a separator or the end, +/// since Azure names gpt-3.5 `gpt-35-turbo`. +fn gpt_version(model: &str) -> Option<(u32, Option)> { + let rest = base_id(model).strip_prefix("gpt-")?; + let mut chars = rest.chars(); + let major = chars.next()?.to_digit(10)?; + match chars.next() { + None | Some('-') => Some((major, None)), + Some('.') => { + let minor: String = chars.take_while(char::is_ascii_digit).collect(); + Some((major, minor.parse().ok())) + } + _ => None, + } +} + +/// Sonnet or Opus 5.x with x >= 1. The version match stops at one digit so a dated id +/// (`claude-sonnet-5-20260101`) stays Sonnet 5. +fn is_claude_5_point_release(model: &str) -> bool { + let model = model.replace('.', "-"); + ["claude-opus-5-", "claude-sonnet-5-"].iter().any(|prefix| { + model.split(prefix).skip(1).any(|rest| { + let mut chars = rest.chars(); + matches!(chars.next(), Some('1'..='9')) && !matches!(chars.next(), Some('0'..='9')) + }) + }) +} + /// Whether a Claude model removed the sampling params (`temperature`, `top_p`, /// `top_k`). On these, any value is a hard 400 — `temperature is deprecated for /// this model` — whatever the thinking mode, so the param has to be dropped on @@ -119,6 +243,46 @@ pub fn remember_chat_completions_only(base_url: &str, model: &str) { CHAT_COMPLETIONS_ONLY.insert((base_url.to_string(), model.to_string()), Instant::now()); } +#[cfg(test)] +mod reasoning_rule_tests { + use super::*; + + /// The frontend registry's test reads the same file. These rules see the model id + /// alone, so it holds only rows whose answer doesn't depend on the provider. + #[test] + fn agrees_with_the_frontend_registry() { + let rows: Vec = serde_json::from_str(include_str!( + "../../../../frontend/src/lib/components/copilot/reasoningParity.json" + )) + .unwrap(); + for row in rows { + let model = row["model"].as_str().unwrap(); + let can_disable = row["canDisable"].as_bool().unwrap(); + let tools_need_off = row["completionsToolsNeedOff"].as_bool().unwrap(); + let sent = effective_reasoning_effort(model, Some(REASONING_OFF_SENTINEL)); + assert_eq!(sent.is_some(), can_disable, "{model}"); + assert_eq!( + effective_reasoning_effort(model, Some("low")), + Some("low"), + "{model}" + ); + assert_eq!( + completions_tools_need_reasoning_off(model), + tools_need_off && can_disable, + "{model}" + ); + } + } + + #[test] + fn reads_ids_the_frontend_resolves_first() { + // Azure's gpt-3.5 is not major 35, and a gateway prefix is not part of the id. + assert_eq!(gpt_version("gpt-35-turbo"), None); + assert!(completions_tools_need_reasoning_off("openai/gpt-6-sol")); + assert_eq!(effective_reasoning_effort("openai/o3", Some("none")), None); + } +} + #[cfg(test)] mod chat_completions_only_tests { use super::*; diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index b6040d8d79..1ea54a7e79 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -1,3 +1,4 @@ +use super::{completions_tools_need_reasoning_off, REASONING_OFF_SENTINEL}; use crate::{ ai_providers::AIProvider, image_handler::prepare_messages_for_api, @@ -143,6 +144,15 @@ impl OtherQueryBuilder { let (reasoning_effort, thinking, temperature) = provider_reasoning_fields(&self.provider_kind, args.reasoning_effort, args.temperature); + // With tools, gpt-5.5+ only run here with reasoning off: turn it off where the model + // can, rather than failing every turn. + let reasoning_effort = if args.tools.is_some_and(|tools| !tools.is_empty()) + && completions_tools_need_reasoning_off(args.model) + { + Some(REASONING_OFF_SENTINEL) + } else { + reasoning_effort + }; // Build request with stream_options for usage tracking let request_with_usage = OpenAICompletionRequest { diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 03476240a3..7e2de09e01 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -294,7 +294,10 @@ impl ProviderWithResource { /// The reasoning effort to thread to the provider, treating an empty string /// (e.g. a cleared flow input) as unset. pub fn get_reasoning_effort(&self) -> Option<&str> { - self.reasoning_effort.as_deref().filter(|s| !s.is_empty()) + crate::providers::effective_reasoning_effort( + &self.model, + self.reasoning_effort.as_deref().filter(|s| !s.is_empty()), + ) } pub async fn get_base_url(&self, db: &DB) -> Result { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 655a872f96..ba718bf760 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -22,6 +22,8 @@ import { } from './modelConfig' import { applyReasoningToConfig, + completionsRejectsToolsWithReasoning, + explicitOffToken, requestsReasoning, stripLegacyThinkingSuffix, type ReasoningEffort @@ -69,6 +71,9 @@ interface AIProviderDetails { // the frontier model. The gpt-5 family is deprecated (retires 2026-12-11) but // still served, so it stays in the list below the 5.6 models. const OPENAI_MODELS = [ + 'gpt-6-sol', + 'gpt-6-astra', + 'gpt-6-luna', 'gpt-5.6-terra', 'gpt-5.6-sol', 'gpt-5.6-luna', @@ -87,7 +92,14 @@ export const AI_PROVIDERS: Record = { }, anthropic: { label: 'Anthropic', - defaultModels: ['claude-sonnet-5', 'claude-opus-5', 'claude-opus-4-8', 'claude-haiku-4-5'] + defaultModels: [ + 'claude-sonnet-5-5', + 'claude-opus-5-5', + 'claude-sonnet-5', + 'claude-opus-5', + 'claude-opus-4-8', + 'claude-haiku-4-5' + ] }, googleai: { label: 'Google AI', @@ -1160,6 +1172,12 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() + // gpt-5.5+ refuse function tools here unless reasoning is off, so the Responses API + // fallback turns it off where the model can rather than failing the turn. + const reasoningEffort = + tools?.length && completionsRejectsToolsWithReasoning(provider, modelProvider.model) + ? (explicitOffToken(provider, modelProvider.model) ?? options?.reasoningEffort) + : options?.reasoningEffort const completionConfig = applyReasoningToConfig( config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { @@ -1175,7 +1193,7 @@ export async function getCompletion( } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', - options?.reasoningEffort + reasoningEffort ) const completion = client.chat.completions.create(completionConfig, { signal: abortController.signal, diff --git a/frontend/src/lib/components/copilot/modelConfig.test.ts b/frontend/src/lib/components/copilot/modelConfig.test.ts index 4f54eb223a..8bcdf70f54 100644 --- a/frontend/src/lib/components/copilot/modelConfig.test.ts +++ b/frontend/src/lib/components/copilot/modelConfig.test.ts @@ -21,6 +21,13 @@ describe('workspace context window overrides', () => { expect(getConfiguredModelContextWindow('openai', 'qwen-local', overrides)).toBeUndefined() expect(getEffectiveModelContextWindow('openai', 'qwen-local', overrides)).toBe(128_000) }) + + it('knows the gpt-6 and Claude 5.5 windows, so they do not compact at the assumed one', () => { + expect(getConfiguredModelContextWindow('openai', 'gpt-6-sol', undefined)).toBe(1_050_000) + expect(getConfiguredModelContextWindow('anthropic', 'claude-sonnet-5-5', undefined)).toBe( + 1_000_000 + ) + }) }) describe('usesAnthropicMessagesApi', () => { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index 83e7771873..f9653bff25 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -58,7 +58,7 @@ export function usesOpenRouterPromptCaching(provider: AIProvider, model: string) // so it does not catch unrelated ids like Mistral's "open-mistral-*" or "optimus-*". export function requiresMaxCompletionTokens(model: string) { const baseModel = parseModelId(model).base - return baseModel.startsWith('gpt-5') || /^o\d/.test(baseModel) + return Number(/^gpt-(\d)(?:[.-]|$)/.exec(baseModel)?.[1] ?? 0) >= 5 || /^o\d/.test(baseModel) } // Context windows of the models we know, most specific entry first — the first @@ -87,6 +87,7 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['claude', 200_000], // OpenAI — gpt-5 covers the base family (-mini / -nano) and the 5.1/5.2 // revisions, all 400K; 5.4/5.5 moved to 1M and 5.6 to 1.05M + ['gpt-6', 1_050_000], ['gpt-5.6', 1_050_000], ['gpt-5.5', 1_000_000], ['gpt-5.4', 1_000_000], @@ -140,6 +141,7 @@ const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ ['claude-fable', 64_000], ['claude-mythos', 64_000], // OpenAI + ['gpt-6', 128_000], ['gpt-5', 128_000], ['gpt-4.1', 32_768], ['gpt-4o', 16_384], diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts index e295682187..4eded135ee 100644 --- a/frontend/src/lib/components/copilot/modelPricing.test.ts +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -35,6 +35,16 @@ describe('resolveModelPrice', () => { expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined() }) + it('prices Claude 5.5 flat and leaves the tiered gpt-6 unpriced', () => { + expect(resolveModelPrice('anthropic', 'claude-opus-5-5', undefined)?.price).toMatchObject({ + input: 4, + output: 20, + cacheRead: 0.2 + }) + expect(resolveModelPrice('anthropic', 'claude-sonnet-5-5', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-6-sol', undefined)).toBeUndefined() + }) + it('reports an unknown model as unpriced rather than guessing', () => { expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() }) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts index 1e5b63247b..b191a0bcaa 100644 --- a/frontend/src/lib/components/copilot/modelPricing.ts +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -69,6 +69,8 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ // fallback sits below the explicit entries rather than covering them. ['claude-fable-5', { input: 10, output: 50 }], ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5-5', { input: 4, output: 20, cacheRead: 0.2 }], + ['claude-sonnet-5-5', { input: 2, output: 10, cacheRead: 0.2 }], ['claude-opus-5', { input: 5, output: 25 }], ['claude-opus-4-8', { input: 5, output: 25 }], ['claude-opus-4-7', { input: 5, output: 25 }], @@ -98,6 +100,9 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ // Revisions past gpt-5 are priced separately by OpenAI and are not tracked here. // The matcher's revision guard already keeps them off the family rate; these // entries stay so a revision the guard admits still resolves to no rate. + // gpt-6 bills the whole request at a higher rate above 272K input tokens, which a + // per-model rate cannot express. + ['gpt-6', null], ['gpt-5.6', null], ['gpt-5.5', null], ['gpt-5.4', null], diff --git a/frontend/src/lib/components/copilot/reasoningParity.json b/frontend/src/lib/components/copilot/reasoningParity.json new file mode 100644 index 0000000000..93b8ac7eb2 --- /dev/null +++ b/frontend/src/lib/components/copilot/reasoningParity.json @@ -0,0 +1,84 @@ +[ + { + "provider": "anthropic", + "model": "claude-sonnet-5-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-5-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-fable-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "aws_bedrock", + "model": "global.anthropic.claude-opus-5-5-v1:0", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-sonnet-5", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-5-20260101", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-4-8", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "openai", + "model": "gpt-6-astra", + "canDisable": false, + "completionsToolsNeedOff": true + }, + { + "provider": "openai", + "model": "gpt-6-sol", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { + "provider": "azure_openai", + "model": "gpt-6-luna", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { + "provider": "openai", + "model": "gpt-5.6-sol", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { "provider": "openai", "model": "gpt-5.5", "canDisable": true, "completionsToolsNeedOff": true }, + { "provider": "openai", "model": "gpt-5.7", "canDisable": true, "completionsToolsNeedOff": true }, + { + "provider": "openai", + "model": "gpt-5.1", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { "provider": "openai", "model": "gpt-5", "canDisable": false, "completionsToolsNeedOff": false }, + { + "provider": "openai", + "model": "gpt-5-mini", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { "provider": "openai", "model": "o3", "canDisable": false, "completionsToolsNeedOff": false } +] diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts index faf6f60264..30543e6c2f 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest' import { applyReasoningToConfig, + completionsRejectsToolsWithReasoning, + explicitOffToken, getReasoningCapability, REASONING_OFF, resolveEffectiveReasoning, @@ -8,6 +10,8 @@ import { stripLegacyThinkingSuffix, supportsReasoning } from './reasoningRegistry' +import type { AIProvider } from '$lib/gen' +import parity from './reasoningParity.json' describe('stripLegacyThinkingSuffix', () => { it('removes the deprecated /thinking suffix', () => { @@ -261,6 +265,42 @@ describe('supportsReasoning (static registry)', () => { expect(getReasoningCapability('openrouter', 'x-ai/grok-4').canDisable).toBe(false) expect(getReasoningCapability('openrouter', 'deepseek/deepseek-r1').canDisable).toBe(false) }) + it('never sends a disable the 5.5 point releases and gpt-6-astra reject', () => { + // Live-verified: Claude 5.5 rejects `thinking: disabled`, gpt-6-astra rejects `none`. + for (const [provider, model] of [ + ['anthropic', 'claude-sonnet-5-5'], + ['anthropic', 'claude-opus-5-5'], + ['aws_bedrock', 'global.anthropic.claude-opus-5-5-v1:0'], + ['openai', 'gpt-6-astra'] + ] as const) { + expect(getReasoningCapability(provider, model).canDisable, model).toBe(false) + expect(explicitOffToken(provider, model), model).toBeUndefined() + } + expect(getReasoningCapability('openrouter', 'anthropic/claude-sonnet-5.5').canDisable).toBe( + false + ) + expect(explicitOffToken('openrouter', 'anthropic/claude-sonnet-5.5')).toBeUndefined() + // A dated Claude 5 id is not a point release. + expect(explicitOffToken('anthropic', 'claude-sonnet-5-20260101')).toBe('none') + expect(explicitOffToken('openai', 'gpt-6-sol')).toBe('none') + expect(getReasoningCapability('openai', 'gpt-6-luna')).toMatchObject({ + supported: true, + canDisable: true, + levels: ['low', 'medium', 'high', 'xhigh', 'max'] + }) + }) + it("reads Azure's gpt-35-turbo as gpt-3.5, not a gpt-5+ reasoning model", () => { + expect(supportsReasoning('azure_openai', 'gpt-35-turbo')).toBe(false) + expect(supportsReasoning('openai', 'gpt-35-turbo-16k')).toBe(false) + }) + it('finds the models that refuse function tools with reasoning on Chat Completions', () => { + for (const model of ['gpt-5.5', 'gpt-5.6-sol', 'gpt-6-astra']) { + expect(completionsRejectsToolsWithReasoning('openai', model), model).toBe(true) + } + for (const model of ['gpt-5', 'gpt-5.1', 'gpt-35-turbo', 'o3']) { + expect(completionsRejectsToolsWithReasoning('azure_openai', model), model).toBe(false) + } + }) it('forwards an explicit off as effort none through OpenRouter', () => { expect( resolveRequestReasoning({ @@ -353,6 +393,23 @@ describe('Azure AI Foundry reasoning follows the model family', () => { }) }) +describe('backend parity', () => { + // windmill-ai's `providers/mod.rs` test reads the same file. The backend rules see the + // model id alone, so the file holds only rows whose answer doesn't depend on the + // provider: a Bedrock- or Gemini-Pro-specific row belongs in the tests above. + it.each(parity)( + '$provider $model', + ({ provider, model, canDisable, completionsToolsNeedOff }) => { + const capability = getReasoningCapability(provider as AIProvider, model) + expect(capability.supported).toBe(true) + expect(capability.canDisable).toBe(canDisable) + expect(completionsRejectsToolsWithReasoning(provider as AIProvider, model)).toBe( + completionsToolsNeedOff + ) + } + ) +}) + describe('resolveEffectiveReasoning', () => { it('defaults capable models to high when unset', () => { expect(resolveEffectiveReasoning({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe( diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 9915e9c42f..68934c22f3 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -1,5 +1,5 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' -import { parseModelId, usesAnthropicMessagesApi } from './modelConfig' +import { usesAnthropicMessagesApi } from './modelConfig' /** * Reasoning effort is provider/model-specific. We never normalize a single @@ -33,11 +33,6 @@ export function stripLegacyThinkingSuffix(model: string): string { : model } -/** Bare model id without any provider/gateway prefix (e.g. OpenRouter's `openai/o3`). */ -function baseModelId(model: string): string { - return parseModelId(model).base -} - /** * Azure AI Foundry hosts multiple model families under one provider, so reasoning * support follows the underlying model rather than the provider: Claude deployments @@ -53,172 +48,218 @@ function reasoningProviderFamily(provider: AIProvider, model: string): AIProvide } /** - * Suggested effort levels per provider, sourced from each provider SDK's own - * vocabulary. + * Sentinel sent for the deepseek off case. It never reaches the wire as an + * effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to + * the provider's `thinking: {type: "disabled"}` param (`reasoning_effort: + * "none"` is rejected by their API). */ -const PROVIDER_REASONING_LEVELS: Partial> = { - // DeepSeek accepts the full five-token vocabulary but only two levels are - // real: low/medium are server-mapped to high and xhigh to max — offering - // them would be a no-op knob. - deepseek: ['high', 'max'], - // Mistral's only effort token besides the 'none' disable is 'high' - // (anything else is rejected), so the knob is effectively on/off. - mistral: ['high'] -} +export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' /** - * OpenRouter validates effort against its own vocabulary - * (minimal..xhigh + none) and translates per underlying provider, so the - * real ladder depends on the model family: Anthropic gets all five as - * distinct budget ratios (minimal 10% .. xhigh 95% of max_tokens); Gemini - * maps to thinkingLevel with xhigh clamped to high (a no-op vs high); - * OpenAI gets the token passed through verbatim, so the per-model OpenAI - * scoping applies; DeepSeek server-maps low/medium to high and xhigh to max. + * Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches + * the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig` + * translates it to `thinking: {type: "disabled"}`, the only off that Opus and + * Sonnet 5 respect. */ -function openrouterReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - const base = baseModelId(model) - if (/claude-(opus|sonnet)-(4|5)/.test(m)) { - return ['minimal', 'low', 'medium', 'high', 'xhigh'] - } - if (m.includes('gemini-')) { - return geminiReasoningLevels(m) - } - if (base.startsWith('gpt-5') || /^o\d/.test(base)) { - return openaiReasoningLevels(base) - } - if (m.includes('deepseek-v4')) { - return ['high', 'xhigh'] - } - return ['low', 'medium', 'high'] -} +export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none' /** - * OpenAI's effort vocabulary is model-dependent: `minimal` exists on gpt-5 but - * not on gpt-5.1+, `xhigh` arrived on gpt-5.5 and `max` on gpt-5.6; o-series - * take low/medium/high. An unsupported level is rejected, so scope the list to - * the model. (`none` is the disable token, handled by `explicitOffToken`.) + * What the registry knows about one group of models. The rows below are matched in + * order against the lowercased model id, first match wins, so a narrower row goes above + * the row it overrides. A model no row matches does not reason, as far as we know. */ -function openaiReasoningLevels(model: string): ReasoningEffort[] { - const base = baseModelId(model) - if (/^gpt-5\.6/.test(base)) { - return ['low', 'medium', 'high', 'xhigh', 'max'] - } - if (/^gpt-5\.5/.test(base)) { - return ['low', 'medium', 'high', 'xhigh'] - } - if (/^gpt-5\./.test(base)) { - return ['low', 'medium', 'high'] - } - if (/^gpt-5/.test(base)) { - return ['minimal', 'low', 'medium', 'high'] - } - return ['low', 'medium', 'high'] +type ReasoningRule = { + match: RegExp + /** The levels the UI offers. An unsupported level is a 400, so this is per model. */ + levels: readonly ReasoningEffort[] + /** + * Whether "off" really stops the model reasoning. When false the UI offers no off: + * the provider would reject it, or coerce it to the lowest level. + */ + canDisable: boolean + /** + * The effort to send for off on a model that reasons when the field is omitted. + * Unset where omission is already off. Always `'none'`: `requestsReasoning` reads + * that as off, and each wire format translates it (see `applyReasoningToConfig`). + */ + offToken?: ReasoningEffort + /** + * gpt-5.5 and later refuse function tools on Chat Completions while they reason, + * even with the effort omitted (live-verified); the Responses API has no such limit. + */ + completionsToolsNeedOff?: boolean } +const LOW_TO_HIGH = ['low', 'medium', 'high'] +const LOW_TO_XHIGH = ['low', 'medium', 'high', 'xhigh'] +const LOW_TO_MAX = ['low', 'medium', 'high', 'xhigh', 'max'] +const MINIMAL_TO_HIGH = ['minimal', 'low', 'medium', 'high'] + /** - * Gemini's level ladder is model-dependent: Gemini 3+ Flash / Flash-Lite accept - * `minimal`, while 3.x Pro does not (and cannot disable thinking). Gemini 2.5 - * uses numeric budgets — the proxy maps the three tiers to budget values, so - * `minimal` is not offered there. + * Claude models whose thinking cannot be turned off: an explicit disable 400s. Fable, + * Mythos, and the 5.x point releases (Sonnet 5.5, Opus 5.5), whose lowest setting is + * adaptive thinking at `low` (live-verified). The version match stops at one digit so a + * dated id (`claude-sonnet-5-20260101`) stays Sonnet 5. */ -function geminiReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - const isGemini3Plus = !m.includes('gemini-2.5') - if (isGemini3Plus && (m.includes('flash') || m.includes('lite'))) { - return ['minimal', 'low', 'medium', 'high'] - } - return ['low', 'medium', 'high'] +const CLAUDE_ALWAYS_THINKING = /fable|mythos|claude-(opus|sonnet)-5[-.][1-9](?!\d)/ + +// Anthropic ids are matched anywhere in the id: Bedrock prefixes them +// (`us.anthropic.claude-opus-4-6-v1`). Opus 4.5 and older reject adaptive thinking. +const ANTHROPIC_RULES: ReasoningRule[] = [ + { match: CLAUDE_ALWAYS_THINKING, levels: LOW_TO_MAX, canDisable: false }, + // The 5 family thinks when the field is absent, so off is an explicit disable. + { + match: /claude-(opus|sonnet)-5/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: ANTHROPIC_OFF_SENTINEL + }, + // 4.6-4.8 only think when asked, so omission is already off. + { match: /claude-opus-4-[78]/, levels: LOW_TO_MAX, canDisable: true }, + { match: /claude-(opus|sonnet)-4-6/, levels: ['low', 'medium', 'high', 'max'], canDisable: true } +] + +const BEDROCK_RULES: ReasoningRule[] = [ + ANTHROPIC_RULES[0], + // AWS documents Sonnet 5 on Bedrock as always thinking, where the native API + // accepts a disable for it. + { match: /claude-sonnet-5/, levels: LOW_TO_MAX, canDisable: false }, + ...ANTHROPIC_RULES.slice(1) +] + +// Anchored at the start or after a gateway's `vendor/`, and the major is one digit: +// Azure names gpt-3.5 `gpt-35-turbo`. `minimal` exists on gpt-5 only, `xhigh` from +// gpt-5.5, `max` from gpt-5.6. +const OPENAI_RULES: ReasoningRule[] = [ + // Live-verified: astra takes low..max only, where sol and luna also take `none`. + { + match: /(?:^|\/)gpt-6-astra/, + levels: LOW_TO_MAX, + canDisable: false, + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-[6-9](?:[.:-]|$)/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-5\.6/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-5\.5/, + levels: LOW_TO_XHIGH, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + // A later gpt-5 minor keeps the tools limit, which holds for every version from 5.5, + // but only the levels every gpt-5.x takes until it has a row of its own. + { + match: /(?:^|\/)gpt-5\.(?:[5-9]|\d{2,})/, + levels: LOW_TO_HIGH, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + // gpt-5.1+ are off only through `none`: omitted, they reason at medium. + { match: /(?:^|\/)gpt-5\./, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' }, + // gpt-5 and the o-series reject `none` and reason when it is omitted. + { match: /(?:^|\/)gpt-5(?:[:-]|$)/, levels: MINIMAL_TO_HIGH, canDisable: false }, + { match: /(?:^|\/)o\d/, levels: LOW_TO_HIGH, canDisable: false } +] + +// Gemini 2.5/3 think by default; the backend proxy maps `none` to off on Flash, or to +// the floor on Pro, which enforces one (level `low` on 3.x, 128 tokens on 2.5). +// Gemini 3+ Flash / Flash-Lite accept `minimal`; 2.5 takes numeric budgets the proxy +// maps from three tiers. +const GEMINI_RULES: ReasoningRule[] = [ + { match: /gemini-2\.5.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' }, + { match: /gemini-2\.5/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' }, + { match: /gemini-3.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' }, + { match: /gemini-3.*(flash|lite)/, levels: MINIMAL_TO_HIGH, canDisable: true, offToken: 'none' }, + { match: /gemini-3/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' } +] + +const REASONING_RULES: Partial> = { + anthropic: ANTHROPIC_RULES, + aws_bedrock: BEDROCK_RULES, + openai: OPENAI_RULES, + azure_openai: OPENAI_RULES, + googleai: GEMINI_RULES, + deepseek: [ + // Every current API model takes reasoning_effort, but only two levels are real: + // low/medium are server-mapped to high, xhigh to max. The retired + // `deepseek-chat` alias means "non-thinking mode", so a saved selection on it + // must not silently become a thinking request. Off is a separate `thinking` param. + { + match: /(?:^|\/)deepseek(?!-chat(?::|$))/, + levels: ['high', 'max'], + canDisable: true, + offToken: DEEPSEEK_OFF_SENTINEL + } + ], + mistral: [ + // Only the ids verified to accept reasoning_effort (large, magistral, ministral + // and pinned versions reject it), whose only effort besides off is `high`. + { + match: /(?:^|\/)mistral-(?:(?:small|medium)-latest(?::|$)|medium-3[-.]5)/, + levels: ['high'], + canDisable: true + } + ], + // OpenRouter validates effort against its own vocabulary (minimal..xhigh + none) and + // translates it per underlying provider, which scopes the ladder: Anthropic gets all + // five as budget ratios, OpenAI gets the token verbatim, DeepSeek server-maps. `none` + // is its documented off, more reliable than omission; it can only disable a model + // whose upstream can. + openrouter: [ + { + match: /claude-(opus|sonnet)-5[-.][1-9](?!\d)/, + levels: ['minimal', ...LOW_TO_XHIGH], + canDisable: false + }, + { + match: /claude-(opus|sonnet)-(4|5)/, + levels: ['minimal', ...LOW_TO_XHIGH], + canDisable: true, + offToken: 'none' + }, + ...GEMINI_RULES, + ...OPENAI_RULES.map(({ completionsToolsNeedOff: _, ...rule }) => ({ + ...rule, + offToken: 'none' + })), + { match: /deepseek-v4/, levels: LOW_TO_XHIGH.slice(2), canDisable: true, offToken: 'none' }, + // deepseek-r1, grok-4 and :thinking variants reason unconditionally. + { + match: /deepseek-r|grok-4|:thinking/, + levels: LOW_TO_HIGH, + canDisable: false, + offToken: 'none' + } + ] } -/** - * Gemini Pro models cannot turn thinking off — the API enforces a floor - * (level `low` on 3.x Pro, a 128-token budget on 2.5 Pro), so an off option - * would silently mean "lowest". Flash / Flash-Lite can truly disable - * (budget 0 / level `minimal`). - */ -function geminiCanDisable(model: string): boolean { - return !model.toLowerCase().includes('pro') +/** The row that describes a model, and whether its provider family has rows at all. */ +function findReasoningRule( + provider: AIProvider, + model: string +): { rule: ReasoningRule | undefined; known: boolean } { + const rules = REASONING_RULES[reasoningProviderFamily(provider, model)] + const id = stripLegacyThinkingSuffix(model).toLowerCase() + return { rule: rules?.find((rule) => rule.match.test(id)), known: rules !== undefined } } -/** - * Anthropic's effort ladder is model-dependent: `xhigh` exists on Opus 4.7/4.8, - * the 5 family and Fable/Mythos; `max` also on Opus 4.6 and Sonnet 4.6. - * Offering an unsupported level would 400, so scope the list to the model. - */ -function anthropicReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - if ( - /claude-(opus|sonnet)-5/.test(m) || - /claude-opus-4-(7|8)/.test(m) || - m.includes('fable') || - m.includes('mythos') - ) { - return ['low', 'medium', 'high', 'xhigh', 'max'] - } - return ['low', 'medium', 'high', 'max'] -} - -/** Mistral writes both `mistral-medium-3.5` and `mistral-medium-3-5`. */ -function normalizeMistralId(model: string): string { - return baseModelId(model).replace(/\./g, '-') -} - -/** - * Conservative static predicate for whether a model accepts an effort knob. - * Kept tight to avoid 400s on models that reject reasoning params. - */ -function supportsReasoningStatic(provider: AIProvider, model: string): boolean { - const m = model.toLowerCase() - const base = baseModelId(model) - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Bedrock serves the same Claude models under prefixed ids - // (e.g. `us.anthropic.claude-opus-4-6-v1`), so match on the full string. - case 'aws_bedrock': - // 4.6+ only: Opus 4.5 rejects adaptive thinking (and, on Bedrock, - // the whole output_config surface) — live-verified hard 400. - return ( - /claude-opus-(4-(6|7|8)|5)/.test(m) || - /claude-sonnet-(4-6|5)/.test(m) || - m.includes('fable') || - m.includes('mythos') - ) - case 'openai': - case 'azure_openai': - return base.startsWith('gpt-5') || /^o\d/.test(base) - case 'openrouter': - // Best-effort markers for models whose `supported_parameters` include - // `reasoning` in OpenRouter's catalog; OpenRouter translates the effort - // per underlying provider. - return ( - base.startsWith('gpt-5') || - /^o\d/.test(base) || - /claude-(opus|sonnet)-(4|5)/.test(m) || - /gemini-(2\.5|3)/.test(m) || - m.includes('deepseek-r') || - m.includes('deepseek-v4') || - m.includes('grok-4') || - m.includes(':thinking') - ) - case 'googleai': - return /gemini-(2\.5|3)/.test(m) - case 'deepseek': - // All current API models take reasoning_effort (live-verified). The - // retired `deepseek-chat` alias stays excluded: its documented meaning - // is "non-thinking mode", so a saved selection on it must not silently - // become a thinking request. - return base.startsWith('deepseek') && base !== 'deepseek-chat' - case 'mistral': - // Only the ids verified to accept reasoning_effort; other models - // (large, magistral, ministral, pinned versions) reject the param. - return ( - /^mistral-(small|medium)-latest$/.test(base) || - normalizeMistralId(model).startsWith('mistral-medium-3-5') - ) - default: - return false - } +/** Whether Chat Completions needs reasoning off for this model to take function tools. */ +export function completionsRejectsToolsWithReasoning(provider: AIProvider, model: string): boolean { + return findReasoningRule(provider, model).rule?.completionsToolsNeedOff ?? false } export type ReasoningCapability = { @@ -241,89 +282,12 @@ export type ReasoningCapability = { known: boolean } -/** Provider families the registry has real rules for; everything else is a shrug. */ -const KNOWN_REASONING_FAMILIES: ReadonlySet = new Set([ - 'anthropic', - 'aws_bedrock', - 'openai', - 'azure_openai', - 'openrouter', - 'googleai', - 'deepseek', - 'mistral' -]) - /** Resolve the reasoning capability of a model from the static registry. */ export function getReasoningCapability(provider: AIProvider, model: string): ReasoningCapability { - const bareModel = stripLegacyThinkingSuffix(model) - const known = KNOWN_REASONING_FAMILIES.has(reasoningProviderFamily(provider, bareModel)) - const supported = supportsReasoningStatic(provider, bareModel) - if (!supported) { - return { supported: false, levels: [], canDisable: false, known } - } - const family = reasoningProviderFamily(provider, bareModel) - const levels = - family === 'anthropic' || family === 'aws_bedrock' - ? anthropicReasoningLevels(bareModel) - : family === 'googleai' - ? geminiReasoningLevels(bareModel) - : family === 'openai' || family === 'azure_openai' - ? openaiReasoningLevels(bareModel) - : family === 'openrouter' - ? openrouterReasoningLevels(bareModel) - : (PROVIDER_REASONING_LEVELS[family] ?? ['low', 'medium', 'high']) - return { supported, levels, canDisable: canDisableReasoning(provider, bareModel), known } -} - -/** - * Whether selecting "off" truly disables reasoning for the model. Off is - * sent either as an explicit provider disable (see `explicitOffToken`) or by - * omitting the effort — which only works where the model doesn't reason by - * default. - */ -function canDisableReasoning(provider: AIProvider, model: string): boolean { - const m = model.toLowerCase() - const base = baseModelId(model) - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Every Claude but Fable and Mythos can stop thinking: 4.6-4.8 by - // omission, and the 5 family through the explicit disable that - // `explicitOffToken` sends. - return !ANTHROPIC_ALWAYS_THINKING.test(m) - case 'aws_bedrock': - // Same models, different answer: AWS documents Sonnet 5 on Bedrock as - // always thinking, where the native API accepts a disable for it. - return !(ANTHROPIC_ALWAYS_THINKING.test(m) || m.includes('claude-sonnet-5')) - case 'googleai': - return geminiCanDisable(model) - case 'openai': - case 'azure_openai': - // gpt-5.1+ accept effort 'none'; gpt-5 and o-series reject it and - // reason at `medium` by default, so omission isn't off either. - return /^gpt-5\./.test(base) - case 'openrouter': - // 'none' is in OpenRouter's vocabulary, but the gateway can't - // disable a model whose upstream can't — scope off per underlying - // family, like the levels. - // The 5 family thinks by default, but its upstream takes an explicit - // disable, so the gateway's 'none' has something to translate to. - if (/claude-(opus|sonnet)-(4|5)/.test(m)) { - return true - } - if (m.includes('gemini-')) { - return geminiCanDisable(m) - } - if (base.startsWith('gpt-5') || /^o\d/.test(base)) { - return /^gpt-5\./.test(base) - } - if (m.includes('deepseek-v4')) { - return true - } - // grok-4, deepseek-r1 and :thinking variants reason unconditionally. - return false - default: - return true - } + const { rule, known } = findReasoningRule(provider, model) + return rule + ? { supported: true, levels: [...rule.levels], canDisable: rule.canDisable, known } + : { supported: false, levels: [], canDisable: false, known } } export function supportsReasoning(provider: AIProvider, model: string): boolean { @@ -352,64 +316,14 @@ export function resolveEffectiveReasoning( : undefined } -/** - * Sentinel sent for the deepseek off case. It never reaches the wire as an - * effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to - * the provider's `thinking: {type: "disabled"}` param (`reasoning_effort: - * "none"` is rejected by their API). - */ -export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' - -/** - * Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches - * the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig` - * translates it to `thinking: {type: "disabled"}`, which is the only off the - * always-on 5 family respects. - */ -export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none' - -/** Claude models whose thinking cannot be turned off — an explicit disable 400s. */ -const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/ - /** * Disable token to forward when the user explicitly turns reasoning off on a * model that reasons *by default* — omitting the field would silently keep - * the default-on behavior. Undefined means omission is the correct off. Every - * token is `'none'`: `requestsReasoning` reads that value as off. + * the default-on behavior. Undefined means omission is the correct off, or that + * the model cannot be turned off at all. */ export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Claude 4.6-4.8 only think when asked, so omission is already a - // real off there and stays the wire form. Only the 5 family, which - // thinks when the field is absent, needs the explicit disable — - // Fable and Mythos reject it outright and get no off token at all. - return /claude-(opus|sonnet)-5/.test(model.toLowerCase()) ? ANTHROPIC_OFF_SENTINEL : undefined - case 'aws_bedrock': - // Bedrock's Sonnet 5 cannot be disabled at all, so only Opus 5 gets - // the sentinel; the rest keep omission. - return model.toLowerCase().includes('claude-opus-5') ? ANTHROPIC_OFF_SENTINEL : undefined - case 'googleai': - // Gemini 2.5/3 think by default (dynamic budget / level). The backend - // proxy maps 'none' to off on Flash, or the floor on Pro (only - // reachable via a stale persisted preference — see canDisableReasoning). - return 'none' - case 'deepseek': - return DEEPSEEK_OFF_SENTINEL - case 'openai': - case 'azure_openai': - // gpt-5.1+ reasoning is off only via the explicit 'none' effort - // (gpt-5.5 defaults to medium when the field is omitted). - return /^gpt-5\./.test(baseModelId(model)) ? 'none' : undefined - case 'openrouter': - // OpenRouter validates effort against xhigh..minimal|none and - // documents 'none' as disabling reasoning, translated per the - // underlying provider — more reliable than omission, which keeps - // reasoning-by-default models thinking. - return 'none' - default: - return undefined - } + return findReasoningRule(provider, model).rule?.offToken } /**