From 1055617caeac6971fa5af8c47a7973063d6449fe Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Nov 2026 13:10:23 +0000 Subject: [PATCH] feat: web search fallback for AI chat via Exa, Tavily or Linkup (#12229) * feat: add web search fallback for AI chat via Exa, Tavily or Linkup Co-Authored-By: Claude Opus 5.5 * fix: read the web search key from token, api_key or apiKey Co-Authored-By: Claude Opus 5.5 * fix: offer the search fallback on the retry after native search is refused Co-Authored-By: Claude Opus 5.5 * fix: offer the search fallback on the Completions retry, reuse formatToolError Co-Authored-By: Claude Opus 5.5 * feat: count web search calls per backend in feature usage Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to 8990719c0e4a6e4632d812e32b37ca5c2b341a44 This commit updates the EE repository reference after PR #955 was merged in windmill-ee-private. Previous ee-repo-ref: db18639beb83fec910769e6370d77f72bb2ff2d4 New ee-repo-ref: 8990719c0e4a6e4632d812e32b37ca5c2b341a44 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-ai/src/lib.rs | 1 + backend/windmill-ai/src/web_search.rs | 312 ++++++++++++++++++ backend/windmill-api/openapi.yaml | 116 +++++++ backend/windmill-api/src/ai.rs | 130 ++++++++ backend/windmill-api/src/workspaces.rs | 6 + docs/feature-telemetry.md | 2 +- frontend/src/lib/aiStore.ts | 5 + .../lib/components/InstanceSettings.svelte | 10 +- .../copilot/chat/AIChatManager.svelte.ts | 7 +- .../components/copilot/chat/chatLoop.test.ts | 61 ++++ .../lib/components/copilot/chat/chatLoop.ts | 20 +- .../lib/components/copilot/chat/web/core.ts | 119 +++++++ .../workspaceSettings/AISettings.svelte | 62 +++- 14 files changed, 839 insertions(+), 14 deletions(-) create mode 100644 backend/windmill-ai/src/web_search.rs create mode 100644 frontend/src/lib/components/copilot/chat/web/core.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 69b1654e24..f2a645c01e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -496b0fc99d3980b83599a842267f3192090b4fb0 +8990719c0e4a6e4632d812e32b37ca5c2b341a44 diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index 22eeb95449..39ef1eee4d 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -16,3 +16,4 @@ pub mod retry; pub mod sse; pub mod types; pub mod utils; +pub mod web_search; diff --git a/backend/windmill-ai/src/web_search.rs b/backend/windmill-ai/src/web_search.rs new file mode 100644 index 0000000000..72f1ddfccd --- /dev/null +++ b/backend/windmill-ai/src/web_search.rs @@ -0,0 +1,312 @@ +//! Web search and page fetching through a third-party search API, for models whose provider +//! has no built-in search. Every backend reaches the web from its own servers, so a URL a +//! model picks never makes Windmill itself open a connection to it. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use windmill_common::error::{Error, Result}; + +use crate::utils::AI_HTTP_CLIENT; + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum WebSearchProvider { + Exa, + Tavily, + Linkup, +} + +pub const MAX_SEARCH_RESULTS: usize = 10; +const DEFAULT_SEARCH_RESULTS: usize = 5; +/// Per result: enough for the passage that answers a query, short enough that ten +/// results stay well inside a tool result. +const SEARCH_CONTENT_CHARS: usize = 1_500; +const FETCH_CONTENT_CHARS: usize = 40_000; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(45); + +#[derive(Serialize, Debug, PartialEq)] +pub struct WebSearchResult { + pub title: String, + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub published_date: Option, + pub content: String, +} + +#[derive(Serialize, Debug, PartialEq)] +pub struct WebPage { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + pub content: String, + pub truncated: bool, +} + +impl WebSearchProvider { + fn base_url(self) -> &'static str { + match self { + WebSearchProvider::Exa => "https://api.exa.ai", + WebSearchProvider::Tavily => "https://api.tavily.com", + WebSearchProvider::Linkup => "https://api.linkup.so/v1", + } + } + + pub fn id(self) -> &'static str { + match self { + WebSearchProvider::Exa => "exa", + WebSearchProvider::Tavily => "tavily", + WebSearchProvider::Linkup => "linkup", + } + } + + fn name(self) -> &'static str { + match self { + WebSearchProvider::Exa => "Exa", + WebSearchProvider::Tavily => "Tavily", + WebSearchProvider::Linkup => "Linkup", + } + } + + async fn post(self, api_key: &str, path: &str, body: Value) -> Result { + let request = AI_HTTP_CLIENT + .post(format!("{}{path}", self.base_url())) + .timeout(REQUEST_TIMEOUT) + .json(&body); + let request = match self { + WebSearchProvider::Exa => request.header("x-api-key", api_key), + WebSearchProvider::Tavily | WebSearchProvider::Linkup => request.bearer_auth(api_key), + }; + let response = request + .send() + .await + .map_err(|e| Error::BadGateway(format!("{} request failed: {e}", self.name())))?; + let status = response.status(); + let text = response.text().await.map_err(|e| { + Error::BadGateway(format!("{} response could not be read: {e}", self.name())) + })?; + if !status.is_success() { + return Err(Error::BadGateway(format!( + "{} returned {status}: {}", + self.name(), + truncate_chars(&text, 500).0 + ))); + } + serde_json::from_str(&text) + .map_err(|e| Error::BadGateway(format!("{} returned invalid JSON: {e}", self.name()))) + } + + pub async fn search( + self, + api_key: &str, + query: &str, + max_results: Option, + ) -> Result> { + let n = max_results + .unwrap_or(DEFAULT_SEARCH_RESULTS) + .clamp(1, MAX_SEARCH_RESULTS); + let (path, body) = match self { + WebSearchProvider::Exa => ( + "/search", + json!({ + "query": query, + "numResults": n, + "contents": { "text": { "maxCharacters": SEARCH_CONTENT_CHARS } } + }), + ), + WebSearchProvider::Tavily => ( + "/search", + json!({ "query": query, "max_results": n, "include_published_date": true }), + ), + WebSearchProvider::Linkup => ( + "/search", + json!({ + "q": query, + "depth": "standard", + "outputType": "searchResults", + "maxResults": n + }), + ), + }; + let response = self.post(api_key, path, body).await?; + let mut results = parse_search(self, &response); + results.truncate(n); + Ok(results) + } + + pub async fn fetch(self, api_key: &str, url: &str) -> Result { + let (path, body) = match self { + WebSearchProvider::Exa => ( + "/contents", + json!({ "urls": [url], "text": { "maxCharacters": FETCH_CONTENT_CHARS + 1 } }), + ), + WebSearchProvider::Tavily => { + ("/extract", json!({ "urls": [url], "format": "markdown" })) + } + WebSearchProvider::Linkup => ("/fetch", json!({ "url": url })), + }; + let response = self.post(api_key, path, body).await?; + parse_fetch(self, url, &response) + } +} + +fn str_field(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) +} + +/// The first `max` chars of `text`, and whether anything was cut. +fn truncate_chars(text: &str, max: usize) -> (String, bool) { + match text.char_indices().nth(max) { + Some((cut, _)) => (text[..cut].to_string(), true), + None => (text.to_string(), false), + } +} + +fn parse_search(provider: WebSearchProvider, response: &Value) -> Vec { + let items = response + .get("results") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + items + .iter() + .filter_map(|item| { + let (title, content, published_date) = match provider { + WebSearchProvider::Exa => ( + str_field(item, "title"), + str_field(item, "text"), + str_field(item, "publishedDate"), + ), + WebSearchProvider::Tavily => ( + str_field(item, "title"), + str_field(item, "content"), + str_field(item, "published_date"), + ), + WebSearchProvider::Linkup => { + if item.get("type").and_then(Value::as_str) == Some("image") { + return None; + } + (str_field(item, "name"), str_field(item, "content"), None) + } + }; + let url = str_field(item, "url")?; + Some(WebSearchResult { + title: title.unwrap_or_else(|| url.clone()), + published_date, + content: truncate_chars(&content.unwrap_or_default(), SEARCH_CONTENT_CHARS).0, + url, + }) + }) + .collect() +} + +fn parse_fetch(provider: WebSearchProvider, url: &str, response: &Value) -> Result { + let not_fetched = |reason: String| { + Error::BadGateway(format!( + "{} could not fetch {url}: {reason}", + provider.name() + )) + }; + let first_result = || { + response + .get("results") + .and_then(Value::as_array) + .and_then(|r| r.first()) + }; + let (title, content) = match provider { + WebSearchProvider::Exa => { + let failed = response + .get("statuses") + .and_then(Value::as_array) + .and_then(|s| s.first()) + .filter(|s| s.get("status").and_then(Value::as_str) == Some("error")); + if let Some(status) = failed { + let tag = status + .pointer("/error/tag") + .and_then(Value::as_str) + .unwrap_or("unknown error"); + return Err(not_fetched(tag.to_string())); + } + let result = first_result().ok_or_else(|| not_fetched("no content".into()))?; + (str_field(result, "title"), str_field(result, "text")) + } + WebSearchProvider::Tavily => match first_result() { + Some(result) => (None, str_field(result, "raw_content")), + None => { + let reason = response + .pointer("/failed_results/0/error") + .and_then(Value::as_str) + .unwrap_or("no content"); + return Err(not_fetched(reason.to_string())); + } + }, + WebSearchProvider::Linkup => (None, str_field(response, "markdown")), + }; + let content = content.ok_or_else(|| not_fetched("the page has no readable text".into()))?; + let (content, truncated) = truncate_chars(&content, FETCH_CONTENT_CHARS); + Ok(WebPage { url: url.to_string(), title, content, truncated }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_each_provider_search_shape() { + let exa = json!({ "results": [ + { "title": "Exa", "url": "https://a", "publishedDate": "2026-01-01", "text": "body" }, + { "title": "no url" } + ]}); + assert_eq!( + parse_search(WebSearchProvider::Exa, &exa), + vec![WebSearchResult { + title: "Exa".into(), + url: "https://a".into(), + published_date: Some("2026-01-01".into()), + content: "body".into(), + }] + ); + + let linkup = json!({ "results": [ + { "type": "image", "name": "img", "url": "https://img" }, + { "type": "text", "name": "Linkup", "url": "https://b", "content": "text" } + ]}); + let parsed = parse_search(WebSearchProvider::Linkup, &linkup); + assert_eq!(parsed.len(), 1); + assert_eq!( + (parsed[0].title.as_str(), parsed[0].url.as_str()), + ("Linkup", "https://b") + ); + + let tavily = json!({ "results": [ + { "title": "", "url": "https://c", "content": "é".repeat(SEARCH_CONTENT_CHARS + 10) } + ]}); + let parsed = parse_search(WebSearchProvider::Tavily, &tavily); + assert_eq!(parsed[0].title, "https://c"); + assert_eq!(parsed[0].content.chars().count(), SEARCH_CONTENT_CHARS); + } + + #[test] + fn reports_a_page_the_backend_could_not_fetch() { + let exa = json!({ "results": [], "statuses": [ + { "id": "https://x", "status": "error", "error": { "tag": "CRAWL_NOT_FOUND" } } + ]}); + let err = parse_fetch(WebSearchProvider::Exa, "https://x", &exa).unwrap_err(); + assert!(err.to_string().contains("CRAWL_NOT_FOUND"), "{err}"); + + let tavily = json!({ "results": [], "failed_results": [{ "url": "https://x", "error": "blocked" }] }); + let err = parse_fetch(WebSearchProvider::Tavily, "https://x", &tavily).unwrap_err(); + assert!(err.to_string().contains("blocked"), "{err}"); + + let linkup = json!({ "markdown": "a".repeat(FETCH_CONTENT_CHARS + 1) }); + let page = parse_fetch(WebSearchProvider::Linkup, "https://x", &linkup).unwrap(); + assert!(page.truncated); + assert_eq!(page.content.len(), FETCH_CONTENT_CHARS); + } +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bb8107740c..bb813c310a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14099,6 +14099,104 @@ paths: - text - model + /w/{workspace}/ai/web_search: + post: + summary: search the web through the workspace's web search API + description: > + Runs a query against the search API (Exa, Tavily or Linkup) set in the workspace's AI + settings, with the key of its resource. The AI chat uses it when the model's provider + has no built-in web search. + operationId: aiWebSearch + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + query: + type: string + max_results: + type: integer + minimum: 1 + maximum: 10 + required: + - query + responses: + "200": + description: the results, each with an excerpt of its page + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: object + properties: + title: + type: string + url: + type: string + published_date: + type: string + content: + type: string + required: + - title + - url + - content + required: + - results + + /w/{workspace}/ai/web_fetch: + post: + summary: read a web page through the workspace's web search API + description: > + Has the search API set in the workspace's AI settings fetch a page and returns its + text as markdown. The page is fetched from the search API's servers, not Windmill's. + operationId: aiWebFetch + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + required: + - url + responses: + "200": + description: the page's text + content: + application/json: + schema: + type: object + properties: + url: + type: string + title: + type: string + content: + type: string + truncated: + type: boolean + required: + - url + - content + - truncated + /w/{workspace}/ai_evals/datasets/list: get: summary: list eval datasets @@ -31121,6 +31219,24 @@ components: The server deletes the backup of a session no push has reached for this many days. Unset keeps backups until the user deletes the session. Read from the workspace's own settings like `copilot_disabled`. + web_search: + $ref: "#/components/schemas/AIWebSearchConfig" + + AIWebSearchConfig: + type: object + description: >- + The search API the AI chat searches the web and reads pages through when the model's + provider has no built-in web search. Read from the workspace's own settings like + `copilot_disabled`. + properties: + provider: + type: string + enum: [exa, tavily, linkup] + resource_path: + type: string + required: + - provider + - resource_path AISessionBackupListing: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 03ca21a41f..81c6e46451 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -47,6 +47,7 @@ use windmill_ai::proxy::{ }; use windmill_ai::retry::send_with_retries; use windmill_ai::types::{OpenAIContent, OpenAIMessage}; +use windmill_ai::web_search::{WebPage, WebSearchProvider, WebSearchResult}; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::client::AuthedClient; use windmill_common::db::UserDB; @@ -752,6 +753,17 @@ pub struct AIConfig { /// like `copilot_disabled`. #[serde(default, skip_serializing_if = "Option::is_none")] pub sessions_retention_days: Option, + /// The search API the AI chat searches the web and reads pages through when the model's + /// provider has no built-in search, or the workspace turned it off. Read from the + /// workspace's own row like `copilot_disabled`, its resource from that workspace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub web_search: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct WebSearchConfig { + pub provider: WebSearchProvider, + pub resource_path: String, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -928,6 +940,122 @@ fn parse_ai_resource( serde_json::from_str::(value.0.get()).map_err(|e| Error::BadRequest(e.to_string())) } +/// The workspace's search API and its key. The resource was picked by a workspace admin, so, +/// like a provider's, it is read with the raw pool: a member searches without read access to +/// the key. +async fn web_search_credentials( + authed: &ApiAuthed, + db: &DB, + w_id: &str, +) -> Result<(WebSearchProvider, String)> { + let config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_one(db) + .await? + .and_then(|v| serde_json::from_value::(v).ok()) + .and_then(|c| c.web_search) + .ok_or_else(|| { + Error::BadRequest( + "No web search API is configured for this workspace: a workspace admin can set one in the workspace's AI settings".to_string(), + ) + })?; + check_scopes(authed, || { + format!("resources:read:{}", config.resource_path) + })?; + let value = sqlx::query_scalar::<_, Option>>( + "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + ) + .bind(&config.resource_path) + .bind(w_id) + .fetch_optional(db) + .await? + .flatten() + .ok_or_else(|| { + Error::NotFound(format!( + "Could not find the web search resource {}, update it in the workspace's AI settings", + config.resource_path + )) + })?; + // `exa` predates the hub's snake_case keys; newer API-key types name the key `token`. + let api_key = ["token", "api_key", "apiKey"] + .iter() + .find_map(|key| { + value + .get(*key) + .and_then(serde_json::Value::as_str) + .filter(|k| !k.is_empty()) + }) + .ok_or_else(|| { + Error::BadRequest(format!( + "The web search resource {} has no API key", + config.resource_path + )) + })?; + let api_key = get_variable_or_self(api_key.to_string(), db, w_id).await?; + Ok((config.provider, api_key)) +} + +fn log_web_search_usage(provider: WebSearchProvider, action: &str) { + windmill_common::feature_usage::log_feature_usage( + "ai_chat", + "web_search_backend", + &format!("{}:{action}", provider.id()), + ); +} + +#[derive(Deserialize)] +struct WebSearchRequest { + query: String, + #[serde(default)] + max_results: Option, +} + +#[derive(Serialize)] +struct WebSearchResponse { + results: Vec, +} + +async fn web_search( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult { + let query = request.query.trim(); + if query.is_empty() { + return Err(Error::BadRequest("The search query is empty".to_string())); + } + let (provider, api_key) = web_search_credentials(&authed, &db, &w_id).await?; + let results = provider + .search(&api_key, query, request.max_results) + .await?; + log_web_search_usage(provider, "search"); + Ok(Json(WebSearchResponse { results })) +} + +#[derive(Deserialize)] +struct WebFetchRequest { + url: String, +} + +async fn web_fetch( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult { + let url = reqwest::Url::parse(request.url.trim()) + .ok() + .filter(|u| matches!(u.scheme(), "http" | "https")) + .ok_or_else(|| Error::BadRequest("The URL must be an http(s) URL".to_string()))?; + let (provider, api_key) = web_search_credentials(&authed, &db, &w_id).await?; + let page = provider.fetch(&api_key, url.as_str()).await?; + log_web_search_usage(provider, "fetch"); + Ok(Json(page)) +} + pub fn global_service() -> Router { Router::new().route("/proxy/{*ai}", post(global_proxy).get(global_proxy)) } @@ -936,6 +1064,8 @@ pub fn workspaced_service() -> Router { let router = Router::new() .route("/proxy/{*ai}", post(proxy).get(proxy)) .route("/complete", post(complete)) + .route("/web_search", post(web_search)) + .route("/web_fetch", post(web_fetch)) .route( "/usage", post(record_ai_usage) diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 6edf18ff07..b3a7160630 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -155,6 +155,7 @@ pub(crate) async fn edit_copilot_config( let copilot_disabled = ai_config.copilot_disabled; let sessions_storage_disabled = ai_config.sessions_storage_disabled; let sessions_retention_days = ai_config.sessions_retention_days; + let web_search = ai_config.web_search.clone(); let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -184,6 +185,7 @@ pub(crate) async fn edit_copilot_config( effective_ai_config.copilot_disabled = copilot_disabled; effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; effective_ai_config.sessions_retention_days = sessions_retention_days; + effective_ai_config.web_search = web_search; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -228,6 +230,9 @@ async fn get_copilot_info( let sessions_retention_days = workspace_ai_config .as_ref() .and_then(|c| c.0.sessions_retention_days); + let web_search = workspace_ai_config + .as_ref() + .and_then(|c| c.0.web_search.clone()); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -254,6 +259,7 @@ async fn get_copilot_info( effective.copilot_disabled = copilot_disabled; effective.sessions_storage_disabled = sessions_storage_disabled; effective.sessions_retention_days = sessions_retention_days; + effective.web_search = web_search; Ok(Json(effective)) } diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 00f4406816..e55350a300 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,7 +4,7 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 98 registered actions across thirty-nine features (`access_view`, `ai_session`, `ai_chat`, +It currently carries 99 registered actions across thirty-nine features (`access_view`, `ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_decision`, `ai_agent_eval`, `app_sandbox`, `datatable`, `db_manager`, `flow_editor`, `flow_run`, `flow_step`, `home`, `header_nav`, `run_form`, `sharing`, `debugger`, `trigger`, `command_script`, `mcp`, `mcp_workspace`, `mcp_external_client`, `hub_script`, `usage_meter`, `sso_groups_claim`, diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index ff02157e4a..ac9320f8a3 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type AIWebSearchConfig, type FreeTierInfo, type ModelPriceOverride } from './gen' @@ -56,6 +57,9 @@ export const copilotInfo = writable<{ /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + /** The workspace's search API, which the chat searches through when the model's provider + * does not search on its own (`ai_config.web_search`). */ + webSearchBackend?: AIWebSearchConfig['provider'] // Set only when the workspace has no AI provider of its own and is running on // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but // that is a different state from "never configured" and the UI must say so. @@ -149,6 +153,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, contextWindowPerModel: aiConfig.context_window_per_model ?? {}, webSearchEnabledProviders, + webSearchBackend: aiConfig.web_search?.provider, modelPricing: aiConfig.model_pricing ?? {}, freeTier: aiConfig.free_tier }) diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 5d98fcdcf1..d0b25f474a 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1121,7 +1121,10 @@
  • Job counts and durations per language
  • Git sync repository counts
  • -
  • Feature usage and adoption counts, including AI model and public Hub names
  • +
  • Feature usage and adoption counts, including AI model, web search provider and public + Hub names
  • Counts of workspaces, scripts, flows and apps
  • Database, runtime and object storage details
@@ -1157,7 +1160,10 @@
  • Login types, user and seat counts
  • Worker count, vCPUs and memory
  • Development instance status
  • -
  • Feature usage and adoption counts, including AI model and public Hub names
  • +
  • Feature usage and adoption counts, including AI model, web search provider and public + Hub names
  • Counts of workspaces, scripts, flows and apps
  • diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 17058e490f..6c8142fa72 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -102,6 +102,7 @@ import { copilotInfo, copilotSessionModel } from '$lib/aiStore' import { copilotWorkspaceRequested, loadCopilot } from '$lib/components/copilot/loadCopilot' import { askTools, prepareAskSystemMessage, prepareAskUserMessage } from './ask/core' import { readDocsPageTool, searchDocsTool } from './docs/core' +import { withWebSearchFallback } from './web/core' import { prefersInstantReveal, TypewriterReveal } from './typewriterReveal' import { triggerablesByAi } from './sharedChatState.svelte' import { @@ -3499,7 +3500,11 @@ export class AIChatManager implements ChatViewHost { return base }, get tools() { - return self.tools + return withWebSearchFallback( + self.tools, + !!get(copilotInfo).webSearchBackend, + self.providerTools.length > 0 + ) }, get helpers() { return self.helpers diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index af4b648b61..e79c0d16b4 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -4,6 +4,7 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/completio import { closeInterruptedToolBatch, runChatLoop, + sendsWebSearch, truncateToToolPairedPrefix, type ChatLoopConfig } from './chatLoop' @@ -283,6 +284,66 @@ describe('runChatLoop web search fallback', () => { expect(callbacks.setToolStatus).not.toHaveBeenCalled() }) + it('sends the workspace search tools on the retry once the provider refuses its own', async () => { + const workspace = `workspace-${randomUUID()}` + const modelProvider: ReasoningProviderModel = { + provider: 'anthropic', + model: 'claude-3-5-sonnet-latest' + } + const searchWeb = { + def: { type: 'function', function: { name: 'search_web', parameters: {} } }, + fn: vi.fn() + } as unknown as ChatLoopConfig['tools'][number] + const config = createConfig({ workspace, modelProvider }) + Object.defineProperty(config, 'tools', { + get: () => (sendsWebSearch(workspace, modelProvider, true) ? [] : [searchWeb]) + }) + + mocks.getAnthropicCompletion + .mockRejectedValueOnce( + Object.assign(new Error('Your organization must enable web search in the Claude Console'), { + status: 400, + error: { type: 'invalid_request_error' } + }) + ) + .mockResolvedValue({}) + + await runChatLoop(config) + + const toolNames = (call: unknown[]) => + (call[2] as { function: { name: string } }[]).map((t) => t.function.name) + expect(toolNames(mocks.getAnthropicCompletion.mock.calls[0])).toEqual([]) + expect(toolNames(mocks.getAnthropicCompletion.mock.calls[1])).toEqual(['search_web']) + }) + + it('sends the workspace search tools when OpenAI falls back to Completions', async () => { + const workspace = `workspace-${randomUUID()}` + const searchWeb = { + def: { type: 'function', function: { name: 'search_web', parameters: {} } }, + fn: vi.fn() + } as unknown as ChatLoopConfig['tools'][number] + let skipResponsesApi = false + const config = createConfig({ workspace }) + config.onSkipResponsesApi = () => { + skipResponsesApi = true + } + Object.defineProperty(config, 'tools', { + get: () => (skipResponsesApi ? [searchWeb] : []) + }) + + mocks.getOpenAIResponsesCompletion.mockRejectedValueOnce( + new Error('Responses API is not enabled for this deployment') + ) + mocks.getCompletion.mockResolvedValue({}) + + await runChatLoop(config) + + const toolNames = (call: unknown[]) => + (call[2] as { function: { name: string } }[]).map((t) => t.function.name) + expect(toolNames(mocks.getOpenAIResponsesCompletion.mock.calls[0])).toEqual([]) + expect(toolNames(mocks.getCompletion.mock.calls[0])).toEqual(['search_web']) + }) + it('does not retry Anthropic request errors that only mention web search input', async () => { const callbacks = createCallbacks() const workspace = `workspace-${randomUUID()}` diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 09232debbb..4f579fd2ba 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -427,9 +427,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise (t.schemaFor ? { ...t, def: await t.schemaFor(helpers) } : t)) - ) + const resolveTools = () => + Promise.all( + config.tools.map(async (t) => (t.schemaFor ? { ...t, def: await t.schemaFor(helpers) } : t)) + ) + let tools = await resolveTools() const pendingUserMessage = getPendingUserMessage?.() @@ -454,7 +456,14 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) + let toolDefs = tools.map((t) => t.def) + // Losing the provider's own search (it refused the tool, or the Responses API is off) + // makes the toolset offer the workspace's search tools, and the retry of this same + // request must already carry them. + const reloadToolsWithoutWebSearch = async () => { + tools = await resolveTools() + toolDefs = tools.map((t) => t.def) + } const parseOptions = { workspace, provider: modelProvider.provider, @@ -534,6 +543,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise = { + requires: NONE, + def: SEARCH_WEB_TOOL, + planModeSafe: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const query = typeof args?.query === 'string' ? args.query.trim() : '' + if (!query) return 'No search query was provided. Provide a `query`.' + toolCallbacks.setToolStatus(toolId, { content: `Searching the web for "${query}"...` }) + try { + const { results } = await AiService.aiWebSearch({ workspace, requestBody: { query } }) + toolCallbacks.setToolStatus(toolId, { + content: `Searched the web for "${query}"`, + ...(results.length + ? { + webSearchSources: results.map((r) => ({ url: r.url, title: r.title })), + showDetails: true + } + : {}) + }) + if (results.length === 0) return `No web results for "${query}".` + const formatted = results + .map((r, i) => + [ + `${i + 1}. ${r.title}`, + `URL: ${r.url}`, + ...(r.published_date ? [`Published: ${r.published_date}`] : []), + r.content + ].join('\n') + ) + .join('\n\n') + return `${UNTRUSTED}\n\n${formatted}` + } catch (error) { + const message = formatToolError(error) + toolCallbacks.setToolStatus(toolId, { content: 'Web search failed', error: message }) + return `Failed to search the web: ${message}` + } + } +} + +export const fetchWebPageTool: SessionTool<{}> = { + requires: NONE, + def: FETCH_WEB_PAGE_TOOL, + planModeSafe: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const url = typeof args?.url === 'string' ? args.url.trim() : '' + if (!url) return 'No URL was provided. Provide a `url`.' + toolCallbacks.setToolStatus(toolId, { content: `Reading ${url}...` }) + try { + const page = await AiService.aiWebFetch({ workspace, requestBody: { url } }) + toolCallbacks.setToolStatus(toolId, { + content: `Read ${page.title ?? url}`, + webSearchSources: [{ url: page.url, title: page.title }] + }) + const cut = page.truncated ? '\n\n[The page was cut at 40,000 characters.]' : '' + return `${UNTRUSTED}\n\n# ${page.title ?? page.url}\n\n${page.content}${cut}` + } catch (error) { + const message = formatToolError(error) + toolCallbacks.setToolStatus(toolId, { content: 'Could not read the page', error: message }) + return `Failed to read ${url}: ${message}` + } + } +} + +/** The web tools, appended to `tools` when the workspace configured a search API and the + * model's provider does not search on its own this request. Decided per request, as the + * provider's own search is: a model switch, or a provider refusing its search tool, + * changes the answer mid-session. */ +export function withWebSearchFallback( + tools: T[], + backendConfigured: boolean, + providerSearches: boolean +): T[] { + if (!backendConfigured || providerSearches) return tools + return [...tools, searchWebTool as T, fetchWebPageTool as T] +} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 3839119676..fdd1c311be 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -4,6 +4,7 @@ WorkspaceService, type AIConfig, type AIProvider, + type AIWebSearchConfig, type GetCopilotSettingsStateResponse, type InstanceAISummary, type ModelPriceOverride @@ -92,6 +93,7 @@ let copilotDisabled = $state(false) let sessionsStorageDisabled = $state(false) let sessionsRetentionDays: number | undefined = $state(undefined) + let webSearch: AIWebSearchConfig | undefined = $state(undefined) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -106,6 +108,7 @@ let initialCopilotDisabled = $state(false) let initialSessionsStorageDisabled = $state(false) let initialSessionsRetentionDays: number | undefined = $state(undefined) + let initialWebSearch: AIWebSearchConfig | undefined = $state(undefined) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -137,6 +140,7 @@ copilotDisabled = config?.copilot_disabled === true sessionsStorageDisabled = config?.sessions_storage_disabled === true sessionsRetentionDays = config?.sessions_retention_days + webSearch = config?.web_search ? clone(config.web_search) : undefined for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -157,6 +161,7 @@ initialCopilotDisabled = copilotDisabled initialSessionsStorageDisabled = sessionsStorageDisabled initialSessionsRetentionDays = sessionsRetentionDays + initialWebSearch = webSearch ? clone(webSearch) : undefined } export function loadFromConfig(config: AIConfig | undefined) { @@ -176,6 +181,7 @@ copilotDisabled = initialCopilotDisabled sessionsStorageDisabled = initialSessionsStorageDisabled sessionsRetentionDays = initialSessionsRetentionDays + webSearch = initialWebSearch ? clone(initialWebSearch) : undefined } $effect(() => { @@ -214,7 +220,8 @@ JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || copilotDisabled !== initialCopilotDisabled || sessionsStorageDisabled !== initialSessionsStorageDisabled || - sessionsRetentionDays !== initialSessionsRetentionDays + sessionsRetentionDays !== initialSessionsRetentionDays || + JSON.stringify(webSearch) !== JSON.stringify(initialWebSearch) ) $effect(() => { @@ -319,11 +326,12 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) - // The flags and the retention are what a workspace on instance defaults still stores - // of its own. + // The flags, the retention and the search API are what a workspace on instance + // defaults still stores of its own. const copilot_disabled = copilotDisabled ? true : undefined const sessions_storage_disabled = sessionsStorageDisabled ? true : undefined const sessions_retention_days = sessionsRetentionDays + const web_search = webSearch return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -338,9 +346,10 @@ model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, copilot_disabled, sessions_storage_disabled, - sessions_retention_days + sessions_retention_days, + web_search } - : { copilot_disabled, sessions_storage_disabled, sessions_retention_days } + : { copilot_disabled, sessions_storage_disabled, sessions_retention_days, web_search } } // The server refuses anything outside this range; the input holds the same bounds. @@ -352,11 +361,18 @@ sessionsRetentionDays > MAX_SESSIONS_RETENTION_DAYS) ) + const WEB_SEARCH_PROVIDERS: { label: string; value: AIWebSearchConfig['provider'] }[] = [ + { label: 'Exa', value: 'exa' }, + { label: 'Tavily', value: 'tavily' }, + { label: 'Linkup', value: 'linkup' } + ] + function isSaveDisabled(): boolean { return ( !Object.values(aiProviders).every((p) => p.resource_path) || (metadataModel != undefined && metadataModel.length === 0) || (codeCompletionModel != undefined && codeCompletionModel.length === 0) || + (webSearch != undefined && !webSearch.resource_path) || retentionInvalid ) } @@ -710,6 +726,42 @@ {/if} {#if promptScope === 'workspace'} + +
    +