Files
windmill/backend/windmill-api/src/token.rs
T
9d61e4e59e feat: self-host docs search for chat, mcp, cli; drop inkeep (#9772)
* feat: self-host docs search for chat, mcp and cli; remove inkeep

Embed a vendored docs snapshot (llms.txt/llms-full.txt) in the backend and
serve ranking + page rendering from GET /api/docs/{search,page}. The AI chat,
the MCP searchDocs/readDocsPage tools, and 'wmill docs' all consume it, so docs
search works with no runtime egress and is no longer EE-gated. Removes the
inkeep proxy. EE companion deletes inkeep_ee.rs (ee-repo-ref bumped).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* refactor: name read_docs_page param `url` instead of `path`

search_docs returns each hit's `Source` URL, so the read tool now takes a
`url` argument to match — the AI/MCP loop reads "search gives a Source URL,
read takes that url" rather than copying a `Source:` URL into a `path` slot.
A bare `/docs/...` path is still accepted and canonicalized before lookup.

Regenerated openapi-deref, the MCP endpoint tools, and the frontend client.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* ci: add scheduled workflow to refresh the vendored docs snapshot

The backend embeds docs_snapshot/*.gz at build time, so the in-product docs
corpus is otherwise only as fresh as the last manual fetch.sh run. This adds a
weekly (and manually dispatchable) job that re-runs fetch.sh, sanity-checks the
result against truncation/garbage, and opens a PR via the internal app when the
snapshot changed — so a human reviews the docs diff before it rides into the
next release build.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* refactor: make docs tool-result strings caller-neutral

The search/page endpoints back three differently-named consumers (the AI chat
`read_docs_page` tool, the MCP `readDocsPage` tool, and the `wmill docs` CLI),
so the shared rendered text shouldn't name one of them. Refer to "the docs
page-reading tool" and its `url` argument instead, and add tests pinning the
caller-neutral follow-up guidance.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* chore: point ee-repo-ref at inkeep-removal companion rebased on EE main

The companion branch now carries only the inkeep_ee.rs deletion on top of EE
main (was based on the native-job-retry EE line, which polluted the EE PR diff).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(docs): expose docs:read in token catalog; precompute lowercased corpus

Addresses two review nits on the self-hosted docs PR:

- docs:read was enforced (ScopeDomain::Docs) but missing from the token scope
  catalog (token.rs ALL_SCOPES), so it couldn't be selected when creating a
  standard scoped token in the UI — leaving scope-restricted CLI/MCP docs use
  effectively ungrantable. Add a read-only "Documentation" group (no write
  surface) and a test asserting it is exposed.
- search ran page.body.to_lowercase() on the whole corpus per query. Lowercase
  body/title/description once at parse time (into the OnceLock corpus) and scan
  the precomputed copies instead.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* chore: update ee-repo-ref to 27a4f41b8e5603d6e444efcfc420bd1c44a07eed

This commit updates the EE repository reference after PR #630 was merged in windmill-ee-private.

Previous ee-repo-ref: c7ec3a0c2fa38d4cb5e50bf0265eef4710de4860

New ee-repo-ref: 27a4f41b8e5603d6e444efcfc420bd1c44a07eed

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
2026-06-25 16:32:07 +02:00

242 lines
7.8 KiB
Rust

use axum::{routing::get, Json, Router};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use windmill_common::error::JsonResult;
#[derive(Default, Serialize, Deserialize, Clone)]
pub struct ScopeOption {
pub value: String,
pub label: String,
pub requires_resource_path: bool,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct ScopeDomain {
pub name: String,
pub description: Option<String>,
pub scopes: Vec<ScopeOption>,
}
fn build_trigger_scope_domains() -> Vec<ScopeDomain> {
const TRIGGER_DOMAINS: &[(&str, &str)] = &[
("http_triggers", "HTTP"),
("websocket_triggers", "WebSocket"),
("kafka_triggers", "Kafka"),
("nats_triggers", "NATS"),
("mqtt_triggers", "MQTT"),
("sqs_triggers", "AWS SQS"),
("gcp_triggers", "GCP Pub/Sub"),
("azure_triggers", "Azure Event Grid"),
("postgres_triggers", "PostgreSQL"),
("email_triggers", "Email"),
];
TRIGGER_DOMAINS
.iter()
.map(|(domain, display_name)| ScopeDomain {
name: format!("{} Triggers", display_name),
description: Some(format!("{} trigger management", display_name)),
scopes: vec![
ScopeOption {
value: format!("{domain}:read"),
label: "Read".to_string(),
requires_resource_path: true,
},
ScopeOption {
value: format!("{domain}:write"),
label: "Write".to_string(),
requires_resource_path: true,
},
],
})
.collect()
}
fn build_standard_scope_domains() -> Vec<ScopeDomain> {
const STANDARD_DOMAINS: &[(&str, &str, &str, bool)] = &[
(
"scripts",
"Scripts",
"Access to automation scripts and workflows",
true,
),
(
"flows",
"Flows",
"Access to automation scripts and workflows",
true,
),
(
"flow_conversations",
"Flow Conversations",
"Flow conversation management",
false,
),
("apps", "Apps", "App management", true),
("raw_apps", "RawApps", "Raw app management", true),
("resources", "Resources", "Resource management", true),
("variables", "Variables", "", true),
(
"schedules",
"Schedules",
"Scheduled tasks and automated triggers",
true,
),
("folders", "Folders", "Folder management", true),
("users", "Users", "User account management", false),
("groups", "Groups", "Group management", false),
("workspaces", "Workspaces", "Workspace management", false),
("audit", "Audit", "Audit log management", false),
("workers", "Workers", "Worker management", false),
("settings", "Settings", "System settings management", false),
(
"service_logs",
"Service Logs",
"Service log management",
false,
),
("configs", "Configs", "Configuration management", false),
("oauth", "OAuth", "OAuth management", false),
("ai", "AI", "AI feature management", false),
("ai_skills", "AI Skills", "AI skill management", false),
(
"agent_workers",
"Agent Workers",
"Agent worker management",
false,
),
("drafts", "Drafts", "Draft management", false),
("favorites", "Favorites", "Favorite items management", false),
("inputs", "Inputs", "Input management", false),
("job_helpers", "Job Helpers", "Job helper utilities", false),
(
"openapi",
"OpenAPI",
"OpenAPI documentation management",
false,
),
("capture", "Capture", "Request capture management", false),
(
"concurrency_groups",
"Concurrency Groups",
"Concurrency group management",
false,
),
("oidc", "OIDC", "OIDC management", false),
("acls", "ACLs", "Access Control List management", false),
("indexer", "Indexer", "Search indexer management", false),
("teams", "Teams", "Team management", false),
(
"git_sync",
"Git Sync",
"Git synchronization management",
false,
),
(
"native_triggers",
"Native Triggers",
"Native triggers management",
true,
),
];
STANDARD_DOMAINS
.iter()
.map(|(key, name, desc, req)| ScopeDomain {
name: name.to_string(),
description: if desc.is_empty() {
None
} else {
Some(desc.to_string())
},
scopes: vec![
ScopeOption {
value: format!("{key}:read"),
label: "Read".to_string(),
requires_resource_path: *req,
},
ScopeOption {
value: format!("{key}:write"),
label: "Write".to_string(),
requires_resource_path: *req,
},
],
})
.collect()
}
lazy_static! {
static ref ALL_SCOPES: Vec<ScopeDomain> = {
let mut groups = vec![ScopeDomain {
name: "Jobs".to_string(),
description: Some("Job management".to_string()),
scopes: vec![
ScopeOption {
value: "jobs:read".to_string(),
label: "Read".to_string(),
requires_resource_path: false,
},
ScopeOption {
value: "jobs:write".to_string(),
label: "Write".to_string(),
requires_resource_path: false,
},
ScopeOption {
value: "jobs:run:scripts".to_string(),
label: "Run scripts".to_string(),
requires_resource_path: true,
},
ScopeOption {
value: "jobs:run:flows".to_string(),
label: "Run flows".to_string(),
requires_resource_path: true,
},
],
}];
// Read-only: `/api/docs/*` exposes only GET routes, so there is no
// `docs:write`. Kept out of build_standard_scope_domains (which mints a
// read+write pair) for that reason.
groups.push(ScopeDomain {
name: "Documentation".to_string(),
description: Some("Read-only documentation search".to_string()),
scopes: vec![ScopeOption {
value: "docs:read".to_string(),
label: "Read".to_string(),
requires_resource_path: false,
}],
});
groups.extend(build_standard_scope_domains());
groups.extend(build_trigger_scope_domains());
groups
};
}
pub fn global_service() -> Router {
Router::new().route("/list/scopes", get(get_all_available_scopes))
}
async fn get_all_available_scopes() -> JsonResult<Vec<ScopeDomain>> {
Ok(Json(ALL_SCOPES.clone()))
}
#[cfg(test)]
mod tests {
use super::*;
/// The token-scope picker is driven by this catalog, so a scope that is
/// enforced but absent here can't be granted through the supported UI.
#[test]
fn docs_read_scope_is_exposed_read_only() {
let values: Vec<&str> = ALL_SCOPES
.iter()
.flat_map(|d| d.scopes.iter())
.map(|s| s.value.as_str())
.collect();
assert!(values.contains(&"docs:read"), "docs:read must be selectable");
assert!(!values.contains(&"docs:write"), "docs has no write surface");
}
}