mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-29 08:12:56 +00:00
Merge remote-tracking branch 'origin/main' into migrate-session-to-fork
# Conflicts: # backend/ee-repo-ref.txt # frontend/src/lib/components/sessions/SessionWrapper.svelte # frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte
This commit is contained in:
@@ -2,8 +2,9 @@
|
||||
//!
|
||||
//! The run is built here from the stored agent rather than sent by the client, which is what lets
|
||||
//! anyone who can read the agent run it, operators included, the way a deployed flow runs by path:
|
||||
//! a preview takes arbitrary code, this takes only the run's inputs. It runs as the caller, who
|
||||
//! needs access to the agent's AI resource and to whatever its tools use.
|
||||
//! a preview takes arbitrary code, this takes only the run's inputs. It runs as the caller, or as
|
||||
//! the agent's `on_behalf_of` when its `run_on_behalf_of` is set; the server resolves
|
||||
//! `on_behalf_of` on every write of the agent.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -20,13 +21,16 @@ use windmill_common::{
|
||||
jobs::JobPayload,
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, StripPath},
|
||||
variables::untrusted_arg_interpolates,
|
||||
};
|
||||
use windmill_queue::{push, PushArgs, PushIsolationLevel};
|
||||
|
||||
use crate::ai_evals::run::config_to_draft;
|
||||
use crate::ai_evals::subject::AgentDraft;
|
||||
use crate::db::{ApiAuthed, DB};
|
||||
use crate::jobs::{handle_chat_conversation_messages, set_flow_memory_id, RunJobQuery};
|
||||
use crate::jobs::{
|
||||
handle_chat_conversation_messages, run_end_user_email, set_flow_memory_id, RunJobQuery,
|
||||
};
|
||||
|
||||
/// The id the agent's step carries in the run, as the editor's chat names it. A token scoped to
|
||||
/// `jobs:run:agents:<path>` reads its runs back by it (`require_job_within_read_scope`).
|
||||
@@ -42,62 +46,79 @@ pub(crate) async fn run_agent(
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
Json(args): Json<HashMap<String, Box<RawValue>>>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
crate::jobs::check_license_key_valid().await?;
|
||||
|
||||
let path = path.to_path();
|
||||
check_scopes(&authed, || format!("jobs:run:agents:{path}"))?;
|
||||
|
||||
let uuid = run_agent_as(&authed, &db, &user_db, &w_id, path, &run_query, &args).await?;
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
}
|
||||
|
||||
/// [`run_agent`] for a caller already authenticated, such as the identity of an agent shared in a
|
||||
/// Slack channel.
|
||||
pub(crate) async fn run_agent_as(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
path: &str,
|
||||
run_query: &RunJobQuery,
|
||||
args: &HashMap<String, Box<RawValue>>,
|
||||
) -> Result<uuid::Uuid> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
crate::jobs::check_license_key_valid().await?;
|
||||
|
||||
// Read through the caller's own permissions: reading the agent is what allows running it.
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
let value = sqlx::query_scalar!(
|
||||
"SELECT value AS \"value: sqlx::types::Json<serde_json::Value>\"
|
||||
FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'",
|
||||
w_id,
|
||||
path
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
let Some(sqlx::types::Json(value)) = not_found_if_none(value, "Agent", path)? else {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Agent {path} has no configuration"
|
||||
)));
|
||||
};
|
||||
let config = config_to_draft(value)?;
|
||||
|
||||
// A chat turn is filed where the agent's chat lists its conversations, which a flow cannot
|
||||
// name (flow paths carry no `.`), so the two never share a conversation list.
|
||||
let chat = run_query.memory_id.is_some();
|
||||
let run_path = if chat {
|
||||
format!("{path}.chat")
|
||||
} else {
|
||||
path.to_string()
|
||||
};
|
||||
// A step memory id would replace the conversation's, and the agent would forget the turns
|
||||
// before; history comes from the conversation alone.
|
||||
let flow_value = agent_step_flow(
|
||||
&config,
|
||||
AGENT_NODE_ID,
|
||||
chat,
|
||||
&["memory_id", "previous_messages"],
|
||||
)?;
|
||||
let AgentTurn { flow_value, run_path, agent_obo } =
|
||||
agent_turn(authed, user_db, w_id, path, chat).await?;
|
||||
|
||||
let push_authed = authed.clone().into();
|
||||
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
|
||||
let on_behalf_of =
|
||||
windmill_common::on_behalf_of_from_permissioned_as(agent_obo.as_deref(), w_id, db).await?;
|
||||
// As `run_flow` picks the identity of a deployed flow: the agent's own when it has one, the
|
||||
// caller's otherwise, which lends nobody's permissions.
|
||||
let (email, permissioned_as, push_authed, isolation) = if let Some(obo) = on_behalf_of.as_ref()
|
||||
{
|
||||
(
|
||||
&obo.email,
|
||||
obo.permissioned_as.clone(),
|
||||
None,
|
||||
PushIsolationLevel::IsolatedRoot(db.clone()),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
Some(authed.clone().into()),
|
||||
PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()),
|
||||
)
|
||||
};
|
||||
if let Some(obo) = on_behalf_of.as_ref() {
|
||||
// The worker would interpolate such an input with the job's token, which is the
|
||||
// identity's.
|
||||
if args.values().any(|v| untrusted_arg_interpolates(v.get())) {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Agent {path} runs on behalf of {}, so its inputs cannot reference variables or resources",
|
||||
obo.permissioned_as
|
||||
)));
|
||||
}
|
||||
#[cfg(all(feature = "parquet", feature = "private"))]
|
||||
check_attachments_readable(db, authed, w_id, args).await?;
|
||||
}
|
||||
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?;
|
||||
let end_user_email = run_end_user_email(db, w_id, authed, None).await?;
|
||||
let (uuid, mut tx) = push(
|
||||
&db,
|
||||
PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()),
|
||||
&w_id,
|
||||
db,
|
||||
isolation,
|
||||
w_id,
|
||||
JobPayload::RawFlow {
|
||||
value: flow_value,
|
||||
path: Some(run_path.clone()),
|
||||
restarted_from: None,
|
||||
},
|
||||
PushArgs::from(&args),
|
||||
PushArgs::from(args),
|
||||
authed.display_username(),
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
email,
|
||||
permissioned_as,
|
||||
authed.token_prefix.as_deref(),
|
||||
authed.username_override.as_deref(),
|
||||
None,
|
||||
@@ -114,42 +135,173 @@ pub(crate) async fn run_agent(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&push_authed),
|
||||
push_authed.as_ref(),
|
||||
false,
|
||||
None,
|
||||
end_user_email,
|
||||
authed.trigger_or_fallback(None),
|
||||
None,
|
||||
scope_ceiling.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(memory_id) = run_query.memory_key(&w_id, &run_path) {
|
||||
if let Some(memory_id) = run_query.memory_key(w_id, &run_path) {
|
||||
set_flow_memory_id(&mut tx, uuid, memory_id).await?;
|
||||
}
|
||||
if chat {
|
||||
// A real conversation, not a test one: this is the agent as deployed, not a draft.
|
||||
// The caller's conversation, whoever the agent runs as, and a real one: this is the
|
||||
// agent as deployed, not a draft under test.
|
||||
handle_chat_conversation_messages(
|
||||
&mut tx,
|
||||
&authed,
|
||||
&w_id,
|
||||
authed,
|
||||
w_id,
|
||||
&run_path,
|
||||
&run_query,
|
||||
run_query,
|
||||
args.get("user_message"),
|
||||
uuid,
|
||||
false,
|
||||
&args,
|
||||
args,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
Ok(uuid)
|
||||
}
|
||||
|
||||
/// A turn of a saved agent, as the flow that runs it.
|
||||
pub(crate) struct AgentTurn {
|
||||
pub flow_value: windmill_common::flows::FlowValue,
|
||||
/// Where the turn is filed: a chat turn under `<path>.chat`, which a flow cannot name (flow
|
||||
/// paths carry no `.`), so the agent's conversations never share a list with a flow's.
|
||||
pub run_path: String,
|
||||
/// The identity the agent runs as when it opts in (`run_on_behalf_of`). A caller that runs it
|
||||
/// as some other identity, such as an app's, ignores it.
|
||||
pub agent_obo: Option<String>,
|
||||
}
|
||||
|
||||
/// Reads the agent through `authed`'s own permissions and builds the flow of one of its turns.
|
||||
/// Who it may see and which inputs or slack settings it carries are not inputs of the run.
|
||||
pub(crate) async fn agent_turn(
|
||||
authed: &ApiAuthed,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
path: &str,
|
||||
chat: bool,
|
||||
) -> Result<AgentTurn> {
|
||||
let mut tx = user_db.clone().begin(authed).await?;
|
||||
let value = sqlx::query_scalar!(
|
||||
"SELECT value AS \"value: sqlx::types::Json<serde_json::Value>\"
|
||||
FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'",
|
||||
w_id,
|
||||
path
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
let Some(sqlx::types::Json(mut value)) = not_found_if_none(value, "Agent", path)? else {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Agent {path} has no configuration"
|
||||
)));
|
||||
};
|
||||
let obj = value.as_object_mut();
|
||||
let (agent_obo, run_obo) = obj
|
||||
.map(|o| {
|
||||
o.remove("slack_anyone");
|
||||
(o.remove("on_behalf_of"), o.remove("run_on_behalf_of"))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let agent_obo = agent_obo
|
||||
.filter(|_| run_obo.and_then(|v| v.as_bool()).unwrap_or(false))
|
||||
.and_then(|v| v.as_str().map(str::to_string));
|
||||
let config = config_to_draft(value)?;
|
||||
let run_path = if chat {
|
||||
format!("{path}.chat")
|
||||
} else {
|
||||
path.to_string()
|
||||
};
|
||||
// A step memory id would replace the conversation's, and the agent would forget the turns
|
||||
// before; history comes from the conversation alone.
|
||||
let flow_value = agent_step_flow(
|
||||
&config,
|
||||
AGENT_NODE_ID,
|
||||
chat,
|
||||
&["memory_id", "previous_messages"],
|
||||
)?;
|
||||
Ok(AgentTurn { flow_value, run_path, agent_obo })
|
||||
}
|
||||
|
||||
/// The step fetches the message's files with its job's token, so a run as the agent's identity
|
||||
/// would read any file that identity can: a caller attaches only what they could download
|
||||
/// themselves, checked as `sign_s3_objects` checks a key before signing it.
|
||||
#[cfg(all(feature = "parquet", feature = "private"))]
|
||||
pub(crate) async fn check_attachments_readable(
|
||||
db: &DB,
|
||||
authed: &ApiAuthed,
|
||||
w_id: &str,
|
||||
args: &HashMap<String, Box<RawValue>>,
|
||||
) -> Result<()> {
|
||||
use crate::job_helpers_oss::{
|
||||
get_large_file_storage, get_workspace_s3_resource_and_check_paths,
|
||||
};
|
||||
use windmill_common::db::DbWithOptAuthed;
|
||||
use windmill_types::s3::S3Permission;
|
||||
|
||||
for attachment in run_attachments(args)? {
|
||||
// The path check is skipped, not failed, when the storage does not resolve.
|
||||
if get_large_file_storage(db, w_id, attachment.storage.clone())
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Err(windmill_object_store::workspace_storage_not_found(
|
||||
attachment.storage.as_deref(),
|
||||
));
|
||||
}
|
||||
get_workspace_s3_resource_and_check_paths(
|
||||
&DbWithOptAuthed::from_authed(authed, db.clone(), None),
|
||||
Some(authed),
|
||||
w_id,
|
||||
attachment.storage.clone(),
|
||||
&[(&attachment.s3, S3Permission::READ)],
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every file the step fetches with its job's token: the message's attachments, and the
|
||||
/// `s3_object` parts of the history a caller supplies, which the worker downloads alike.
|
||||
pub(crate) fn run_attachments(
|
||||
args: &HashMap<String, Box<RawValue>>,
|
||||
) -> Result<Vec<windmill_types::s3::S3Object>> {
|
||||
use windmill_ai::ai_types::{ContentPart, OpenAIContent, OpenAIMessage};
|
||||
|
||||
let mut files: Vec<windmill_types::s3::S3Object> = match args.get("user_attachments") {
|
||||
Some(raw) => serde_json::from_str::<Option<Vec<_>>>(raw.get())
|
||||
.map_err(|e| Error::BadRequest(format!("invalid user_attachments: {e}")))?
|
||||
.unwrap_or_default(),
|
||||
None => vec![],
|
||||
};
|
||||
if let Some(raw) = args.get("previous_messages") {
|
||||
let messages = serde_json::from_str::<Option<Vec<OpenAIMessage>>>(raw.get())
|
||||
.map_err(|e| Error::BadRequest(format!("invalid previous_messages: {e}")))?;
|
||||
for message in messages.into_iter().flatten() {
|
||||
if let Some(OpenAIContent::Parts(parts)) = message.content {
|
||||
files.extend(parts.into_iter().filter_map(|part| match part {
|
||||
ContentPart::S3Object { s3_object } => Some(s3_object),
|
||||
_ => None,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
files.retain(|f| !f.s3.is_empty());
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// The agent as a one-step flow, validated by deserializing through `FlowValue` rather than
|
||||
/// trusted as raw JSON. Its own transforms are the step's, minus `without`, and the run's inputs
|
||||
/// supply the message and the attachments over the top. Always inlined, never a link to the
|
||||
/// resource: a linked step would resolve the agent when it runs, not as it was read here.
|
||||
/// supply the message, the attachments and any history over the top. Always inlined, never a link
|
||||
/// to the resource: a linked step would resolve the agent when it runs, not as it was read here.
|
||||
pub(crate) fn agent_step_flow(
|
||||
config: &AgentDraft,
|
||||
id: &str,
|
||||
@@ -163,7 +315,7 @@ pub(crate) fn agent_step_flow(
|
||||
for key in without {
|
||||
input_transforms.remove(*key);
|
||||
}
|
||||
for key in ["user_message", "user_attachments"] {
|
||||
for key in ["user_message", "user_attachments", "previous_messages"] {
|
||||
input_transforms.insert(
|
||||
key.to_string(),
|
||||
serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }),
|
||||
@@ -184,3 +336,25 @@ pub(crate) fn agent_step_flow(
|
||||
}
|
||||
Ok(serde_json::from_value(flow)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn run_attachments_include_the_files_a_supplied_history_quotes() {
|
||||
let args: HashMap<String, Box<RawValue>> = serde_json::from_value(serde_json::json!({
|
||||
"user_attachments": [{ "s3": "u/a/attached.png" }],
|
||||
"previous_messages": [
|
||||
{ "role": "user", "content": [
|
||||
{ "type": "text", "text": "look" },
|
||||
{ "type": "s3_object", "s3_object": { "s3": "u/b/quoted.png" } }
|
||||
]},
|
||||
{ "role": "assistant", "content": "ok" }
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let keys: Vec<_> = run_attachments(&args).unwrap().into_iter().map(|f| f.s3).collect();
|
||||
assert_eq!(keys, ["u/a/attached.png", "u/b/quoted.png"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ use windmill_ai::providers::{
|
||||
},
|
||||
};
|
||||
use windmill_ai::proxy::{
|
||||
fim::maybe_transform_fim_request, proxy_execution_mode, ProxyBuildArgs, ProxyExecutionMode,
|
||||
ProxyRequest,
|
||||
fim::maybe_transform_fim_request, proxy_execution_mode, validate_proxy_path, ProxyBuildArgs,
|
||||
ProxyExecutionMode, ProxyRequest,
|
||||
};
|
||||
use windmill_ai::retry::send_with_retries;
|
||||
use windmill_audit::{audit_oss::audit_log, ActionKind};
|
||||
@@ -166,27 +166,14 @@ fn ai_http_client_builder() -> reqwest::ClientBuilder {
|
||||
/// after the check (DNS-rebinding TOCTOU).
|
||||
///
|
||||
/// Returns the shared pooled [`HTTP_CLIENT`] unchanged when there is nothing to
|
||||
/// pin — an IP-literal host, or a deployment that opted into private AI
|
||||
/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint
|
||||
/// as `get_base_url` apply, so the guard here is consistent with save-time
|
||||
/// pin — an IP-literal host, or a host `ALLOW_PRIVATE_AI_BASE_URLS` allows
|
||||
/// by name or wholesale. The same allowlist and error hint as `get_base_url`
|
||||
/// apply, so the guard here is consistent with save-time
|
||||
/// validation while additionally closing the connect-time window.
|
||||
async fn pinned_ai_client_for(url: &str) -> Result<std::borrow::Cow<'static, Client>> {
|
||||
use std::borrow::Cow;
|
||||
use windmill_common::ssrf::SsrfValidationError;
|
||||
|
||||
if *windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS {
|
||||
return Ok(Cow::Borrowed(&HTTP_CLIENT));
|
||||
}
|
||||
|
||||
let target = windmill_common::ssrf::validate_url_for_ssrf(url)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!(
|
||||
"{e}. If you need to use private/internal AI endpoints, \
|
||||
set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable"
|
||||
)),
|
||||
e => Error::from(e),
|
||||
})?;
|
||||
let target = windmill_ai::ai_providers::validate_ai_url(url).await?;
|
||||
|
||||
if target.pinned_addrs().is_empty() {
|
||||
return Ok(Cow::Borrowed(&HTTP_CLIENT));
|
||||
@@ -713,6 +700,10 @@ pub fn workspaced_service() -> Router {
|
||||
.nest(
|
||||
"/shared_artifacts",
|
||||
crate::ai_shared_artifacts::workspaced_service(),
|
||||
)
|
||||
.nest(
|
||||
"/shared_sessions",
|
||||
crate::ai_shared_sessions::workspaced_service(),
|
||||
);
|
||||
|
||||
#[cfg(feature = "bedrock")]
|
||||
@@ -1099,6 +1090,7 @@ async fn global_proxy(
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> impl IntoResponse {
|
||||
validate_proxy_path(&ai_path)?;
|
||||
let provider = headers
|
||||
.get("X-Provider")
|
||||
.map(|v| v.to_str().unwrap_or("").to_string());
|
||||
@@ -1217,6 +1209,7 @@ async fn proxy(
|
||||
headers: HeaderMap,
|
||||
mut body: Bytes,
|
||||
) -> impl IntoResponse {
|
||||
validate_proxy_path(&ai_path)?;
|
||||
let provider = headers
|
||||
.get("X-Provider")
|
||||
.map(|v| v.to_str().unwrap_or("").to_string());
|
||||
|
||||
@@ -123,17 +123,15 @@ pub async fn list_datasets(
|
||||
|
||||
pub async fn create_dataset(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(payload): Json<CreateDataset>,
|
||||
) -> Result<String> {
|
||||
check_proper_path(&payload.path)?;
|
||||
check_summary(payload.summary.as_deref())?;
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot create eval datasets".to_string(),
|
||||
));
|
||||
}
|
||||
windmill_common::workspaces::check_role_can_code(&db, &w_id, &authed, "create eval datasets")
|
||||
.await?;
|
||||
check_case_set(
|
||||
payload
|
||||
.cases
|
||||
@@ -207,15 +205,13 @@ pub async fn get_dataset(
|
||||
/// or not at all.
|
||||
pub async fn update_dataset(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, String)>,
|
||||
Json(payload): Json<EditDataset>,
|
||||
) -> Result<String> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot modify eval datasets".to_string(),
|
||||
));
|
||||
}
|
||||
windmill_common::workspaces::check_role_can_code(&db, &w_id, &authed, "modify eval datasets")
|
||||
.await?;
|
||||
check_summary(payload.summary.as_deref())?;
|
||||
let new_path = match payload.path.filter(|p| *p != path) {
|
||||
Some(new_path) => {
|
||||
@@ -301,15 +297,13 @@ pub async fn update_dataset(
|
||||
/// own retention.
|
||||
pub async fn delete_dataset(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, String)>,
|
||||
) -> Result<String> {
|
||||
check_proper_path(&path)?;
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot delete eval datasets".to_string(),
|
||||
));
|
||||
}
|
||||
windmill_common::workspaces::check_role_can_code(&db, &w_id, &authed, "delete eval datasets")
|
||||
.await?;
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
let deleted = sqlx::query_scalar!(
|
||||
"DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path",
|
||||
|
||||
@@ -267,16 +267,14 @@ pub(crate) async fn read_dataset_and_cases(
|
||||
/// being mirrored here.
|
||||
async fn require_dataset_writable(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
path: &str,
|
||||
) -> Result<()> {
|
||||
check_proper_path(path)?;
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot modify eval datasets".to_string(),
|
||||
));
|
||||
}
|
||||
windmill_common::workspaces::check_role_can_code(db, w_id, authed, "modify eval datasets")
|
||||
.await?;
|
||||
let mut tx = user_db.clone().begin(authed).await?;
|
||||
let writable = sqlx::query_scalar!(
|
||||
"SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
|
||||
|
||||
@@ -255,6 +255,8 @@ fn agent_module(config: &AgentDraft) -> Result<serde_json::Value> {
|
||||
.entry("input_transforms")
|
||||
.or_insert_with(|| serde_json::json!({}));
|
||||
if let Some(transforms) = transforms.as_object_mut() {
|
||||
// A case carries no history: each one is asked fresh.
|
||||
transforms.remove("previous_messages");
|
||||
for key in ["user_message", "user_attachments"] {
|
||||
transforms.insert(
|
||||
key.to_string(),
|
||||
@@ -561,7 +563,7 @@ pub async fn run_experiment(
|
||||
) -> Result<String> {
|
||||
check_scopes(&authed, || "jobs:run".to_string())?;
|
||||
// A write, not a read: it persists an experiment into the dataset.
|
||||
require_dataset_writable(&authed, &user_db, &w_id, &payload.dataset).await?;
|
||||
require_dataset_writable(&authed, &db, &user_db, &w_id, &payload.dataset).await?;
|
||||
|
||||
let mut subject = payload.subject;
|
||||
let draft = validate_subject(&subject)?;
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2026
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
//! Read-only snapshots of AI session transcripts, shared by their author.
|
||||
//!
|
||||
//! Sessions live in the author's browser; a row here exists only because the author asked for
|
||||
//! a link. A share is readable by the members of its workspace, or by anyone holding the link
|
||||
//! once its author makes it public. The snapshot is whatever the author's browser sent, so the
|
||||
//! viewer treats it as untrusted data.
|
||||
|
||||
use crate::db::{ApiAuthed, DB};
|
||||
use axum::{
|
||||
extract::{DefaultBodyLimit, Extension, Json, Path, Query},
|
||||
routing::{delete, get, post},
|
||||
Router,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use uuid::Uuid;
|
||||
use windmill_api_auth::{is_effectively_unscoped, OptAuthed};
|
||||
use windmill_audit::{audit_oss::audit_log, ActionKind};
|
||||
use windmill_common::error::{Error, JsonResult, Result};
|
||||
|
||||
const MAX_SNAPSHOT_BYTES: usize = 10 * 1024 * 1024;
|
||||
const MAX_NAME_CHARS: usize = 255;
|
||||
const MAX_SESSION_ID_CHARS: usize = 255;
|
||||
const SHARE_BODY_LIMIT: usize = MAX_SNAPSHOT_BYTES + 64 * 1024;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
"/share",
|
||||
post(share_session).layer(DefaultBodyLimit::max(SHARE_BODY_LIMIT)),
|
||||
)
|
||||
.route("/status", get(get_share_status))
|
||||
.route("/set_public/{id}", post(set_public))
|
||||
.route("/delete/{id}", delete(unshare_session))
|
||||
}
|
||||
|
||||
/// Mounted outside the authenticated router: a public share is served to anyone.
|
||||
pub fn workspaced_unauthed_service() -> Router {
|
||||
Router::new().route("/get/{id}", get(get_shared_session))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ShareSession {
|
||||
session_id: String,
|
||||
name: String,
|
||||
/// Left as it is on an existing share when absent; a new share starts private.
|
||||
is_public: Option<bool>,
|
||||
message_count: i32,
|
||||
snapshot: Box<RawValue>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SharedSessionInfo {
|
||||
id: Uuid,
|
||||
name: String,
|
||||
is_public: bool,
|
||||
message_count: i32,
|
||||
created_by: String,
|
||||
shared_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SharedSession {
|
||||
#[serde(flatten)]
|
||||
info: SharedSessionInfo,
|
||||
snapshot: Box<RawValue>,
|
||||
/// Whether the caller may stop sharing it: its author, or a workspace admin.
|
||||
can_unshare: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ShareStatus {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
share: Option<SharedSessionInfo>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ShareStatusQuery {
|
||||
session_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SetPublic {
|
||||
is_public: bool,
|
||||
}
|
||||
|
||||
/// Sharing exposes the author's own browser state, so only the author's own unscoped token may
|
||||
/// do it: a job token can carry an `on_behalf_of` identity, and every scoped token (guest,
|
||||
/// embed, app policy, MCP) was minted for something narrower.
|
||||
fn require_plain_user_token(authed: &ApiAuthed) -> Result<()> {
|
||||
if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) {
|
||||
return Err(Error::PermissionDenied(
|
||||
"Shared AI sessions are only reachable with an unscoped user token".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// It is compared against a `VARCHAR(255)` column, and Postgres answers a NUL in a text
|
||||
/// parameter with an opaque 500.
|
||||
fn check_session_id(session_id: &str) -> Result<()> {
|
||||
if session_id.is_empty() || session_id.chars().count() > MAX_SESSION_ID_CHARS {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Session id must be between 1 and {MAX_SESSION_ID_CHARS} characters"
|
||||
)));
|
||||
}
|
||||
if session_id.contains('\0') {
|
||||
return Err(Error::BadRequest(
|
||||
"Session id cannot contain NUL characters".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Share a session, or move the caller's existing link for it to this snapshot.
|
||||
async fn share_session(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(payload): Json<ShareSession>,
|
||||
) -> JsonResult<SharedSessionInfo> {
|
||||
require_plain_user_token(&authed)?;
|
||||
check_session_id(&payload.session_id)?;
|
||||
let name = payload.name.trim();
|
||||
if name.is_empty() || name.chars().count() > MAX_NAME_CHARS || name.contains('\0') {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Session name must be between 1 and {MAX_NAME_CHARS} characters, without NUL"
|
||||
)));
|
||||
}
|
||||
let snapshot = payload.snapshot.get();
|
||||
if snapshot.len() > MAX_SNAPSHOT_BYTES {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Session snapshot is {} bytes, above the {MAX_SNAPSHOT_BYTES} byte limit",
|
||||
snapshot.len()
|
||||
)));
|
||||
}
|
||||
if !snapshot.trim_start().starts_with('{') {
|
||||
return Err(Error::BadRequest(
|
||||
"Session snapshot must be a JSON object".to_string(),
|
||||
));
|
||||
}
|
||||
if payload.message_count < 0 {
|
||||
return Err(Error::BadRequest(
|
||||
"Message count cannot be negative".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
let row = sqlx::query!(
|
||||
r#"INSERT INTO ai_shared_session
|
||||
(workspace_id, session_id, email, created_by, name, is_public, message_count, snapshot)
|
||||
VALUES ($1, $2, $3, $4, $5, COALESCE($6, false), $7, $8)
|
||||
ON CONFLICT (workspace_id, email, session_id) DO UPDATE
|
||||
SET created_by = EXCLUDED.created_by,
|
||||
name = EXCLUDED.name,
|
||||
is_public = COALESCE($6, ai_shared_session.is_public),
|
||||
message_count = EXCLUDED.message_count,
|
||||
snapshot = EXCLUDED.snapshot,
|
||||
shared_at = now()
|
||||
RETURNING id, is_public, shared_at, (xmax = 0) AS "inserted!""#,
|
||||
&w_id,
|
||||
&payload.session_id,
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
name,
|
||||
payload.is_public,
|
||||
payload.message_count,
|
||||
snapshot,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let id = row.id.to_string();
|
||||
let public = row.is_public.to_string();
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
"ai.shared_sessions.share",
|
||||
if row.inserted {
|
||||
ActionKind::Create
|
||||
} else {
|
||||
ActionKind::Update
|
||||
},
|
||||
&w_id,
|
||||
Some(&id),
|
||||
Some([("name", name), ("public", public.as_str())].into()),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(SharedSessionInfo {
|
||||
id: row.id,
|
||||
name: name.to_string(),
|
||||
is_public: row.is_public,
|
||||
message_count: payload.message_count,
|
||||
created_by: authed.username.clone(),
|
||||
shared_at: row.shared_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// The caller's own share of one of their sessions, if any.
|
||||
async fn get_share_status(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(query): Query<ShareStatusQuery>,
|
||||
) -> JsonResult<ShareStatus> {
|
||||
require_plain_user_token(&authed)?;
|
||||
check_session_id(&query.session_id)?;
|
||||
let share = sqlx::query_as!(
|
||||
SharedSessionInfo,
|
||||
"SELECT id, name, is_public, message_count, created_by, shared_at FROM ai_shared_session
|
||||
WHERE workspace_id = $1 AND email = $2 AND session_id = $3",
|
||||
&w_id,
|
||||
&authed.email,
|
||||
&query.session_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
Ok(Json(ShareStatus { share }))
|
||||
}
|
||||
|
||||
/// Only the author changes who can open their link; an admin may only take it down.
|
||||
async fn set_public(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
Json(payload): Json<SetPublic>,
|
||||
) -> JsonResult<SharedSessionInfo> {
|
||||
require_plain_user_token(&authed)?;
|
||||
let mut tx = db.begin().await?;
|
||||
let info = sqlx::query_as!(
|
||||
SharedSessionInfo,
|
||||
"UPDATE ai_shared_session SET is_public = $4
|
||||
WHERE workspace_id = $1 AND id = $2 AND email = $3
|
||||
RETURNING id, name, is_public, message_count, created_by, shared_at",
|
||||
&w_id,
|
||||
id,
|
||||
&authed.email,
|
||||
payload.is_public,
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
Error::NotFound(format!(
|
||||
"Shared session {id} not found, or not shared by you"
|
||||
))
|
||||
})?;
|
||||
|
||||
let id_str = id.to_string();
|
||||
let public = info.is_public.to_string();
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
"ai.shared_sessions.set_public",
|
||||
ActionKind::Update,
|
||||
&w_id,
|
||||
Some(&id_str),
|
||||
Some([("name", info.name.as_str()), ("public", public.as_str())].into()),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(info))
|
||||
}
|
||||
|
||||
/// A public share is served to anyone with the link; any other one only to a member of its
|
||||
/// workspace, which is what an `ApiAuthed` resolved on a `/w/{workspace_id}` path is.
|
||||
async fn get_shared_session(
|
||||
OptAuthed(opt_authed): OptAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> JsonResult<SharedSession> {
|
||||
let row = sqlx::query!(
|
||||
"SELECT id, email, name, is_public, message_count, created_by, snapshot, shared_at
|
||||
FROM ai_shared_session
|
||||
WHERE workspace_id = $1 AND id = $2",
|
||||
&w_id,
|
||||
id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
let not_found = || {
|
||||
Error::NotFound(format!(
|
||||
"Shared session {id} not found: it may have been unshared, or be visible only to members of {w_id}"
|
||||
))
|
||||
};
|
||||
let member = opt_authed
|
||||
.as_ref()
|
||||
.filter(|a| a.job_id.is_none() && is_effectively_unscoped(a.scopes.as_deref()));
|
||||
let row = match row {
|
||||
Some(row) if row.is_public || member.is_some() => row,
|
||||
// A missing share and a private one answer a stranger alike, so the link does not
|
||||
// confirm that a private share exists.
|
||||
_ if member.is_none() => {
|
||||
return Err(Error::NotAuthorized(format!(
|
||||
"Shared session {id} is not public: sign in as a member of {w_id} to open it"
|
||||
)))
|
||||
}
|
||||
_ => return Err(not_found()),
|
||||
};
|
||||
|
||||
Ok(Json(SharedSession {
|
||||
can_unshare: member.is_some_and(|a| row.email == a.email || a.is_admin),
|
||||
snapshot: RawValue::from_string(row.snapshot).map_err(|e| {
|
||||
Error::internal_err(format!("Stored session snapshot is not valid JSON: {e}"))
|
||||
})?,
|
||||
info: SharedSessionInfo {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
is_public: row.is_public,
|
||||
message_count: row.message_count,
|
||||
created_by: row.created_by,
|
||||
shared_at: row.shared_at,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn unshare_session(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> Result<String> {
|
||||
require_plain_user_token(&authed)?;
|
||||
let mut tx = db.begin().await?;
|
||||
let name = sqlx::query_scalar!(
|
||||
"DELETE FROM ai_shared_session
|
||||
WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool)
|
||||
RETURNING name",
|
||||
&w_id,
|
||||
id,
|
||||
&authed.email,
|
||||
authed.is_admin,
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
Error::NotFound(format!(
|
||||
"Shared session {id} not found, or not shared by you"
|
||||
))
|
||||
})?;
|
||||
|
||||
let id_str = id.to_string();
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
"ai.shared_sessions.unshare",
|
||||
ActionKind::Delete,
|
||||
&w_id,
|
||||
Some(&id_str),
|
||||
Some([("name", name.as_str())].into()),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(format!("Stopped sharing {name}"))
|
||||
}
|
||||
+432
-179
File diff suppressed because it is too large
Load Diff
@@ -135,7 +135,7 @@ export async function main(
|
||||
const nonEmpty = (v: unknown) => typeof v === 'string' && v.length > 0
|
||||
// The prefixes `execute_component` resolves a run against; anything else is a
|
||||
// grant no run can match.
|
||||
const RUN_TYPES = ['script', 'flow', 'hubscript']
|
||||
const RUN_TYPES = ['script', 'flow', 'hubscript', 'agent']
|
||||
const malformed = Object.entries(runnables ?? {})
|
||||
.filter(([, r]) => r != null)
|
||||
.filter(([, r]) => {
|
||||
|
||||
@@ -139,13 +139,13 @@ impl RawWebhookArgs {
|
||||
get_random_file_name, get_workspace_s3_resource, upload_file_internal,
|
||||
};
|
||||
use futures::TryStreamExt;
|
||||
use windmill_object_store::build_object_store_client;
|
||||
use windmill_object_store::fork_store::workspace_object_store;
|
||||
use windmill_object_store::object_store_reexports::{Attribute, Attributes};
|
||||
|
||||
let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, w_id, None).await?;
|
||||
|
||||
if let Some(s3_resource) = s3_resource {
|
||||
let s3_client = build_object_store_client(&s3_resource).await?;
|
||||
let s3_client = workspace_object_store(db, w_id, None, &s3_resource).await?;
|
||||
|
||||
let mut parts: Vec<(String, MultipartValue)> = Vec::new();
|
||||
|
||||
|
||||
@@ -113,6 +113,12 @@ lazy_static::lazy_static! {
|
||||
(20260909163047, include_str!(
|
||||
"../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql"
|
||||
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
|
||||
(20261009173546, include_str!(
|
||||
"../../migrations/20261009173546_agent_owner_prefix_index.up.sql"
|
||||
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")),
|
||||
(20261015114712, include_str!(
|
||||
"../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql"
|
||||
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
|
||||
// The widening fails every query the previous release's still-running
|
||||
// servers prepared on the column; `variables::widen_variable_value` does it
|
||||
// once they are gone.
|
||||
|
||||
@@ -16,6 +16,7 @@ use axum::{
|
||||
use serde::{Deserialize, Serialize};
|
||||
use windmill_api_flows::flows::validate_operator_flow;
|
||||
use windmill_audit::{audit_oss::audit_log, ActionKind};
|
||||
use windmill_common::roles::AuthorLevel;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{Error, Result},
|
||||
@@ -24,7 +25,7 @@ use windmill_common::{
|
||||
users::resolve_username_to_email,
|
||||
utils::{check_proper_path, require_admin, strip_json_nul},
|
||||
variables::{build_crypt, encrypt},
|
||||
workspaces::{operator_builder_rights, BuilderKind},
|
||||
workspaces::operator_can_draft,
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
@@ -110,12 +111,6 @@ async fn list_drafts(
|
||||
Path(w_id): Path<String>,
|
||||
Query(query): Query<ListDraftsQuery>,
|
||||
) -> Result<Json<Vec<DraftListItem>>> {
|
||||
// Without builder rights an operator has no drafts of their own (they can't write any, see
|
||||
// `require_can_write_path`), so this list is always empty for them. They can still READ some
|
||||
// collaborators' drafts via `/drafts/get`.
|
||||
if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
let all_users = query.all_users.unwrap_or(false);
|
||||
// Only honor `compare_to_workspace` when it is genuinely this workspace's
|
||||
// parent (a fork comparing against its source). Any other value is dropped
|
||||
@@ -476,7 +471,10 @@ async fn update_draft(
|
||||
}
|
||||
}
|
||||
|
||||
if authed.is_operator && kind == UserDraftItemKind::Flow {
|
||||
// A role that composes but does not author in full is held to the composition-only rules on
|
||||
// its drafts as on its deploys.
|
||||
let caps = windmill_common::roles::caller_capabilities(&db, &w_id, &authed).await?;
|
||||
if caps.flows == AuthorLevel::Compose && kind == UserDraftItemKind::Flow {
|
||||
if let Some(value) = &req.value {
|
||||
#[derive(Deserialize)]
|
||||
struct FlowDraft {
|
||||
@@ -503,7 +501,7 @@ async fn update_draft(
|
||||
}
|
||||
|
||||
// A raw-app draft carries the app value and its policy side by side.
|
||||
if authed.is_operator && kind == UserDraftItemKind::RawApp {
|
||||
if caps.apps == AuthorLevel::Compose && kind == UserDraftItemKind::RawApp {
|
||||
if let Some(value) = &req.value {
|
||||
let draft: serde_json::Value = serde_json::from_str(&strip_json_nul(value.0.get()))
|
||||
.map_err(|e| Error::BadRequest(format!("Invalid app draft: {e}")))?;
|
||||
@@ -1437,8 +1435,7 @@ fn table_for_kind(kind: UserDraftItemKind) -> Option<&'static str> {
|
||||
}
|
||||
|
||||
/// Resolves to `Ok(())` if `authed` may SAVE a draft at `path`. Operators are
|
||||
/// rejected, except for flow drafts in a workspace that granted them builder
|
||||
/// rights. Two layers:
|
||||
/// rejected, except for a kind they could deploy (`operator_can_draft`). Two layers:
|
||||
/// 1. Claim-based namespace rules (admin, own `u/`, member `g/`, writable
|
||||
/// `f/`) — mirror what RLS reads from the same JWT claims, and are the
|
||||
/// ENTIRE check for draft-only paths (no deployed row for RLS to use).
|
||||
@@ -1455,26 +1452,17 @@ pub(crate) async fn require_can_write_path(
|
||||
if authed.is_admin {
|
||||
return Ok(());
|
||||
}
|
||||
// Operators are read-only and never WRITE drafts, except of a kind the workspace granted them
|
||||
// the matching builder right for: a flows-only workspace must not get raw-app drafts through
|
||||
// here. Read access is deliberately asymmetric:
|
||||
// `require_can_read_path` has no operator block, so an operator can still READ a draft they
|
||||
// can read via `/drafts/get`, mirroring their read access to deployed content. Intended.
|
||||
if authed.is_operator {
|
||||
let allowed = match kind {
|
||||
UserDraftItemKind::Flow => Some(BuilderKind::Flows),
|
||||
UserDraftItemKind::RawApp => Some(BuilderKind::Apps),
|
||||
_ => None,
|
||||
};
|
||||
let granted = match allowed {
|
||||
Some(kind) => operator_builder_rights(db, w_id).await?.has(kind),
|
||||
None => false,
|
||||
};
|
||||
if !granted {
|
||||
return Err(Error::PermissionDenied(
|
||||
"operators cannot save drafts".to_string(),
|
||||
));
|
||||
}
|
||||
// Operators WRITE drafts only of a kind they could deploy: resources and variables always,
|
||||
// flows and full-code apps under their builder right, schedules and triggers under the manage
|
||||
// rights. The kind has to be checked, or one right would open drafts of kinds it says nothing
|
||||
// about. Read access is deliberately asymmetric: `require_can_read_path` has no operator block,
|
||||
// so an operator can still READ a draft they can read via `/drafts/get`, mirroring their read
|
||||
// access to deployed content. Intended.
|
||||
if !operator_can_draft(db, w_id, authed, kind).await? {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"your role cannot save {} drafts in this workspace",
|
||||
kind.as_str()
|
||||
)));
|
||||
}
|
||||
// Cheap claim-based namespace checks first: they evaluate the same JWT
|
||||
// claims RLS reads, so the outcome matches the policies while sparing the
|
||||
|
||||
@@ -32,6 +32,7 @@ pub enum FavoriteKind {
|
||||
#[allow(non_camel_case_types)]
|
||||
Raw_App,
|
||||
Asset,
|
||||
Agent,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct Favorite {
|
||||
|
||||
@@ -42,6 +42,7 @@ use windmill_common::jobs::{
|
||||
use windmill_common::jobs::{
|
||||
InlineScriptTarget, RunInlinePreviewScriptFnParams, RunInlineScriptFnParams,
|
||||
};
|
||||
use windmill_common::roles::AuthorLevel;
|
||||
use windmill_common::runnable_settings::{
|
||||
ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings,
|
||||
};
|
||||
@@ -115,7 +116,9 @@ use windmill_common::{
|
||||
add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue,
|
||||
Suspend,
|
||||
},
|
||||
jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode},
|
||||
jobs::{
|
||||
script_path_to_payload, CompletedJob, JobKind, JobPayload, OnBehalfOf, QueuedJob, RawCode,
|
||||
},
|
||||
oauth2::HmacSha256,
|
||||
query_builders,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
@@ -6736,6 +6739,12 @@ struct Preview {
|
||||
/// preview job resolves those imports from not-yet-deployed local content
|
||||
/// (uploaded to raw_script_temp) instead of the deployed script.
|
||||
temp_script_refs: Option<HashMap<String, String>>,
|
||||
/// Run the preview as this identity (an email or a `u/`/`g/` principal) instead of the
|
||||
/// caller. See `preview_on_behalf_of`.
|
||||
on_behalf_of: Option<String>,
|
||||
/// Data table and object storage access as the caller rather than as the job's identity,
|
||||
/// as a full-code app's inline runnable flagged `viewer_data_access` does for its viewer.
|
||||
viewer_data_access: Option<bool>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "run_inline")]
|
||||
@@ -6768,6 +6777,8 @@ struct PreviewFlow {
|
||||
/// each flow step so inline-script relative imports resolve from
|
||||
/// not-yet-deployed local content instead of the deployed script.
|
||||
temp_script_refs: Option<HashMap<String, String>>,
|
||||
/// See `Preview::on_behalf_of`.
|
||||
on_behalf_of: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -7048,6 +7059,7 @@ async fn batch_rerun_handle_job(
|
||||
)
|
||||
.await;
|
||||
if let Ok((uuid, _, _, _)) = result {
|
||||
resolve_batch_rerun_source(authed, db, user_db, w_id, job.id, uuid).await;
|
||||
return Ok(uuid.to_string());
|
||||
}
|
||||
}
|
||||
@@ -7081,6 +7093,7 @@ async fn batch_rerun_handle_job(
|
||||
.map(|r| r.0)
|
||||
};
|
||||
if let Ok(uuid) = result {
|
||||
resolve_batch_rerun_source(authed, db, user_db, w_id, job.id, uuid).await;
|
||||
return Ok(uuid.to_string());
|
||||
}
|
||||
}
|
||||
@@ -7091,6 +7104,68 @@ async fn batch_rerun_handle_job(
|
||||
))
|
||||
}
|
||||
|
||||
/// Marks a failed or canceled source of a batch re-run as handled, with the re-run as its note.
|
||||
/// This is the only moment the link exists (the re-run is pushed with no back-pointer), and it
|
||||
/// is what lets "hide resolved" exclude already re-run jobs from the next batch. Never fails the
|
||||
/// re-run, which is already queued: a source that cannot be resolved is logged and left as is.
|
||||
async fn resolve_batch_rerun_source(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
source: Uuid,
|
||||
rerun: Uuid,
|
||||
) {
|
||||
if !role_can_code(db, w_id, authed).await.unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let result: error::Result<()> = async {
|
||||
let mut tx = user_db.clone().begin(authed).await?;
|
||||
let tags = get_scope_tags(authed);
|
||||
let (resolved_by, _) = resolution_attribution(authed, None);
|
||||
// Same authorization and eligibility as `resolve_completed_jobs`. DO NOTHING keeps a
|
||||
// resolution someone already explained.
|
||||
let resolved = sqlx::query_scalar!(
|
||||
"INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)
|
||||
SELECT c.id, c.workspace_id, $4, $5, false
|
||||
FROM v2_job_completed c
|
||||
JOIN v2_job j ON j.id = c.id
|
||||
WHERE c.id = $1
|
||||
AND c.workspace_id = $2
|
||||
AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))
|
||||
AND c.status IN ('failure', 'canceled')
|
||||
AND j.flow_step_id IS NULL
|
||||
ON CONFLICT (job_id) DO NOTHING
|
||||
RETURNING job_id",
|
||||
source,
|
||||
w_id,
|
||||
tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>,
|
||||
resolved_by,
|
||||
format!("Re-run as {rerun}"),
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if resolved.is_some() {
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
authed,
|
||||
"jobs.resolve",
|
||||
ActionKind::Update,
|
||||
w_id,
|
||||
Some(&source.to_string()),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
tracing::error!("Could not resolve job {source} after re-running it as {rerun}: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_flow_by_path(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
@@ -8916,20 +8991,138 @@ async fn operator_may_run_datatable_query(
|
||||
.unwrap_or(false))
|
||||
}
|
||||
|
||||
/// The refusal an operator gets from a preview route. Inside a job the caller never ran a
|
||||
/// preview themselves, so name the one thing the job's token may do.
|
||||
/// Whether the caller's role may run code of its own: previews, dependency jobs, inline dynamic
|
||||
/// selects, resolving jobs.
|
||||
async fn role_can_code(db: &DB, w_id: &str, authed: &ApiAuthed) -> error::Result<bool> {
|
||||
Ok(
|
||||
windmill_common::roles::caller_capabilities(db, w_id, authed)
|
||||
.await?
|
||||
.code,
|
||||
)
|
||||
}
|
||||
|
||||
/// The refusal a member whose role cannot code gets from a preview route. Inside a job the caller
|
||||
/// never ran a preview themselves, so name the one thing the job's token may do.
|
||||
fn operator_preview_refusal(job_id: Option<Uuid>) -> error::Error {
|
||||
let reason = if job_id.is_some() {
|
||||
"Operators cannot run preview jobs for security reasons: from inside a job, an \
|
||||
operator may only run a wmill.datatable() query while that job is running"
|
||||
"Your role cannot run preview jobs in this workspace: from inside a job, a member \
|
||||
whose role cannot code may only run a wmill.datatable() query while that job is running"
|
||||
} else {
|
||||
"Operators cannot run preview jobs for security reasons"
|
||||
"Your role cannot run preview jobs in this workspace"
|
||||
};
|
||||
// 403, not 401: the frontend reads a 401 as a dead session and logs the user out, and the
|
||||
// flow editor builders open reaches this route.
|
||||
error::Error::PermissionDenied(reason.to_string())
|
||||
}
|
||||
|
||||
/// The identity a preview asks to run as, or `None` when it runs as the caller.
|
||||
///
|
||||
/// A preview is request-supplied code, so running it as someone else is impersonation, not
|
||||
/// "running their runnable": only those who may set a runnable's `on_behalf_of` on deploy
|
||||
/// (admins and `wm_deployers`) may ask for it. Naming oneself is always allowed. Operators are
|
||||
/// refused even in `wm_deployers`: the datatable-query exemption lets them reach this route
|
||||
/// without being able to deploy anything.
|
||||
pub(crate) async fn preview_on_behalf_of(
|
||||
authed: &ApiAuthed,
|
||||
requested: Option<&str>,
|
||||
w_id: &str,
|
||||
db: &DB,
|
||||
) -> error::Result<Option<OnBehalfOf>> {
|
||||
let Some(requested) = requested.filter(|s| !s.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let own = username_to_permissioned_as(&authed.username);
|
||||
if requested == own || requested == authed.email {
|
||||
return Ok(None);
|
||||
}
|
||||
if !role_can_code(db, w_id, authed).await?
|
||||
|| !windmill_common::can_preserve_on_behalf_of(authed)
|
||||
{
|
||||
return Err(Error::PermissionDenied(
|
||||
"Only admins and members of wm_deployers can run a preview on behalf of another user"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
let permissioned_as =
|
||||
windmill_common::resolve_on_behalf_of(None, Some(requested), true, authed, w_id, db)
|
||||
.await?
|
||||
.unwrap_or(own.clone());
|
||||
if permissioned_as == own {
|
||||
return Ok(None);
|
||||
}
|
||||
let email =
|
||||
windmill_common::users::get_email_from_permissioned_as_uncached(&permissioned_as, w_id, db)
|
||||
.await?;
|
||||
Ok(Some(OnBehalfOf { email, permissioned_as }))
|
||||
}
|
||||
|
||||
/// Audits the impersonation and opens the job's own logs with it, so whoever reads the run sees
|
||||
/// who it ran as and who asked, not only the identity columns.
|
||||
pub(crate) async fn audit_preview_on_behalf_of(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
authed: &ApiAuthed,
|
||||
w_id: &str,
|
||||
job_id: Uuid,
|
||||
path: Option<&str>,
|
||||
obo: &OnBehalfOf,
|
||||
) -> error::Result<()> {
|
||||
let mut params = HashMap::from([("on_behalf_of", obo.permissioned_as.as_str())]);
|
||||
if let Some(path) = path {
|
||||
params.insert("path", path);
|
||||
}
|
||||
audit_log(
|
||||
&mut **tx,
|
||||
authed,
|
||||
"jobs.run.preview_on_behalf_of",
|
||||
ActionKind::Execute,
|
||||
w_id,
|
||||
Some(&job_id.to_string()),
|
||||
Some(params),
|
||||
)
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)",
|
||||
format!(
|
||||
"Preview run on behalf of {} ({}), requested by {}\n",
|
||||
obo.permissioned_as, obo.email, authed.email
|
||||
),
|
||||
job_id,
|
||||
w_id,
|
||||
)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The push identity and isolation level for a preview: the caller's own, or the requested
|
||||
/// `on_behalf_of`'s, pushed through the root pool like a deployed `on_behalf_of` run.
|
||||
fn preview_push_identity(
|
||||
authed: &ApiAuthed,
|
||||
user_db: &UserDB,
|
||||
db: &DB,
|
||||
obo: Option<&OnBehalfOf>,
|
||||
) -> (
|
||||
String,
|
||||
String,
|
||||
Option<windmill_common::db::Authed>,
|
||||
PushIsolationLevel<'static>,
|
||||
) {
|
||||
match obo {
|
||||
Some(obo) => (
|
||||
obo.email.clone(),
|
||||
obo.permissioned_as.clone(),
|
||||
None,
|
||||
PushIsolationLevel::IsolatedRoot(db.clone()),
|
||||
),
|
||||
None => (
|
||||
authed.email.clone(),
|
||||
username_to_permissioned_as(&authed.username),
|
||||
Some(authed.clone().into()),
|
||||
PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_preview_script(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
@@ -8940,7 +9133,7 @@ async fn run_preview_script(
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
if authed.is_operator {
|
||||
if !role_can_code(&db, &w_id, &authed).await? {
|
||||
// A deferred run would outlive the running job the exemption keys off.
|
||||
if run_query.get_scheduled_for(&db).await?.is_some()
|
||||
|| !operator_may_run_datatable_query(
|
||||
@@ -8964,7 +9157,13 @@ async fn run_preview_script(
|
||||
require_path_read_access_for_preview(&authed, &preview.path)?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let tag = run_query.tag.clone().or(preview.tag.clone());
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
|
||||
let obo = preview_on_behalf_of(&authed, preview.on_behalf_of.as_deref(), &w_id, &db).await?;
|
||||
let (email, permissioned_as, push_authed, tx) =
|
||||
preview_push_identity(&authed, &user_db, &db, obo.as_ref());
|
||||
let audited_path = preview.path.clone();
|
||||
// A job token's end user is whoever ran the calling job, who never consented to that job's
|
||||
// code reading data as them: only a direct caller, reading as themselves, may ask for it.
|
||||
let viewer_data_access = authed.job_id.is_none() && preview.viewer_data_access.unwrap_or(false);
|
||||
|
||||
let preview_args = preview.args.unwrap_or_default();
|
||||
let mut extra = HashMap::new();
|
||||
@@ -9010,8 +9209,8 @@ async fn run_preview_script(
|
||||
},
|
||||
push_args,
|
||||
authed.display_username(),
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
&email,
|
||||
permissioned_as,
|
||||
authed.token_prefix.as_deref(),
|
||||
authed.username_override.as_deref(),
|
||||
scheduled_for,
|
||||
@@ -9028,7 +9227,7 @@ async fn run_preview_script(
|
||||
run_query.timeout,
|
||||
None,
|
||||
None,
|
||||
Some(&authed.clone().into()),
|
||||
push_authed.as_ref(),
|
||||
false,
|
||||
end_user_email,
|
||||
authed.trigger_or_fallback(None),
|
||||
@@ -9036,6 +9235,19 @@ async fn run_preview_script(
|
||||
scope_ceiling.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let mut tx = tx;
|
||||
if let Some(obo) = obo.as_ref() {
|
||||
audit_preview_on_behalf_of(&mut tx, &authed, &w_id, uuid, audited_path.as_deref(), obo)
|
||||
.await?;
|
||||
}
|
||||
if viewer_data_access {
|
||||
sqlx::query!(
|
||||
"UPDATE job_perms SET viewer_data_access = true WHERE job_id = $1",
|
||||
uuid
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
@@ -9052,7 +9264,7 @@ async fn run_inline_preview_script(
|
||||
// Same arbitrary-code class as run_preview_script, and every worker and standalone
|
||||
// server exposes this route, so an operator is refused on the same terms. A
|
||||
// narrowly-scoped token must not escape its scope through inline preview either.
|
||||
if authed.is_operator
|
||||
if !role_can_code(&db, &w_id, &authed).await?
|
||||
&& !operator_may_run_datatable_query(
|
||||
&db,
|
||||
&w_id,
|
||||
@@ -9066,6 +9278,7 @@ async fn run_inline_preview_script(
|
||||
return Err(operator_preview_refusal(job_id));
|
||||
}
|
||||
check_scopes(&authed, || format!("jobs:run"))?;
|
||||
windmill_queue::tags::check_inline_run_allowed(&w_id, &db).await?;
|
||||
if let Some(job_id) = job_id {
|
||||
register_potential_assets_on_inline_execution(job_id, &w_id, &preview);
|
||||
}
|
||||
@@ -9084,7 +9297,6 @@ async fn run_inline_preview_script(
|
||||
end_user_email,
|
||||
viewer_data_access,
|
||||
lang: preview.language,
|
||||
job_dir: "".to_string(),
|
||||
worker_name: "".to_string(),
|
||||
worker_dir: "".to_string(),
|
||||
client: AuthedClient {
|
||||
@@ -9186,6 +9398,7 @@ async fn run_inline_script_inner(
|
||||
args: Option<HashMap<String, Box<JsonRawValue>>>,
|
||||
user_db: Option<UserDB>,
|
||||
) -> error::Result<Response> {
|
||||
windmill_queue::tags::check_inline_run_allowed(&w_id, &db).await?;
|
||||
// An inline run executes with the caller's own token, so it cannot honour a script's
|
||||
// `job_token_scopes`: such a script only runs as a job. A path is resolved here, once, and
|
||||
// the version checked is the version run.
|
||||
@@ -9235,7 +9448,6 @@ async fn run_inline_script_inner(
|
||||
permissioned_as: username_to_permissioned_as(&authed.username),
|
||||
permissioned_as_email: authed.email.clone(),
|
||||
end_user_email,
|
||||
job_dir: "".to_string(),
|
||||
worker_name: "".to_string(),
|
||||
worker_dir: "".to_string(),
|
||||
client: AuthedClient {
|
||||
@@ -9358,10 +9570,8 @@ async fn run_bundle_preview_script(
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
mut multipart: axum::extract::Multipart,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
if authed.is_operator {
|
||||
return Err(error::Error::NotAuthorized(
|
||||
"Operators cannot run preview jobs for security reasons".to_string(),
|
||||
));
|
||||
if !role_can_code(&db, &w_id, &authed).await? {
|
||||
return Err(operator_preview_refusal(None));
|
||||
}
|
||||
// Bundle preview runs arbitrary, request-supplied code; require the broad jobs:run
|
||||
// scope so a narrowly-scoped token cannot escape its scope. See run_preview_script.
|
||||
@@ -9387,7 +9597,11 @@ async fn run_bundle_preview_script(
|
||||
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let tag = run_query.tag.clone().or(preview.tag.clone());
|
||||
let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
|
||||
let obo =
|
||||
preview_on_behalf_of(&authed, preview.on_behalf_of.as_deref(), &w_id, &db).await?;
|
||||
let (email, permissioned_as, push_authed, ltx) =
|
||||
preview_push_identity(&authed, &user_db, &db, obo.as_ref());
|
||||
let audited_path = preview.path.clone();
|
||||
|
||||
let args = preview.args.unwrap_or_default();
|
||||
|
||||
@@ -9435,8 +9649,8 @@ async fn run_bundle_preview_script(
|
||||
}),
|
||||
push_args,
|
||||
authed.display_username(),
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
&email,
|
||||
permissioned_as,
|
||||
authed.token_prefix.as_deref(),
|
||||
authed.username_override.as_deref(),
|
||||
scheduled_for,
|
||||
@@ -9453,7 +9667,7 @@ async fn run_bundle_preview_script(
|
||||
run_query.timeout,
|
||||
None,
|
||||
None,
|
||||
Some(&authed.clone().into()),
|
||||
push_authed.as_ref(),
|
||||
false,
|
||||
end_user_email,
|
||||
authed.trigger_or_fallback(None),
|
||||
@@ -9461,6 +9675,18 @@ async fn run_bundle_preview_script(
|
||||
scope_ceiling.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let mut ntx = ntx;
|
||||
if let Some(obo) = obo.as_ref() {
|
||||
audit_preview_on_behalf_of(
|
||||
&mut ntx,
|
||||
&authed,
|
||||
&w_id,
|
||||
uuid,
|
||||
audited_path.as_deref(),
|
||||
obo,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
job_id = Some(uuid);
|
||||
tx = Some(ntx);
|
||||
}
|
||||
@@ -9552,9 +9778,9 @@ async fn push_dependencies_job(
|
||||
req: RunDependenciesRequest,
|
||||
) -> error::Result<Uuid> {
|
||||
check_dependencies_scope(authed, &req.entrypoint)?;
|
||||
if authed.is_operator {
|
||||
return Err(error::Error::NotAuthorized(
|
||||
"Operators cannot run dependencies jobs for security reasons".to_string(),
|
||||
if !role_can_code(db, w_id, authed).await? {
|
||||
return Err(error::Error::PermissionDenied(
|
||||
"Your role cannot run dependencies jobs in this workspace".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -9675,10 +9901,10 @@ async fn push_flow_dependencies_job(
|
||||
req: RunFlowDependenciesRequest,
|
||||
) -> error::Result<Uuid> {
|
||||
check_dependencies_scope(authed, &req.path)?;
|
||||
check_operator_can_build(
|
||||
let flows_level = check_operator_can_build(
|
||||
db,
|
||||
w_id,
|
||||
authed.is_operator,
|
||||
authed,
|
||||
BuilderKind::Flows,
|
||||
"run dependencies jobs",
|
||||
)
|
||||
@@ -9686,7 +9912,7 @@ async fn push_flow_dependencies_job(
|
||||
// The dependency job locks whatever inline code this request carries, on a worker. A
|
||||
// composition-only flow has none, so validating here costs a builder nothing and keeps the
|
||||
// lock step from becoming the way to run code the write path refuses.
|
||||
if authed.is_operator {
|
||||
if flows_level == AuthorLevel::Compose {
|
||||
validate_operator_composed_flow(&req.flow_value, &None, authed, db, user_db, w_id).await?;
|
||||
// The job rewrites bookkeeping stored under `req.path` (dependency map, asset usages,
|
||||
// lock error) even with `skip_flow_update`, so a builder must be able to write there.
|
||||
@@ -9971,17 +10197,22 @@ async fn add_batch_jobs(
|
||||
|
||||
let language = language.unwrap_or(ScriptLang::Deno);
|
||||
|
||||
let tag = if let Some(dedicated_worker) = dedicated_worker {
|
||||
let tag = match (dedicated_worker, batch_info.tag) {
|
||||
// Same rule as `push`: a dedicated worker would run it with its own unscoped token.
|
||||
if dedicated_worker && path.is_some() && job_token_scopes.is_none() {
|
||||
(Some(true), _) if path.is_some() && job_token_scopes.is_none() => {
|
||||
windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
|
||||
} else {
|
||||
format!("{}", language.as_str())
|
||||
}
|
||||
} else if let Some(tag) = batch_info.tag {
|
||||
tag
|
||||
} else {
|
||||
format!("{}", language.as_str())
|
||||
(None, Some(tag)) => {
|
||||
windmill_queue::tags::apply_dev_workspace_suffix(tag, &w_id, &db).await?
|
||||
}
|
||||
_ => {
|
||||
windmill_queue::tags::apply_dev_workspace_suffix(
|
||||
language.as_str().to_string(),
|
||||
&w_id,
|
||||
&db,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
@@ -10097,14 +10328,9 @@ async fn run_preview_flow_job(
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
Json(raw_flow): Json<PreviewFlow>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"run preview jobs",
|
||||
)
|
||||
.await?;
|
||||
let flows_level =
|
||||
check_operator_can_build(&db, &w_id, &authed, BuilderKind::Flows, "run preview jobs")
|
||||
.await?;
|
||||
// Flow preview runs an arbitrary, request-supplied flow definition; require the broad
|
||||
// jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script.
|
||||
check_scopes(&authed, || format!("jobs:run"))?;
|
||||
@@ -10118,7 +10344,7 @@ async fn run_preview_flow_job(
|
||||
}
|
||||
// A builder must be able to test what it composes, but the submitted value is not the stored
|
||||
// one: without this the preview is a way to run inline code the write path refuses.
|
||||
if authed.is_operator {
|
||||
if flows_level == AuthorLevel::Compose {
|
||||
validate_operator_composed_flow(
|
||||
&raw_flow.value,
|
||||
&raw_flow.tag,
|
||||
@@ -10138,7 +10364,9 @@ async fn run_preview_flow_job(
|
||||
}
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let tag = run_query.tag.clone().or(raw_flow.tag.clone());
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
|
||||
let obo = preview_on_behalf_of(&authed, raw_flow.on_behalf_of.as_deref(), &w_id, &db).await?;
|
||||
let (email, permissioned_as, push_authed, tx) =
|
||||
preview_push_identity(&authed, &user_db, &db, obo.as_ref());
|
||||
|
||||
let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false);
|
||||
let flow_path = raw_flow.path.clone().unwrap_or_default();
|
||||
@@ -10171,8 +10399,8 @@ async fn run_preview_flow_job(
|
||||
},
|
||||
PushArgs::from(&flow_args),
|
||||
authed.display_username(),
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
&email,
|
||||
permissioned_as,
|
||||
authed.token_prefix.as_deref(),
|
||||
authed.username_override.as_deref(),
|
||||
scheduled_for,
|
||||
@@ -10189,7 +10417,7 @@ async fn run_preview_flow_job(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&authed.clone().into()),
|
||||
push_authed.as_ref(),
|
||||
false,
|
||||
end_user_email,
|
||||
authed.trigger_or_fallback(None),
|
||||
@@ -10198,6 +10426,11 @@ async fn run_preview_flow_job(
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(obo) = obo.as_ref() {
|
||||
let audited_path = Some(flow_path.as_str()).filter(|p| !p.is_empty());
|
||||
audit_preview_on_behalf_of(&mut tx, &authed, &w_id, uuid, audited_path, obo).await?;
|
||||
}
|
||||
|
||||
// Set memory_id if provided (for agent memory)
|
||||
if let Some(memory_id) = run_query.memory_key(&w_id, &flow_path) {
|
||||
set_flow_memory_id(&mut tx, uuid, memory_id).await?;
|
||||
@@ -10263,7 +10496,7 @@ async fn run_dynamic_select(
|
||||
if matches!(
|
||||
request.runnable_ref,
|
||||
DynamicSelectRunnableRef::Inline { .. }
|
||||
) && authed.is_operator
|
||||
) && !role_can_code(&db, &w_id, &authed).await?
|
||||
{
|
||||
return Err(operator_preview_refusal(None));
|
||||
}
|
||||
@@ -12346,14 +12579,16 @@ fn resolution_attribution<'a>(
|
||||
(None, None)
|
||||
}
|
||||
|
||||
fn check_resolution_request(
|
||||
async fn check_resolution_request(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
w_id: &str,
|
||||
job_ids: &[Uuid],
|
||||
note: Option<&str>,
|
||||
) -> error::Result<()> {
|
||||
if authed.is_operator {
|
||||
return Err(error::Error::NotAuthorized(
|
||||
"Operators cannot resolve jobs".to_string(),
|
||||
if !role_can_code(db, w_id, authed).await? {
|
||||
return Err(error::Error::PermissionDenied(
|
||||
"Your role cannot resolve jobs in this workspace".to_string(),
|
||||
));
|
||||
}
|
||||
if job_ids.len() > MAX_RESOLUTION_BATCH {
|
||||
@@ -12373,23 +12608,24 @@ fn check_resolution_request(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Marks failed jobs as handled. Returns the ids actually affected: an id that is not
|
||||
/// visible to the caller, carries an out-of-scope tag, or did not fail is silently
|
||||
/// Marks failed or canceled jobs as handled. Returns the ids actually affected: an id that is
|
||||
/// not visible to the caller, carries an out-of-scope tag, or did not fail nor get canceled is silently
|
||||
/// absent rather than an error, so a bulk selection never fails as a whole.
|
||||
async fn resolve_completed_jobs(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(req): Json<ResolveJobsRequest>,
|
||||
) -> error::JsonResult<Vec<Uuid>> {
|
||||
check_resolution_request(&authed, &req.job_ids, req.note.as_deref())?;
|
||||
check_resolution_request(&authed, &db, &w_id, &req.job_ids, req.note.as_deref()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let tags = get_scope_tags(&authed);
|
||||
let (resolved_by, typed_note) = resolution_attribution(&authed, req.note.as_deref());
|
||||
let system_note = req.superseded_by.map(|_| SUPERSEDED_NOTE);
|
||||
// The join on v2_job is what authorizes this write: v2_job_completed has RLS
|
||||
// disabled, so v2_job's policies are the only thing scoping rows to the caller.
|
||||
// `status = 'failure'` keeps the invariant that only a failure can be resolved.
|
||||
// Only a failed or canceled run can be resolved: a success has nothing to handle.
|
||||
let resolved = sqlx::query_scalar!(
|
||||
"INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)
|
||||
SELECT c.id, c.workspace_id, $4, COALESCE($5, $7), false
|
||||
@@ -12398,7 +12634,7 @@ async fn resolve_completed_jobs(
|
||||
WHERE c.id = ANY($1)
|
||||
AND c.workspace_id = $2
|
||||
AND ($3::TEXT[] IS NULL OR j.tag = ANY($3))
|
||||
AND c.status = 'failure'
|
||||
AND c.status IN ('failure', 'canceled')
|
||||
-- Resolution is a top-level triage state: a step resolved on its own
|
||||
-- would render orange inside a flow whose status is still red.
|
||||
AND j.flow_step_id IS NULL
|
||||
@@ -12464,11 +12700,12 @@ async fn resolve_completed_jobs(
|
||||
|
||||
async fn unresolve_completed_jobs(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(req): Json<ResolveJobsRequest>,
|
||||
) -> error::JsonResult<Vec<Uuid>> {
|
||||
check_resolution_request(&authed, &req.job_ids, None)?;
|
||||
check_resolution_request(&authed, &db, &w_id, &req.job_ids, None).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let tags = get_scope_tags(&authed);
|
||||
let unresolved = sqlx::query_scalar!(
|
||||
|
||||
@@ -9,8 +9,6 @@
|
||||
use crate::db::ApiAuthed;
|
||||
#[cfg(feature = "embedding")]
|
||||
use crate::embeddings::load_embeddings_db;
|
||||
#[cfg(feature = "oauth2")]
|
||||
use crate::oauth2_oss::SlackVerifier;
|
||||
#[cfg(feature = "smtp")]
|
||||
use crate::smtp_server_oss::SmtpServer;
|
||||
use axum::middleware::from_fn_with_state;
|
||||
@@ -77,6 +75,7 @@ mod ai_sessions;
|
||||
#[cfg(feature = "parquet")]
|
||||
pub use ai_sessions::sweep_expired_ai_session_backups;
|
||||
mod ai_shared_artifacts;
|
||||
mod ai_shared_sessions;
|
||||
mod apps;
|
||||
mod apps_raw_bundle;
|
||||
pub use apps::invalidate_app_policy_cache;
|
||||
@@ -163,6 +162,9 @@ mod scim_oss;
|
||||
mod scripts;
|
||||
mod secret_backend_ext;
|
||||
mod service_logs;
|
||||
// Answering in Slack is only reached from the EE Slack event handlers.
|
||||
#[cfg_attr(not(all(feature = "oauth2", feature = "private")), allow(dead_code))]
|
||||
mod slack_agents;
|
||||
mod slack_approvals;
|
||||
#[cfg(all(feature = "smtp", feature = "private"))]
|
||||
pub mod smtp_server_ee;
|
||||
@@ -209,6 +211,7 @@ mod workspaces;
|
||||
#[cfg(feature = "private")]
|
||||
pub mod workspaces_ee;
|
||||
mod workspaces_export;
|
||||
mod workspace_settings_document;
|
||||
|
||||
#[cfg(feature = "mcp")]
|
||||
mod mcp_tools;
|
||||
@@ -245,10 +248,6 @@ pub use windmill_oauth::OAUTH_CLIENTS;
|
||||
|
||||
#[cfg(feature = "oauth2")]
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref SLACK_SIGNING_SECRET: Option<SlackVerifier> = std::env::var("SLACK_SIGNING_SECRET")
|
||||
.ok()
|
||||
.map(|x| SlackVerifier::new(x).unwrap());
|
||||
|
||||
// Comma-separated Slack v2 OAuth bot scopes requested when connecting a workspace.
|
||||
// Must be a subset of the bot scopes declared in the Slack app manifest. Default matches
|
||||
// Windmill's recommended manifest at docs.windmill.dev/docs/misc/setup_oauth.
|
||||
@@ -256,6 +255,11 @@ lazy_static::lazy_static! {
|
||||
.unwrap_or_else(|_| "commands,chat:write,chat:write.public,channels:join,files:write,app_mentions:read,im:history,im:read".to_string());
|
||||
}
|
||||
|
||||
/// Added to `SLACK_OAUTH_SCOPES` when a workspace connects with AI agents enabled: whether a
|
||||
/// message's sender is a person of the team and, for agents that run as their asker, their email,
|
||||
/// and the channel picker. Opt-in, as a Slack app must declare every scope it is asked for.
|
||||
pub const SLACK_AGENT_SCOPES: &[&str] = &["users:read", "users:read.email", "channels:read"];
|
||||
|
||||
// Compliance with cloud events spec.
|
||||
pub async fn add_webhook_allowed_origin(
|
||||
req: axum::extract::Request,
|
||||
@@ -566,14 +570,14 @@ pub async fn run_server(
|
||||
|
||||
// Setup MCP server
|
||||
#[allow(unused_variables)]
|
||||
let (mcp_router, gateway_mcp_router, mcp_cancellation_token) = {
|
||||
let (mcp_router, gateway_mcp_router, gateway_session_router, mcp_cancellation_token) = {
|
||||
#[cfg(feature = "mcp")]
|
||||
if server_mode || mcp_mode {
|
||||
use mcp::{
|
||||
add_www_authenticate_header, add_www_authenticate_header_gateway,
|
||||
extract_workspace_from_token, reject_token_query_param,
|
||||
};
|
||||
let (mcp_router, mcp_cancellation_token) = setup_mcp_server(
|
||||
let (mcp_router, session_router, mcp_cancellation_token) = setup_mcp_server(
|
||||
db.clone(),
|
||||
user_db,
|
||||
_base_internal_url.clone(),
|
||||
@@ -595,17 +599,32 @@ pub async fn run_server(
|
||||
.layer(axum::middleware::from_fn(
|
||||
add_www_authenticate_header_gateway,
|
||||
));
|
||||
// The workspace server. Its tools name their workspace; the token bounds which.
|
||||
let gateway_session_router = session_router
|
||||
.route_layer(from_extractor::<ApiAuthed>())
|
||||
.layer(axum::middleware::from_fn(extract_workspace_from_token))
|
||||
.layer(axum::middleware::from_fn(reject_token_query_param))
|
||||
.layer(axum::middleware::from_fn(
|
||||
add_www_authenticate_header_gateway,
|
||||
))
|
||||
.layer(Extension(mcp::McpEndpoint("/api/mcp")));
|
||||
(
|
||||
workspaced_mcp_router,
|
||||
gateway_mcp_router,
|
||||
gateway_session_router,
|
||||
Some(mcp_cancellation_token),
|
||||
)
|
||||
} else {
|
||||
(Router::new(), Router::new(), None)
|
||||
(Router::new(), Router::new(), Router::new(), None)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "mcp"))]
|
||||
(Router::new(), Router::new(), Option::<()>::None)
|
||||
(
|
||||
Router::new(),
|
||||
Router::new(),
|
||||
Router::new(),
|
||||
Option::<()>::None,
|
||||
)
|
||||
};
|
||||
|
||||
// Workers block on this before pulling their first job, so it is released ahead of
|
||||
@@ -618,15 +637,15 @@ pub async fn run_server(
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let mcp_list_tools_service = {
|
||||
let (mcp_list_tools_service, mcp_toolset_service) = {
|
||||
#[cfg(feature = "mcp")]
|
||||
{
|
||||
mcp::list_tools_service()
|
||||
(mcp::list_tools_service(), mcp::toolset_service())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "mcp"))]
|
||||
{
|
||||
Router::new()
|
||||
(Router::new(), Router::new())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -680,6 +699,7 @@ pub async fn run_server(
|
||||
.nest("/embeddings", embeddings::workspaced_service())
|
||||
.nest("/favorites", favorite::workspaced_service())
|
||||
.nest("/flows", flows::workspaced_service())
|
||||
.nest("/slack_agents", slack_agents::workspaced_service())
|
||||
.nest("/runnables", runnables::workspaced_service())
|
||||
.nest(
|
||||
"/workspace_dependencies",
|
||||
@@ -843,6 +863,7 @@ pub async fn run_server(
|
||||
.nest("/docs", docs::global_service())
|
||||
.nest("/indexer", indexer_oss::management_service())
|
||||
.nest("/mcp/w/{workspace_id}/list_tools", mcp_list_tools_service)
|
||||
.nest("/mcp/toolset", mcp_toolset_service)
|
||||
.nest("/db_health", db_health::global_service())
|
||||
.nest("/health/detailed", health::detailed_service())
|
||||
.nest(
|
||||
@@ -923,6 +944,10 @@ pub async fn run_server(
|
||||
.layer(from_extractor::<OptAuthed>())
|
||||
.layer(cors.clone()),
|
||||
)
|
||||
.nest(
|
||||
"/w/{workspace_id}/ai_u/shared_sessions",
|
||||
ai_shared_sessions::workspaced_unauthed_service(),
|
||||
)
|
||||
.layer(from_extractor::<OptAuthed>())
|
||||
// Deprecated, here for backwards compatibility: user should use /mcp/w/{workspace_id}/mcp instead
|
||||
.nest(
|
||||
@@ -934,6 +959,7 @@ pub async fn run_server(
|
||||
mcp_router.clone().layer(mcp_cors.clone()),
|
||||
)
|
||||
.nest("/mcp/gateway", gateway_mcp_router.layer(mcp_cors.clone()))
|
||||
.merge(gateway_session_router.layer(mcp_cors.clone()))
|
||||
.nest("/agent_workers", {
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
{
|
||||
@@ -1171,6 +1197,16 @@ pub async fn run_server(
|
||||
get(|| async { axum::http::StatusCode::NOT_FOUND })
|
||||
}
|
||||
})
|
||||
.route("/.well-known/oauth-protected-resource/api/mcp", {
|
||||
#[cfg(feature = "mcp")]
|
||||
{
|
||||
get(mcp::oauth_server::gateway_session_protected_resource_metadata)
|
||||
}
|
||||
#[cfg(not(feature = "mcp"))]
|
||||
{
|
||||
get(|| async { axum::http::StatusCode::NOT_FOUND })
|
||||
}
|
||||
})
|
||||
// JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix)
|
||||
.route("/.well-known/jwks.json", {
|
||||
#[cfg(all(feature = "private", feature = "enterprise", feature = "openidconnect"))]
|
||||
@@ -1244,6 +1280,9 @@ pub async fn run_server(
|
||||
if let Some(name) = name.as_ref() {
|
||||
tracing::info!("server starting for name={name}");
|
||||
}
|
||||
#[cfg(feature = "mcp")]
|
||||
mcp::register_router(&_base_internal_url, Some(app.clone()));
|
||||
|
||||
let listener = listener.tap_io(move |tcp_stream| {
|
||||
let _ = tcp_stream.set_nodelay(!server_mode);
|
||||
});
|
||||
@@ -1281,7 +1320,10 @@ pub async fn run_server(
|
||||
}
|
||||
});
|
||||
|
||||
server.await?;
|
||||
let served = server.await;
|
||||
#[cfg(feature = "mcp")]
|
||||
mcp::register_router(&_base_internal_url, None);
|
||||
served?;
|
||||
let forks = &windmill_api_workspaces::workspaces::BACKGROUND_FORKS;
|
||||
forks.close();
|
||||
if !forks.is_empty() {
|
||||
|
||||
@@ -56,6 +56,46 @@ pub fn all_tools() -> Vec<EndpointTool> {
|
||||
query_field_renames: None,
|
||||
body_field_renames: None,
|
||||
},
|
||||
EndpointTool {
|
||||
name: Cow::Borrowed("getWorkspaceSettingsDocument"),
|
||||
description: Cow::Borrowed("get the workspace settings document: The workspace's settings in the layout of the settings.yaml that `wmill sync` writes. Admin only"),
|
||||
instructions: Cow::Borrowed(""),
|
||||
path: Cow::Borrowed("/w/{workspace}/workspaces/settings_document"),
|
||||
method: Cow::Borrowed("GET"),
|
||||
path_params_schema: None,
|
||||
query_params_schema: None,
|
||||
body_schema: None,
|
||||
query_field_renames: None,
|
||||
body_field_renames: None,
|
||||
},
|
||||
EndpointTool {
|
||||
name: Cow::Borrowed("applyWorkspaceSettingsDocument"),
|
||||
description: Cow::Borrowed("apply a workspace settings document: Applies a settings document, each setting through the endpoint that owns it. Admin only"),
|
||||
instructions: Cow::Borrowed("Send only the top-level keys to change, each whole as getWorkspaceSettingsDocument returns it: a key the document omits is left as it is, and a key set to null clears it. Keep `__SENSITIVE_AND_UNCHANGED__` for a secret you were not given. Call with dry_run true first, show the user the changes and apply only once they agree."),
|
||||
path: Cow::Borrowed("/w/{workspace}/workspaces/settings_document"),
|
||||
method: Cow::Borrowed("POST"),
|
||||
path_params_schema: None,
|
||||
query_params_schema: None,
|
||||
body_schema: Some(serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"document": {
|
||||
"type": "object",
|
||||
"description": "The settings to apply, in the layout of the document. A secret given as `__SENSITIVE_AND_UNCHANGED__` keeps its stored value; one prefixed with `$encrypted:` is decrypted with the workspace key."
|
||||
},
|
||||
"dry_run": {
|
||||
"description": "Only compute the changes, without applying them.",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"document"
|
||||
],
|
||||
"minProperties": 1
|
||||
})),
|
||||
query_field_renames: None,
|
||||
body_field_renames: None,
|
||||
},
|
||||
EndpointTool {
|
||||
name: Cow::Borrowed("listDataMetrics"),
|
||||
description: Cow::Borrowed("list declared measures and dimensions on DuckLake tables: Call this before writing any aggregate query over a DuckLake table. A declared measure is the canonical definition of that number, and reproducing it yourself will silently disagree with it (a `revenue` measure typically excludes refunds or test rows). Filter by `table` for one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for everything declared under a folder; omit both to browse the whole catalog. Results are keyset-paged: a full page may mean more remain, so continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates can share one GROUP BY. If a number you need has no declared measure, write your own aggregate as usual. Results are limited to declarations whose producing script the caller can read"),
|
||||
@@ -559,6 +599,10 @@ pub fn all_tools() -> Vec<EndpointTool> {
|
||||
"include_draft_only": {
|
||||
"type": "boolean",
|
||||
"description": "When true, append per-user draft resources whose path has\nno deployed resource. Synthesized rows carry\n`draft_only: true`.\n"
|
||||
},
|
||||
"starred_only": {
|
||||
"type": "boolean",
|
||||
"description": "Only the `ai_agent` resources the caller starred"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
@@ -1296,7 +1340,7 @@ is, a different one moves it there and archives the old path"),
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n"
|
||||
"description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1413,7 +1457,7 @@ is, a different one moves it there and archives the old path"),
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n"
|
||||
"description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1802,7 +1846,7 @@ is, a different one moves it there and archives the old path"),
|
||||
},
|
||||
"resolved": {
|
||||
"type": "boolean",
|
||||
"description": "filter on whether a failure has been marked as handled. true keeps only resolved failures, false hides them"
|
||||
"description": "filter on whether a failed or canceled run has been marked as handled. true keeps only resolved runs, false hides them"
|
||||
},
|
||||
"is_flow_step": {
|
||||
"type": "boolean",
|
||||
|
||||
@@ -13,7 +13,8 @@ use windmill_mcp::common::types::{
|
||||
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo,
|
||||
};
|
||||
use windmill_mcp::server::{
|
||||
BackendResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter,
|
||||
BackendResult, CallToolResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter,
|
||||
SessionAccess, Tool,
|
||||
};
|
||||
|
||||
use crate::auth::AuthCache;
|
||||
@@ -23,10 +24,11 @@ use crate::jobs::{
|
||||
};
|
||||
|
||||
use super::auto_generated_endpoints::all_tools;
|
||||
use super::session;
|
||||
use super::utils::{
|
||||
build_query_string, build_request_body, create_http_request, get_hub_script_schema,
|
||||
get_item_schema, get_items, get_resources, get_resources_types, get_scripts_from_hub,
|
||||
parse_response_body, prepare_push_args, substitute_path_params,
|
||||
get_item_paths, get_item_schema, get_items, get_resources, get_resources_types,
|
||||
get_scripts_from_hub, parse_response_body, prepare_push_args, substitute_path_params,
|
||||
};
|
||||
|
||||
use std::sync::Arc;
|
||||
@@ -34,8 +36,8 @@ use std::time::Duration;
|
||||
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use windmill_mcp::server::{
|
||||
LocalSessionManager, McpToken, MultiWorkspaceMcp, Runner, StreamableHttpServerConfig,
|
||||
StreamableHttpService,
|
||||
LocalSessionManager, McpToken, MultiWorkspaceMcp, NoEndpointTools, Runner, SessionRunner,
|
||||
StreamableHttpServerConfig, StreamableHttpService,
|
||||
};
|
||||
use windmill_mcp::WorkspaceId;
|
||||
|
||||
@@ -64,6 +66,32 @@ pub struct WindmillBackend {
|
||||
}
|
||||
|
||||
impl WindmillBackend {
|
||||
/// The instructions only read, so they are built at the token's own read-only flag.
|
||||
fn session_ctx<'a>(&'a self, auth: &'a ApiAuthed, w_id: &'a str) -> session::Ctx<'a> {
|
||||
let access = if auth.read_only {
|
||||
SessionAccess::ReadOnly
|
||||
} else {
|
||||
SessionAccess::Full
|
||||
};
|
||||
self.session_ctx_with(auth, w_id, access)
|
||||
}
|
||||
|
||||
fn session_ctx_with<'a>(
|
||||
&'a self,
|
||||
auth: &'a ApiAuthed,
|
||||
w_id: &'a str,
|
||||
access: SessionAccess,
|
||||
) -> session::Ctx<'a> {
|
||||
session::Ctx {
|
||||
auth,
|
||||
w_id,
|
||||
base_internal_url: &self.base_internal_url,
|
||||
access,
|
||||
origin: None,
|
||||
jwt: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(
|
||||
db: DB,
|
||||
user_db: UserDB,
|
||||
@@ -118,6 +146,27 @@ impl McpBackend for WindmillBackend {
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))
|
||||
}
|
||||
|
||||
async fn list_runnable_paths(
|
||||
&self,
|
||||
auth: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
item_type: &str,
|
||||
favorites_only: bool,
|
||||
path_filter: Option<PathFilter<'_>>,
|
||||
) -> BackendResult<Vec<String>> {
|
||||
let scope_type = if favorites_only { "favorites" } else { "all" };
|
||||
get_item_paths(
|
||||
&self.user_db,
|
||||
auth,
|
||||
workspace_id,
|
||||
scope_type,
|
||||
item_type,
|
||||
path_filter,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))
|
||||
}
|
||||
|
||||
async fn list_resource_types(
|
||||
&self,
|
||||
auth: &ApiAuthed,
|
||||
@@ -221,6 +270,7 @@ impl McpBackend for WindmillBackend {
|
||||
args: Value,
|
||||
request: &McpRequest<'_>,
|
||||
) -> BackendResult<Value> {
|
||||
windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "script");
|
||||
let push_args = prepare_push_args(&self.db, workspace_id, path, false, args, request)
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
|
||||
@@ -248,6 +298,7 @@ impl McpBackend for WindmillBackend {
|
||||
args: Value,
|
||||
request: &McpRequest<'_>,
|
||||
) -> BackendResult<Value> {
|
||||
windmill_common::feature_usage::log_feature_usage("mcp", "runnable_tool_call", "flow");
|
||||
let push_args = prepare_push_args(&self.db, workspace_id, path, true, args, request)
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
|
||||
@@ -274,6 +325,12 @@ impl McpBackend for WindmillBackend {
|
||||
endpoint_tool: &EndpointTool,
|
||||
args: Value,
|
||||
) -> BackendResult<Value> {
|
||||
// Whether the generated endpoint tools are used at all, and which ones.
|
||||
windmill_common::feature_usage::log_feature_usage(
|
||||
"mcp",
|
||||
"endpoint_tool_call",
|
||||
&endpoint_tool.name,
|
||||
);
|
||||
let args_map = match &args {
|
||||
Value::Object(map) => map,
|
||||
_ => {
|
||||
@@ -400,8 +457,57 @@ impl McpBackend for WindmillBackend {
|
||||
fn all_endpoint_tools(&self) -> Vec<EndpointTool> {
|
||||
all_tools()
|
||||
}
|
||||
|
||||
async fn session_instructions(
|
||||
&self,
|
||||
auth: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
) -> BackendResult<String> {
|
||||
Ok(session::session_instructions(&self.session_ctx(auth, workspace_id)).await)
|
||||
}
|
||||
|
||||
fn generic_session_instructions(&self) -> String {
|
||||
session::generic_session_instructions()
|
||||
}
|
||||
|
||||
async fn workspace_context(&self, auth: &ApiAuthed, workspace_id: &str) -> String {
|
||||
session::workspace_context(&self.session_ctx(auth, workspace_id)).await
|
||||
}
|
||||
|
||||
fn session_tools(&self, access: SessionAccess) -> Vec<Tool> {
|
||||
session::session_tools(access)
|
||||
}
|
||||
|
||||
async fn call_session_tool(
|
||||
&self,
|
||||
auth: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
name: &str,
|
||||
args: Value,
|
||||
access: SessionAccess,
|
||||
origin: Option<&str>,
|
||||
) -> BackendResult<CallToolResult> {
|
||||
let ctx = session::Ctx { origin, ..self.session_ctx_with(auth, workspace_id, access) };
|
||||
session::call_session_tool(&ctx, name, args).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Set on a request whose token is scoped to a selection of scripts, flows and endpoints
|
||||
/// rather than `mcp:all`: on `/api/mcp`, the tools server answers it.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct McpSelection;
|
||||
|
||||
/// MCP scopes without `mcp:all`: favorites, chosen scripts, flows, endpoints or hub apps.
|
||||
fn scoped_to_selection(scopes: &[String]) -> bool {
|
||||
let mut mcp = scopes.iter().filter(|scope| scope.starts_with("mcp:"));
|
||||
mcp.clone().next().is_some() && !mcp.any(|scope| scope == "mcp:all")
|
||||
}
|
||||
|
||||
/// The path of a workspace-less MCP URL, when it is not the tools server's
|
||||
/// `/api/mcp/gateway`: each URL is its own OAuth protected resource.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct McpEndpoint(pub &'static str);
|
||||
|
||||
/// Extract workspace ID from path and store it in request extensions
|
||||
pub async fn extract_and_store_workspace_id(
|
||||
Path(params): Path<String>,
|
||||
@@ -433,7 +539,6 @@ pub async fn add_www_authenticate_header(
|
||||
.body(axum::body::Body::from("Missing workspace_id in request"))
|
||||
.unwrap();
|
||||
};
|
||||
|
||||
let response = next.run(request).await;
|
||||
|
||||
// Only add header to 401 Unauthorized responses
|
||||
@@ -520,27 +625,32 @@ pub async fn extract_workspace_from_token(
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
{
|
||||
Ok(Some(row)) => match row.workspace_id {
|
||||
Some(workspace_id) => {
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(GatewayWorkspaceId(workspace_id.clone()));
|
||||
request.extensions_mut().insert(WorkspaceId(workspace_id));
|
||||
Ok(Some(row)) => {
|
||||
if scoped_to_selection(row.scopes.as_deref().unwrap_or_default()) {
|
||||
request.extensions_mut().insert(McpSelection);
|
||||
}
|
||||
None => {
|
||||
// Only enter multi-workspace mode for genuine MCP tokens; a
|
||||
// full-privilege global token without mcp scope is rejected
|
||||
// by the runner's mcp-scope check anyway.
|
||||
let is_mcp = row
|
||||
.scopes
|
||||
.as_deref()
|
||||
.is_some_and(|s| s.iter().any(|scope| scope.starts_with("mcp:")));
|
||||
if is_mcp {
|
||||
request.extensions_mut().insert(MultiWorkspaceMcp);
|
||||
request.extensions_mut().insert(McpToken(token));
|
||||
match row.workspace_id {
|
||||
Some(workspace_id) => {
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(GatewayWorkspaceId(workspace_id.clone()));
|
||||
request.extensions_mut().insert(WorkspaceId(workspace_id));
|
||||
}
|
||||
None => {
|
||||
// Only enter multi-workspace mode for genuine MCP tokens; a
|
||||
// full-privilege global token without mcp scope is rejected
|
||||
// by the runner's mcp-scope check anyway.
|
||||
let is_mcp = row
|
||||
.scopes
|
||||
.as_deref()
|
||||
.is_some_and(|s| s.iter().any(|scope| scope.starts_with("mcp:")));
|
||||
if is_mcp {
|
||||
request.extensions_mut().insert(MultiWorkspaceMcp);
|
||||
request.extensions_mut().insert(McpToken(token));
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
tracing::error!("Gateway token workspace lookup failed: {}", e);
|
||||
@@ -559,11 +669,16 @@ pub async fn add_www_authenticate_header_gateway(
|
||||
use axum::http::StatusCode;
|
||||
use windmill_common::BASE_URL;
|
||||
|
||||
let endpoint = request
|
||||
.extensions()
|
||||
.get::<McpEndpoint>()
|
||||
.map_or("/api/mcp/gateway", |e| e.0);
|
||||
|
||||
let response = next.run(request).await;
|
||||
|
||||
if response.status() == StatusCode::UNAUTHORIZED {
|
||||
let base_url = BASE_URL.load();
|
||||
let resource_url = format!("{}/api/mcp/gateway", base_url);
|
||||
let resource_url = format!("{}{}", base_url, endpoint);
|
||||
let www_authenticate = format!("Bearer resource=\"{}\"", resource_url);
|
||||
|
||||
let (mut parts, body) = response.into_parts();
|
||||
@@ -585,11 +700,12 @@ pub async fn setup_mcp_server(
|
||||
user_db: UserDB,
|
||||
base_internal_url: String,
|
||||
auth_cache: Arc<AuthCache>,
|
||||
) -> anyhow::Result<(Router, CancellationToken)> {
|
||||
) -> anyhow::Result<(Router, Router, CancellationToken)> {
|
||||
let cancellation_token = CancellationToken::new();
|
||||
let session_manager = Arc::new(LocalSessionManager::default());
|
||||
|
||||
let backend = WindmillBackend::new(db, user_db, base_internal_url, auth_cache);
|
||||
let session_runner = SessionRunner::new(backend.clone());
|
||||
let runner = Runner::new(backend);
|
||||
|
||||
let service_config = StreamableHttpServerConfig::default()
|
||||
@@ -611,11 +727,38 @@ pub async fn setup_mcp_server(
|
||||
// much smaller default, which would 413 them with no way to raise it.
|
||||
.with_max_request_body_bytes(*crate::REQUEST_SIZE_LIMIT.read().await);
|
||||
|
||||
let session_service = StreamableHttpService::new(
|
||||
move || Ok(session_runner.clone()),
|
||||
session_manager.clone(),
|
||||
service_config.clone(),
|
||||
);
|
||||
let service =
|
||||
StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config);
|
||||
|
||||
let router = Router::new().route_service("/", service);
|
||||
Ok((router, cancellation_token))
|
||||
let router = Router::new().route_service("/", service.clone());
|
||||
// `/api/mcp` serves both servers: a token scoped to a selection of scripts, flows and
|
||||
// endpoints gets the tools server, any other the session's toolset. At `/mcp` itself,
|
||||
// beside the `/mcp/...` routes, so it is merged rather than nested.
|
||||
let session_router = Router::new().route(
|
||||
"/mcp",
|
||||
axum::routing::any(move |mut request: Request<axum::body::Body>| {
|
||||
let (service, session_service) = (service.clone(), session_service.clone());
|
||||
async move {
|
||||
use tower::ServiceExt;
|
||||
let response = if request.extensions().get::<McpSelection>().is_some() {
|
||||
request.extensions_mut().insert(NoEndpointTools);
|
||||
service.oneshot(request).await
|
||||
} else {
|
||||
session_service.oneshot(request).await
|
||||
};
|
||||
match response {
|
||||
Ok(response) => response.into_response(),
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
Ok((router, session_router, cancellation_token))
|
||||
}
|
||||
|
||||
/// HTTP handler to list MCP tools as JSON
|
||||
@@ -628,3 +771,62 @@ async fn list_mcp_tools_handler() -> JsonResult<Vec<EndpointTool>> {
|
||||
pub fn list_tools_service() -> Router {
|
||||
Router::new().route("/", get(list_mcp_tools_handler))
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ToolsetQuery {
|
||||
access: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct ToolsetEntry {
|
||||
name: String,
|
||||
description: String,
|
||||
}
|
||||
|
||||
/// The tools `/api/mcp` offers an `mcp:all` token at each access level, for the token
|
||||
/// form to show: the server's own filter, so the list cannot drift from it.
|
||||
async fn list_toolset_handler(
|
||||
axum::extract::Query(query): axum::extract::Query<ToolsetQuery>,
|
||||
) -> JsonResult<Vec<ToolsetEntry>> {
|
||||
let access = match query.access.as_deref() {
|
||||
Some("read") => SessionAccess::ReadOnly,
|
||||
Some("read_run") => SessionAccess::ReadAndRun,
|
||||
_ => SessionAccess::Full,
|
||||
};
|
||||
Ok(Json(
|
||||
session::session_tools(access)
|
||||
.into_iter()
|
||||
.map(|tool| ToolsetEntry {
|
||||
name: tool.name.to_string(),
|
||||
description: tool.description.map(|d| d.to_string()).unwrap_or_default(),
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn toolset_service() -> Router {
|
||||
Router::new().route("/", get(list_toolset_handler))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::scoped_to_selection;
|
||||
|
||||
#[test]
|
||||
fn only_a_token_without_mcp_all_is_a_selection() {
|
||||
let selection = |scopes: &[&str]| {
|
||||
scoped_to_selection(&scopes.iter().map(|s| s.to_string()).collect::<Vec<_>>())
|
||||
};
|
||||
assert!(selection(&["mcp:favorites"]));
|
||||
assert!(selection(&["mcp:scripts:f/x/*", "mcp:endpoints:listJobs"]));
|
||||
assert!(selection(&["mcp:all:f/x/*"]));
|
||||
assert!(!selection(&["mcp:all"]));
|
||||
assert!(!selection(&[
|
||||
"mcp:all",
|
||||
"mcp:read_run",
|
||||
"mcp:workspaces:a,b"
|
||||
]));
|
||||
assert!(!selection(&[]));
|
||||
assert!(!selection(&["variables:read"]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
mod auto_generated_endpoints;
|
||||
mod core;
|
||||
mod session;
|
||||
mod utils;
|
||||
|
||||
// Re-export only what's needed externally
|
||||
@@ -12,5 +13,6 @@ pub mod oauth_server;
|
||||
pub use core::{
|
||||
add_www_authenticate_header, add_www_authenticate_header_gateway,
|
||||
extract_and_store_workspace_id, extract_workspace_from_token, list_tools_service,
|
||||
reject_token_query_param, setup_mcp_server,
|
||||
reject_token_query_param, setup_mcp_server, toolset_service, McpEndpoint,
|
||||
};
|
||||
pub(crate) use session::register_router;
|
||||
|
||||
@@ -201,7 +201,7 @@ struct AuthorizationCode {
|
||||
code: String,
|
||||
client_id: String,
|
||||
user_email: String,
|
||||
workspace_id: String,
|
||||
workspace_id: Option<String>,
|
||||
scopes: Vec<String>,
|
||||
redirect_uri: String,
|
||||
code_challenge: Option<String>,
|
||||
@@ -216,7 +216,7 @@ struct RefreshTokenRow {
|
||||
access_token_hash: String,
|
||||
client_id: String,
|
||||
user_email: String,
|
||||
workspace_id: String,
|
||||
workspace_id: Option<String>,
|
||||
scopes: Vec<String>,
|
||||
token_family: sqlx::types::Uuid,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
@@ -232,6 +232,9 @@ fn supported_scopes() -> Vec<String> {
|
||||
"mcp:scripts:*".to_string(),
|
||||
"mcp:flows:*".to_string(),
|
||||
"mcp:endpoints:*".to_string(),
|
||||
"mcp:read_only".to_string(),
|
||||
"mcp:read_run".to_string(),
|
||||
"mcp:runnables_as_tools".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -365,18 +368,59 @@ pub async fn oauth_token(
|
||||
Extension(db): Extension<DB>,
|
||||
Form(req): Form<TokenRequest>,
|
||||
) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
|
||||
match req.grant_type.as_str() {
|
||||
"authorization_code" => handle_authorization_code_grant(&db, &req).await,
|
||||
"refresh_token" => handle_refresh_token_grant(&db, &req).await,
|
||||
_ => Err(OAuthTokenError::unsupported_grant_type(
|
||||
"Supported grant types: authorization_code, refresh_token",
|
||||
)),
|
||||
let db_error = |e: sqlx::Error| {
|
||||
tracing::error!("Database error issuing MCP OAuth token: {}", e);
|
||||
OAuthTokenError::server_error("Database error")
|
||||
};
|
||||
let owner = match req.grant_type.as_str() {
|
||||
"authorization_code" => sqlx::query_scalar!(
|
||||
"SELECT user_email FROM mcp_oauth_server_code WHERE code = $1",
|
||||
req.code
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
.map_err(db_error)?,
|
||||
"refresh_token" => sqlx::query_scalar!(
|
||||
"SELECT user_email FROM mcp_oauth_refresh_token WHERE refresh_token = $1",
|
||||
req.refresh_token.as_deref().unwrap_or_default()
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
.map_err(db_error)?,
|
||||
_ => {
|
||||
return Err(OAuthTokenError::unsupported_grant_type(
|
||||
"Supported grant types: authorization_code, refresh_token",
|
||||
))
|
||||
}
|
||||
};
|
||||
let mut tx = db.begin().await.map_err(db_error)?;
|
||||
if let Some(email) = &owner {
|
||||
crate::users::lock_mcp_oauth_grants(&mut tx, email, false)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
}
|
||||
let res = if req.grant_type == "authorization_code" {
|
||||
handle_authorization_code_grant(&mut tx, &req).await
|
||||
} else {
|
||||
handle_refresh_token_grant(&mut tx, &req).await
|
||||
};
|
||||
// Committed on error too, so a rejected exchange still consumes its code or refresh token.
|
||||
let committed = tx.commit().await;
|
||||
match (res, committed) {
|
||||
(Ok(res), Ok(())) => Ok(res),
|
||||
(Err(e), committed) => {
|
||||
if let Err(ce) = committed {
|
||||
tracing::error!("Failed to commit rejected MCP OAuth exchange: {}", ce);
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
(Ok(_), Err(e)) => Err(db_error(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle authorization_code grant type
|
||||
async fn handle_authorization_code_grant(
|
||||
db: &DB,
|
||||
tx: &mut sqlx::PgConnection,
|
||||
req: &TokenRequest,
|
||||
) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
|
||||
let auth_code = match sqlx::query_as!(
|
||||
@@ -387,7 +431,7 @@ async fn handle_authorization_code_grant(
|
||||
code_challenge, code_challenge_method",
|
||||
req.code
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
{
|
||||
Ok(Some(code)) => code,
|
||||
@@ -458,7 +502,7 @@ async fn handle_authorization_code_grant(
|
||||
&scopes,
|
||||
auth_code.workspace_id,
|
||||
)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create access token: {}", e);
|
||||
@@ -471,7 +515,7 @@ async fn handle_authorization_code_grant(
|
||||
}
|
||||
|
||||
// Create refresh token — store the hash of the access token so we can delete it later
|
||||
let refresh_token_result = sqlx::query!(
|
||||
sqlx::query!(
|
||||
"INSERT INTO mcp_oauth_refresh_token
|
||||
(refresh_token, access_token_hash, client_id, user_email, workspace_id, scopes, token_family, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
|
||||
@@ -484,29 +528,25 @@ async fn handle_authorization_code_grant(
|
||||
token_family,
|
||||
MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
|
||||
)
|
||||
.execute(db)
|
||||
.await;
|
||||
|
||||
let refresh_token_value = match refresh_token_result {
|
||||
Ok(_) => Some(refresh_token),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to create refresh token: {}", e);
|
||||
None // Don't include invalid refresh token
|
||||
}
|
||||
};
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create refresh token: {}", e);
|
||||
OAuthTokenError::server_error("Failed to create refresh token")
|
||||
})?;
|
||||
|
||||
Ok(Json(TokenResponse {
|
||||
access_token,
|
||||
token_type: "Bearer".to_string(),
|
||||
expires_in: MCP_OAUTH_TOKEN_EXPIRATION_SECS,
|
||||
scope: Some(scopes.join(" ")),
|
||||
refresh_token: refresh_token_value,
|
||||
refresh_token: Some(refresh_token),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Handle refresh_token grant type with token rotation and theft detection
|
||||
async fn handle_refresh_token_grant(
|
||||
db: &DB,
|
||||
tx: &mut sqlx::PgConnection,
|
||||
req: &TokenRequest,
|
||||
) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
|
||||
let refresh_token_value = req
|
||||
@@ -533,7 +573,7 @@ async fn handle_refresh_token_grant(
|
||||
refresh_token_value,
|
||||
req.client_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
{
|
||||
Ok(Some(row)) => row,
|
||||
@@ -544,7 +584,7 @@ async fn handle_refresh_token_grant(
|
||||
WHERE refresh_token = $1 AND used_at IS NOT NULL",
|
||||
refresh_token_value
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Refresh token reuse detected, revoking family {:?}", family);
|
||||
@@ -552,7 +592,7 @@ async fn handle_refresh_token_grant(
|
||||
"UPDATE mcp_oauth_refresh_token SET revoked = TRUE WHERE token_family = $1",
|
||||
family
|
||||
)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await;
|
||||
}
|
||||
return Err(OAuthTokenError::invalid_grant("Invalid refresh token"));
|
||||
@@ -564,16 +604,16 @@ async fn handle_refresh_token_grant(
|
||||
};
|
||||
|
||||
// Delete old access token using the stored hash
|
||||
if let Err(e) = sqlx::query!(
|
||||
sqlx::query!(
|
||||
"DELETE FROM token WHERE token_hash = $1",
|
||||
token_row.access_token_hash
|
||||
)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
{
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to delete old access token: {}", e);
|
||||
// Non-fatal, continue with token creation
|
||||
}
|
||||
OAuthTokenError::server_error("Database error")
|
||||
})?;
|
||||
|
||||
// Generate new tokens
|
||||
let new_access_token = rd_string(32);
|
||||
@@ -605,7 +645,7 @@ async fn handle_refresh_token_grant(
|
||||
&scopes,
|
||||
token_row.workspace_id,
|
||||
)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create new access token: {}", e);
|
||||
@@ -618,7 +658,7 @@ async fn handle_refresh_token_grant(
|
||||
}
|
||||
|
||||
// Create new refresh token (same token family for tracking) — store hash of access token
|
||||
if let Err(e) = sqlx::query!(
|
||||
sqlx::query!(
|
||||
"INSERT INTO mcp_oauth_refresh_token
|
||||
(refresh_token, access_token_hash, client_id, user_email, workspace_id, scopes, token_family, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
|
||||
@@ -631,12 +671,12 @@ async fn handle_refresh_token_grant(
|
||||
token_row.token_family,
|
||||
MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
|
||||
)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
{
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create new refresh token: {}", e);
|
||||
// Access token was created, return success without refresh token
|
||||
}
|
||||
OAuthTokenError::server_error("Failed to create refresh token")
|
||||
})?;
|
||||
|
||||
Ok(Json(TokenResponse {
|
||||
access_token: new_access_token,
|
||||
@@ -758,10 +798,31 @@ async fn oauth_authorize_inner(
|
||||
}
|
||||
|
||||
/// Shared approve logic — validates form, checks membership, stores auth code.
|
||||
async fn ensure_member(db: &DB, workspace_id: &str, email: &str) -> Result<()> {
|
||||
let is_member = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
|
||||
workspace_id,
|
||||
email
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
|
||||
.unwrap_or(false);
|
||||
|
||||
if !is_member {
|
||||
return Err(Error::NotAuthorized(format!(
|
||||
"User is not a member of workspace '{workspace_id}'"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `workspace_id` is the workspace the minted token is bound to; without one, the token
|
||||
/// reaches what its scopes allow (`mcp:workspaces:`), or every workspace of its owner.
|
||||
async fn oauth_approve_inner(
|
||||
db: &DB,
|
||||
authed: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
workspace_id: Option<&str>,
|
||||
form: ApprovalForm,
|
||||
) -> Result<Json<ApprovalResponse>> {
|
||||
// The code approved here is exchanged for a database token carrying only this
|
||||
@@ -771,21 +832,8 @@ async fn oauth_approve_inner(
|
||||
// the gateway approve route reach the exchange through here.
|
||||
forbid_elevated_job_token(db, &authed.email, authed.job_id).await?;
|
||||
|
||||
// Verify user is a member of the workspace
|
||||
let is_member = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
|
||||
workspace_id,
|
||||
authed.email
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
|
||||
.unwrap_or(false);
|
||||
|
||||
if !is_member {
|
||||
return Err(Error::NotAuthorized(
|
||||
"User is not a member of this workspace".to_string(),
|
||||
));
|
||||
if let Some(workspace_id) = workspace_id {
|
||||
ensure_member(db, workspace_id, &authed.email).await?;
|
||||
}
|
||||
|
||||
// Verify client exists and redirect_uri is registered
|
||||
@@ -908,7 +956,7 @@ pub async fn workspaced_oauth_approve(
|
||||
authed: ApiAuthed,
|
||||
Json(form): Json<ApprovalForm>,
|
||||
) -> Result<Json<ApprovalResponse>> {
|
||||
oauth_approve_inner(&db, &authed, &workspace_id, form).await
|
||||
oauth_approve_inner(&db, &authed, Some(&workspace_id), form).await
|
||||
}
|
||||
|
||||
/// PKCE validation (S256 only)
|
||||
@@ -990,6 +1038,16 @@ pub async fn gateway_protected_resource_metadata() -> Json<ProtectedResourceMeta
|
||||
))
|
||||
}
|
||||
|
||||
/// GET /.well-known/oauth-protected-resource/api/mcp
|
||||
pub async fn gateway_session_protected_resource_metadata() -> Json<ProtectedResourceMetadata> {
|
||||
let base_url = BASE_URL.load();
|
||||
Json(build_protected_resource_metadata(
|
||||
"/api/mcp",
|
||||
"/api/mcp/gateway/oauth/server",
|
||||
&base_url,
|
||||
))
|
||||
}
|
||||
|
||||
/// GET /api/mcp/gateway/oauth/server/authorize — gateway mode (no workspace in path)
|
||||
pub async fn gateway_oauth_authorize(
|
||||
Extension(db): Extension<DB>,
|
||||
@@ -998,23 +1056,44 @@ pub async fn gateway_oauth_authorize(
|
||||
oauth_authorize_inner(&db, params, None).await
|
||||
}
|
||||
|
||||
/// POST /api/mcp/gateway/oauth/server/approve — workspace_id comes from JSON body
|
||||
/// POST /api/mcp/gateway/oauth/server/approve — the workspaces come from the JSON body
|
||||
pub async fn gateway_oauth_approve(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: ApiAuthed,
|
||||
Json(form): Json<GatewayApprovalForm>,
|
||||
) -> Result<Json<ApprovalResponse>> {
|
||||
if form.workspace_id.is_empty() {
|
||||
return Err(Error::BadRequest("workspace_id is required".to_string()));
|
||||
let mut scope = form.scope;
|
||||
let mut workspace_ids = form.workspace_ids;
|
||||
if workspace_ids.is_empty() {
|
||||
workspace_ids.extend(form.workspace_id);
|
||||
}
|
||||
let workspace_id = if form.all_workspaces {
|
||||
None
|
||||
} else {
|
||||
match workspace_ids.as_slice() {
|
||||
[] => {
|
||||
return Err(Error::BadRequest(
|
||||
"Pick at least one workspace, or all of them".to_string(),
|
||||
))
|
||||
}
|
||||
[only] => Some(only.as_str()),
|
||||
several => {
|
||||
for workspace_id in several {
|
||||
ensure_member(&db, workspace_id, &authed.email).await?;
|
||||
}
|
||||
scope.push_str(&format!(" mcp:workspaces:{}", several.join(",")));
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
oauth_approve_inner(
|
||||
&db,
|
||||
&authed,
|
||||
&form.workspace_id,
|
||||
workspace_id,
|
||||
ApprovalForm {
|
||||
client_id: form.client_id,
|
||||
redirect_uri: form.redirect_uri,
|
||||
scope: form.scope,
|
||||
scope,
|
||||
state: form.state,
|
||||
code_challenge: form.code_challenge,
|
||||
code_challenge_method: form.code_challenge_method,
|
||||
@@ -1023,7 +1102,8 @@ pub async fn gateway_oauth_approve(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Gateway approval form — same as ApprovalForm but includes workspace_id
|
||||
/// Gateway approval form — same as ApprovalForm, plus the workspaces the token reaches:
|
||||
/// one binds the token to it, several are listed in its scopes.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct GatewayApprovalForm {
|
||||
pub client_id: String,
|
||||
@@ -1032,7 +1112,14 @@ pub struct GatewayApprovalForm {
|
||||
pub state: String,
|
||||
pub code_challenge: String,
|
||||
pub code_challenge_method: String,
|
||||
pub workspace_id: String,
|
||||
#[serde(default)]
|
||||
pub workspace_ids: Vec<String>,
|
||||
/// The single workspace a consent page loaded before `workspace_ids` existed sends.
|
||||
#[serde(default)]
|
||||
pub workspace_id: Option<String>,
|
||||
/// Every workspace the user is a member of, including ones they join later.
|
||||
#[serde(default)]
|
||||
pub all_workspaces: bool,
|
||||
}
|
||||
|
||||
// ── Router constructors ──
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,969 @@
|
||||
//! Data tables, DuckLake and database resource schemas.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::runs::{backgrounded_summary, run_script_preview, wait_budget, wait_job};
|
||||
use super::workspace::{api_message, path_enc};
|
||||
use super::{array, enc, js_str, non_null, opt_str_arg, pretty, str_arg, truthy, Ctx, ToolResult};
|
||||
|
||||
pub(super) const TOOLS: &[&str] = &[
|
||||
"list_datatables",
|
||||
"get_datatable_table_schema",
|
||||
"exec_datatable_sql",
|
||||
"list_data_metrics",
|
||||
"list_ducklakes",
|
||||
"get_db_schema",
|
||||
"list_datatable_migrations",
|
||||
"create_datatable_migration",
|
||||
"run_datatable_migrations",
|
||||
"rollback_datatable_migration",
|
||||
];
|
||||
|
||||
pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult {
|
||||
match name {
|
||||
"list_datatables" => Ok(list_datatables(ctx, args)
|
||||
.await
|
||||
.unwrap_or_else(|e| format!("Error listing datatables: {e}"))),
|
||||
"get_datatable_table_schema" => get_datatable_table_schema(ctx, args).await,
|
||||
"exec_datatable_sql" => exec_datatable_sql(ctx, args).await,
|
||||
"list_data_metrics" => list_data_metrics(ctx, args).await,
|
||||
"list_ducklakes" => Ok(list_ducklakes(ctx)
|
||||
.await
|
||||
.unwrap_or_else(|e| format!("Error listing DuckLake catalogs: {e}"))),
|
||||
"get_db_schema" => get_db_schema(ctx, args).await,
|
||||
"list_datatable_migrations" => list_datatable_migrations(ctx, args).await,
|
||||
"create_datatable_migration" => create_datatable_migration(ctx, args).await,
|
||||
"run_datatable_migrations" => run_datatable_migrations(ctx, args).await,
|
||||
"rollback_datatable_migration" => rollback_datatable_migration(ctx, args).await,
|
||||
_ => unreachable!("{name} is not a data tool"),
|
||||
}
|
||||
}
|
||||
|
||||
const NO_DATATABLES_CONFIGURED_MESSAGE: &str = "No datatables are configured in this workspace. Datatable operations (querying data, creating or altering tables) are blocked until a datatable exists. Datatables are not created by SQL — the user must set one up in the workspace settings (Workspace settings → Data Tables) first. Do not call exec_datatable_sql or assume a \"main\" datatable exists; instead tell the user this is a required prerequisite and ask them to configure a datatable in their workspace settings.";
|
||||
|
||||
const NO_DUCKLAKES_CONFIGURED_MESSAGE: &str = "No DuckLake catalogs are configured in this workspace. A data pipeline that materializes DuckLake tables or reads/writes S3 assets cannot run until object storage and a DuckLake catalog are set up. You can still draft the pipeline scripts, but tell the user how to enable it by role: a workspace admin adds object storage under Workspace settings → Object Storage (S3/Azure/GCS), then a DuckLake catalog on top of it; a user without admin rights should ask a workspace admin. Do not assume a default 'main' DuckLake exists.";
|
||||
|
||||
const NO_DATA_METRICS_NOTE: &str = "Nothing matched. That does not establish that nothing is declared: declarations whose producing script you cannot read are omitted from this list, not flagged. You may write your own aggregate, but tell the user you found no declared measure you can read rather than stating none is declared.";
|
||||
|
||||
const DEFAULT_DATA_METRICS_LIMIT: u64 = 200;
|
||||
/// Rows of a SELECT handed to the model.
|
||||
const MAX_ROWS: usize = 100;
|
||||
/// Seconds a query is awaited before it is left running for `get_run`.
|
||||
const MAX_SCHEMA_LENGTH: usize = 350_000;
|
||||
|
||||
/// The worker's error for a data table that is not configured in the workspace.
|
||||
static DATATABLE_NOT_FOUND: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)datatable\s+\S+\s+not found").expect("valid regex"));
|
||||
|
||||
fn datatable_not_configured_message(name: &str) -> String {
|
||||
format!(
|
||||
"Datatable \"{name}\" is not configured in this workspace, so this operation cannot run. \
|
||||
Datatables are not created by SQL — they must be set up first by the user in the workspace settings \
|
||||
(Workspace settings → Data Tables) before any table can be queried or created. \
|
||||
This is a required, blocking prerequisite: do not retry on this or another name. \
|
||||
Tell the user they need to configure a datatable (e.g. named \"{name}\") in their workspace settings, then try again."
|
||||
)
|
||||
}
|
||||
|
||||
/// The optional `role` argument, held to the rule the server applies to a role name.
|
||||
fn role_arg(args: &Value) -> Result<Option<&str>, String> {
|
||||
let role = opt_str_arg(args, "role");
|
||||
match role {
|
||||
Some(r)
|
||||
if r.is_empty()
|
||||
|| r.len() > 63
|
||||
|| !r
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') =>
|
||||
{
|
||||
Err(format!(
|
||||
"Invalid `role` '{r}': only letters, digits, '_' and '-' are allowed (1 to 63 characters)"
|
||||
))
|
||||
}
|
||||
_ => Ok(role),
|
||||
}
|
||||
}
|
||||
|
||||
/// `path?k=v&...` over the parameters that are set.
|
||||
fn with_query(path: String, params: &[(&str, Option<&str>)]) -> String {
|
||||
let query: Vec<String> = params
|
||||
.iter()
|
||||
.filter_map(|(k, v)| v.map(|v| format!("{k}={}", enc(v))))
|
||||
.collect();
|
||||
if query.is_empty() {
|
||||
path
|
||||
} else {
|
||||
format!("{path}?{}", query.join("&"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_datatables(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let role = role_arg(args)?;
|
||||
let datatable_name = opt_str_arg(args, "datatable_name");
|
||||
if role.is_some() && datatable_name.is_none() {
|
||||
return Err("`role` needs `datatable_name`, the datatable it is a role of".to_string());
|
||||
}
|
||||
let path = with_query(
|
||||
ctx.ws("workspaces/list_datatable_tables"),
|
||||
&[
|
||||
("datatable_name", datatable_name),
|
||||
("role_for", role.and(datatable_name)),
|
||||
("role", role),
|
||||
],
|
||||
);
|
||||
let metadata = ctx.get(&path).await?;
|
||||
let metadata = array(Some(&metadata));
|
||||
if metadata.is_empty() {
|
||||
return Ok(NO_DATATABLES_CONFIGURED_MESSAGE.to_string());
|
||||
}
|
||||
let listed: Vec<Value> = metadata
|
||||
.iter()
|
||||
.map(|d| {
|
||||
let mut out = Map::new();
|
||||
out.insert("datatable_name".into(), d["datatable_name"].clone());
|
||||
out.insert("schemas".into(), d["schemas"].clone());
|
||||
if truthy(d.get("error")) {
|
||||
out.insert("error".into(), d["error"].clone());
|
||||
}
|
||||
if truthy(d.get("permissioned")) {
|
||||
for key in ["usable_roles", "default_role"] {
|
||||
if let Some(v) = d.get(key) {
|
||||
out.insert(key.into(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::Object(out)
|
||||
})
|
||||
.collect();
|
||||
Ok(pretty(&Value::Array(listed)))
|
||||
}
|
||||
|
||||
async fn get_datatable_table_schema(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let datatable_name = str_arg(args, "datatable_name")?;
|
||||
let schema_name = str_arg(args, "schema_name")?;
|
||||
let table_name = str_arg(args, "table_name")?;
|
||||
let role = role_arg(args)?;
|
||||
let path = with_query(
|
||||
ctx.ws("workspaces/get_datatable_table_schema"),
|
||||
&[
|
||||
("datatable_name", Some(datatable_name)),
|
||||
("schema_name", Some(schema_name)),
|
||||
("table_name", Some(table_name)),
|
||||
("role", role),
|
||||
],
|
||||
);
|
||||
Ok(match ctx.get(&path).await {
|
||||
Ok(schema) => pretty(&json!({
|
||||
"datatable_name": datatable_name,
|
||||
"schema_name": schema_name,
|
||||
"table_name": table_name,
|
||||
"columns": schema["columns"],
|
||||
})),
|
||||
Err(e) if DATATABLE_NOT_FOUND.is_match(&e) => {
|
||||
datatable_not_configured_message(datatable_name)
|
||||
}
|
||||
Err(e) => format!("Error getting table schema: {e}"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_ducklakes(ctx: &Ctx<'_>) -> ToolResult {
|
||||
let ducklakes = ctx.get(&ctx.ws("workspaces/list_ducklakes")).await?;
|
||||
if ducklakes.as_array().is_none_or(Vec::is_empty) {
|
||||
return Ok(NO_DUCKLAKES_CONFIGURED_MESSAGE.to_string());
|
||||
}
|
||||
Ok(pretty(&json!({ "ducklakes": ducklakes })))
|
||||
}
|
||||
|
||||
async fn list_data_metrics(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let limit = match args.get("limit") {
|
||||
None | Some(Value::Null) => DEFAULT_DATA_METRICS_LIMIT,
|
||||
Some(v) => v
|
||||
.as_u64()
|
||||
.filter(|n| (1..=1000).contains(n))
|
||||
.ok_or("`limit` must be an integer between 1 and 1000")?,
|
||||
};
|
||||
let table = opt_str_arg(args, "table");
|
||||
let path = with_query(
|
||||
ctx.ws("data_metrics/list"),
|
||||
&[
|
||||
("table", table),
|
||||
("path_prefix", opt_str_arg(args, "path_prefix")),
|
||||
("per_page", Some(&limit.to_string())),
|
||||
],
|
||||
);
|
||||
let page = ctx.get(&path).await?;
|
||||
let metrics = page.get("metrics").cloned().unwrap_or_else(|| json!([]));
|
||||
let note = if truthy(page.get("next_cursor")) {
|
||||
Some(format!("More declarations exist beyond the first {limit}. Re-call with table/path_prefix to target what you are looking for, or with a higher limit (max 1000) for the rest of the list, rather than concluding a measure is undeclared."))
|
||||
} else if metrics.as_array().is_some_and(Vec::is_empty) {
|
||||
// The endpoint expands a table filter only once it contains a `/`, so a bare
|
||||
// table name matches nothing whatever is declared.
|
||||
let bare = table.map(|t| t.replacen("ducklake://", "", 1));
|
||||
Some(match (table, bare) {
|
||||
(Some(table), Some(bare)) if !bare.is_empty() && !bare.contains('/') => format!("Nothing matched `{table}`: a table filter must name its lake, as `<lake>/{bare}`. Re-call with the lake before concluding anything about what is declared."),
|
||||
_ => NO_DATA_METRICS_NOTE.to_string(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut out = Map::new();
|
||||
out.insert("metrics".into(), metrics);
|
||||
if let Some(note) = note {
|
||||
out.insert("note".into(), note.into());
|
||||
}
|
||||
Ok(pretty(&Value::Object(out)))
|
||||
}
|
||||
|
||||
async fn exec_datatable_sql(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let name = str_arg(args, "datatable_name")?;
|
||||
let sql = str_arg(args, "sql")?;
|
||||
let role = role_arg(args)?;
|
||||
let failed = |e: String| format!("Failed to execute test run: {e}");
|
||||
|
||||
let database = match role {
|
||||
None => format!("datatable://{name}"),
|
||||
// A stored name holding a `?` is read whole by the server, so a role cannot follow it.
|
||||
Some(role) if name.contains('?') => {
|
||||
return Err(failed(format!("Data table '{name}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '{role}'.")));
|
||||
}
|
||||
Some(role) => format!("datatable://{name}?role={role}"),
|
||||
};
|
||||
let id = run_script_preview(
|
||||
ctx,
|
||||
None,
|
||||
"postgresql",
|
||||
sql,
|
||||
json!({ "database": database }),
|
||||
)
|
||||
.await
|
||||
.map_err(failed)?;
|
||||
|
||||
match wait_job(ctx, &id, wait_budget(args), false).await {
|
||||
Some(job) => Ok(format_datatable_sql_completion(&job, name)),
|
||||
None => Ok(backgrounded_summary(&id, &format!("SQL · {name}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_datatable_sql_completion(job: &Value, datatable_name: &str) -> String {
|
||||
if job["success"] == true {
|
||||
let rows = array(job.get("result"));
|
||||
let row_count = rows.len();
|
||||
return pretty(&if row_count > MAX_ROWS {
|
||||
json!({
|
||||
"success": true,
|
||||
"rowCount": row_count,
|
||||
"result": &rows[..MAX_ROWS],
|
||||
"note": format!("Showing first {MAX_ROWS} of {row_count} rows"),
|
||||
})
|
||||
} else {
|
||||
json!({ "success": true, "rowCount": row_count, "result": rows })
|
||||
});
|
||||
}
|
||||
let error = match job.get("result") {
|
||||
None => json!("Unknown error"),
|
||||
Some(result) => match result.pointer("/error/message") {
|
||||
Some(message) if !message.is_null() => message.clone(),
|
||||
_ => match result {
|
||||
Value::String(_) => result.clone(),
|
||||
other => Value::String(other.to_string()),
|
||||
},
|
||||
},
|
||||
};
|
||||
let error = match error.as_str() {
|
||||
Some(raw) if DATATABLE_NOT_FOUND.is_match(raw) => {
|
||||
Value::String(datatable_not_configured_message(datatable_name))
|
||||
}
|
||||
_ => error,
|
||||
};
|
||||
json!({ "success": false, "error": error }).to_string()
|
||||
}
|
||||
|
||||
/// The optional `version` argument: a migration's timestamp.
|
||||
fn version_arg(args: &Value) -> Result<Option<i64>, String> {
|
||||
match args.get("version") {
|
||||
None | Some(Value::Null) => Ok(None),
|
||||
Some(v) => v
|
||||
.as_i64()
|
||||
.or_else(|| v.as_f64().filter(|f| f.fract() == 0.0).map(|f| f as i64))
|
||||
.map(Some)
|
||||
.ok_or_else(|| "`version` must be an integer".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `workspaces/<endpoint>/<datatable>`, the shape of every migration route.
|
||||
fn migrations_path(ctx: &Ctx<'_>, endpoint: &str, datatable_name: &str) -> String {
|
||||
ctx.ws(&format!("workspaces/{endpoint}/{}", enc(datatable_name)))
|
||||
}
|
||||
|
||||
async fn migrations_status(ctx: &Ctx<'_>, datatable_name: &str) -> Result<Value, String> {
|
||||
ctx.get(&migrations_path(
|
||||
ctx,
|
||||
"datatable_migrations_status",
|
||||
datatable_name,
|
||||
))
|
||||
.await
|
||||
.map_err(|e| api_message(&e).to_string())
|
||||
}
|
||||
|
||||
/// `POST` without a body: applying and rolling back take their target from the query.
|
||||
async fn post_migrations(ctx: &Ctx<'_>, path: &str) -> Result<Value, String> {
|
||||
let text = ctx
|
||||
.request("POST", path, None)
|
||||
.await
|
||||
.map_err(|e| api_message(&e).to_string())?;
|
||||
serde_json::from_str(&text).map_err(|e| format!("unexpected response: {e}"))
|
||||
}
|
||||
|
||||
fn migration_label(version: &Value, name: &Value) -> String {
|
||||
format!("{} ({})", js_str(version), js_str(name))
|
||||
}
|
||||
|
||||
fn labels(migrations: Option<&Value>) -> String {
|
||||
array(migrations)
|
||||
.iter()
|
||||
.map(|m| migration_label(&m["version"], &m["name"]))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
fn migration_status(status: &Value) -> &'static str {
|
||||
match status.as_str() {
|
||||
Some("ran") => "applied",
|
||||
Some("not_run") => "pending",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// The rest of a leading comment line whose first word is `keyword`: an annotation attempt.
|
||||
fn annotation_attempt<'a>(line: &'a str, keyword: &str) -> Option<&'a str> {
|
||||
let body = line.strip_prefix("--")?.trim_start();
|
||||
let after = body
|
||||
.get(..keyword.len())
|
||||
.filter(|w| w.eq_ignore_ascii_case(keyword))
|
||||
.map(|_| &body[keyword.len()..])?;
|
||||
match after.chars().next() {
|
||||
Some(c) if !(c.is_whitespace() || c == ':' || c == '=') => None,
|
||||
_ => Some(after),
|
||||
}
|
||||
}
|
||||
|
||||
/// The lines of `sql`'s leading comment block, where the server reads its annotations.
|
||||
fn leading_block(sql: &str) -> impl Iterator<Item = &str> {
|
||||
sql.split('\n')
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.take_while(|l| l.starts_with("--"))
|
||||
}
|
||||
|
||||
/// `sql` with every `keyword` attempt of its leading comment block removed, and `line`, if
|
||||
/// any, on top.
|
||||
fn with_annotation(sql: &str, keyword: &str, line: Option<String>) -> String {
|
||||
let lines: Vec<&str> = sql.split('\n').collect();
|
||||
let leading = lines
|
||||
.iter()
|
||||
.position(|l| !l.trim().is_empty() && !l.trim().starts_with("--"))
|
||||
.unwrap_or(lines.len());
|
||||
let rest = lines[..leading]
|
||||
.iter()
|
||||
.filter(|l| annotation_attempt(l.trim(), keyword).is_none())
|
||||
.chain(&lines[leading..])
|
||||
.skip_while(|l| l.trim().is_empty())
|
||||
.map(|l| l.to_string());
|
||||
line.into_iter().chain(rest).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
/// `sql` declaring `role`: the server reads the role a migration runs as from a `-- role <name>`
|
||||
/// annotation in its leading comment block, so any attempt there is replaced by one on top.
|
||||
fn with_migration_role(sql: &str, role: Option<&str>) -> String {
|
||||
match role {
|
||||
Some(role) => with_annotation(sql, "role", Some(format!("-- role {role}"))),
|
||||
None => sql.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_annotatable_schema_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 63
|
||||
&& name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '$' | '-'))
|
||||
}
|
||||
|
||||
/// The scope `sql`'s `-- schemas a, b` annotation declares, lowercased, as the server's
|
||||
/// `parse_migration_schemas` reads it. None for the whole database, which is also what a
|
||||
/// malformed annotation stands for here.
|
||||
fn declared_schemas(sql: &str) -> Option<Vec<String>> {
|
||||
let after = leading_block(sql).find_map(|l| annotation_attempt(l, "schemas"))?;
|
||||
let rest = after.trim_start();
|
||||
let rest = rest.strip_prefix([':', '=']).unwrap_or(rest).trim();
|
||||
let rest = rest.strip_suffix(';').unwrap_or(rest);
|
||||
let mut schemas: Vec<String> = Vec::new();
|
||||
for name in rest.split(',').map(str::trim) {
|
||||
if !is_annotatable_schema_name(name) {
|
||||
return None;
|
||||
}
|
||||
let name = name.to_ascii_lowercase();
|
||||
if !schemas.contains(&name) {
|
||||
schemas.push(name);
|
||||
}
|
||||
}
|
||||
Some(schemas)
|
||||
}
|
||||
|
||||
/// Whether a listed migration's `schemas` overlaps `own`; an absent scope is the whole database.
|
||||
fn scopes_overlap(theirs: &Value, own: Option<&[String]>) -> bool {
|
||||
match (theirs.as_array(), own) {
|
||||
(Some(theirs), Some(own)) => theirs
|
||||
.iter()
|
||||
.any(|s| own.iter().any(|o| s.as_str() == Some(o))),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The optional `schemas` argument, held to the rule the server applies to a scope name.
|
||||
fn schemas_arg(args: &Value) -> Result<Option<Vec<String>>, String> {
|
||||
let Some(value) = non_null(args.get("schemas")) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let names = value
|
||||
.as_array()
|
||||
.ok_or("`schemas` must be an array of schema names")?;
|
||||
names
|
||||
.iter()
|
||||
.map(|name| match name.as_str() {
|
||||
Some(name) if is_annotatable_schema_name(name) => Ok(name.to_ascii_lowercase()),
|
||||
_ => Err(format!(
|
||||
"Invalid schema name '{}' in `schemas`: only letters, digits, '_', '$' and '-' are allowed (1 to 63 characters)",
|
||||
js_str(name)
|
||||
)),
|
||||
})
|
||||
.collect::<Result<_, _>>()
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
async fn list_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let datatable_name = str_arg(args, "datatable_name")?;
|
||||
let version = version_arg(args)?;
|
||||
let status = match migrations_status(ctx, datatable_name).await {
|
||||
Ok(status) => status,
|
||||
Err(e) => return Ok(format!("Error listing migrations: {e}")),
|
||||
};
|
||||
if !truthy(status.get("enabled")) {
|
||||
return Ok(format!("Migrations are not enabled on datatable \"{datatable_name}\": its schema is changed with plain SQL and there is no history to list. Enabling them is a workspace admin's decision, made in Workspace settings → Data Tables."));
|
||||
}
|
||||
let migrations = array(status.get("migrations"));
|
||||
let Some(version) = version else {
|
||||
let mut out = Map::new();
|
||||
out.insert(
|
||||
"migrations".into(),
|
||||
migrations
|
||||
.iter()
|
||||
.map(|m| {
|
||||
json!({
|
||||
"version": m["timestamp"],
|
||||
"name": m["name"],
|
||||
"status": migration_status(&m["status"]),
|
||||
"schemas": non_null(m.get("schemas")).cloned().unwrap_or_else(|| json!("whole database")),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
if truthy(status.get("error")) {
|
||||
out.insert("error".into(), status["error"].clone());
|
||||
}
|
||||
return Ok(pretty(&Value::Object(out)));
|
||||
};
|
||||
let Some(migration) = migrations
|
||||
.iter()
|
||||
.find(|m| m["timestamp"].as_i64() == Some(version))
|
||||
else {
|
||||
return Ok(format!(
|
||||
"No migration {version} on datatable \"{datatable_name}\""
|
||||
));
|
||||
};
|
||||
let mut out = Map::new();
|
||||
out.insert("version".into(), version.into());
|
||||
out.insert("name".into(), migration["name"].clone());
|
||||
out.insert(
|
||||
"status".into(),
|
||||
migration_status(&migration["status"]).into(),
|
||||
);
|
||||
out.insert("code_up".into(), migration["code_up"].clone());
|
||||
if let Some(code_down) = non_null(migration.get("code_down")) {
|
||||
out.insert("code_down".into(), code_down.clone());
|
||||
}
|
||||
Ok(pretty(&Value::Object(out)))
|
||||
}
|
||||
|
||||
/// Saves the definition and answers its version, or the message the tool returns instead.
|
||||
async fn save_migration(
|
||||
ctx: &Ctx<'_>,
|
||||
datatable_name: &str,
|
||||
version: Option<i64>,
|
||||
mut definition: Map<String, Value>,
|
||||
) -> Result<Result<i64, String>, String> {
|
||||
let Some(version) = version else {
|
||||
let created = ctx
|
||||
.post(
|
||||
&migrations_path(ctx, "create_datatable_migration", datatable_name),
|
||||
Value::Object(definition),
|
||||
)
|
||||
.await?;
|
||||
return created["timestamp"]
|
||||
.as_i64()
|
||||
.map(Ok)
|
||||
.ok_or_else(|| "the created migration has no version".to_string());
|
||||
};
|
||||
// The server inserts at a version it does not know, which would wedge a new migration
|
||||
// into the middle of the history, and stores a down left out as none.
|
||||
let status = migrations_status(ctx, datatable_name).await?;
|
||||
let existing = array(status.get("migrations"))
|
||||
.iter()
|
||||
.find(|m| m["timestamp"].as_i64() == Some(version));
|
||||
let Some(existing) = existing.filter(|m| m["status"] == "not_run") else {
|
||||
return Ok(Err(format!("Migration {version} is not a pending migration of datatable \"{datatable_name}\", so it cannot be rewritten. Omit `version` to add a new migration.")));
|
||||
};
|
||||
definition.insert("timestamp".into(), version.into());
|
||||
if let Some(code_down) = non_null(existing.get("code_down")) {
|
||||
definition
|
||||
.entry("code_down")
|
||||
.or_insert_with(|| code_down.clone());
|
||||
}
|
||||
ctx.post(
|
||||
&migrations_path(ctx, "upsert_datatable_migration", datatable_name),
|
||||
Value::Object(definition),
|
||||
)
|
||||
.await?;
|
||||
Ok(Ok(version))
|
||||
}
|
||||
|
||||
/// The labels of the migrations before `version`, in a scope overlapping `scope`, that are
|
||||
/// not applied yet.
|
||||
async fn earlier_pending(
|
||||
ctx: &Ctx<'_>,
|
||||
datatable_name: &str,
|
||||
version: i64,
|
||||
scope: Option<&[String]>,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let status = migrations_status(ctx, datatable_name).await?;
|
||||
let earlier: Vec<&Value> = array(status.get("migrations"))
|
||||
.iter()
|
||||
.filter(|m| m["timestamp"].as_i64().is_some_and(|t| t < version))
|
||||
.filter(|m| scopes_overlap(&m["schemas"], scope))
|
||||
.collect();
|
||||
// Unreadable is not pending: reporting it so would send the user to apply migrations
|
||||
// that already ran.
|
||||
if truthy(status.get("error")) {
|
||||
return Err(js_str(&status["error"]));
|
||||
}
|
||||
if earlier.iter().any(|m| m["status"] == "unknown") {
|
||||
return Err("the applied migrations could not be read".to_string());
|
||||
}
|
||||
Ok(earlier
|
||||
.into_iter()
|
||||
.filter(|m| m["status"] == "not_run")
|
||||
.map(|m| migration_label(&m["timestamp"], &m["name"]))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn create_datatable_migration(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let datatable_name = str_arg(args, "datatable_name")?;
|
||||
let name = str_arg(args, "name")?;
|
||||
if name.is_empty()
|
||||
|| !name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
|
||||
{
|
||||
return Err(format!(
|
||||
"Invalid `name` '{name}': only letters, digits, '_' and '-' are allowed"
|
||||
));
|
||||
}
|
||||
let role = role_arg(args)?;
|
||||
let version = version_arg(args)?;
|
||||
let code_up = str_arg(args, "code_up")?;
|
||||
// What the caller declares, else what `code_up` does; none is the whole database.
|
||||
let scope = schemas_arg(args)?
|
||||
.or_else(|| declared_schemas(code_up))
|
||||
.unwrap_or_default();
|
||||
let header = (!scope.is_empty()).then(|| format!("-- schemas {}", scope.join(", ")));
|
||||
|
||||
let mut definition = Map::new();
|
||||
definition.insert("name".into(), name.into());
|
||||
definition.insert(
|
||||
"code_up".into(),
|
||||
with_annotation(&with_migration_role(code_up, role), "schemas", header).into(),
|
||||
);
|
||||
if let Some(code_down) = opt_str_arg(args, "code_down") {
|
||||
// A rollback has to run as the role that created what it drops.
|
||||
definition.insert(
|
||||
"code_down".into(),
|
||||
with_migration_role(code_down, role).into(),
|
||||
);
|
||||
}
|
||||
let version = match save_migration(ctx, datatable_name, version, definition).await {
|
||||
Ok(Ok(version)) => version,
|
||||
Ok(Err(message)) => return Ok(message),
|
||||
Err(e) => return Ok(format!("Error saving migration: {}", api_message(&e))),
|
||||
};
|
||||
let label = format!("{version} ({name})");
|
||||
if args.get("apply") == Some(&Value::Bool(false)) {
|
||||
return Ok(format!(
|
||||
"Saved migration {label} as pending. It is not applied yet."
|
||||
));
|
||||
}
|
||||
|
||||
// Applying it alone would run it ahead of migrations it may depend on.
|
||||
let own_scope = (!scope.is_empty()).then_some(scope.as_slice());
|
||||
let earlier = match earlier_pending(ctx, datatable_name, version, own_scope).await {
|
||||
Ok(earlier) => earlier,
|
||||
Err(e) => return Ok(format!("Saved migration {label} as pending, but could not check what else is pending, so it was not applied: {e}")),
|
||||
};
|
||||
if !earlier.is_empty() {
|
||||
return Ok(format!("Saved migration {label} as pending without applying it: earlier migrations are not applied yet ({}) and it may depend on them. Tell the user, then apply them all in order with run_datatable_migrations.", earlier.join(", ")));
|
||||
}
|
||||
|
||||
let run = format!(
|
||||
"{}?only={version}",
|
||||
migrations_path(ctx, "run_datatable_migrations", datatable_name)
|
||||
);
|
||||
Ok(match post_migrations(ctx, &run).await {
|
||||
Ok(_) => format!("Applied migration {label} on datatable \"{datatable_name}\"."),
|
||||
Err(e) => format!("Migration {label} is saved but failed to apply: {e}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version {version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables."),
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let datatable_name = str_arg(args, "datatable_name")?;
|
||||
let path = migrations_path(ctx, "run_datatable_migrations", datatable_name);
|
||||
Ok(match post_migrations(ctx, &path).await {
|
||||
Ok(result) => {
|
||||
let applied = array(result.get("applied"));
|
||||
if applied.is_empty() {
|
||||
format!("No pending migration to apply on datatable \"{datatable_name}\".")
|
||||
} else {
|
||||
format!(
|
||||
"Applied {} migration(s) on datatable \"{datatable_name}\": {}.",
|
||||
applied.len(),
|
||||
labels(result.get("applied"))
|
||||
)
|
||||
}
|
||||
}
|
||||
Err(e) => format!("Error applying migrations: {e}. Migrations before the failing one may have been applied: list them to see."),
|
||||
})
|
||||
}
|
||||
|
||||
async fn rollback_datatable_migration(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let datatable_name = str_arg(args, "datatable_name")?;
|
||||
let version = version_arg(args)?.ok_or("Missing required number argument `version`")?;
|
||||
let path = format!(
|
||||
"{}?only={version}",
|
||||
migrations_path(ctx, "rollback_datatable_migrations", datatable_name)
|
||||
);
|
||||
Ok(match post_migrations(ctx, &path).await {
|
||||
Ok(result) if array(result.get("rolled_back")).is_empty() => format!("Migration {version} is not applied on datatable \"{datatable_name}\", so there is nothing to roll back."),
|
||||
Ok(result) => format!(
|
||||
"Rolled back migration {} on datatable \"{datatable_name}\". It is pending again.",
|
||||
labels(result.get("rolled_back"))
|
||||
),
|
||||
Err(e) => format!("Error rolling back: {e}"),
|
||||
})
|
||||
}
|
||||
|
||||
const POSTGRESQL_SCHEMA_QUERY: &str = "
|
||||
SELECT table_name, column_name, udt_name, column_default, is_nullable, nsp.nspname AS table_schema FROM information_schema.columns
|
||||
RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')
|
||||
AND NOT starts_with(nsp.nspname, 'pg_') AND has_schema_privilege(nsp.oid, 'USAGE')";
|
||||
const MYSQL_SCHEMA_QUERY: &str = "SELECT DATABASE() AS default_db_name, TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT FROM information_schema.columns WHERE table_schema = DATABASE() OR table_schema NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys', '_vt');";
|
||||
const SNOWFLAKE_SCHEMA_QUERY: &str = "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'";
|
||||
const MSSQL_SCHEMA_QUERY: &str = "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'sys'";
|
||||
const BIGQUERY_SCHEMA_SCRIPT: &str = r#"import { BigQuery } from '@google-cloud/bigquery@7.5.0';
|
||||
export async function main(args: bigquery) {
|
||||
const bq = new BigQuery({
|
||||
credentials: args
|
||||
})
|
||||
const [datasets] = await bq.getDatasets();
|
||||
if (!datasets) return {}
|
||||
const schema = {} as any
|
||||
let queries = datasets.map(dataset => `
|
||||
SELECT
|
||||
table_name,
|
||||
'${dataset.id}' as dataset,
|
||||
ARRAY_AGG(STRUCT(
|
||||
if(is_nullable = 'YES', true, false) AS required,
|
||||
column_name AS name,
|
||||
data_type AS type,
|
||||
if(column_default = 'NULL', null, column_default) AS \`default\`
|
||||
) ORDER BY ordinal_position)
|
||||
AS schema
|
||||
FROM \`${dataset.id}\`.INFORMATION_SCHEMA.COLUMNS
|
||||
GROUP BY table_name`
|
||||
)
|
||||
let query = queries.join('\nUNION ALL \n')
|
||||
const [rows] = await bq.query(query)
|
||||
for (const row of rows) {
|
||||
schema[row.dataset] ??= {}
|
||||
schema[row.dataset][row.table_name] = {}
|
||||
for (const col of row.schema) {
|
||||
const colName = col.name
|
||||
delete col.name
|
||||
if (col.default === null) {
|
||||
delete col.default
|
||||
}
|
||||
schema[row.dataset][row.table_name][colName] = col
|
||||
}
|
||||
}
|
||||
return schema
|
||||
}"#;
|
||||
|
||||
/// How long a schema introspection is awaited.
|
||||
const DB_SCHEMA_WAIT_SECS: u64 = 120;
|
||||
|
||||
async fn get_db_schema(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let path = opt_str_arg(args, "resourcePath")
|
||||
.filter(|p| !p.is_empty())
|
||||
.ok_or("Database path not provided")?;
|
||||
let resource = ctx
|
||||
.get(&ctx.ws(&format!("resources/get/{}", path_enc(path))))
|
||||
.await?;
|
||||
let resource_type = resource["resource_type"].as_str().unwrap_or_default();
|
||||
let not_found = || "Database not found".to_string();
|
||||
let (language, content, arg_name) = match resource_type {
|
||||
"postgresql" => ("postgresql", POSTGRESQL_SCHEMA_QUERY, "database"),
|
||||
"mysql" => ("mysql", MYSQL_SCHEMA_QUERY, "database"),
|
||||
"ms_sql_server" | "mssql" => ("mssql", MSSQL_SCHEMA_QUERY, "database"),
|
||||
"snowflake" | "snowflake_oauth" => ("snowflake", SNOWFLAKE_SCHEMA_QUERY, "database"),
|
||||
"bigquery" => ("bun", BIGQUERY_SCHEMA_SCRIPT, "args"),
|
||||
_ => return Err(not_found()),
|
||||
};
|
||||
let id = run_script_preview(
|
||||
ctx,
|
||||
None,
|
||||
language,
|
||||
content,
|
||||
json!({ arg_name: format!("$res:{path}") }),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| not_found())?;
|
||||
let job = wait_job(ctx, &id, DB_SCHEMA_WAIT_SECS, false)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"The schema query did not finish within {DB_SCHEMA_WAIT_SECS} seconds (job {id})"
|
||||
)
|
||||
})?;
|
||||
if job["success"] != true {
|
||||
return Err(not_found());
|
||||
}
|
||||
let mut stringified =
|
||||
stringify_db_schema(resource_type, &job["result"]).ok_or_else(not_found)?;
|
||||
if stringified.chars().count() > MAX_SCHEMA_LENGTH {
|
||||
stringified = stringified
|
||||
.chars()
|
||||
.take(MAX_SCHEMA_LENGTH)
|
||||
.collect::<String>()
|
||||
+ "...";
|
||||
}
|
||||
Ok(format!(
|
||||
"DATABASE SCHEMA (each column is in the format [name, type, required, default?]):\n{stringified}"
|
||||
))
|
||||
}
|
||||
|
||||
/// The object under `key`, created empty where a JavaScript object would first get it.
|
||||
fn slot(map: &mut Map<String, Value>, key: String) -> &mut Map<String, Value> {
|
||||
map.entry(key)
|
||||
.or_insert_with(|| json!({}))
|
||||
.as_object_mut()
|
||||
.expect("only objects are inserted")
|
||||
}
|
||||
|
||||
/// The introspection result as `{schema: {table: [[name, type, required, default?]]}}`,
|
||||
/// narrowed to the default schema's tables when the database has one.
|
||||
fn stringify_db_schema(resource_type: &str, result: &Value) -> Option<String> {
|
||||
let mut schemas: Map<String, Value> = Map::new();
|
||||
let column = |name: String, ty: &Value, required: Value, default: Option<&Value>| {
|
||||
let mut col = vec![Value::String(name), ty.clone(), required];
|
||||
if truthy(default) {
|
||||
col.extend(default.cloned());
|
||||
}
|
||||
Value::Array(col)
|
||||
};
|
||||
if resource_type == "bigquery" {
|
||||
for (dataset, tables) in result.as_object()? {
|
||||
let dataset = slot(&mut schemas, dataset.clone());
|
||||
for (table, columns) in tables.as_object()? {
|
||||
let table = slot(dataset, table.clone());
|
||||
for (name, col) in columns.as_object()? {
|
||||
let entry = column(
|
||||
name.clone(),
|
||||
&col["type"],
|
||||
col["required"].clone(),
|
||||
col.get("default"),
|
||||
);
|
||||
table.insert(name.clone(), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for row in result.as_array()? {
|
||||
let (schema, table, name, ty, required, default) = match resource_type {
|
||||
"postgresql" => (
|
||||
"table_schema",
|
||||
"table_name",
|
||||
"column_name",
|
||||
"udt_name",
|
||||
row["is_nullable"] == "NO",
|
||||
row.get("column_default"),
|
||||
),
|
||||
// `required` reads IS_NULLABLE the way the session does: "YES" is true.
|
||||
"snowflake" | "snowflake_oauth" => (
|
||||
"TABLE_SCHEMA",
|
||||
"TABLE_NAME",
|
||||
"COLUMN_NAME",
|
||||
"DATA_TYPE",
|
||||
row["IS_NULLABLE"] == "YES",
|
||||
row.get("COLUMN_DEFAULT"),
|
||||
),
|
||||
// These queries select neither nullability nor a default the session reads.
|
||||
_ => (
|
||||
"TABLE_SCHEMA",
|
||||
"TABLE_NAME",
|
||||
"COLUMN_NAME",
|
||||
"DATA_TYPE",
|
||||
false,
|
||||
None,
|
||||
),
|
||||
};
|
||||
let schema = slot(&mut schemas, js_str(&row[schema]));
|
||||
// A schema with no table comes back as one row of nulls.
|
||||
if resource_type == "postgresql" && !truthy(row.get(table)) && !truthy(row.get(name)) {
|
||||
continue;
|
||||
}
|
||||
let name = js_str(&row[name]);
|
||||
let entry = column(name.clone(), &row[ty], json!(required), default);
|
||||
slot(schema, js_str(&row[table])).insert(name, entry);
|
||||
}
|
||||
}
|
||||
|
||||
for table in schemas.values_mut().filter_map(Value::as_object_mut) {
|
||||
for columns in table.values_mut() {
|
||||
if let Some(by_name) = columns.as_object_mut() {
|
||||
*columns = Value::Array(
|
||||
std::mem::take(by_name)
|
||||
.into_iter()
|
||||
.map(|(_, c)| c)
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let default_schema = ["public", "PUBLIC", "dbo"]
|
||||
.into_iter()
|
||||
.find(|k| schemas.contains_key(*k))
|
||||
.or_else(|| {
|
||||
(resource_type == "mysql" && schemas.len() == 1)
|
||||
.then(|| schemas.keys().next().map(String::as_str))
|
||||
.flatten()
|
||||
})
|
||||
.map(str::to_string);
|
||||
Some(match default_schema {
|
||||
Some(key) => schemas[&key].to_string(),
|
||||
None => Value::Object(schemas).to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn postgres_schema_is_compact_and_narrowed_to_public() {
|
||||
let rows = json!([
|
||||
{ "table_schema": "empty", "table_name": null, "column_name": null },
|
||||
{ "table_schema": "public", "table_name": "t", "column_name": "id", "udt_name": "int4", "is_nullable": "NO", "column_default": "nextval('s')" },
|
||||
{ "table_schema": "public", "table_name": "t", "column_name": "name", "udt_name": "text", "is_nullable": "YES", "column_default": null },
|
||||
{ "table_schema": "other", "table_name": "u", "column_name": "x", "udt_name": "bool", "is_nullable": "YES", "column_default": "" },
|
||||
]);
|
||||
assert_eq!(
|
||||
stringify_db_schema("postgresql", &rows).unwrap(),
|
||||
r#"{"t":[["id","int4",true,"nextval('s')"],["name","text",false]]}"#
|
||||
);
|
||||
let rows = json!([rows[0], rows[3]]);
|
||||
assert_eq!(
|
||||
stringify_db_schema("postgresql", &rows).unwrap(),
|
||||
r#"{"empty":{},"other":{"u":[["x","bool",false]]}}"#
|
||||
);
|
||||
assert_eq!(stringify_db_schema("postgresql", &json!({})), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_role_replaces_a_leading_annotation_only() {
|
||||
assert_eq!(with_migration_role("BEGIN;\nEND;", None), "BEGIN;\nEND;");
|
||||
assert_eq!(
|
||||
with_migration_role(
|
||||
"\n-- note\n--ROLE: old;\n-- roles kept\nBEGIN;\n-- role below\nEND;",
|
||||
Some("app_rw")
|
||||
),
|
||||
"-- role app_rw\n-- note\n-- roles kept\nBEGIN;\n-- role below\nEND;"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_scope_is_read_from_the_leading_block_and_overlaps() {
|
||||
let sql = "-- role r\n--SCHEMAS: Sales, billing, sales;\nBEGIN;\n-- schemas other\nEND;";
|
||||
let scope = declared_schemas(sql).unwrap();
|
||||
assert_eq!(scope, ["sales", "billing"]);
|
||||
assert_eq!(declared_schemas("BEGIN;\n-- schemas a\nEND;"), None);
|
||||
assert_eq!(declared_schemas("-- schemas a b\nBEGIN;"), None);
|
||||
assert_eq!(declared_schemas("-- schemasx a\nBEGIN;"), None);
|
||||
assert_eq!(
|
||||
with_annotation(
|
||||
&with_migration_role(sql, Some("w")),
|
||||
"schemas",
|
||||
Some("-- schemas sales".into())
|
||||
),
|
||||
"-- schemas sales\n-- role w\nBEGIN;\n-- schemas other\nEND;"
|
||||
);
|
||||
assert_eq!(
|
||||
with_annotation(sql, "schemas", None),
|
||||
"-- role r\nBEGIN;\n-- schemas other\nEND;"
|
||||
);
|
||||
|
||||
assert!(scopes_overlap(&json!(["billing", "x"]), Some(&scope)));
|
||||
assert!(!scopes_overlap(&json!(["x"]), Some(&scope)));
|
||||
assert!(scopes_overlap(&Value::Null, Some(&scope)));
|
||||
assert!(scopes_overlap(&json!(["x"]), None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_completion_caps_rows_and_rewrites_the_missing_datatable_error() {
|
||||
let rows: Vec<Value> = (0..101).map(|i| json!({ "i": i })).collect();
|
||||
let out: Value = serde_json::from_str(&format_datatable_sql_completion(
|
||||
&json!({ "success": true, "result": rows }),
|
||||
"main",
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(out["rowCount"], 101);
|
||||
assert_eq!(out["result"].as_array().unwrap().len(), 100);
|
||||
assert_eq!(out["note"], "Showing first 100 of 101 rows");
|
||||
|
||||
assert_eq!(
|
||||
format_datatable_sql_completion(&json!({ "success": true, "result": [] }), "main"),
|
||||
"{\n \"success\": true,\n \"rowCount\": 0,\n \"result\": []\n}"
|
||||
);
|
||||
assert_eq!(
|
||||
format_datatable_sql_completion(
|
||||
&json!({ "success": false, "result": { "error": { "message": "boom" } } }),
|
||||
"main"
|
||||
),
|
||||
r#"{"success":false,"error":"boom"}"#
|
||||
);
|
||||
let missing = format_datatable_sql_completion(
|
||||
&json!({ "success": false, "result": { "error": { "message": "datatable main not found" } } }),
|
||||
"main",
|
||||
);
|
||||
assert!(missing.contains("Datatable \\\"main\\\" is not configured"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
//! `get_instructions` and `read_skill`: guidance the model pulls on demand.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{opt_str_arg, str_arg, Ctx, ToolResult};
|
||||
|
||||
pub(super) const TOOLS: &[&str] = &["get_instructions", "read_skill"];
|
||||
|
||||
pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult {
|
||||
match name {
|
||||
"get_instructions" => get_instructions(ctx, args),
|
||||
"read_skill" => read_skill(ctx, args).await,
|
||||
_ => unreachable!("{name} is not a guidance tool"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Each subject's text per language, as the session renders it; `""` is the text for
|
||||
/// a call that names no language.
|
||||
#[derive(Deserialize)]
|
||||
struct Instructions {
|
||||
index: HashMap<String, HashMap<String, String>>,
|
||||
texts: HashMap<String, String>,
|
||||
}
|
||||
|
||||
static INSTRUCTIONS: LazyLock<Instructions> = LazyLock::new(|| {
|
||||
serde_json::from_str(include_str!("gen/instructions.json"))
|
||||
.expect("gen/instructions.json is valid")
|
||||
});
|
||||
|
||||
fn get_instructions(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let subject = str_arg(args, "subject")?;
|
||||
if subject == "workspace_settings" && !ctx.auth.is_admin {
|
||||
return Ok("Only a workspace admin can read or change the workspace settings. Tell the user plainly that you cannot, and suggest asking a workspace admin.".to_string());
|
||||
}
|
||||
let by_language = INSTRUCTIONS
|
||||
.index
|
||||
.get(subject)
|
||||
.ok_or_else(|| format!("Unknown subject `{subject}`"))?;
|
||||
let language = opt_str_arg(args, "language").unwrap_or("");
|
||||
let id = by_language
|
||||
.get(language)
|
||||
.ok_or_else(|| format!("Unknown language `{language}`"))?;
|
||||
Ok(INSTRUCTIONS.texts[id].clone())
|
||||
}
|
||||
|
||||
const SKILL_RESOURCE_TYPE: &str = "ai_skill";
|
||||
const SKILLS_PAGE_SIZE: usize = 100;
|
||||
/// Guard against a paging bug looping forever, not a product cap.
|
||||
const MAX_SKILLS_PAGES: usize = 100;
|
||||
const MAX_SKILL_DESCRIPTION_CHARS: usize = 1024;
|
||||
const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024;
|
||||
const TRUNCATED: &str = "… [truncated]";
|
||||
|
||||
pub(super) struct Skill {
|
||||
pub path: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Every skill the caller can read. The session lists only the ones its user left
|
||||
/// switched on, a preference kept in the browser; here they are all in play.
|
||||
pub(super) async fn list_skills(ctx: &Ctx<'_>) -> Result<Vec<Skill>, String> {
|
||||
let mut skills = Vec::new();
|
||||
for page in 1..=MAX_SKILLS_PAGES {
|
||||
let rows = ctx
|
||||
.get(&ctx.ws(&format!(
|
||||
"resources/list?resource_type={SKILL_RESOURCE_TYPE}&page={page}&per_page={SKILLS_PAGE_SIZE}"
|
||||
)))
|
||||
.await?;
|
||||
let rows = rows.as_array().map(Vec::as_slice).unwrap_or_default();
|
||||
for row in rows {
|
||||
let Some(path) = row.get("path").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
let description = row
|
||||
.get("description")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
let description = if description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS {
|
||||
let cut: String = description
|
||||
.chars()
|
||||
.take(MAX_SKILL_DESCRIPTION_CHARS)
|
||||
.collect();
|
||||
format!("{cut}{TRUNCATED}")
|
||||
} else {
|
||||
description.to_string()
|
||||
};
|
||||
skills.push(Skill { path: path.to_string(), description });
|
||||
}
|
||||
if rows.len() < SKILLS_PAGE_SIZE {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(skills)
|
||||
}
|
||||
|
||||
async fn read_skill(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let path = str_arg(args, "path")?;
|
||||
// Checked against the listing: the path is the model's, and without this the tool
|
||||
// reads any resource holding a string `content`.
|
||||
let listed = match list_skills(ctx).await {
|
||||
Ok(skills) => skills.iter().any(|skill| skill.path == path),
|
||||
Err(e) => return Ok(read_failed(path, &e)),
|
||||
};
|
||||
if !listed {
|
||||
return Ok(format!(
|
||||
"\"{path}\" is not one of the skills available here. Only the paths listed under \"Skills\" in the instructions can be read."
|
||||
));
|
||||
}
|
||||
let value = match ctx
|
||||
.get(&ctx.ws(&format!("resources/get_value/{path}")))
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(e) => return Ok(read_failed(path, &e)),
|
||||
};
|
||||
let Some(body) = value.get("content").and_then(Value::as_str) else {
|
||||
return Ok(read_failed(
|
||||
path,
|
||||
&format!("resource {path} has no string \"content\" — is it an {SKILL_RESOURCE_TYPE}?"),
|
||||
));
|
||||
};
|
||||
let body = if body.len() > MAX_SKILL_INSTRUCTIONS_BYTES {
|
||||
let mut end = MAX_SKILL_INSTRUCTIONS_BYTES;
|
||||
while !body.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
format!("{}{TRUNCATED}", &body[..end])
|
||||
} else {
|
||||
body.to_string()
|
||||
};
|
||||
Ok(format!("Skill: {path}\n\nInstructions:\n{body}"))
|
||||
}
|
||||
|
||||
fn read_failed(path: &str, message: &str) -> String {
|
||||
format!(
|
||||
"Failed to read skill \"{path}\": {message}. Check the path against the Skills list in the instructions."
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! The instructions a client receives on connecting: the session's system prompt as
|
||||
//! rendered for an external client (`gen/prompt.json`), filled in for the caller. What
|
||||
//! differs per workspace is cut out of it, so a connection spanning several workspaces
|
||||
//! gets the rest up front and each workspace's part from `get_workspace_context`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde::Deserialize;
|
||||
use windmill_mcp::server::McpAuth;
|
||||
|
||||
use super::{guidance::list_skills, Ctx};
|
||||
|
||||
/// The prompt with a `{{folder_guidance}}` slot, the guidance once per folder situation,
|
||||
/// and the sections appended when the workspace has something to put there.
|
||||
#[derive(Deserialize)]
|
||||
struct PromptParts {
|
||||
base: String,
|
||||
folder_guidance: FolderGuidance,
|
||||
skills_header: String,
|
||||
skill_line: String,
|
||||
workspace_instructions: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct FolderGuidance {
|
||||
admin: String,
|
||||
admin_no_folders: String,
|
||||
member: String,
|
||||
member_no_folders: String,
|
||||
member_readonly: String,
|
||||
member_no_folders_readonly: String,
|
||||
}
|
||||
|
||||
static PARTS: LazyLock<PromptParts> = LazyLock::new(|| {
|
||||
serde_json::from_str(include_str!("gen/prompt.json")).expect("gen/prompt.json is valid")
|
||||
});
|
||||
|
||||
/// Folders named in the prompt before the rest are counted rather than listed, as the
|
||||
/// session does, so a folder-heavy workspace cannot dominate the instructions.
|
||||
const MAX_LISTED_FOLDERS: usize = 40;
|
||||
|
||||
fn folder_list(names: &[&str]) -> String {
|
||||
let shown = names
|
||||
.iter()
|
||||
.take(MAX_LISTED_FOLDERS)
|
||||
.map(|n| format!("`f/{n}`"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
if names.len() > MAX_LISTED_FOLDERS {
|
||||
format!("{shown} (+{} more)", names.len() - MAX_LISTED_FOLDERS)
|
||||
} else {
|
||||
shown
|
||||
}
|
||||
}
|
||||
|
||||
fn folder_guidance(auth: &crate::db::ApiAuthed) -> String {
|
||||
let writable: Vec<&str> = auth
|
||||
.folders()
|
||||
.iter()
|
||||
.filter(|(_, write, _)| *write)
|
||||
.map(|(name, _, _)| name.as_str())
|
||||
.collect();
|
||||
let readonly: Vec<&str> = auth
|
||||
.folders()
|
||||
.iter()
|
||||
.filter(|(_, write, _)| !*write)
|
||||
.map(|(name, _, _)| name.as_str())
|
||||
.collect();
|
||||
|
||||
let guidance = &PARTS.folder_guidance;
|
||||
let template = match (auth.is_admin(), writable.is_empty(), readonly.is_empty()) {
|
||||
(true, false, _) => &guidance.admin,
|
||||
(true, true, _) => &guidance.admin_no_folders,
|
||||
(false, false, true) => &guidance.member,
|
||||
(false, false, false) => &guidance.member_readonly,
|
||||
(false, true, true) => &guidance.member_no_folders,
|
||||
(false, true, false) => &guidance.member_no_folders_readonly,
|
||||
};
|
||||
template
|
||||
.replace("{{username}}", auth.username())
|
||||
.replace("{{writable_folders}}", &folder_list(&writable))
|
||||
.replace("{{readonly_folders}}", &folder_list(&readonly))
|
||||
}
|
||||
|
||||
/// The workspace's skills and its admin-written instructions.
|
||||
async fn appendix(ctx: &Ctx<'_>) -> String {
|
||||
let mut text = String::new();
|
||||
// Either lookup failing leaves its section out: a client that cannot connect at all
|
||||
// is worse than one that connects without the skills list.
|
||||
if let Ok(skills) = list_skills(ctx).await {
|
||||
if !skills.is_empty() {
|
||||
text.push_str(&PARTS.skills_header);
|
||||
let lines = skills
|
||||
.iter()
|
||||
.map(|skill| {
|
||||
PARTS
|
||||
.skill_line
|
||||
.replace("{{skill_path}}", &skill.path)
|
||||
.replace("{{skill_description}}", &skill.description)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
text.push_str(&lines);
|
||||
}
|
||||
}
|
||||
if let Ok(info) = ctx.get(&ctx.ws("workspaces/get_copilot_info")).await {
|
||||
let workspace = info
|
||||
.pointer("/custom_prompts/global")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
if let Some(workspace) = workspace {
|
||||
text.push_str(
|
||||
&PARTS
|
||||
.workspace_instructions
|
||||
.replace("{{workspace_instructions}}", workspace),
|
||||
);
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// The whole prompt, for a connection that reaches one workspace.
|
||||
pub(in crate::mcp) async fn session_instructions(ctx: &Ctx<'_>) -> String {
|
||||
let mut prompt = PARTS
|
||||
.base
|
||||
.replace("{{folder_guidance}}", &folder_guidance(ctx.auth))
|
||||
.replace("{{username}}", ctx.auth.username());
|
||||
prompt.push_str(&appendix(ctx).await);
|
||||
prompt
|
||||
}
|
||||
|
||||
/// The prompt without anything specific to a workspace, for a connection that reaches
|
||||
/// several: usernames are per workspace, so paths read `u/<username>/...`.
|
||||
pub(in crate::mcp) fn generic_session_instructions() -> String {
|
||||
PARTS
|
||||
.base
|
||||
.replace("{{folder_guidance}}", "")
|
||||
.replace("{{username}}", "<username>")
|
||||
}
|
||||
|
||||
/// What `generic_session_instructions` leaves out, for one workspace.
|
||||
pub(in crate::mcp) async fn workspace_context(ctx: &Ctx<'_>) -> String {
|
||||
let username = ctx.auth.username();
|
||||
format!(
|
||||
"Workspace `{}`. Your username here is \"{username}\", so your personal paths are \
|
||||
`u/{username}/<name>`.\n{}{}",
|
||||
ctx.w_id,
|
||||
folder_guidance(ctx.auth).trim_end(),
|
||||
appendix(ctx).await
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,720 @@
|
||||
//! Draft writes for the item kinds that are plain records: schedules, triggers, resources, variables.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::workspace::{finish_draft_write, persist_draft, write_base, Persist};
|
||||
use super::{non_null, put, tool_parameters, Ctx, ToolResult, SCHEMAS};
|
||||
|
||||
pub(super) const TOOLS: &[&str] = &[
|
||||
"write_schedule",
|
||||
"write_trigger",
|
||||
"write_resource",
|
||||
"write_variable",
|
||||
];
|
||||
|
||||
pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult {
|
||||
match name {
|
||||
"write_schedule" => write_schedule(ctx, args).await,
|
||||
"write_trigger" => write_trigger(ctx, args).await,
|
||||
"write_resource" => write_resource(ctx, args).await,
|
||||
"write_variable" => write_variable(ctx, args).await,
|
||||
_ => unreachable!("{name} is not an items tool"),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Argument parsing. The session parses arguments with zod schemas; the same schemas
|
||||
// reach this server as JSON Schema (`gen/tools.json`, `gen/schemas.json`), and `parse`
|
||||
// applies them with zod's semantics rather than JSON Schema's: an undeclared key is
|
||||
// dropped instead of refused, a `default` fills a missing key, a union yields the
|
||||
// first branch that accepts the value.
|
||||
// ============================================================================
|
||||
|
||||
fn tool_schema(name: &str) -> Value {
|
||||
tool_parameters(name)
|
||||
.map(|parameters| Value::Object(parameters.clone()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// A schedule's whole schema: the fields `write_schedule` lists plus the ones it takes
|
||||
/// through `advanced`, which the session spreads to the top level before parsing.
|
||||
static SCHEDULE_SCHEMA: LazyLock<Value> = LazyLock::new(|| {
|
||||
let mut schema = tool_schema("write_schedule");
|
||||
let advanced: Value = serde_json::from_str(&SCHEMAS.schedule).unwrap_or_default();
|
||||
if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) {
|
||||
properties.remove("advanced");
|
||||
if let Some(advanced) = advanced.get("properties").and_then(Value::as_object) {
|
||||
properties.extend(advanced.clone());
|
||||
}
|
||||
}
|
||||
schema
|
||||
});
|
||||
|
||||
fn received(value: Option<&Value>) -> &'static str {
|
||||
match value {
|
||||
None => "undefined",
|
||||
Some(Value::Null) => "null",
|
||||
Some(Value::Bool(_)) => "boolean",
|
||||
Some(Value::Number(_)) => "number",
|
||||
Some(Value::String(_)) => "string",
|
||||
Some(Value::Array(_)) => "array",
|
||||
Some(Value::Object(_)) => "object",
|
||||
}
|
||||
}
|
||||
|
||||
fn join(path: &str, key: &str) -> String {
|
||||
if path.is_empty() {
|
||||
key.to_string()
|
||||
} else {
|
||||
format!("{path}.{key}")
|
||||
}
|
||||
}
|
||||
|
||||
/// `value` as `schema` accepts it, `None` standing for a missing one. Each violation is
|
||||
/// pushed to `issues` as `<dotted path>: <message>`.
|
||||
fn parse(
|
||||
schema: &Value,
|
||||
value: Option<&Value>,
|
||||
path: &str,
|
||||
issues: &mut Vec<String>,
|
||||
) -> Option<Value> {
|
||||
let mut issue = |message: String| {
|
||||
let at = if path.is_empty() { "<root>" } else { path };
|
||||
issues.push(format!("{at}: {message}"));
|
||||
None
|
||||
};
|
||||
if value.is_none() {
|
||||
if let Some(default) = schema.get("default") {
|
||||
return Some(default.clone());
|
||||
}
|
||||
}
|
||||
if let Some(branches) = schema.get("anyOf").and_then(Value::as_array) {
|
||||
let is_null = |branch: &&Value| branch.get("type").and_then(Value::as_str) == Some("null");
|
||||
if value.is_some_and(Value::is_null) && branches.iter().any(|b| is_null(&b)) {
|
||||
return Some(Value::Null);
|
||||
}
|
||||
let branches: Vec<&Value> = branches.iter().filter(|b| !is_null(b)).collect();
|
||||
// A nullable field reports what its one real type refuses.
|
||||
if let [only] = branches[..] {
|
||||
return parse(only, value, path, issues);
|
||||
}
|
||||
for branch in branches {
|
||||
let mut refused = Vec::new();
|
||||
let parsed = parse(branch, value, path, &mut refused);
|
||||
if refused.is_empty() {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
return issue("Invalid input".to_string());
|
||||
}
|
||||
if let Some(options) = schema.get("enum").and_then(Value::as_array) {
|
||||
if value.is_some_and(|v| options.contains(v)) {
|
||||
return value.cloned();
|
||||
}
|
||||
let options: Vec<String> = options.iter().map(Value::to_string).collect();
|
||||
return issue(format!(
|
||||
"Invalid option: expected one of {}",
|
||||
options.join("|")
|
||||
));
|
||||
}
|
||||
let Some(kind) = schema.get("type").and_then(Value::as_str) else {
|
||||
return value.cloned();
|
||||
};
|
||||
match (kind, value) {
|
||||
("boolean", Some(Value::Bool(_))) | ("null", Some(Value::Null)) => value.cloned(),
|
||||
("string", Some(Value::String(text))) => {
|
||||
let matches = schema
|
||||
.get("pattern")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|pattern| regex::Regex::new(pattern).ok())
|
||||
.is_none_or(|pattern| pattern.is_match(text));
|
||||
if matches {
|
||||
value.cloned()
|
||||
} else if schema.get("format").and_then(Value::as_str) == Some("date-time") {
|
||||
issue("Invalid ISO datetime".to_string())
|
||||
} else {
|
||||
issue("Invalid string".to_string())
|
||||
}
|
||||
}
|
||||
("number" | "integer", Some(Value::Number(number))) => {
|
||||
let n = number.as_f64().unwrap_or_default();
|
||||
let bound = |key: &str| schema.get(key).and_then(Value::as_f64);
|
||||
if kind == "integer" && n.fract() != 0.0 {
|
||||
issue("Invalid input: expected int, received number".to_string())
|
||||
} else if let Some(min) = bound("minimum").filter(|min| n < *min) {
|
||||
issue(format!("Too small: expected number to be >={min}"))
|
||||
} else if let Some(max) = bound("maximum").filter(|max| n > *max) {
|
||||
issue(format!("Too big: expected number to be <={max}"))
|
||||
} else {
|
||||
value.cloned()
|
||||
}
|
||||
}
|
||||
("array", Some(Value::Array(items))) => {
|
||||
let any = json!({});
|
||||
let item_schema = schema.get("items").unwrap_or(&any);
|
||||
Some(Value::Array(
|
||||
items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, item)| {
|
||||
parse(item_schema, Some(item), &join(path, &i.to_string()), issues)
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
("object", Some(Value::Object(fields))) => {
|
||||
let Some(properties) = schema.get("properties").and_then(Value::as_object) else {
|
||||
return value.cloned();
|
||||
};
|
||||
let required = schema.get("required").and_then(Value::as_array);
|
||||
let mut out = Map::new();
|
||||
for (key, property) in properties {
|
||||
let field = fields.get(key);
|
||||
let optional = !required.is_some_and(|r| r.iter().any(|k| k == key));
|
||||
if field.is_none() && optional && property.get("default").is_none() {
|
||||
continue;
|
||||
}
|
||||
if let Some(parsed) = parse(property, field, &join(path, key), issues) {
|
||||
out.insert(key.clone(), parsed);
|
||||
}
|
||||
}
|
||||
if schema.get("additionalProperties") != Some(&Value::Bool(false)) {
|
||||
for (key, field) in fields {
|
||||
if !properties.contains_key(key) {
|
||||
out.insert(key.clone(), field.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Value::Object(out))
|
||||
}
|
||||
_ => {
|
||||
let expected = match kind {
|
||||
"integer" => "number",
|
||||
"object" if schema.get("properties").is_none() => "record",
|
||||
other => other,
|
||||
};
|
||||
issue(format!(
|
||||
"Invalid input: expected {expected}, received {}",
|
||||
received(value)
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_args(tool: &str, args: &Value) -> Result<Value, String> {
|
||||
parse_with(&tool_schema(tool), args)
|
||||
}
|
||||
|
||||
fn parse_with(schema: &Value, args: &Value) -> Result<Value, String> {
|
||||
let mut issues = Vec::new();
|
||||
let parsed = parse(schema, Some(args), "", &mut issues);
|
||||
match parsed {
|
||||
Some(parsed) if issues.is_empty() => Ok(parsed),
|
||||
_ => Err(format!("Invalid arguments. Issues: {}", issues.join("; "))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Key paths `supplied` carries that parsing dropped. Every sub-field of a schedule's
|
||||
/// `retry` is optional, so a guessed shape parses clean, loses its misspelled keys and
|
||||
/// would save a policy that does nothing.
|
||||
fn dropped_keys(supplied: &Value, parsed: Option<&Value>, prefix: &str, out: &mut Vec<String>) {
|
||||
let Value::Object(supplied) = supplied else {
|
||||
if parsed.is_none() && !prefix.is_empty() {
|
||||
out.push(prefix.to_string());
|
||||
}
|
||||
return;
|
||||
};
|
||||
match parsed {
|
||||
Some(parsed @ (Value::Object(_) | Value::Array(_))) => {
|
||||
for (key, value) in supplied {
|
||||
dropped_keys(value, parsed.get(key), &join(prefix, key), out);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if !supplied.is_empty() && !prefix.is_empty() {
|
||||
out.push(prefix.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn describe_dropped_schedule_options(dropped: &[String]) -> String {
|
||||
let is_field = |path: &&String| {
|
||||
let field = path.split('.').next().unwrap_or_default();
|
||||
field != "override" && SCHEDULE_SCHEMA["properties"].get(field).is_some()
|
||||
};
|
||||
let (misshaped, unknown): (Vec<&String>, Vec<&String>) = dropped.iter().partition(is_field);
|
||||
let list = |paths: &[&String]| {
|
||||
paths
|
||||
.iter()
|
||||
.map(|p| p.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
};
|
||||
let mut parts = Vec::new();
|
||||
if !misshaped.is_empty() {
|
||||
parts.push(format!("These schedule options do not match their schema and would have been dropped: {}. Call get_schedule_schema for their exact shape.", list(&misshaped)));
|
||||
}
|
||||
if !unknown.is_empty() {
|
||||
parts.push(format!(
|
||||
"These are not schedule fields: {}.",
|
||||
list(&unknown)
|
||||
));
|
||||
}
|
||||
parts.join(" ")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Draft values
|
||||
// ============================================================================
|
||||
|
||||
fn into_object(value: Option<Value>) -> Map<String, Value> {
|
||||
match value {
|
||||
Some(Value::Object(map)) => map,
|
||||
_ => Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A schedule's or trigger's draft: the base without what the server stamps on a
|
||||
/// deployed row, then the fields of this write.
|
||||
fn merge_config(base: Option<Value>, config: Value, path: &str) -> Map<String, Value> {
|
||||
let mut draft = into_object(base);
|
||||
for key in ["workspace_id", "edited_by", "edited_at", "email", "error"] {
|
||||
draft.remove(key);
|
||||
}
|
||||
draft.extend(into_object(Some(config)));
|
||||
draft.insert("path".into(), json!(path));
|
||||
draft
|
||||
}
|
||||
|
||||
fn resource_to_draft_state(resource: &Value) -> Value {
|
||||
let mut state = Map::new();
|
||||
state.insert("path".into(), resource["path"].clone());
|
||||
state.insert(
|
||||
"description".into(),
|
||||
non_null(resource.get("description"))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("")),
|
||||
);
|
||||
state.insert(
|
||||
"args".into(),
|
||||
non_null(resource.get("value"))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!({})),
|
||||
);
|
||||
put(
|
||||
&mut state,
|
||||
"labels",
|
||||
non_null(resource.get("labels")).cloned(),
|
||||
);
|
||||
state.insert(
|
||||
"wsSpecific".into(),
|
||||
non_null(resource.get("ws_specific"))
|
||||
.cloned()
|
||||
.unwrap_or(json!(false)),
|
||||
);
|
||||
state.insert("resource_type".into(), resource["resource_type"].clone());
|
||||
Value::Object(state)
|
||||
}
|
||||
|
||||
fn resource_draft(args: &Value, base: Option<Value>) -> Value {
|
||||
let mut draft = into_object(base);
|
||||
let merged = |arg: &str, key: &str| {
|
||||
non_null(args.get(arg))
|
||||
.or(non_null(draft.get(key)))
|
||||
.cloned()
|
||||
};
|
||||
let description = merged("description", "description").unwrap_or_else(|| json!(""));
|
||||
let value = merged("value", "args").unwrap_or_else(|| json!({}));
|
||||
let labels = merged("labels", "labels");
|
||||
let ws_specific = merged("ws_specific", "wsSpecific").unwrap_or(json!(false));
|
||||
let resource_type = merged("resource_type", "resource_type");
|
||||
draft.insert("path".into(), args["path"].clone());
|
||||
draft.insert("description".into(), description);
|
||||
draft.insert("args".into(), value);
|
||||
put(&mut draft, "labels", labels);
|
||||
draft.insert("wsSpecific".into(), ws_specific);
|
||||
put(&mut draft, "resource_type", resource_type);
|
||||
Value::Object(draft)
|
||||
}
|
||||
|
||||
fn variable_to_draft_state(variable: &Value) -> Value {
|
||||
let field = |key: &str| non_null(variable.get(key)).cloned();
|
||||
// An OAuth variable's value belongs to the refresh flow, and the API hands back a
|
||||
// freshly refreshed live token for an expired one even undecrypted: carried into a
|
||||
// draft, it would pin a rotating value to a static one at the next deploy.
|
||||
let oauth_managed = variable.get("is_oauth").and_then(Value::as_bool) == Some(true)
|
||||
|| field("account").is_some();
|
||||
let value = if oauth_managed { None } else { field("value") };
|
||||
let mut state = Map::new();
|
||||
state.insert("path".into(), variable["path"].clone());
|
||||
state.insert(
|
||||
"variable".into(),
|
||||
json!({
|
||||
"value": value.unwrap_or_else(|| json!("")),
|
||||
"is_secret": variable["is_secret"],
|
||||
"description": field("description").unwrap_or_else(|| json!("")),
|
||||
}),
|
||||
);
|
||||
put(&mut state, "labels", field("labels"));
|
||||
state.insert(
|
||||
"wsSpecific".into(),
|
||||
field("ws_specific").unwrap_or(json!(false)),
|
||||
);
|
||||
for key in ["account", "is_oauth", "expires_at"] {
|
||||
put(&mut state, key, field(key));
|
||||
}
|
||||
Value::Object(state)
|
||||
}
|
||||
|
||||
/// The draft a `write_variable` call leaves. A variable's value is never read back, so
|
||||
/// an omitted `value` keeps whatever the base holds: `""` for "nothing staged", or the
|
||||
/// `$encrypted:` marker the draft store turned a staged secret into.
|
||||
fn variable_draft(args: &Value, base: Option<Value>) -> Result<Value, String> {
|
||||
let path = args["path"].as_str().unwrap_or_default();
|
||||
let arg = |key: &str| non_null(args.get(key)).cloned();
|
||||
let stored = |key: &str| {
|
||||
base.as_ref()
|
||||
.and_then(|b| b.get("variable"))
|
||||
.and_then(|v| non_null(v.get(key)).cloned())
|
||||
};
|
||||
let value = arg("value");
|
||||
if base.is_none() && (value.is_none() || arg("is_secret").is_none()) {
|
||||
return Err(format!("Variable \"{path}\" does not exist yet, so creating it requires both value and is_secret."));
|
||||
}
|
||||
let was_secret = stored("is_secret").and_then(|s| s.as_bool());
|
||||
let is_secret = arg("is_secret")
|
||||
.and_then(|s| s.as_bool())
|
||||
.or(was_secret)
|
||||
.unwrap_or(false);
|
||||
let empty = json!("");
|
||||
// "" is how a secret draft says "nothing staged", so it cannot also mean "set the
|
||||
// secret to empty".
|
||||
if is_secret && value.as_ref() == Some(&empty) {
|
||||
return Err(format!("An empty string is not a valid value for secret variable \"{path}\". Omit value to keep the stored secret, or pass the real new one."));
|
||||
}
|
||||
let next_value = value.clone().or_else(|| stored("value"));
|
||||
if is_secret && was_secret == Some(false) && next_value.as_ref() == Some(&empty) {
|
||||
return Err(format!("Cannot make variable \"{path}\" secret without a value: it currently holds an empty one, so there would be nothing to encrypt. Pass the value it should hold."));
|
||||
}
|
||||
// The stored value of a secret cannot be read, and an `$encrypted:` marker is only
|
||||
// decrypted at deploy while the target stays secret.
|
||||
if !is_secret && was_secret == Some(true) && value.is_none() {
|
||||
return Err(format!("Cannot turn secret variable \"{path}\" into a non-secret one without a value: its stored value cannot be read, so it would be replaced by an empty one. Pass the new plaintext value, or leave is_secret unset to keep it secret."));
|
||||
}
|
||||
let description = arg("description").or_else(|| stored("description"));
|
||||
|
||||
let mut draft = into_object(base);
|
||||
let merged = |arg_key: &str, key: &str| {
|
||||
non_null(args.get(arg_key))
|
||||
.or(non_null(draft.get(key)))
|
||||
.cloned()
|
||||
};
|
||||
let labels = merged("labels", "labels");
|
||||
let ws_specific = merged("ws_specific", "wsSpecific").unwrap_or(json!(false));
|
||||
let extras = ["account", "is_oauth", "expires_at"].map(|key| (key, merged(key, key)));
|
||||
draft.insert("path".into(), json!(path));
|
||||
draft.insert(
|
||||
"variable".into(),
|
||||
json!({
|
||||
"value": next_value.unwrap_or(empty),
|
||||
"is_secret": is_secret,
|
||||
"description": description.unwrap_or_else(|| json!("")),
|
||||
}),
|
||||
);
|
||||
put(&mut draft, "labels", labels);
|
||||
draft.insert("wsSpecific".into(), ws_specific);
|
||||
for (key, value) in extras {
|
||||
put(&mut draft, key, value);
|
||||
}
|
||||
Ok(Value::Object(draft))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools
|
||||
// ============================================================================
|
||||
|
||||
/// The shared write: build on the caller's draft, else on the deployed item (shaped
|
||||
/// into a draft value by `from_deployed`), else from nothing.
|
||||
async fn write_draft(
|
||||
ctx: &Ctx<'_>,
|
||||
item_type: &str,
|
||||
trigger_kind: Option<&str>,
|
||||
path: &str,
|
||||
force: bool,
|
||||
from_deployed: impl FnOnce(Value) -> Value,
|
||||
build: impl FnOnce(Option<Value>) -> Result<Value, String>,
|
||||
) -> ToolResult {
|
||||
let base = write_base(ctx, item_type, path, trigger_kind).await?;
|
||||
let existed = base.base.is_some();
|
||||
let from_draft = base.from_draft;
|
||||
let draft = build(
|
||||
base.base
|
||||
.map(|b| if from_draft { b } else { from_deployed(b) }),
|
||||
)?;
|
||||
let persisted = persist_draft(
|
||||
ctx,
|
||||
item_type,
|
||||
path,
|
||||
trigger_kind,
|
||||
&draft,
|
||||
base.last_sync.as_deref(),
|
||||
force,
|
||||
)
|
||||
.await?;
|
||||
Ok(match persisted {
|
||||
Persist::Saved(item) => finish_draft_write(ctx, &item, existed),
|
||||
Persist::Failed(text) => text,
|
||||
})
|
||||
}
|
||||
|
||||
fn take_override(parsed: &mut Value) -> bool {
|
||||
parsed
|
||||
.as_object_mut()
|
||||
.and_then(|o| o.remove("override"))
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn write_schedule(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
// A named argument outranks the same key inside `advanced`.
|
||||
let mut merged = into_object(args.get("advanced").cloned());
|
||||
for (key, value) in args.as_object().into_iter().flatten() {
|
||||
if key != "advanced" {
|
||||
merged.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
let merged = Value::Object(merged);
|
||||
let mut parsed = parse_with(&SCHEDULE_SCHEMA, &merged)?;
|
||||
let mut dropped = Vec::new();
|
||||
dropped_keys(&merged, Some(&parsed), "", &mut dropped);
|
||||
if !dropped.is_empty() {
|
||||
return Err(describe_dropped_schedule_options(&dropped));
|
||||
}
|
||||
let force = take_override(&mut parsed);
|
||||
let path = parsed["path"].as_str().unwrap_or_default().to_string();
|
||||
write_draft(
|
||||
ctx,
|
||||
"schedule",
|
||||
None,
|
||||
&path,
|
||||
force,
|
||||
|deployed| deployed,
|
||||
|base| Ok(Value::Object(merge_config(base, parsed, &path))),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_trigger(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let mut parsed = parse_args("write_trigger", args)?;
|
||||
let force = take_override(&mut parsed);
|
||||
let kind = parsed["kind"].as_str().unwrap_or_default();
|
||||
let schema: Value = SCHEMAS
|
||||
.trigger
|
||||
.get(kind)
|
||||
.and_then(|schema| serde_json::from_str(schema).ok())
|
||||
.ok_or_else(|| format!("Unknown trigger kind `{kind}`"))?;
|
||||
// The draft is filed under `kind`, so a config of another kind is refused here
|
||||
// rather than stored as a draft no editor or deploy can read.
|
||||
let mut issues = Vec::new();
|
||||
let config = parse(&schema, parsed.get("config"), "", &mut issues);
|
||||
let config = match config {
|
||||
Some(config) if issues.is_empty() => config,
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"Invalid config for a \"{kind}\" trigger. Call get_trigger_schema with kind \"{kind}\" for its exact fields. Issues: {}",
|
||||
issues.join("; ")
|
||||
))
|
||||
}
|
||||
};
|
||||
let path = config["path"].as_str().unwrap_or_default().to_string();
|
||||
write_draft(
|
||||
ctx,
|
||||
"trigger",
|
||||
Some(kind),
|
||||
&path,
|
||||
force,
|
||||
|deployed| deployed,
|
||||
|base| {
|
||||
let mut draft = merge_config(base, config, &path);
|
||||
// Optional in the schema, but a NOT NULL column. Defaulted on the merged
|
||||
// draft so an omitted field keeps the existing trigger's value.
|
||||
if kind == "email" && non_null(draft.get("workspaced_local_part")).is_none() {
|
||||
draft.insert("workspaced_local_part".into(), json!(false));
|
||||
}
|
||||
Ok(Value::Object(draft))
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_resource(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let mut parsed = parse_args("write_resource", args)?;
|
||||
let force = take_override(&mut parsed);
|
||||
let path = parsed["path"].as_str().unwrap_or_default().to_string();
|
||||
write_draft(
|
||||
ctx,
|
||||
"resource",
|
||||
None,
|
||||
&path,
|
||||
force,
|
||||
|deployed| resource_to_draft_state(&deployed),
|
||||
|base| Ok(resource_draft(&parsed, base)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_variable(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let mut parsed = parse_args("write_variable", args)?;
|
||||
let force = take_override(&mut parsed);
|
||||
let path = parsed["path"].as_str().unwrap_or_default().to_string();
|
||||
// A variable's value is never interpolated, so its own `$var:` reference is the
|
||||
// reference syntax echoed back, not a value.
|
||||
if let Some(value) = parsed.get("value").and_then(Value::as_str) {
|
||||
if value == format!("$var:{path}") {
|
||||
return Err(format!("\"{value}\" is not a valid value for variable \"{path}\" — it is a self-reference. Omit value to keep the current one."));
|
||||
}
|
||||
}
|
||||
write_draft(
|
||||
ctx,
|
||||
"variable",
|
||||
None,
|
||||
&path,
|
||||
force,
|
||||
|deployed| variable_to_draft_state(&deployed),
|
||||
|base| variable_draft(&parsed, base),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn issues_of(schema: &Value, value: Value) -> (Option<Value>, Vec<String>) {
|
||||
let mut issues = Vec::new();
|
||||
let parsed = parse(schema, Some(&value), "", &mut issues);
|
||||
(parsed, issues)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trigger_config_parses_like_the_session() {
|
||||
let kafka: Value = serde_json::from_str(&SCHEMAS.trigger["kafka"]).unwrap();
|
||||
let (parsed, issues) = issues_of(
|
||||
&kafka,
|
||||
json!({
|
||||
"path": "p", "script_path": "s", "is_flow": false, "kafka_resource_path": "r",
|
||||
"group_id": "g", "topics": ["a"], "unknown_field": 1,
|
||||
"filters": [{"key": "k", "value": 1, "zzz": 2}, {"any_of": [{"a": 1}]}],
|
||||
}),
|
||||
);
|
||||
assert_eq!(issues, Vec::<String>::new());
|
||||
let parsed = parsed.unwrap();
|
||||
assert_eq!(parsed.get("unknown_field"), None);
|
||||
assert_eq!(
|
||||
parsed["filters"],
|
||||
json!([{"key": "k", "value": 1}, {"any_of": [{"a": 1}]}])
|
||||
);
|
||||
assert_eq!(parsed["filter_logic"], "and");
|
||||
assert_eq!(parsed["auto_commit"], true);
|
||||
|
||||
let (_, issues) = issues_of(
|
||||
&kafka,
|
||||
json!({
|
||||
"path": 5, "is_flow": "x", "topics": "a", "filters": [3], "group_id": null,
|
||||
"auto_offset_reset": "x", "kafka_resource_path": "r",
|
||||
"retry": {"exponential": {"seconds": 0, "random_factor": 1.5}, "retry_if": {}},
|
||||
}),
|
||||
);
|
||||
assert_eq!(
|
||||
issues,
|
||||
[
|
||||
"path: Invalid input: expected string, received number",
|
||||
"script_path: Invalid input: expected string, received undefined",
|
||||
"is_flow: Invalid input: expected boolean, received string",
|
||||
"group_id: Invalid input: expected string, received null",
|
||||
"topics: Invalid input: expected array, received string",
|
||||
"filters.0: Invalid input",
|
||||
"auto_offset_reset: Invalid option: expected one of \"latest\"|\"earliest\"",
|
||||
"retry.exponential.seconds: Too small: expected number to be >=1",
|
||||
"retry.exponential.random_factor: Invalid input: expected int, received number",
|
||||
"retry.retry_if.expr: Invalid input: expected string, received undefined",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schedule_reports_what_parsing_would_drop() {
|
||||
let merged = json!({
|
||||
"path": "p", "schedule": "s", "timezone": "UTC", "script_path": "x",
|
||||
"is_flow": false, "args": null, "bogus": 1, "override": true,
|
||||
"retry": {"constant": {"attempts": 2, "secs": 3}},
|
||||
});
|
||||
let parsed = parse_with(&SCHEDULE_SCHEMA, &merged).unwrap();
|
||||
let mut dropped = Vec::new();
|
||||
dropped_keys(&merged, Some(&parsed), "", &mut dropped);
|
||||
assert_eq!(
|
||||
describe_dropped_schedule_options(&dropped),
|
||||
"These schedule options do not match their schema and would have been dropped: retry.constant.secs. Call get_schedule_schema for their exact shape. These are not schedule fields: bogus."
|
||||
);
|
||||
assert!(parse_with(
|
||||
&SCHEDULE_SCHEMA,
|
||||
&json!({"path": "p", "paused_until": "tomorrow"})
|
||||
)
|
||||
.unwrap_err()
|
||||
.contains("paused_until: Invalid ISO datetime"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn variable_value_rules() {
|
||||
let secret = json!({
|
||||
"path": "p",
|
||||
"variable": {"value": "$encrypted:abc", "is_secret": true, "description": "d"},
|
||||
"wsSpecific": false,
|
||||
});
|
||||
let plain_empty =
|
||||
json!({"path": "p", "variable": {"value": "", "is_secret": false, "description": ""}});
|
||||
|
||||
assert!(variable_draft(&json!({"path": "p", "value": "v"}), None).is_err());
|
||||
let kept = variable_draft(
|
||||
&json!({"path": "p", "description": "new"}),
|
||||
Some(secret.clone()),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
kept["variable"],
|
||||
json!({"value": "$encrypted:abc", "is_secret": true, "description": "new"})
|
||||
);
|
||||
assert!(variable_draft(&json!({"path": "p", "value": ""}), Some(secret.clone())).is_err());
|
||||
assert!(variable_draft(
|
||||
&json!({"path": "p", "is_secret": false}),
|
||||
Some(secret.clone())
|
||||
)
|
||||
.is_err());
|
||||
let revealed = variable_draft(
|
||||
&json!({"path": "p", "is_secret": false, "value": "v"}),
|
||||
Some(secret),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(revealed["variable"]["value"], "v");
|
||||
assert!(
|
||||
variable_draft(&json!({"path": "p", "is_secret": true}), Some(plain_empty)).is_err()
|
||||
);
|
||||
|
||||
let oauth = variable_to_draft_state(&json!({
|
||||
"path": "p", "value": "live-token", "is_secret": true, "account": 3,
|
||||
"description": null, "expires_at": null,
|
||||
}));
|
||||
assert_eq!(
|
||||
oauth,
|
||||
json!({
|
||||
"path": "p",
|
||||
"variable": {"value": "", "is_secret": true, "description": ""},
|
||||
"wsSpecific": false,
|
||||
"account": 3,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,728 @@
|
||||
//! Lookups that touch no draft: hub, docs, npm, resource types, workers, folders.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use futures::future::join_all;
|
||||
use serde_json::{json, Map, Value};
|
||||
use windmill_common::utils::HTTP_CLIENT;
|
||||
|
||||
use super::{
|
||||
array, enc, is_not_found, opt_str_arg, pretty, str_arg, truthy, Ctx, ToolResult, SCHEMAS,
|
||||
};
|
||||
|
||||
pub(super) const TOOLS: &[&str] = &[
|
||||
"search_hub_scripts",
|
||||
"get_hub_integration",
|
||||
"search_npm_packages",
|
||||
"search_docs",
|
||||
"read_docs_page",
|
||||
"search_resource_types",
|
||||
"create_resource_type",
|
||||
"list_workers",
|
||||
"create_folder",
|
||||
"get_trigger_schema",
|
||||
"get_schedule_schema",
|
||||
];
|
||||
|
||||
pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult {
|
||||
match name {
|
||||
"search_hub_scripts" => search_hub_scripts(ctx, args).await,
|
||||
"get_hub_integration" => get_hub_integration(ctx, args).await,
|
||||
"search_npm_packages" => search_npm_packages(args).await,
|
||||
"search_docs" => search_docs(ctx, args).await,
|
||||
"read_docs_page" => read_docs_page(ctx, args).await,
|
||||
"search_resource_types" => search_resource_types(ctx, args).await,
|
||||
"create_resource_type" => create_resource_type(ctx, args).await,
|
||||
"list_workers" => list_workers(ctx).await,
|
||||
"create_folder" => create_folder(ctx, args).await,
|
||||
"get_trigger_schema" => get_trigger_schema(args),
|
||||
"get_schedule_schema" => Ok(SCHEMAS.schedule.clone()),
|
||||
_ => unreachable!("{name} is not a misc tool"),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_trigger_schema(args: &Value) -> ToolResult {
|
||||
let kind = str_arg(args, "kind")?;
|
||||
SCHEMAS
|
||||
.trigger
|
||||
.get(kind)
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("Unknown trigger kind `{kind}`"))
|
||||
}
|
||||
|
||||
/// Copies `keys` from `from` into `to`, each only when truthy.
|
||||
fn copy_truthy(to: &mut Map<String, Value>, from: &Value, keys: &[&str]) {
|
||||
for key in keys {
|
||||
if truthy(from.get(key)) {
|
||||
to.insert(key.to_string(), from[key].clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies `keys` from `from` into `to`, each only when present (`null` included).
|
||||
fn copy_present(to: &mut Map<String, Value>, from: &Value, keys: &[&str]) {
|
||||
for key in keys {
|
||||
if let Some(value) = from.get(key) {
|
||||
to.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn words(text: &str) -> impl Iterator<Item = String> + '_ {
|
||||
text.split(|c: char| !c.is_ascii_alphanumeric())
|
||||
.filter(|w| !w.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hub
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MAX_BROWSED_HUB_SCRIPTS: usize = 20;
|
||||
const MAX_SUGGESTED_INTEGRATIONS: usize = 5;
|
||||
const MAX_INTEGRATION_EXAMPLES: usize = 5;
|
||||
|
||||
/// An unreachable hub degrades to "no suggestions" rather than a tool error.
|
||||
async fn hub_integrations(ctx: &Ctx<'_>) -> Vec<Value> {
|
||||
match ctx.get("/integrations/hub/list?kind=script").await {
|
||||
Ok(Value::Array(integrations)) => integrations,
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A token matches a whole word of the slug or display name, or extends one from four
|
||||
/// characters on: a bare substring test makes `for` hit sales*for*ce.
|
||||
fn token_matches_integration(token: &str, integration: &Value) -> bool {
|
||||
let name = integration.get("name").and_then(Value::as_str);
|
||||
let display_name = integration.get("display_name").and_then(Value::as_str);
|
||||
let words: Vec<String> = [name, display_name]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flat_map(words)
|
||||
.collect();
|
||||
if words.iter().any(|w| w == token) {
|
||||
return true;
|
||||
}
|
||||
token.len() >= 4
|
||||
&& words
|
||||
.iter()
|
||||
.any(|w| w.len() >= 4 && (w.starts_with(token) || token.starts_with(w.as_str())))
|
||||
}
|
||||
|
||||
/// Ranked by how many query words each integration accounts for, so "google drive"
|
||||
/// puts `gdrive` ahead of every other `g*` integration matching on "google" alone.
|
||||
fn suggest_hub_integrations(integrations: &[Value], query: &str) -> Vec<String> {
|
||||
let tokens: Vec<String> = words(query).filter(|t| t.len() >= 2).collect();
|
||||
let mut ranked: Vec<(usize, &str)> = integrations
|
||||
.iter()
|
||||
.filter_map(|integration| {
|
||||
let matched = tokens
|
||||
.iter()
|
||||
.filter(|t| token_matches_integration(t, integration))
|
||||
.count();
|
||||
let name = integration.get("name")?.as_str()?;
|
||||
(matched > 0).then_some((matched, name))
|
||||
})
|
||||
.collect();
|
||||
ranked.sort_by(|a, b| b.0.cmp(&a.0));
|
||||
ranked
|
||||
.into_iter()
|
||||
.take(MAX_SUGGESTED_INTEGRATIONS)
|
||||
.map(|(_, name)| name.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The hub resolves a script by its version id alone; the other segments are descriptive.
|
||||
fn hub_script_path(script: &Value) -> String {
|
||||
let summary = script
|
||||
.get("summary")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.to_lowercase();
|
||||
format!(
|
||||
"hub/{}/{}/{}",
|
||||
script.get("version_id").unwrap_or(&Value::Null),
|
||||
script
|
||||
.get("app")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default(),
|
||||
summary.split_whitespace().collect::<Vec<_>>().join("_")
|
||||
)
|
||||
}
|
||||
|
||||
async fn search_hub_scripts(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let query = opt_str_arg(args, "query").filter(|s| !s.is_empty());
|
||||
let app = opt_str_arg(args, "integration").filter(|s| !s.is_empty());
|
||||
if query.is_none() && app.is_none() {
|
||||
return Ok(
|
||||
"Pass query (what the script should do), integration (a slug to list), or both."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
let app_param = app
|
||||
.map(|app| format!("&app={}", enc(app)))
|
||||
.unwrap_or_default();
|
||||
// Listing an integration goes through the top-scripts endpoint: it applies no
|
||||
// similarity floor, so near-misses worth reading as examples survive.
|
||||
let scripts = match query {
|
||||
Some(query) => {
|
||||
ctx.get(&format!(
|
||||
"/embeddings/query_hub_scripts?text={}&kind=script{app_param}",
|
||||
enc(query)
|
||||
))
|
||||
.await?
|
||||
}
|
||||
None => ctx
|
||||
.get(&format!(
|
||||
"/scripts/hub/top?limit={MAX_BROWSED_HUB_SCRIPTS}&kind=script{app_param}"
|
||||
))
|
||||
.await?
|
||||
.get("asks")
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
let scripts = array(Some(&scripts));
|
||||
let integrations = hub_integrations(ctx).await;
|
||||
|
||||
if scripts.is_empty() {
|
||||
let subject = [app, query]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
return Ok(json!({
|
||||
"results": [],
|
||||
"suggested_integrations": suggest_hub_integrations(&integrations, &subject)
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
|
||||
// Lowercased because a hub slug is case-sensitive and both casings can exist.
|
||||
let documented: HashSet<String> = integrations
|
||||
.iter()
|
||||
.filter(|i| i.get("documented") == Some(&Value::Bool(true)))
|
||||
.filter_map(|i| Some(i.get("name")?.as_str()?.to_lowercase()))
|
||||
.collect();
|
||||
let results: Vec<Value> = scripts
|
||||
.iter()
|
||||
.map(|script| {
|
||||
let mut result = Map::new();
|
||||
result.insert("path".to_string(), hub_script_path(script).into());
|
||||
copy_present(&mut result, script, &["summary"]);
|
||||
let app = script.get("app").cloned().unwrap_or_default();
|
||||
let is_documented =
|
||||
documented.contains(&app.as_str().unwrap_or_default().to_lowercase());
|
||||
result.insert("integration".to_string(), app);
|
||||
if is_documented {
|
||||
result.insert("documented".to_string(), true.into());
|
||||
}
|
||||
copy_truthy(&mut result, script, &["description"]);
|
||||
Value::Object(result)
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "results": results }).to_string())
|
||||
}
|
||||
|
||||
/// How the authored notes were checked is provenance for a human reviewing the hub;
|
||||
/// the verdict and the fields it flags as shaky are what a caller can act on.
|
||||
fn without_provenance(meta: &Value) -> Value {
|
||||
let Value::Object(meta) = meta else {
|
||||
return meta.clone();
|
||||
};
|
||||
let mut rest = meta.clone();
|
||||
let validation = rest.shift_remove("validation");
|
||||
let kept: Map<String, Value> = validation
|
||||
.as_ref()
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|(key, _)| *key == "status" || *key == "low_confidence_fields")
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect();
|
||||
if !kept.is_empty() {
|
||||
rest.insert("validation".to_string(), Value::Object(kept));
|
||||
}
|
||||
Value::Object(rest)
|
||||
}
|
||||
|
||||
async fn get_hub_integration(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let integration = str_arg(args, "integration")?;
|
||||
let doc = match ctx
|
||||
.get(&format!("/integrations/hub/{}/meta", enc(integration)))
|
||||
.await
|
||||
{
|
||||
Ok(doc) => doc,
|
||||
Err(e) => {
|
||||
// Only a 404 means the integration is absent. Reporting a timeout the same
|
||||
// way would teach the model that a real integration does not exist.
|
||||
let error = if is_not_found(&e) {
|
||||
format!("No hub metadata for \"{integration}\".")
|
||||
} else {
|
||||
format!("Could not reach the hub for \"{integration}\"; it may still exist. Read its scripts instead, or try again.")
|
||||
};
|
||||
let integrations = hub_integrations(ctx).await;
|
||||
return Ok(json!({
|
||||
"error": error,
|
||||
"suggested_integrations": suggest_hub_integrations(&integrations, integration)
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
// The hub owns this response's shape and can be older than this server, so every
|
||||
// section is optional: a hub that sends less returns less.
|
||||
let mut out = Map::new();
|
||||
if let Some(app) = doc.get("app") {
|
||||
out.insert("integration".to_string(), app.clone());
|
||||
}
|
||||
copy_present(&mut out, &doc, &["display_name"]);
|
||||
copy_truthy(&mut out, &doc, &["description", "docs_url"]);
|
||||
if truthy(doc.get("meta")) {
|
||||
out.insert(
|
||||
"verified_provider_notes".to_string(),
|
||||
without_provenance(&doc["meta"]),
|
||||
);
|
||||
}
|
||||
let derived = doc.get("derived").filter(|d| truthy(Some(d)));
|
||||
if let Some(derived) = derived {
|
||||
let mut observed = Map::new();
|
||||
let hosts: Vec<Value> = array(derived.get("api_hosts"))
|
||||
.iter()
|
||||
.filter_map(|h| h.get("host").cloned())
|
||||
.collect();
|
||||
observed.insert("api_hosts".to_string(), hosts.into());
|
||||
copy_present(&mut observed, derived, &["style"]);
|
||||
let languages: Vec<&String> = derived
|
||||
.get("languages")
|
||||
.and_then(Value::as_object)
|
||||
.map(|l| l.keys().collect())
|
||||
.unwrap_or_default();
|
||||
observed.insert("languages".to_string(), json!(languages));
|
||||
copy_present(&mut observed, derived, &["script_counts"]);
|
||||
out.insert("observed_from_scripts".to_string(), observed.into());
|
||||
}
|
||||
if doc.get("curated") == Some(&Value::Bool(true)) {
|
||||
out.insert(
|
||||
"scripts_note".to_string(),
|
||||
"These scripts were pruned to idiomatic actions rather than generated one per API endpoint, so they are worth following as style examples.".into(),
|
||||
);
|
||||
}
|
||||
let resource_types: Vec<Value> = array(doc.get("resource_types"))
|
||||
.iter()
|
||||
.map(|rt| {
|
||||
let mut entry = Map::new();
|
||||
copy_present(&mut entry, rt, &["name"]);
|
||||
copy_truthy(&mut entry, rt, &["description"]);
|
||||
// The hub keeps a schema in a text column and hands it back as a JSON string.
|
||||
match rt.get("schema") {
|
||||
Some(Value::String(schema)) => {
|
||||
let parsed = serde_json::from_str(schema)
|
||||
.unwrap_or_else(|_| Value::String(schema.clone()));
|
||||
entry.insert("schema".to_string(), parsed);
|
||||
}
|
||||
Some(schema) => {
|
||||
entry.insert("schema".to_string(), schema.clone());
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
Value::Object(entry)
|
||||
})
|
||||
.collect();
|
||||
out.insert("resource_types".to_string(), resource_types.into());
|
||||
let examples: Vec<Value> = array(derived.and_then(|d| d.get("top_scripts")))
|
||||
.iter()
|
||||
.take(MAX_INTEGRATION_EXAMPLES)
|
||||
.map(|script| {
|
||||
let mut entry = Map::new();
|
||||
copy_present(&mut entry, script, &["path", "summary"]);
|
||||
copy_truthy(&mut entry, script, &["description", "language"]);
|
||||
Value::Object(entry)
|
||||
})
|
||||
.collect();
|
||||
out.insert("example_scripts".to_string(), examples.into());
|
||||
Ok(Value::Object(out).to_string())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// npm
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const NPM_SCORE_THRESHOLD: f64 = 1000.0;
|
||||
/// The session caps a package's readme and types at 1% of its model's context window
|
||||
/// each. The caller's model is unknown here, so this is 1% of the window the session
|
||||
/// assumes for an unknown model (128000).
|
||||
const NPM_TEXT_LIMIT: usize = 1280;
|
||||
|
||||
fn take_chars(text: &str, limit: usize) -> String {
|
||||
text.chars().take(limit).collect()
|
||||
}
|
||||
|
||||
async fn fetch(url: &str) -> Result<reqwest::Response, reqwest::Error> {
|
||||
HTTP_CLIENT.get(url).send().await
|
||||
}
|
||||
|
||||
/// The package's declared entry declaration file. The session walks every `.d.ts` of the
|
||||
/// package in the browser and keeps the same number of characters of the result.
|
||||
async fn npm_package_types(name: &str, version: &str) -> Option<String> {
|
||||
let base = format!("https://unpkg.com/{name}@{version}");
|
||||
let manifest: Value = fetch(&format!("{base}/package.json"))
|
||||
.await
|
||||
.ok()?
|
||||
.error_for_status()
|
||||
.ok()?
|
||||
.json()
|
||||
.await
|
||||
.ok()?;
|
||||
let entry = ["types", "typings"]
|
||||
.iter()
|
||||
.find_map(|key| manifest.get(key)?.as_str())
|
||||
.unwrap_or("index.d.ts")
|
||||
.trim_start_matches("./");
|
||||
let types = fetch(&format!("{base}/{entry}"))
|
||||
.await
|
||||
.ok()?
|
||||
.error_for_status()
|
||||
.ok()?
|
||||
.text()
|
||||
.await
|
||||
.ok()?;
|
||||
Some(format!("// /node_modules/{name}/{entry}\n{types}"))
|
||||
}
|
||||
|
||||
async fn npm_search(query: &str) -> Result<Vec<Value>, String> {
|
||||
let data: Value = fetch(&format!(
|
||||
"https://registry.npmjs.org/-/v1/search?text={}&size=2",
|
||||
enc(query)
|
||||
))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let packages = data
|
||||
.get("objects")
|
||||
.and_then(Value::as_array)
|
||||
.ok_or("no `objects` in the registry answer")?
|
||||
.iter()
|
||||
.filter(|o| {
|
||||
o.get("searchScore").and_then(Value::as_f64).unwrap_or(0.0) >= NPM_SCORE_THRESHOLD
|
||||
})
|
||||
.filter_map(|o| {
|
||||
let package = o.get("package")?;
|
||||
Some((
|
||||
package.get("name")?.as_str()?,
|
||||
package.get("version").and_then(Value::as_str),
|
||||
))
|
||||
});
|
||||
Ok(join_all(packages.map(|(name, version)| async move {
|
||||
let documentation = match fetch(&format!("https://unpkg.com/{name}/readme.md")).await {
|
||||
Ok(response) => response.text().await.unwrap_or_default(),
|
||||
Err(_) => String::new(),
|
||||
};
|
||||
let types = npm_package_types(name, version.unwrap_or("latest"))
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
json!({
|
||||
"package": name,
|
||||
"documentation": take_chars(&documentation, NPM_TEXT_LIMIT),
|
||||
"types": take_chars(&types, NPM_TEXT_LIMIT),
|
||||
})
|
||||
}))
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn search_npm_packages(args: &Value) -> ToolResult {
|
||||
let query = str_arg(args, "query")?;
|
||||
Ok(match npm_search(query).await {
|
||||
Ok(results) => Value::Array(results).to_string(),
|
||||
Err(e) => {
|
||||
tracing::warn!("npm package search failed: {e}");
|
||||
"Error searching external integration resources".to_string()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Docs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `text` of a `/docs/*` answer, or the session's wording for a failed lookup.
|
||||
fn docs_text(result: Result<Value, String>, action: &str) -> String {
|
||||
match result {
|
||||
Ok(response) => response
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
Err(e) => format!("Failed to {action}: {e}, pursuing with the user request..."),
|
||||
}
|
||||
}
|
||||
|
||||
async fn search_docs(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let query = opt_str_arg(args, "query").unwrap_or_default().trim();
|
||||
if query.is_empty() {
|
||||
return Ok(
|
||||
"No search query was provided. Provide a `query` of one or more keywords.".to_string(),
|
||||
);
|
||||
}
|
||||
let result = ctx.get(&format!("/docs/search?query={}", enc(query))).await;
|
||||
Ok(docs_text(result, "search documentation"))
|
||||
}
|
||||
|
||||
async fn read_docs_page(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let url = opt_str_arg(args, "url").unwrap_or_default();
|
||||
if url.trim().is_empty() {
|
||||
return Ok("No documentation page URL was provided. Provide a `url` — e.g. a `Source` URL returned by search_docs.".to_string());
|
||||
}
|
||||
let section = opt_str_arg(args, "section")
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.map(|s| format!("§ion={}", enc(s)))
|
||||
.unwrap_or_default();
|
||||
let result = ctx
|
||||
.get(&format!("/docs/page?url={}{section}", enc(url)))
|
||||
.await;
|
||||
Ok(docs_text(result, "read documentation page"))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Resource types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The hub publishes hundreds of resource types and an instance seeds only a few of its
|
||||
/// own, so fewer than this means the instance never synced with the hub.
|
||||
const UNSYNCED_RESOURCE_TYPE_COUNT: usize = 20;
|
||||
|
||||
const HUB_SYNC_INSTRUCTIONS: &str = "Tell the user: no resource of a missing type can be created until a superadmin syncs resource types from the hub, with \"Sync resource types with Hub\" in the Add resource dialog of the Resources page.";
|
||||
|
||||
async fn search_resource_types(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let query = str_arg(args, "query")?;
|
||||
let limit = args.get("limit").and_then(Value::as_u64).unwrap_or(5);
|
||||
let search_route = ctx.ws(&format!(
|
||||
"embeddings/query_resource_types?text={}&limit={limit}",
|
||||
enc(query)
|
||||
));
|
||||
let names_route = ctx.ws("resources/type/listnames");
|
||||
let (results, names) =
|
||||
futures::future::join(ctx.get(&search_route), ctx.get(&names_route)).await;
|
||||
let results = results?;
|
||||
let results = array(Some(&results));
|
||||
let count = names
|
||||
.map(|names| {
|
||||
array(Some(&names))
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.collect::<HashSet<_>>()
|
||||
.len()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let listing: Vec<Value> = results
|
||||
.iter()
|
||||
.map(|rt| {
|
||||
let mut entry = Map::new();
|
||||
copy_present(&mut entry, rt, &["name", "schema"]);
|
||||
Value::Object(entry)
|
||||
})
|
||||
.collect();
|
||||
let listing = pretty(&Value::Array(listing));
|
||||
// The similarity search still returns the nearest of the few types there are, so the
|
||||
// note cannot wait for an empty result; it replaces the "no dedicated type" one, which
|
||||
// would have the model create a duplicate of a type a sync brings.
|
||||
if count > 0 && count < UNSYNCED_RESOURCE_TYPE_COUNT {
|
||||
return Ok(format!("{listing}\n\nOnly {count} resource types exist on this instance, so it has most likely never synced with the Windmill Hub, which has types for most services. {HUB_SYNC_INSTRUCTIONS}"));
|
||||
}
|
||||
// The search is semantic and always returns its closest types, so a miss reads
|
||||
// like a hit unless the absence is spelled out.
|
||||
let words: Vec<String> = words(query).filter(|w| w.len() >= 3).collect();
|
||||
let named = results.iter().any(|rt| {
|
||||
let name = rt
|
||||
.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.to_lowercase();
|
||||
words.iter().any(|w| name.contains(w.as_str()))
|
||||
});
|
||||
Ok(if named || words.is_empty() {
|
||||
listing
|
||||
} else {
|
||||
format!("{listing}\n\nNo resource type name contains \"{query}\": these are only the nearest matches, so no dedicated type exists. Searching again with synonyms will not find one — use a fitting generic type above, or create a custom c_<name> type.")
|
||||
})
|
||||
}
|
||||
|
||||
/// The prefix the Add resource type form adds: a hub type of the same name, synced later,
|
||||
/// would otherwise clash with this one.
|
||||
fn custom_resource_type_name(name: &str) -> String {
|
||||
if name.starts_with("c_") {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("c_{name}")
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_resource_type(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let name = custom_resource_type_name(str_arg(args, "name")?);
|
||||
let description = str_arg(args, "description")?;
|
||||
let schema = args
|
||||
.get("schema")
|
||||
.filter(|s| s.is_object())
|
||||
.ok_or("Missing required object argument `schema`")?;
|
||||
ctx.post(
|
||||
&ctx.ws("resources/type/create"),
|
||||
json!({ "name": name, "description": description, "schema": schema }),
|
||||
)
|
||||
.await?;
|
||||
Ok(json!({
|
||||
"success": true,
|
||||
"message": format!("Created resource type {name}. Refer to it by this name.")
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const WORKER_PAGE_SIZE: usize = 100;
|
||||
// `GET /workers/list` hides workers from a caller without the devops role by returning
|
||||
// an empty list, not an error, when HIDE_WORKERS_FOR_NON_ADMINS is set. The flag is not
|
||||
// visible here, so absence is only provable for a devops or superadmin caller.
|
||||
const NO_WORKERS_VISIBLE_MESSAGE: &str = "No workers came back. This does NOT establish that no workers are running: an instance can hide workers from callers without the devops role, and it does so by returning an empty list rather than an error. Tell the user you cannot see any workers and that worker visibility may be restricted for your account, and suggest they check the Workers page themselves if they can open it, or ask a workspace admin otherwise. Never state that no workers are online or that the instance has none.";
|
||||
const NO_WORKERS_CONNECTED_MESSAGE: &str = "No workers are connected to this instance (none pinged in the last 5 minutes). Queued runs will stay queued until a worker starts.";
|
||||
|
||||
async fn list_workers(ctx: &Ctx<'_>) -> ToolResult {
|
||||
let pings = ctx
|
||||
.get(&format!("/workers/list?per_page={WORKER_PAGE_SIZE}"))
|
||||
.await?;
|
||||
let pings = array(Some(&pings));
|
||||
if pings.is_empty() {
|
||||
let me = ctx.get("/users/whoami").await.unwrap_or_default();
|
||||
let sees_all = truthy(me.get("super_admin")) || truthy(me.get("devops"));
|
||||
return Ok(if sees_all {
|
||||
NO_WORKERS_CONNECTED_MESSAGE
|
||||
} else {
|
||||
NO_WORKERS_VISIBLE_MESSAGE
|
||||
}
|
||||
.to_string());
|
||||
}
|
||||
let workers: Vec<Value> = pings
|
||||
.iter()
|
||||
.map(|ping| {
|
||||
let mut worker = Map::new();
|
||||
copy_present(
|
||||
&mut worker,
|
||||
ping,
|
||||
&[
|
||||
"worker",
|
||||
"worker_group",
|
||||
"custom_tags",
|
||||
"last_ping",
|
||||
"jobs_executed",
|
||||
],
|
||||
);
|
||||
Value::Object(worker)
|
||||
})
|
||||
.collect();
|
||||
let mut out = Map::new();
|
||||
let listed = workers.len();
|
||||
out.insert("workers".to_string(), workers.into());
|
||||
if listed == WORKER_PAGE_SIZE {
|
||||
out.insert(
|
||||
"note".to_string(),
|
||||
format!("Only the first {WORKER_PAGE_SIZE} workers are listed; more may be connected.")
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
Ok(pretty(&Value::Object(out)))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Folders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn create_folder(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let name = str_arg(args, "name")?;
|
||||
// Same rule as the API's own check, so an invalid name gets this wording.
|
||||
if name.is_empty()
|
||||
|| !name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
|
||||
{
|
||||
return Ok(json!({
|
||||
"success": false,
|
||||
"error": "Folder name can only contain alphanumeric characters, underscores, and hyphens."
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
let mut body = Map::new();
|
||||
body.insert("name".to_string(), name.into());
|
||||
copy_present(&mut body, args, &["summary"]);
|
||||
Ok(
|
||||
match ctx.post(&ctx.ws("folders/create"), body.into()).await {
|
||||
Ok(_) => json!({
|
||||
"success": true,
|
||||
"message": format!("Created folder `f/{name}`. You can now write items to `f/{name}/<name>`.")
|
||||
}),
|
||||
Err(error) => json!({ "success": false, "error": error }),
|
||||
}
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn suggestions_rank_by_matched_words() {
|
||||
let integrations = vec![
|
||||
json!({ "name": "gcal", "display_name": "Google Calendar" }),
|
||||
json!({ "name": "gdrive", "display_name": "Google Drive" }),
|
||||
json!({ "name": "salesforce", "display_name": null }),
|
||||
json!({ "name": "s3", "display_name": "S3" }),
|
||||
];
|
||||
assert_eq!(
|
||||
suggest_hub_integrations(&integrations, "google drive"),
|
||||
["gdrive", "gcal"]
|
||||
);
|
||||
// A short token must equal a word; a long one may extend it.
|
||||
assert!(suggest_hub_integrations(&integrations, "for").is_empty());
|
||||
assert_eq!(
|
||||
suggest_hub_integrations(&integrations, "sales"),
|
||||
["salesforce"]
|
||||
);
|
||||
assert_eq!(suggest_hub_integrations(&integrations, "s3 bucket"), ["s3"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_keeps_only_the_verdict() {
|
||||
let meta = json!({
|
||||
"pagination": "cursor",
|
||||
"validation": { "status": "verified", "method": "live", "low_confidence_fields": ["a"] }
|
||||
});
|
||||
assert_eq!(
|
||||
without_provenance(&meta),
|
||||
json!({
|
||||
"pagination": "cursor",
|
||||
"validation": { "status": "verified", "low_confidence_fields": ["a"] }
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
without_provenance(&json!({ "a": 1, "validation": { "method": "live" } })),
|
||||
json!({ "a": 1 })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_resource_types_carry_the_c_prefix_once() {
|
||||
assert_eq!(custom_resource_type_name("acme"), "c_acme");
|
||||
assert_eq!(custom_resource_type_name("c_acme"), "c_acme");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hub_path_from_summary() {
|
||||
let script = json!({ "version_id": 28479, "app": "stripe", "summary": "Cancel Or Reverse a Payout" });
|
||||
assert_eq!(
|
||||
hub_script_path(&script),
|
||||
"hub/28479/stripe/cancel_or_reverse_a_payout"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,609 @@
|
||||
//! The Windmill MCP server's toolset (`/api/mcp`, tokens with `mcp:all`): an AI session's toolset, for a client
|
||||
//! outside the Windmill UI.
|
||||
//!
|
||||
//! Everything a model reads here — tool names, descriptions, input schemas, the
|
||||
//! instructions, the `get_instructions` texts — is generated from the session's own
|
||||
//! definitions into `gen/` (`npm run gen:mcp-session` in `frontend/`). What a tool
|
||||
//! *does* is written here, mirroring its session counterpart in
|
||||
//! `frontend/src/lib/components/copilot/chat/global/core.ts`: a change to one side's
|
||||
//! behaviour belongs on the other.
|
||||
//!
|
||||
//! A tool acts through the API as the caller, the way the session's does, so every
|
||||
//! permission check is the API's own. It calls the server's router in-process rather
|
||||
//! than over a socket.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use windmill_mcp::server::{
|
||||
BackendResult, CallToolResult, ContentBlock, ErrorData, MetaObject, SessionAccess, Tool,
|
||||
ToolAnnotations,
|
||||
};
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
use super::utils::{mint_caller_jwt, send_as_caller};
|
||||
|
||||
/// Each running server's fully layered router, by its `base_internal_url`. Keyed rather
|
||||
/// than global because the backend tests run one server per test in a single process,
|
||||
/// each on its own database: a tool must reach the server it was called on.
|
||||
static IN_PROCESS_ROUTERS: LazyLock<std::sync::RwLock<HashMap<String, axum::Router>>> =
|
||||
LazyLock::new(Default::default);
|
||||
|
||||
/// Let session tools of the server at `base_internal_url` call its API in-process.
|
||||
/// Without a registration they fall back to HTTP.
|
||||
pub(crate) fn register_router(base_internal_url: &str, router: Option<axum::Router>) {
|
||||
let mut routers = IN_PROCESS_ROUTERS.write().unwrap();
|
||||
match router {
|
||||
Some(router) => routers.insert(base_internal_url.to_string(), router),
|
||||
None => routers.remove(base_internal_url),
|
||||
};
|
||||
}
|
||||
|
||||
mod apps;
|
||||
mod data;
|
||||
mod deploy;
|
||||
mod flows;
|
||||
mod guidance;
|
||||
mod instructions;
|
||||
mod items;
|
||||
mod misc;
|
||||
mod runs;
|
||||
mod settings;
|
||||
mod workspace;
|
||||
|
||||
pub(super) use instructions::{
|
||||
generic_session_instructions, session_instructions, workspace_context,
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ToolDef {
|
||||
name: String,
|
||||
description: String,
|
||||
parameters: Map<String, Value>,
|
||||
read_only: bool,
|
||||
}
|
||||
|
||||
static TOOL_DEFS: LazyLock<Vec<ToolDef>> = LazyLock::new(|| {
|
||||
serde_json::from_str(include_str!("gen/tools.json")).expect("gen/tools.json is valid")
|
||||
});
|
||||
|
||||
/// A tool's input schema, as generated from the session's zod schema.
|
||||
pub(super) fn tool_parameters(name: &str) -> Option<&'static Map<String, Value>> {
|
||||
TOOL_DEFS
|
||||
.iter()
|
||||
.find(|def| def.name == name)
|
||||
.map(|def| &def.parameters)
|
||||
}
|
||||
|
||||
/// The JSON Schema text of each trigger kind's config, and of a schedule's `advanced` bag,
|
||||
/// as the session renders them from its request schemas.
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct Schemas {
|
||||
pub trigger: HashMap<String, String>,
|
||||
pub schedule: String,
|
||||
}
|
||||
|
||||
pub(super) static SCHEMAS: LazyLock<Schemas> = LazyLock::new(|| {
|
||||
serde_json::from_str(include_str!("gen/schemas.json")).expect("gen/schemas.json is valid")
|
||||
});
|
||||
|
||||
/// Session tools that need the Windmill UI and have no meaning for an external client.
|
||||
/// Every generated tool is either here or implemented by a group below.
|
||||
#[cfg(test)]
|
||||
const EDITOR_ONLY: &[&str] = &[
|
||||
// The side panel, the question card and the browser-held user instructions.
|
||||
"open_page",
|
||||
"askUserQuestion",
|
||||
"update_user_instructions",
|
||||
// Files attached to the chat.
|
||||
"search_files",
|
||||
"read_file",
|
||||
];
|
||||
|
||||
/// Tools whose effect reaches the deployed workspace and cannot be undone by discarding
|
||||
/// a draft. Stands in for the session's confirmation card.
|
||||
const DESTRUCTIVE: &[&str] = &[
|
||||
"deploy_workspace_item",
|
||||
"delete_workspace_item",
|
||||
"create_resource_type",
|
||||
"set_workspace_settings",
|
||||
];
|
||||
|
||||
/// Tools the session counts as reads that still start a job. A read-only token runs
|
||||
/// nothing on the tools server, so it runs nothing here either.
|
||||
const RUNS_A_JOB: &[&str] = &["get_db_schema"];
|
||||
|
||||
/// Tools that run a deployed script or flow, the one write a read-and-run token keeps.
|
||||
const RUNS_DEPLOYED: &[&str] = &["run_script", "run_flow"];
|
||||
|
||||
fn allowed(def: &ToolDef, access: SessionAccess) -> bool {
|
||||
let read = def.read_only && !RUNS_A_JOB.contains(&def.name.as_str());
|
||||
match access {
|
||||
SessionAccess::Full => true,
|
||||
SessionAccess::ReadAndRun => read || RUNS_DEPLOYED.contains(&def.name.as_str()),
|
||||
SessionAccess::ReadOnly => read,
|
||||
}
|
||||
}
|
||||
|
||||
/// Each group's tools and its dispatcher, tried in order.
|
||||
macro_rules! groups {
|
||||
($($group:ident),*) => {
|
||||
fn implemented() -> impl Iterator<Item = &'static str> {
|
||||
std::iter::empty()$(.chain($group::TOOLS.iter().copied()))*
|
||||
}
|
||||
|
||||
async fn dispatch(ctx: &Ctx<'_>, name: &str, args: &Value) -> Option<ToolResult> {
|
||||
$(if $group::TOOLS.contains(&name) {
|
||||
return Some($group::call(ctx, name, args).await);
|
||||
})*
|
||||
None
|
||||
}
|
||||
};
|
||||
}
|
||||
groups!(guidance, workspace, items, misc, flows, apps, runs, deploy, data, settings);
|
||||
|
||||
/// Claude clients keep an MCP result in context only up to a token budget well under the
|
||||
/// authoring references `get_instructions` returns, unless the tool raises it. Above the
|
||||
/// runner's own cap on a result, so that cap is the one that applies.
|
||||
fn result_size_meta() -> MetaObject {
|
||||
let mut meta = Map::new();
|
||||
meta.insert("anthropic/maxResultSizeChars".to_string(), 100_000.into());
|
||||
MetaObject(meta)
|
||||
}
|
||||
|
||||
pub(super) fn session_tools(access: SessionAccess) -> Vec<Tool> {
|
||||
TOOL_DEFS
|
||||
.iter()
|
||||
.filter(|def| implemented().any(|name| name == def.name))
|
||||
.filter(|def| allowed(def, access))
|
||||
.map(|def| {
|
||||
Tool::new(
|
||||
def.name.clone(),
|
||||
def.description.clone(),
|
||||
def.parameters.clone(),
|
||||
)
|
||||
.with_annotations(
|
||||
ToolAnnotations::new()
|
||||
.read_only(def.read_only)
|
||||
.destructive(DESTRUCTIVE.contains(&def.name.as_str())),
|
||||
)
|
||||
.with_meta(result_size_meta())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// What a tool hands back: the text the model reads, or the message of a failed call.
|
||||
pub(super) type ToolResult = Result<String, String>;
|
||||
|
||||
/// The caller, and the API reached as them.
|
||||
pub(super) struct Ctx<'a> {
|
||||
pub auth: &'a ApiAuthed,
|
||||
pub w_id: &'a str,
|
||||
pub base_internal_url: &'a str,
|
||||
pub access: SessionAccess,
|
||||
/// The scheme and host the MCP client reached, when known.
|
||||
pub origin: Option<&'a str>,
|
||||
/// Minted once per tool call: every JWT the API sees is cached by the auth layer, and a
|
||||
/// tool can make dozens of calls, which would evict other users' cached sessions.
|
||||
pub jwt: tokio::sync::OnceCell<String>,
|
||||
}
|
||||
|
||||
impl Ctx<'_> {
|
||||
/// Call the API as the caller. `path` follows `/api`; a non-2xx answer is the error.
|
||||
pub async fn request(&self, method: &str, path: &str, body: Option<Value>) -> ToolResult {
|
||||
let url = guarded_url(self.base_internal_url, self.w_id, method, path, self.access)?;
|
||||
// No route scope: this server only admits `mcp:all`, several routes (hub, embeddings)
|
||||
// map to none, and `guarded_url` is what keeps a call on its route. The tag fence
|
||||
// is not a route scope and stays: it limits which jobs the caller runs and reads.
|
||||
let tag_scopes: Vec<String> = self
|
||||
.auth
|
||||
.scopes
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|s| s.starts_with("if_jobs:filter_tags:"))
|
||||
.cloned()
|
||||
.collect();
|
||||
let scopes = (!tag_scopes.is_empty()).then_some(tag_scopes);
|
||||
let router = IN_PROCESS_ROUTERS
|
||||
.read()
|
||||
.unwrap()
|
||||
.get(self.base_internal_url)
|
||||
.cloned();
|
||||
let (status, text) = match router {
|
||||
Some(router) => self.call_router(router, method, &url, body, scopes).await?,
|
||||
None => {
|
||||
let response = send_as_caller(method, &url, self.w_id, self.auth, body, scopes)
|
||||
.await
|
||||
.map_err(|e| e.message.to_string())?;
|
||||
let status = response.status();
|
||||
(status, response.text().await.map_err(|e| e.to_string())?)
|
||||
}
|
||||
};
|
||||
if status.is_success() {
|
||||
Ok(text)
|
||||
} else {
|
||||
Err(format!("{}: {}", status.as_u16(), text))
|
||||
}
|
||||
}
|
||||
|
||||
/// `request` through the server's own router: the same layers and handlers as over
|
||||
/// HTTP, without a socket.
|
||||
async fn call_router(
|
||||
&self,
|
||||
router: axum::Router,
|
||||
method: &str,
|
||||
url: &str,
|
||||
body: Option<Value>,
|
||||
scopes: Option<Vec<String>>,
|
||||
) -> Result<(http::StatusCode, String), String> {
|
||||
use tower::ServiceExt;
|
||||
let parsed = reqwest::Url::parse(url).map_err(|e| e.to_string())?;
|
||||
// The router does not see the base URL's own path prefix.
|
||||
let at = reqwest::Url::parse(self.base_internal_url)
|
||||
.map_or(0, |base| base.path().trim_end_matches('/').len());
|
||||
let uri = match parsed.query() {
|
||||
Some(q) => format!("{}?{q}", &parsed.path()[at..]),
|
||||
None => parsed.path()[at..].to_string(),
|
||||
};
|
||||
let token = self
|
||||
.jwt
|
||||
.get_or_try_init(|| mint_caller_jwt(self.w_id, self.auth, scopes))
|
||||
.await
|
||||
.map_err(|e| e.message.to_string())?;
|
||||
let mut request = http::Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header(http::header::AUTHORIZATION, format!("Bearer {token}"));
|
||||
let body = match body {
|
||||
Some(body) => {
|
||||
request = request.header(http::header::CONTENT_TYPE, "application/json");
|
||||
axum::body::Body::from(serde_json::to_vec(&body).map_err(|e| e.to_string())?)
|
||||
}
|
||||
None => axum::body::Body::empty(),
|
||||
};
|
||||
let request = request.body(body).map_err(|e| e.to_string())?;
|
||||
// The total timeout of the HTTP client this replaces.
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
let response = router.oneshot(request).await.map_err(|e| e.to_string())?;
|
||||
let status = response.status();
|
||||
let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok((status, String::from_utf8_lossy(&bytes).into_owned()))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| format!("Failed to execute request: {method} {url} timed out"))?
|
||||
}
|
||||
|
||||
pub async fn get(&self, path: &str) -> Result<Value, String> {
|
||||
let text = self.request("GET", path, None).await?;
|
||||
serde_json::from_str(&text).map_err(|e| format!("unexpected response from {path}: {e}"))
|
||||
}
|
||||
|
||||
/// `POST` a JSON body. Several endpoints answer with plain text (a path, a job id):
|
||||
/// that comes back as a JSON string.
|
||||
pub async fn post(&self, path: &str, body: Value) -> Result<Value, String> {
|
||||
let text = self.request("POST", path, Some(body)).await?;
|
||||
Ok(serde_json::from_str(&text).unwrap_or(Value::String(text)))
|
||||
}
|
||||
|
||||
pub async fn delete(&self, path: &str) -> Result<Value, String> {
|
||||
let text = self.request("DELETE", path, None).await?;
|
||||
Ok(serde_json::from_str(&text).unwrap_or(Value::String(text)))
|
||||
}
|
||||
|
||||
/// A workspace-scoped path: `ws("scripts/list")` is `/w/<workspace>/scripts/list`.
|
||||
pub fn ws(&self, rest: &str) -> String {
|
||||
format!("/w/{}/{}", self.w_id, rest)
|
||||
}
|
||||
|
||||
/// A Windmill page in this workspace, as a link the user can open: an external client
|
||||
/// has no preview panel, so a link is how it shows what a tool made or ran.
|
||||
pub fn page_url(&self, page: &str) -> String {
|
||||
let (page, hash) = page
|
||||
.split_once('#')
|
||||
.map_or((page, None), |(p, h)| (p, Some(h)));
|
||||
let sep = if page.contains('?') { '&' } else { '?' };
|
||||
let hash = hash.map(|h| format!("#{h}")).unwrap_or_default();
|
||||
// An instance that never set its base URL holds this placeholder, which no client
|
||||
// can open; the host the client reached is the better guess then.
|
||||
let base = windmill_common::BASE_URL.load();
|
||||
let base = match self.origin {
|
||||
Some(origin) if base.as_str() == "http://localhost" => origin,
|
||||
_ => base.as_str(),
|
||||
};
|
||||
format!("{base}{page}{sep}workspace={}{hash}", enc(self.w_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Routes outside the workspace that a tool reads.
|
||||
const INSTANCE_ROUTES: &[&str] = &[
|
||||
"/docs/",
|
||||
"/embeddings/",
|
||||
"/integrations/hub/",
|
||||
"/scripts/hub/",
|
||||
"/users/whoami",
|
||||
"/workers/list",
|
||||
"/workspaces/users",
|
||||
];
|
||||
|
||||
/// The URL of an API call, refused unless it lands on the route its handler wrote.
|
||||
///
|
||||
/// The JWT behind a call carries the caller's full rights, and paths embed arguments a
|
||||
/// model chose: a `..` segment, or an id carrying `#`, would otherwise turn a read of
|
||||
/// one job into a run of any script. A read-only or read-and-run token is held to its rule
|
||||
/// here too, since that JWT carries neither.
|
||||
fn guarded_url(
|
||||
base: &str,
|
||||
w_id: &str,
|
||||
method: &str,
|
||||
path: &str,
|
||||
access: SessionAccess,
|
||||
) -> Result<String, String> {
|
||||
let refused = || format!("Refusing to call `{path}`: not a route of this tool");
|
||||
let route = path.split('?').next().unwrap_or_default();
|
||||
let in_workspace = route
|
||||
.strip_prefix("/w/")
|
||||
.and_then(|rest| rest.strip_prefix(w_id))
|
||||
.is_some_and(|rest| rest.starts_with('/'));
|
||||
let workspace_is_plain = w_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
|
||||
if path.contains('#')
|
||||
|| !workspace_is_plain
|
||||
|| !(in_workspace || INSTANCE_ROUTES.iter().any(|r| route.starts_with(r)))
|
||||
{
|
||||
return Err(refused());
|
||||
}
|
||||
let url = format!("{base}/api{path}");
|
||||
let parsed = reqwest::Url::parse(&url).map_err(|_| refused())?;
|
||||
// Parsing resolves dot segments, encoded or not: a path that changed went elsewhere.
|
||||
if !parsed.path().ends_with(&format!("/api{route}")) {
|
||||
return Err(refused());
|
||||
}
|
||||
// Runs of deployed items by path only: the preview routes take code, so they are writes.
|
||||
// A run also stores its secret arguments as expiring variables; the write tools that
|
||||
// could create other variables are not offered to a read-and-run token.
|
||||
let run_writes = method == "POST"
|
||||
&& ["jobs/run/p/", "jobs/run/f/"]
|
||||
.iter()
|
||||
.any(|run| route.starts_with(&format!("/w/{w_id}/{run}")))
|
||||
|| method == "POST" && route == format!("/w/{w_id}/variables/create");
|
||||
let held_to_reads = match access {
|
||||
SessionAccess::Full => false,
|
||||
SessionAccess::ReadAndRun => !run_writes,
|
||||
SessionAccess::ReadOnly => true,
|
||||
};
|
||||
if held_to_reads {
|
||||
windmill_api_auth::scopes::check_read_only_for_route(parsed.path(), method)
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// A string argument, or the error the model gets for leaving it out.
|
||||
pub(super) fn str_arg<'a>(args: &'a Value, name: &str) -> Result<&'a str, String> {
|
||||
args.get(name)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| format!("Missing required string argument `{name}`"))
|
||||
}
|
||||
|
||||
pub(super) fn opt_str_arg<'a>(args: &'a Value, name: &str) -> Option<&'a str> {
|
||||
args.get(name).and_then(Value::as_str)
|
||||
}
|
||||
|
||||
/// `JSON.stringify(value, null, 2)`, the form the session's tools return objects in.
|
||||
pub(super) fn pretty(value: &Value) -> String {
|
||||
serde_json::to_string_pretty(value).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Percent-encode one query-string value.
|
||||
pub(super) fn enc(value: &str) -> String {
|
||||
urlencoding::encode(value).into_owned()
|
||||
}
|
||||
|
||||
pub(super) fn bool_arg(args: &Value, name: &str) -> bool {
|
||||
args.get(name).and_then(Value::as_bool).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether a `Ctx` error is the API answering 404.
|
||||
pub(super) fn is_not_found(error: &str) -> bool {
|
||||
error.starts_with("404: ")
|
||||
}
|
||||
|
||||
/// JavaScript truthiness, for the fields the session keeps only `if (value)`.
|
||||
pub(super) fn truthy(value: Option<&Value>) -> bool {
|
||||
match value {
|
||||
None | Some(Value::Null) => false,
|
||||
Some(Value::Bool(b)) => *b,
|
||||
Some(Value::Number(n)) => n.as_f64().is_some_and(|n| n != 0.0),
|
||||
Some(Value::String(s)) => !s.is_empty(),
|
||||
Some(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// `value ?? undefined`: the value unless it is missing or null.
|
||||
pub(super) fn non_null(value: Option<&Value>) -> Option<&Value> {
|
||||
value.filter(|v| !v.is_null())
|
||||
}
|
||||
|
||||
/// Set `key` unless the value is JS `undefined`, which `JSON.stringify` drops.
|
||||
pub(super) fn put(map: &mut Map<String, Value>, key: &str, value: Option<Value>) {
|
||||
if let Some(value) = value {
|
||||
map.insert(key.to_string(), value);
|
||||
}
|
||||
}
|
||||
|
||||
/// The elements of an array; none for anything else.
|
||||
pub(super) fn array(value: Option<&Value>) -> &[Value] {
|
||||
value
|
||||
.and_then(Value::as_array)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// A value as a JS template literal prints it.
|
||||
pub(super) fn js_str(value: &Value) -> String {
|
||||
match value {
|
||||
Value::String(s) => s.clone(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `String.length`: UTF-16 code units.
|
||||
pub(super) fn js_len(text: &str) -> usize {
|
||||
text.encode_utf16().count()
|
||||
}
|
||||
|
||||
/// `text.slice(0, max)` in UTF-16 units, never splitting a character.
|
||||
pub(super) fn js_head(text: &str, max: usize) -> &str {
|
||||
if text.len() <= max {
|
||||
return text;
|
||||
}
|
||||
let mut units = 0;
|
||||
for (i, c) in text.char_indices() {
|
||||
units += c.len_utf16();
|
||||
if units > max {
|
||||
return &text[..i];
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// `text.slice(-max)` in UTF-16 units, never splitting a character.
|
||||
pub(super) fn js_tail(text: &str, max: usize) -> &str {
|
||||
if text.len() <= max {
|
||||
return text;
|
||||
}
|
||||
let mut units = 0;
|
||||
for (i, c) in text.char_indices().rev() {
|
||||
units += c.len_utf16();
|
||||
if units > max {
|
||||
return &text[i + c.len_utf8()..];
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// `*` any characters but `/`, `**` any characters, `?` one character but `/`. A pattern
|
||||
/// without `/` is matched against the file name alone, so `*.tsx` finds nested files.
|
||||
pub(super) fn matches_glob(subject: &str, glob: &str) -> bool {
|
||||
let subject = if glob.contains('/') {
|
||||
subject
|
||||
} else {
|
||||
subject.rsplit('/').next().unwrap_or(subject)
|
||||
};
|
||||
let mut pattern = String::from("^");
|
||||
let mut chars = glob.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
match c {
|
||||
'*' if chars.peek() == Some(&'*') => {
|
||||
chars.next();
|
||||
pattern.push_str(".*");
|
||||
}
|
||||
'*' => pattern.push_str("[^/]*"),
|
||||
'?' => pattern.push_str("[^/]"),
|
||||
c => pattern.push_str(®ex::escape(c.encode_utf8(&mut [0; 4]))),
|
||||
}
|
||||
}
|
||||
pattern.push('$');
|
||||
regex::Regex::new(&pattern).is_ok_and(|re| re.is_match(subject))
|
||||
}
|
||||
|
||||
pub(super) async fn call_session_tool(
|
||||
ctx: &Ctx<'_>,
|
||||
name: &str,
|
||||
args: Value,
|
||||
) -> BackendResult<CallToolResult> {
|
||||
let Some(def) = TOOL_DEFS.iter().find(|def| def.name == name) else {
|
||||
return Err(ErrorData::invalid_params(
|
||||
format!("Unknown tool '{name}'"),
|
||||
None,
|
||||
));
|
||||
};
|
||||
if !allowed(def, ctx.access) {
|
||||
let token = match ctx.access {
|
||||
SessionAccess::ReadAndRun => "can only read and run deployed scripts and flows",
|
||||
_ => "is read-only",
|
||||
};
|
||||
return Err(ErrorData::invalid_request(
|
||||
format!("Tool '{name}' changes or runs something, and this token {token}"),
|
||||
None,
|
||||
));
|
||||
}
|
||||
let Some(result) = dispatch(ctx, name, &args).await else {
|
||||
return Err(ErrorData::invalid_params(
|
||||
format!("Tool '{name}' is not available on this server"),
|
||||
None,
|
||||
));
|
||||
};
|
||||
windmill_common::feature_usage::log_feature_usage("mcp_workspace", "tool_call", name);
|
||||
Ok(match result {
|
||||
Ok(text) => CallToolResult::success(vec![ContentBlock::text(text)]),
|
||||
// Without the status code `Ctx` prefixes an API failure with, as the session reads it.
|
||||
Err(message) => {
|
||||
let body = workspace::api_message(&message);
|
||||
let text = if body.is_empty() { &message } else { body };
|
||||
CallToolResult::error(vec![ContentBlock::text(text.to_string())])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_call_stays_on_the_route_its_handler_wrote() {
|
||||
use SessionAccess::*;
|
||||
let url = |method, path, access| guarded_url("http://h", "ws", method, path, access);
|
||||
assert!(url("GET", "/w/ws/jobs_u/get/0191?no_logs=true", Full).is_ok());
|
||||
assert!(url("GET", "/docs/search?query=a%2Fb", ReadOnly).is_ok());
|
||||
for path in [
|
||||
"/w/ws/jobs_u/get/../../jobs/run_wait_result/p/u/a/b",
|
||||
"/w/ws/jobs_u/get/%2e%2e/%2E%2E/variables/get_value/u/a/b",
|
||||
"/w/ws/jobs_u/get/x#",
|
||||
"/w/other/scripts/list",
|
||||
"/w/ws",
|
||||
"/settings/global/x",
|
||||
"/docs/../users/tokens/list",
|
||||
] {
|
||||
assert!(url("GET", path, Full).is_err(), "{path} should be refused");
|
||||
}
|
||||
assert!(guarded_url("http://h", "a/../b", "GET", "/w/a/../b/scripts/list", Full).is_err());
|
||||
// A read-only token neither writes nor runs, whatever the tool.
|
||||
assert!(url("POST", "/w/ws/drafts/update", ReadOnly).is_err());
|
||||
assert!(url("GET", "/w/ws/jobs/run_wait_result/p/u/a/b", ReadOnly).is_err());
|
||||
assert!(url("POST", "/w/ws/jobs/run/p/u/a/b", ReadOnly).is_err());
|
||||
// A read-and-run token runs deployed items by path, and writes nothing else: a
|
||||
// preview takes code.
|
||||
assert!(url("POST", "/w/ws/jobs/run/p/u/a/b", ReadAndRun).is_ok());
|
||||
assert!(url("POST", "/w/ws/jobs/run/f/u/a/b", ReadAndRun).is_ok());
|
||||
assert!(url("POST", "/w/ws/jobs/run/preview", ReadAndRun).is_err());
|
||||
assert!(url("POST", "/w/ws/drafts/update", ReadAndRun).is_err());
|
||||
assert!(url("POST", "/w/ws/variables/create", ReadAndRun).is_ok());
|
||||
assert!(url("POST", "/w/ws/variables/update/u/a/b", ReadAndRun).is_err());
|
||||
assert!(url("POST", "/w/ws/variables/create", ReadOnly).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_session_tool_is_implemented_or_editor_only() {
|
||||
let implemented: Vec<&str> = implemented().collect();
|
||||
for def in TOOL_DEFS.iter() {
|
||||
let name = def.name.as_str();
|
||||
assert!(
|
||||
implemented.contains(&name) != EDITOR_ONLY.contains(&name),
|
||||
"session tool `{name}` must be implemented here or listed in EDITOR_ONLY, not both"
|
||||
);
|
||||
}
|
||||
for name in implemented.iter().chain(EDITOR_ONLY) {
|
||||
assert!(
|
||||
TOOL_DEFS.iter().any(|def| def.name == *name),
|
||||
"`{name}` is not a session tool any more"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,140 @@
|
||||
//! The workspace settings document, in the layout of the `settings.yaml` the CLI syncs.
|
||||
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::workspace::api_message;
|
||||
use super::{array, js_head, js_len, js_str, str_arg, Ctx, ToolResult};
|
||||
|
||||
pub(super) const TOOLS: &[&str] = &["get_workspace_settings", "set_workspace_settings"];
|
||||
|
||||
pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult {
|
||||
match name {
|
||||
"get_workspace_settings" => get_workspace_settings(ctx, args).await,
|
||||
"set_workspace_settings" => set_workspace_settings(ctx, args).await,
|
||||
_ => unreachable!("{name} is not a settings tool"),
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_READ_CHARS: usize = 20_000;
|
||||
|
||||
async fn get_workspace_settings(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let doc = ctx
|
||||
.get(&ctx.ws("workspaces/settings_document?secrets=redacted"))
|
||||
.await?;
|
||||
let keys: Vec<&str> = array(args.get("keys"))
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.collect();
|
||||
let mut picked = doc.clone();
|
||||
let mut note = String::new();
|
||||
if !keys.is_empty() {
|
||||
picked = Value::Object(
|
||||
keys.iter()
|
||||
.filter_map(|k| Some((k.to_string(), doc.get(k)?.clone())))
|
||||
.collect::<Map<_, _>>(),
|
||||
);
|
||||
let unset: Vec<&str> = keys
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|k| doc.get(k).is_none())
|
||||
.collect();
|
||||
if !unset.is_empty() {
|
||||
note = format!(
|
||||
"\n# Not in the document (unset, or not a setting): {}",
|
||||
unset.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut text = serde_yml::to_string(&picked).map_err(|e| e.to_string())?;
|
||||
text.push_str(¬e);
|
||||
Ok(if js_len(&text) > MAX_READ_CHARS {
|
||||
format!(
|
||||
"{}\n… truncated at {MAX_READ_CHARS} chars; read the keys you need with `keys`.",
|
||||
js_head(&text, MAX_READ_CHARS)
|
||||
)
|
||||
} else {
|
||||
text
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_settings_yaml(text: &str) -> Result<Value, String> {
|
||||
match serde_yml::from_str::<Value>(text) {
|
||||
Ok(doc @ Value::Object(_)) => Ok(doc),
|
||||
Ok(_) => Err("The yaml must be a mapping of setting keys to values.".to_string()),
|
||||
Err(e) => Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The dry run stands in for the session's confirmation card: it reports a no-op or an
|
||||
/// invalid document before anything is written.
|
||||
async fn set_workspace_settings(ctx: &Ctx<'_>, args: &Value) -> ToolResult {
|
||||
let route = ctx.ws("workspaces/settings_document");
|
||||
let document = match parse_settings_yaml(str_arg(args, "yaml")?) {
|
||||
Ok(document) => document,
|
||||
Err(e) => return Ok(format!("Nothing was changed: {e}")),
|
||||
};
|
||||
let dry_run = json!({ "document": document, "replace": false, "dry_run": true });
|
||||
let dry_run = match ctx.post(&route, dry_run).await {
|
||||
Ok(result) => result,
|
||||
Err(e) => return Ok(format!("Nothing was changed: {}", api_message(&e))),
|
||||
};
|
||||
if array(dry_run.get("changes")).is_empty() {
|
||||
let warnings: Vec<String> = array(dry_run.get("warnings")).iter().map(js_str).collect();
|
||||
let warnings = if warnings.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" Warnings: {}", warnings.join(" "))
|
||||
};
|
||||
return Ok(format!(
|
||||
"The settings already hold these values: nothing to change.{warnings}"
|
||||
));
|
||||
}
|
||||
let applied = match ctx
|
||||
.post(&route, json!({ "document": document, "replace": false }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(e) => return Ok(json!({ "success": false, "error": api_message(&e) }).to_string()),
|
||||
};
|
||||
let keys: Vec<String> = array(applied.get("changes"))
|
||||
.iter()
|
||||
.map(|c| js_str(c.get("key").unwrap_or(&Value::Null)))
|
||||
.collect();
|
||||
let mut out = Map::new();
|
||||
out.insert("success".into(), json!(true));
|
||||
out.insert(
|
||||
"message".into(),
|
||||
json!(format!(
|
||||
"These settings are now applied: {}.",
|
||||
keys.join(", ")
|
||||
)),
|
||||
);
|
||||
if let Some(warnings) = applied
|
||||
.get("warnings")
|
||||
.filter(|w| !array(Some(w)).is_empty())
|
||||
{
|
||||
out.insert("warnings".into(), warnings.clone());
|
||||
}
|
||||
Ok(Value::Object(out).to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn settings_yaml_must_be_a_mapping() {
|
||||
assert_eq!(
|
||||
parse_settings_yaml("webhook: null\ncolor: red").unwrap(),
|
||||
json!({ "webhook": null, "color": "red" })
|
||||
);
|
||||
for text in ["- a\n- b", "just text", "42"] {
|
||||
assert_eq!(
|
||||
parse_settings_yaml(text).unwrap_err(),
|
||||
"The yaml must be a mapping of setting keys to values.",
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
assert!(parse_settings_yaml("a: [").is_err());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -171,18 +171,16 @@ pub async fn get_resources(
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Generic function to get items (scripts or flows) from the database
|
||||
pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>(
|
||||
user_db: &UserDB,
|
||||
/// The scripts or flows a listing covers: the caller's workspace, unarchived,
|
||||
/// callable, and narrowed by favorites and `path_filter`.
|
||||
fn items_query(
|
||||
authed: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
scope_type: &str,
|
||||
item_type: &str,
|
||||
path_filter: Option<PathFilter<'_>>,
|
||||
) -> Result<Vec<T>, ErrorData> {
|
||||
) -> SqlBuilder {
|
||||
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
|
||||
let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
|
||||
sqlb.fields(&fields);
|
||||
if scope_type == "favorites" {
|
||||
sqlb.join("favorite")
|
||||
.on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type)
|
||||
@@ -209,17 +207,29 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
|
||||
sqlb.and_where(cond);
|
||||
}
|
||||
}
|
||||
Some(PathFilter::LastSegment(segments)) => {
|
||||
if segments.is_empty() {
|
||||
sqlb.and_where("false");
|
||||
} else {
|
||||
let conds: Vec<String> = segments
|
||||
.iter()
|
||||
.map(|s| {
|
||||
"o.path LIKE ? ESCAPE '\\'".bind(&format!("%/{}", escape_like(s)))
|
||||
})
|
||||
.collect();
|
||||
sqlb.and_where(format!("({})", conds.join(" OR ")));
|
||||
}
|
||||
}
|
||||
}
|
||||
sqlb
|
||||
}
|
||||
|
||||
sqlb.order_by(
|
||||
if item_type == "flow" {
|
||||
"o.edited_at"
|
||||
} else {
|
||||
"o.created_at"
|
||||
},
|
||||
false,
|
||||
)
|
||||
.limit(ITEMS_FETCH_MAX_LIMIT);
|
||||
async fn fetch_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>(
|
||||
user_db: &UserDB,
|
||||
authed: &ApiAuthed,
|
||||
sqlb: &SqlBuilder,
|
||||
item_type: &str,
|
||||
) -> Result<Vec<T>, ErrorData> {
|
||||
let sql = sqlb.sql().map_err(|e| {
|
||||
tracing::error!("failed to build sql: {}", e);
|
||||
ErrorData::internal_error(format!("failed to build sql: {}", e), None)
|
||||
@@ -242,6 +252,44 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Generic function to get items (scripts or flows) from the database
|
||||
pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>(
|
||||
user_db: &UserDB,
|
||||
authed: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
scope_type: &str,
|
||||
item_type: &str,
|
||||
path_filter: Option<PathFilter<'_>>,
|
||||
) -> Result<Vec<T>, ErrorData> {
|
||||
let mut sqlb = items_query(authed, workspace_id, scope_type, item_type, path_filter);
|
||||
sqlb.fields(&["o.path", "o.summary", "o.description", "o.schema"]);
|
||||
sqlb.order_by(
|
||||
if item_type == "flow" {
|
||||
"o.edited_at"
|
||||
} else {
|
||||
"o.created_at"
|
||||
},
|
||||
false,
|
||||
)
|
||||
.limit(ITEMS_FETCH_MAX_LIMIT);
|
||||
fetch_items(user_db, authed, &sqlb, item_type).await
|
||||
}
|
||||
|
||||
/// Every path `get_items` would return with no fetch cap.
|
||||
pub async fn get_item_paths(
|
||||
user_db: &UserDB,
|
||||
authed: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
scope_type: &str,
|
||||
item_type: &str,
|
||||
path_filter: Option<PathFilter<'_>>,
|
||||
) -> Result<Vec<String>, ErrorData> {
|
||||
let mut sqlb = items_query(authed, workspace_id, scope_type, item_type, path_filter);
|
||||
sqlb.distinct().field("o.path");
|
||||
let rows: Vec<(String,)> = fetch_items(user_db, authed, &sqlb, item_type).await?;
|
||||
Ok(rows.into_iter().map(|(path,)| path).collect())
|
||||
}
|
||||
|
||||
/// Get scripts from the Hub
|
||||
pub async fn get_scripts_from_hub(
|
||||
db: &DB,
|
||||
@@ -652,6 +700,47 @@ pub async fn create_http_request(
|
||||
workspace_id: &str,
|
||||
api_authed: &ApiAuthed,
|
||||
body_json: Option<Value>,
|
||||
) -> BackendResult<reqwest::Response> {
|
||||
// Scope the minted JWT to the proxied route so a scope-restricted MCP token
|
||||
// can't be widened into a full-privilege blank check. See
|
||||
// `jwt_scopes_for_proxied_route`.
|
||||
let parsed = reqwest::Url::parse(url)
|
||||
.map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?;
|
||||
let scopes = jwt_scopes_for_proxied_route(api_authed.scopes.as_deref(), method, parsed.path())?;
|
||||
send_as_caller(method, url, workspace_id, api_authed, body_json, scopes).await
|
||||
}
|
||||
|
||||
/// A JWT that acts as `api_authed`, carrying `scopes`; `None` is the caller's full rights.
|
||||
pub(crate) async fn mint_caller_jwt(
|
||||
workspace_id: &str,
|
||||
api_authed: &ApiAuthed,
|
||||
scopes: Option<Vec<String>>,
|
||||
) -> BackendResult<String> {
|
||||
// Carry the caller's job provenance into the JWT: a job's WM_TOKEN is capped at
|
||||
// workspace admin (GHSA-hfh4-cx4h-3fcr), and dropping `job_id` here would re-mint an
|
||||
// uncapped token that satisfies require_super_admin / require_devops_role.
|
||||
create_jwt_token(
|
||||
Authed::from(api_authed.clone()),
|
||||
workspace_id,
|
||||
3600,
|
||||
api_authed.job_id,
|
||||
None,
|
||||
None,
|
||||
scopes,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.to_string(), None))
|
||||
}
|
||||
|
||||
/// Call the API as `api_authed` with a JWT carrying `scopes`; `None` is the caller's
|
||||
/// full rights, for a server whose own code picks every route it calls.
|
||||
pub async fn send_as_caller(
|
||||
method: &str,
|
||||
url: &str,
|
||||
workspace_id: &str,
|
||||
api_authed: &ApiAuthed,
|
||||
body_json: Option<Value>,
|
||||
scopes: Option<Vec<String>>,
|
||||
) -> BackendResult<reqwest::Response> {
|
||||
let client = &HTTP_CLIENT;
|
||||
let mut request_builder = match method {
|
||||
@@ -668,29 +757,7 @@ pub async fn create_http_request(
|
||||
}
|
||||
};
|
||||
|
||||
// Scope the minted JWT to the proxied route so a scope-restricted MCP token
|
||||
// can't be widened into a full-privilege blank check. See
|
||||
// `jwt_scopes_for_proxied_route`.
|
||||
let parsed = reqwest::Url::parse(url)
|
||||
.map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?;
|
||||
let scopes = jwt_scopes_for_proxied_route(api_authed.scopes.as_deref(), method, parsed.path())?;
|
||||
|
||||
// Add authorization header. Carry the caller's job provenance into the proxy
|
||||
// JWT: a job's WM_TOKEN is capped at workspace admin (GHSA-hfh4-cx4h-3fcr), and
|
||||
// dropping `job_id` here would re-mint an uncapped token that satisfies
|
||||
// require_super_admin / require_devops_role on the proxied route.
|
||||
let authed = Authed::from(api_authed.clone());
|
||||
let token = create_jwt_token(
|
||||
authed,
|
||||
workspace_id,
|
||||
3600,
|
||||
api_authed.job_id,
|
||||
None,
|
||||
None,
|
||||
scopes,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
|
||||
let token = mint_caller_jwt(workspace_id, api_authed, scopes).await?;
|
||||
request_builder = request_builder.header("Authorization", format!("Bearer {}", token));
|
||||
|
||||
// Add body if present
|
||||
|
||||
@@ -10,7 +10,10 @@ use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
utils::{not_found_if_none, StripPath},
|
||||
};
|
||||
use windmill_store::{resources::explain_resource_perm_error, variables::get_value_internal};
|
||||
use windmill_store::{
|
||||
resources::{check_resource_var_ref_scope, explain_resource_perm_error},
|
||||
variables::get_value_internal,
|
||||
};
|
||||
|
||||
/// A connected MCP server is a third party the user chose, reached over a
|
||||
/// connection this request holds open: without a deadline one that never answers
|
||||
@@ -67,12 +70,22 @@ async fn connect_mcp_client(
|
||||
serde_json::from_str::<windmill_mcp::McpResource>(resource_value.0.get())
|
||||
.map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?;
|
||||
|
||||
// The token, URL and header secrets are read on the caller's behalf and sent to a URL
|
||||
// the resource chose, so each needs the same scope as a resource reference. The token
|
||||
// is checked before the OAuth refresh touches its variable.
|
||||
let token_var_path = mcp_resource
|
||||
.token
|
||||
.as_deref()
|
||||
.map(|t| t.trim_start_matches("$var:"))
|
||||
.filter(|p| !p.trim().is_empty());
|
||||
if let Some(token_var_path) = token_var_path {
|
||||
check_resource_var_ref_scope(authed, path, token_var_path)?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "oauth2")]
|
||||
{
|
||||
tracing::info!("Checking if token needs refresh before creating MCP client");
|
||||
if let Some(ref token_path) = mcp_resource.token {
|
||||
let token_var_path = token_path.trim_start_matches("$var:");
|
||||
|
||||
if let Some(token_var_path) = token_var_path {
|
||||
let token_info = sqlx::query!(
|
||||
r#"
|
||||
SELECT
|
||||
@@ -90,13 +103,12 @@ async fn connect_mcp_client(
|
||||
|
||||
if let Some(info) = token_info {
|
||||
if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) {
|
||||
let refresh_tx = user_db.clone().begin(authed).await?;
|
||||
if let Err(e) = crate::oauth2_oss::_refresh_token(
|
||||
refresh_tx,
|
||||
token_var_path,
|
||||
w_id,
|
||||
account_id,
|
||||
db,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -113,14 +125,9 @@ async fn connect_mcp_client(
|
||||
// Resolve the token through the caller's permissioned (RLS + audit) path so
|
||||
// a developer cannot exfiltrate a secret they are not allowed to read by
|
||||
// pointing an MCP resource's token at it.
|
||||
let token = if let Some(token_path) = &mcp_resource.token {
|
||||
let token_var_path = token_path.trim_start_matches("$var:");
|
||||
if token_var_path.trim().is_empty() {
|
||||
None
|
||||
} else {
|
||||
let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone()));
|
||||
Some(get_value_internal(&db_authed, w_id, token_var_path, false).await?)
|
||||
}
|
||||
let token = if let Some(token_var_path) = token_var_path {
|
||||
let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone()));
|
||||
Some(get_value_internal(&db_authed, w_id, token_var_path, false).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -130,7 +137,7 @@ async fn connect_mcp_client(
|
||||
.chain(mcp_resource.headers.iter_mut().flatten().map(|(_, v)| v))
|
||||
{
|
||||
if let Some(var_path) = value.strip_prefix("$var:") {
|
||||
check_scopes(authed, || format!("variables:read:{}", var_path))?;
|
||||
check_resource_var_ref_scope(authed, path, var_path)?;
|
||||
let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone()));
|
||||
*value = get_value_internal(&db_authed, w_id, var_path, false).await?;
|
||||
}
|
||||
|
||||
@@ -22,8 +22,6 @@ use hmac::Mac;
|
||||
use itertools::Itertools;
|
||||
#[cfg(not(feature = "private"))]
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[cfg(not(feature = "private"))]
|
||||
use sqlx::{Postgres, Transaction};
|
||||
#[cfg(all(feature = "oauth2", not(feature = "private")))]
|
||||
use windmill_common::more_serde::maybe_number_opt;
|
||||
#[cfg(all(feature = "oauth2", not(feature = "private")))]
|
||||
@@ -84,6 +82,7 @@ pub async fn build_oauth_clients(
|
||||
logins: HashMap::default(),
|
||||
connects: HashMap::default(),
|
||||
slack: None,
|
||||
slack_signing_secret: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -133,12 +132,12 @@ async fn list_connects() -> windmill_common::error::JsonResult<Vec<String>> {
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub async fn _refresh_token<'c>(
|
||||
_tx: Transaction<'c, Postgres>,
|
||||
pub async fn _refresh_token(
|
||||
_path: &str,
|
||||
_w_id: &str,
|
||||
_id: i32,
|
||||
_db: &DB,
|
||||
_force: bool,
|
||||
) -> error::Result<String> {
|
||||
// Implementation is not open source
|
||||
Err(error::Error::BadRequest(
|
||||
|
||||
@@ -266,6 +266,17 @@ async fn get_offboard_preview(
|
||||
departing.as_deref(), &user_prefix, w_id
|
||||
).fetch_all(db).await?;
|
||||
|
||||
// An agent's identity sits in its value, under any path, the user's own included: those
|
||||
// move with the path but keep running as the departing user until re-pointed.
|
||||
let obo_agents: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT path FROM resource WHERE resource_type = 'ai_agent'
|
||||
AND value->>'on_behalf_of' = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(departing.as_deref())
|
||||
.bind(w_id)
|
||||
.fetch_all(db)
|
||||
.await?;
|
||||
|
||||
let obo_schedules = sqlx::query_scalar!(
|
||||
"SELECT path FROM schedule WHERE permissioned_as = $1 AND NOT path LIKE $2 AND workspace_id = $3",
|
||||
&user_owner, &user_prefix, w_id
|
||||
@@ -354,6 +365,7 @@ async fn get_offboard_preview(
|
||||
scripts: obo_scripts,
|
||||
flows: obo_flows,
|
||||
apps: obo_apps,
|
||||
resources: obo_agents,
|
||||
schedules: obo_schedules,
|
||||
triggers: obo_triggers,
|
||||
..Default::default()
|
||||
@@ -1007,6 +1019,16 @@ async fn offboard_user_from_workspace<'c>(
|
||||
.await?
|
||||
.unwrap_or(0);
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE resource SET value = jsonb_set(value, '{on_behalf_of}', to_jsonb($1::text))
|
||||
WHERE resource_type = 'ai_agent' AND value->>'on_behalf_of' = $2 AND workspace_id = $3",
|
||||
)
|
||||
.bind(new_permissioned_as)
|
||||
.bind(departing.as_deref())
|
||||
.bind(w_id)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
// ---- eval datasets ----
|
||||
// The foreign keys cascade the rename onto cases and experiments; the paths held inside JSONB
|
||||
// (an experiment's subject, a dataset's scorers) are rewritten separately since the cascade
|
||||
@@ -1187,6 +1209,11 @@ async fn offboard_user_from_workspace<'c>(
|
||||
// They are NOT done here for reassign-only (delete_user=false) since the user stays.
|
||||
|
||||
// ---- Related path tables ----
|
||||
sqlx::query!(
|
||||
r#"UPDATE native_trigger SET connection_path = REGEXP_REPLACE(connection_path, 'u/' || $2 || '/(.*)', $1 || '/\1') WHERE connection_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
|
||||
&new_prefix, username, w_id
|
||||
).execute(&mut **tx).await?;
|
||||
|
||||
sqlx::query!(
|
||||
r#"UPDATE workspace_integrations SET resource_path = REGEXP_REPLACE(resource_path, 'u/' || $2 || '/(.*)', $1 || '/\1') WHERE resource_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
|
||||
&new_prefix, username, w_id
|
||||
|
||||
@@ -7,15 +7,17 @@
|
||||
*/
|
||||
|
||||
//! Unified, keyset-paginated listing of a workspace's runnables (scripts,
|
||||
//! flows, apps) merged into one globally-ordered stream. The homepage uses it
|
||||
//! so a chosen order (recently updated / oldest / name) is correct and complete
|
||||
//! across all three kinds at any workspace size, instead of client-sorting a
|
||||
//! flows, apps, and the `ai_agent` resources listed as agents) merged into one
|
||||
//! globally-ordered stream. The homepage uses it so a chosen order (recently
|
||||
//! updated / oldest / name) is correct and complete across every kind at any
|
||||
//! workspace size, instead of client-sorting a
|
||||
//! per-kind capped window.
|
||||
//!
|
||||
//! Efficiency: each kind is a UNION ALL branch ordered by an index on
|
||||
//! `(workspace_id, archived, <sort key>)` (created_at / edited_at, or the lowered
|
||||
//! summary-or-path expression for name orders); Postgres merges the ordered
|
||||
//! branches and stops at the page limit. Pagination is keyset — a
|
||||
//! branches and stops at the page limit. Agents, which are few, are instead
|
||||
//! reached through the partial `idx_resource_agent_owner_prefix` and sorted whole. Pagination is keyset — a
|
||||
//! `(sort_key, path, kind, tiebreak)` cursor, where `tiebreak` (a script's hash,
|
||||
//! 0 for flow/app) is a stable final key that keeps the order total even when rows
|
||||
//! tie on (sort_key, path, kind) — so deep pages don't re-scan. Visibility is
|
||||
@@ -34,7 +36,7 @@ use std::collections::HashMap;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{Error, JsonResult},
|
||||
workspaces::{operator_builder_rights, OperatorBuilderRights},
|
||||
roles::{caller_capabilities, AuthorLevel, RoleCapabilities},
|
||||
};
|
||||
use windmill_types::scripts::ScriptHash;
|
||||
use windmill_types::user_drafts::DraftUserRef;
|
||||
@@ -51,7 +53,7 @@ struct ListRunnablesQuery {
|
||||
order_by: Option<String>,
|
||||
/// Descending when true (default true).
|
||||
order_desc: Option<bool>,
|
||||
/// Comma-separated subset of `script,flow,app`; omitted means all.
|
||||
/// Comma-separated subset of `script,flow,app,agent`; omitted means all.
|
||||
kinds: Option<String>,
|
||||
show_archived: Option<bool>,
|
||||
/// Include library scripts (no runnable main). Ignored for flows/apps.
|
||||
@@ -80,7 +82,7 @@ struct ListRunnablesQuery {
|
||||
#[derive(Serialize, sqlx::FromRow)]
|
||||
struct RunnableItem {
|
||||
#[serde(rename = "type")]
|
||||
kind: String, // 'script' | 'flow' | 'app'
|
||||
kind: String, // 'script' | 'flow' | 'app' | 'agent'
|
||||
path: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
summary: Option<String>,
|
||||
@@ -130,6 +132,11 @@ struct RunnableItem {
|
||||
id: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
version: Option<i64>,
|
||||
// agent-only: the `memory` setting, which decides whether the agent opens as a chat.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
agent_memory: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
ws_specific: Option<bool>,
|
||||
// sort keys, echoed into the cursor (not serialized to the client)
|
||||
#[serde(skip)]
|
||||
sort_time: chrono::DateTime<chrono::Utc>,
|
||||
@@ -247,7 +254,7 @@ fn scope_path_predicate(
|
||||
}
|
||||
}
|
||||
|
||||
/// The three UNION-ALL branch SELECTs, each projecting the shared `RunnableItem`
|
||||
/// The UNION-ALL branch SELECTs, each projecting the shared `RunnableItem`
|
||||
/// column set (NULL for columns that don't apply to that kind). `$1`=workspace,
|
||||
/// `$2`=username (favorites), `$3`=email (drafts). Kind-specific and per-request
|
||||
/// WHERE fragments are appended by the caller.
|
||||
@@ -255,6 +262,7 @@ struct Branches {
|
||||
script: String,
|
||||
flow: String,
|
||||
app: String,
|
||||
agent: String,
|
||||
}
|
||||
|
||||
fn branch_sqls() -> Branches {
|
||||
@@ -281,6 +289,7 @@ fn branch_sqls() -> Branches {
|
||||
o.codebase IS NOT NULL as use_codebase, \
|
||||
(o.lock_error_logs IS NOT NULL) as has_deploy_errors, NULL::bool as chat_input_enabled, \
|
||||
NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific, \
|
||||
o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, o.hash as tiebreak \
|
||||
FROM script o \
|
||||
LEFT JOIN favorite ON favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \
|
||||
@@ -298,6 +307,7 @@ fn branch_sqls() -> Branches {
|
||||
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \
|
||||
(o.value->>'chat_input_enabled')::bool as chat_input_enabled, \
|
||||
NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific, \
|
||||
o.edited_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \
|
||||
FROM flow o \
|
||||
LEFT JOIN favorite ON favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \
|
||||
@@ -315,6 +325,7 @@ fn branch_sqls() -> Branches {
|
||||
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \
|
||||
av.raw_app, o.policy->>'execution_mode' as execution_mode, o.id, \
|
||||
o.versions[array_upper(o.versions, 1)] as version, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific, \
|
||||
COALESCE(av.created_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \
|
||||
FROM app o \
|
||||
LEFT JOIN favorite ON favorite.favorite_kind = 'app' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \
|
||||
@@ -323,25 +334,68 @@ fn branch_sqls() -> Branches {
|
||||
draft_users = draft_users("d.typ IN ('app', 'raw_app')")
|
||||
);
|
||||
|
||||
Branches { script, flow, app }
|
||||
// An agent is an `ai_agent` resource; the subquery renames its description to the
|
||||
// `summary` the shared search and name sort read.
|
||||
let agent = format!(
|
||||
"SELECT 'agent' as kind, o.path, o.summary, o.workspace_id, o.extra_perms, \
|
||||
favorite.path IS NOT NULL as starred, false as archived, \
|
||||
draft.email IS NOT NULL as is_draft, NULL::bool as draft_only, NULL::text as draft_path, \
|
||||
{draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \
|
||||
NULL::bool as ws_error_handler_muted, o.edited_at, \
|
||||
NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
|
||||
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \
|
||||
NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
|
||||
o.agent_memory, ws_specific.path IS NOT NULL as ws_specific, \
|
||||
COALESCE(o.edited_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \
|
||||
FROM (SELECT workspace_id, path, description as summary, extra_perms, labels, edited_at, \
|
||||
value->'memory' as agent_memory \
|
||||
FROM resource WHERE resource_type = 'ai_agent') o \
|
||||
LEFT JOIN favorite ON favorite.favorite_kind = 'agent' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \
|
||||
LEFT JOIN draft ON draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'resource' AND draft.email = $3 \
|
||||
LEFT JOIN ws_specific ON ws_specific.path = o.path AND ws_specific.workspace_id = o.workspace_id AND ws_specific.item_kind = 'resource'",
|
||||
draft_users = draft_users("d.typ = 'resource'")
|
||||
);
|
||||
|
||||
Branches { script, flow, app, agent }
|
||||
}
|
||||
|
||||
/// The draft rows a kind lists and the table whose rows make a draft not draft-only.
|
||||
fn draft_kind_source(kind: &str) -> (&'static str, &'static str) {
|
||||
match kind {
|
||||
"script" => ("d.typ = 'script'", "script"),
|
||||
"flow" => ("d.typ = 'flow'", "flow"),
|
||||
// Typed by the draft itself, as the resource listing filters its draft-only rows.
|
||||
"agent" => (
|
||||
"d.typ = 'resource' AND d.value->>'resource_type' = 'ai_agent'",
|
||||
"resource",
|
||||
),
|
||||
_ => ("d.typ IN ('app', 'raw_app')", "app"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a draft records the path its editor has staged. Scripts and resources bind the
|
||||
/// Path widget to the item's own `path`, so it round-trips through the draft JSON's
|
||||
/// `path`; flows and apps write a separate `draft_path` only when it differs from the
|
||||
/// deployed one. See scripts.rs.
|
||||
fn draft_typed_path(kind: &str) -> &'static str {
|
||||
if kind == "script" || kind == "agent" {
|
||||
"path"
|
||||
} else {
|
||||
"draft_path"
|
||||
}
|
||||
}
|
||||
|
||||
/// The draft-only branch for a kind: the caller's drafts at paths carrying no
|
||||
/// deployed row, projected into the same column set as `branch_sqls` so they
|
||||
/// sort, search and paginate as ordinary rows. Same `$1`/`$2`/`$3` contract.
|
||||
fn draft_branch_sql(kind: &str) -> String {
|
||||
let (typ_pred, deployed) = match kind {
|
||||
"script" => ("d.typ = 'script'", "script"),
|
||||
"flow" => ("d.typ = 'flow'", "flow"),
|
||||
_ => ("d.typ IN ('app', 'raw_app')", "app"),
|
||||
};
|
||||
// Scripts bind the Path widget to `script.path`, so the typed path round-trips
|
||||
// through the draft JSON's own `path`; flows and apps write a separate
|
||||
// `draft_path` only when it differs from the deployed one. See scripts.rs.
|
||||
let typed_path = if kind == "script" {
|
||||
"path"
|
||||
let (typ_pred, deployed) = draft_kind_source(kind);
|
||||
let typed_path = draft_typed_path(kind);
|
||||
// A resource draft carries the description where the other kinds carry a summary.
|
||||
let summary_field = if kind == "agent" {
|
||||
"description"
|
||||
} else {
|
||||
"draft_path"
|
||||
"summary"
|
||||
};
|
||||
// `auto_kind` is only what the editor stamped into the draft — a `// pipeline`
|
||||
// annotation is not re-derived from the content here, unlike the per-kind
|
||||
@@ -351,7 +405,8 @@ fn draft_branch_sql(kind: &str) -> String {
|
||||
"script" => {
|
||||
"d.value->>'language' as language, d.value->>'kind' as script_kind, \
|
||||
d.value->>'auto_kind' as auto_kind, false as raw_app, \
|
||||
NULL::bool as chat_input_enabled"
|
||||
NULL::bool as chat_input_enabled, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific"
|
||||
}
|
||||
// Type-guarded rather than a bare `::bool` cast: draft JSON is stored
|
||||
// unvalidated, so a malformed value must yield NULL, not abort the list,
|
||||
@@ -360,11 +415,20 @@ fn draft_branch_sql(kind: &str) -> String {
|
||||
"NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
|
||||
false as raw_app, \
|
||||
CASE WHEN json_typeof(d.value->'value'->'chat_input_enabled') = 'boolean' \
|
||||
THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled"
|
||||
THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific"
|
||||
}
|
||||
"agent" => {
|
||||
"NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
|
||||
false as raw_app, NULL::bool as chat_input_enabled, \
|
||||
(d.value->'args'->'memory')::jsonb as agent_memory, \
|
||||
CASE WHEN json_typeof(d.value->'wsSpecific') = 'boolean' \
|
||||
THEN (d.value->>'wsSpecific')::bool END as ws_specific"
|
||||
}
|
||||
_ => {
|
||||
"NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
|
||||
(d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled"
|
||||
(d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled, \
|
||||
NULL::jsonb as agent_memory, NULL::bool as ws_specific"
|
||||
}
|
||||
};
|
||||
format!(
|
||||
@@ -378,11 +442,12 @@ fn draft_branch_sql(kind: &str) -> String {
|
||||
NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \
|
||||
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, o.chat_input_enabled, \
|
||||
o.raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
|
||||
o.agent_memory, o.ws_specific, \
|
||||
o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \
|
||||
FROM ( \
|
||||
SELECT DISTINCT ON (d.path) d.workspace_id, d.path, d.created_at, \
|
||||
d.email IS NULL as legacy, \
|
||||
COALESCE(d.value->>'summary', '') as summary, \
|
||||
COALESCE(d.value->>'{summary_field}', '') as summary, \
|
||||
NULLIF(NULLIF(d.value->>'{typed_path}', ''), d.path) as draft_path, \
|
||||
{kind_cols} \
|
||||
FROM draft d \
|
||||
@@ -397,6 +462,8 @@ fn draft_branch_sql(kind: &str) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
const ALL_KINDS: [&str; 4] = ["script", "flow", "app", "agent"];
|
||||
|
||||
async fn list_runnables(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -404,6 +471,7 @@ async fn list_runnables(
|
||||
Path(w_id): Path<String>,
|
||||
Query(q): Query<ListRunnablesQuery>,
|
||||
) -> JsonResult<ListRunnablesResponse> {
|
||||
let caps = caller_capabilities(&db, &w_id, &authed).await?;
|
||||
let order_by_name = q.order_by.as_deref() == Some("name");
|
||||
let desc = q.order_desc.unwrap_or(true);
|
||||
let per_page = q.per_page.unwrap_or(50).clamp(1, 1000);
|
||||
@@ -416,16 +484,16 @@ async fn list_runnables(
|
||||
};
|
||||
|
||||
let mut kinds: Vec<&str> = match q.kinds.as_deref() {
|
||||
None | Some("") => vec!["script", "flow", "app"],
|
||||
None | Some("") => ALL_KINDS.to_vec(),
|
||||
Some(csv) => csv
|
||||
.split(',')
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| ["script", "flow", "app"].contains(s))
|
||||
.filter(|s| ALL_KINDS.contains(s))
|
||||
.collect(),
|
||||
};
|
||||
// Apps carry no `archived` column and are never listed as archived.
|
||||
// Apps and agents carry no `archived` column and are never listed as archived.
|
||||
if show_archived {
|
||||
kinds.retain(|k| *k != "app");
|
||||
kinds.retain(|k| *k != "app" && *k != "agent");
|
||||
}
|
||||
|
||||
let branches = branch_sqls();
|
||||
@@ -533,6 +601,11 @@ async fn list_runnables(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let agent_scope = if kinds.contains(&"agent") {
|
||||
scope_path_predicate(&authed, "resources", "o", 3, &mut binds)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Per-kind archived predicate (scripts/flows have the column; apps don't and
|
||||
// are excluded from the archived view).
|
||||
@@ -542,7 +615,7 @@ async fn list_runnables(
|
||||
"o.archived = false"
|
||||
};
|
||||
let mut script_extras: Vec<String> = vec![];
|
||||
if !q.include_without_main.unwrap_or(false) || authed.is_operator {
|
||||
if !q.include_without_main.unwrap_or(false) || !caps.code {
|
||||
script_extras.push("(o.auto_kind IS NULL OR o.auto_kind <> 'lib')".to_string());
|
||||
}
|
||||
script_extras.push(archived_pred.to_string());
|
||||
@@ -570,18 +643,17 @@ async fn list_runnables(
|
||||
if let Some(s) = &app_scope {
|
||||
app_extras.push(s.clone());
|
||||
}
|
||||
let mut agent_extras: Vec<String> = vec![];
|
||||
if let Some(s) = &agent_scope {
|
||||
agent_extras.push(s.clone());
|
||||
}
|
||||
|
||||
// Draft-only rows are the caller's own work in progress: never archived, so they
|
||||
// have no place in the archived view, and carrying no labels of their own they are
|
||||
// out of scope of a label filter (as in the per-kind endpoints). Operators don't
|
||||
// see other people's drafts and have none of their own, except of what they may build.
|
||||
let builder = if authed.is_operator {
|
||||
operator_builder_rights(&db, &w_id).await?
|
||||
} else {
|
||||
OperatorBuilderRights::default()
|
||||
};
|
||||
let include_drafts = q.include_draft_only.unwrap_or(false)
|
||||
&& (!authed.is_operator || builder.any())
|
||||
&& ALL_KINDS.iter().any(|k| role_drafts_kind(&caps, k))
|
||||
&& !show_archived
|
||||
&& q.label.as_ref().filter(|s| !s.is_empty()).is_none();
|
||||
let draft_extras_for = |kind: &str| -> Vec<String> {
|
||||
@@ -589,11 +661,12 @@ async fn list_runnables(
|
||||
let scope = match kind {
|
||||
"script" => &script_scope,
|
||||
"flow" => &flow_scope,
|
||||
"agent" => &agent_scope,
|
||||
_ => &app_scope,
|
||||
};
|
||||
// The lib filter reads the same projected `auto_kind`, so a draft-only library
|
||||
// script hides with the deployed ones.
|
||||
if kind == "script" && (!q.include_without_main.unwrap_or(false) || authed.is_operator) {
|
||||
if kind == "script" && (!q.include_without_main.unwrap_or(false) || !caps.code) {
|
||||
extras.push("(o.auto_kind IS NULL OR o.auto_kind <> 'lib')".to_string());
|
||||
}
|
||||
if let Some(s) = scope {
|
||||
@@ -649,6 +722,7 @@ async fn list_runnables(
|
||||
"script" => (&branches.script, &script_extras),
|
||||
"flow" => (&branches.flow, &flow_extras),
|
||||
"app" => (&branches.app, &app_extras),
|
||||
"agent" => (&branches.agent, &agent_extras),
|
||||
_ => return None,
|
||||
};
|
||||
Some(build_branch(
|
||||
@@ -667,10 +741,7 @@ async fn list_runnables(
|
||||
keyset: Option<&str>,
|
||||
limit: Option<usize>|
|
||||
-> Option<String> {
|
||||
if !include_drafts
|
||||
|| !kinds.contains(&kind)
|
||||
|| (authed.is_operator && !operator_drafts_kind(builder, kind))
|
||||
{
|
||||
if !include_drafts || !kinds.contains(&kind) || !role_drafts_kind(&caps, kind) {
|
||||
return None;
|
||||
}
|
||||
// `fav` is ignored: with no favorite join there is nothing to filter on, and the
|
||||
@@ -708,7 +779,7 @@ async fn list_runnables(
|
||||
// No draft branch here: a draft-only path has no favorite row (the UI won't let
|
||||
// you star one), so it would scan the caller's whole draft slice per kind to
|
||||
// return nothing. The main stream below takes them unfiltered instead.
|
||||
let starred_branches: Vec<String> = ["script", "flow", "app"]
|
||||
let starred_branches: Vec<String> = ALL_KINDS
|
||||
.iter()
|
||||
.filter_map(|k| branch_for(k, Some(true), None, None))
|
||||
.collect();
|
||||
@@ -728,10 +799,10 @@ async fn list_runnables(
|
||||
// Main paged stream: non-starred rows (starred were pinned on the first page above).
|
||||
let main_fav = Some(false);
|
||||
let ns_branches: Vec<String> =
|
||||
["script", "flow", "app"]
|
||||
ALL_KINDS
|
||||
.iter()
|
||||
.filter_map(|k| branch_for(k, main_fav, keyset_sql.as_deref(), Some(per_page)))
|
||||
.chain(["script", "flow", "app"].iter().filter_map(|k| {
|
||||
.chain(ALL_KINDS.iter().filter_map(|k| {
|
||||
draft_branch_for(k, main_fav, keyset_sql.as_deref(), Some(per_page))
|
||||
}))
|
||||
.collect();
|
||||
@@ -762,7 +833,7 @@ async fn list_runnables(
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CountRunnablesQuery {
|
||||
/// Comma-separated subset of `script,flow,app`; omitted means all.
|
||||
/// Comma-separated subset of `script,flow,app,agent`; omitted means all.
|
||||
kinds: Option<String>,
|
||||
/// Include library scripts (no runnable main). Ignored for flows/apps.
|
||||
include_without_main: Option<bool>,
|
||||
@@ -789,7 +860,7 @@ struct OwnerCount {
|
||||
/// `owner/<rest>` is >= `owner || '/'` and, since '/' (0x2F) is immediately
|
||||
/// followed by '0' (0x30), < `owner || '0'`. `~>=~` / `~<~` are the
|
||||
/// text_pattern_ops operators, which is what lets `idx_<kind>_owner_prefix`
|
||||
/// answer the count with an index-only scan — the default opclass sorts by the
|
||||
/// (`idx_resource_agent_owner_prefix` for agents) answer the count with an index-only scan — the default opclass sorts by the
|
||||
/// database collation and cannot serve a byte-prefix range.
|
||||
fn owner_prefix_range(alias: &str) -> String {
|
||||
format!("{alias}.path ~>=~ (o.owner || '/') AND {alias}.path ~<~ (o.owner || '0')")
|
||||
@@ -817,14 +888,15 @@ async fn count_runnables_by_owner(
|
||||
Path(w_id): Path<String>,
|
||||
Query(q): Query<CountRunnablesQuery>,
|
||||
) -> JsonResult<RunnableCountsResponse> {
|
||||
let caps = caller_capabilities(&db, &w_id, &authed).await?;
|
||||
let kinds: Vec<&str> = match q.kinds.as_deref() {
|
||||
None | Some("") => vec!["script", "flow", "app"],
|
||||
None | Some("") => ALL_KINDS.to_vec(),
|
||||
// Deduplicated: every kind becomes its own count subquery, so a repeated
|
||||
// entry would both double that kind's count and multiply the scans.
|
||||
Some(csv) => {
|
||||
let mut ks: Vec<&str> = vec![];
|
||||
for k in csv.split(',').map(|s| s.trim()) {
|
||||
if ["script", "flow", "app"].contains(&k) && !ks.contains(&k) {
|
||||
if ALL_KINDS.contains(&k) && !ks.contains(&k) {
|
||||
ks.push(k);
|
||||
}
|
||||
}
|
||||
@@ -834,7 +906,7 @@ async fn count_runnables_by_owner(
|
||||
if kinds.is_empty() {
|
||||
return Ok(Json(RunnableCountsResponse { counts: HashMap::new() }));
|
||||
}
|
||||
let with_libs = q.include_without_main.unwrap_or(false) && !authed.is_operator;
|
||||
let with_libs = q.include_without_main.unwrap_or(false) && caps.code;
|
||||
|
||||
// Per-kind predicates shared by both passes. `alias` is the branch's table
|
||||
// alias; `binds` collects the scope values, numbered from `base + 1`.
|
||||
@@ -866,6 +938,13 @@ async fn count_runnables_by_owner(
|
||||
w.push(s);
|
||||
}
|
||||
}
|
||||
"agent" => {
|
||||
w.push(format!("{alias}.resource_type = 'ai_agent'"));
|
||||
if let Some(s) = scope_path_predicate(&authed, "resources", alias, base, binds)
|
||||
{
|
||||
w.push(s);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if let Some(s) = scope_path_predicate(&authed, "apps", alias, base, binds) {
|
||||
w.push(s);
|
||||
@@ -877,6 +956,7 @@ async fn count_runnables_by_owner(
|
||||
let table_of = |kind: &str| match kind {
|
||||
"script" => "script",
|
||||
"flow" => "flow",
|
||||
"agent" => "resource",
|
||||
_ => "app",
|
||||
};
|
||||
|
||||
@@ -920,7 +1000,17 @@ async fn count_runnables_by_owner(
|
||||
for r in query.fetch_all(&db).await? {
|
||||
counts.insert(r.owner, r.count);
|
||||
}
|
||||
add_draft_counts(&authed, &db, &w_id, &kinds, with_libs, &q, &mut counts).await?;
|
||||
add_draft_counts(
|
||||
&caps,
|
||||
&db,
|
||||
&w_id,
|
||||
&authed,
|
||||
&kinds,
|
||||
with_libs,
|
||||
&q,
|
||||
&mut counts,
|
||||
)
|
||||
.await?;
|
||||
counts.retain(|_, c| *c > 0);
|
||||
return Ok(Json(RunnableCountsResponse { counts }));
|
||||
}
|
||||
@@ -993,17 +1083,27 @@ async fn count_runnables_by_owner(
|
||||
counts.entry(r.owner).or_insert(r.count);
|
||||
}
|
||||
|
||||
add_draft_counts(&authed, &db, &w_id, &kinds, with_libs, &q, &mut counts).await?;
|
||||
add_draft_counts(
|
||||
&caps,
|
||||
&db,
|
||||
&w_id,
|
||||
&authed,
|
||||
&kinds,
|
||||
with_libs,
|
||||
&q,
|
||||
&mut counts,
|
||||
)
|
||||
.await?;
|
||||
counts.retain(|_, c| *c > 0);
|
||||
Ok(Json(RunnableCountsResponse { counts }))
|
||||
}
|
||||
|
||||
/// Whether an operator with `builder` rights can have drafts of a `/list` kind.
|
||||
fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool {
|
||||
/// Whether a role can have drafts of a `/list` kind: one it may write.
|
||||
fn role_drafts_kind(caps: &RoleCapabilities, kind: &str) -> bool {
|
||||
match kind {
|
||||
"flow" => builder.flows,
|
||||
"app" => builder.apps,
|
||||
_ => false,
|
||||
"flow" => caps.flows != AuthorLevel::None,
|
||||
"app" => caps.apps != AuthorLevel::None,
|
||||
_ => caps.code,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1016,9 +1116,10 @@ fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool {
|
||||
/// Deployed and draft rows can't overlap (the anti-join is what makes a draft
|
||||
/// "draft-only"), so the two counts add.
|
||||
async fn add_draft_counts(
|
||||
authed: &ApiAuthed,
|
||||
caps: &RoleCapabilities,
|
||||
db: &DB,
|
||||
w_id: &str,
|
||||
authed: &ApiAuthed,
|
||||
kinds: &[&str],
|
||||
with_libs: bool,
|
||||
q: &CountRunnablesQuery,
|
||||
@@ -1027,37 +1128,32 @@ async fn add_draft_counts(
|
||||
if !q.include_draft_only.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
// An operator has no drafts of their own, except of what they may build.
|
||||
let builder_kinds: Vec<&str>;
|
||||
let kinds: &[&str] = if !authed.is_operator {
|
||||
kinds
|
||||
} else {
|
||||
let builder = operator_builder_rights(db, w_id).await?;
|
||||
builder_kinds = kinds
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|k| operator_drafts_kind(builder, k))
|
||||
.collect();
|
||||
if builder_kinds.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
&builder_kinds
|
||||
};
|
||||
// A role has no drafts of its own except of what it may write.
|
||||
let kinds: Vec<&str> = kinds
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|k| role_drafts_kind(caps, k))
|
||||
.collect();
|
||||
if kinds.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let kinds = kinds.as_slice();
|
||||
// $1 = workspace, $2 = the caller's email.
|
||||
let mut binds: Vec<String> = vec![];
|
||||
let branches: Vec<String> = kinds
|
||||
.iter()
|
||||
.map(|kind| {
|
||||
let (typ_pred, deployed, domain) = match *kind {
|
||||
"script" => ("d.typ = 'script'", "script", "scripts"),
|
||||
"flow" => ("d.typ = 'flow'", "flow", "flows"),
|
||||
_ => ("d.typ IN ('app', 'raw_app')", "app", "apps"),
|
||||
let (typ_pred, deployed) = draft_kind_source(kind);
|
||||
let domain = match *kind {
|
||||
"script" => "scripts",
|
||||
"flow" => "flows",
|
||||
"agent" => "resources",
|
||||
_ => "apps",
|
||||
};
|
||||
// The owner a draft counts under is where it says it will live, not the
|
||||
// `u/<caller>/draft_<uuid>` it is parked at — same path `/list` groups and
|
||||
// filters on. Scripts round-trip the typed path through the draft JSON's
|
||||
// own `path`; flows and apps write `draft_path`. See scripts.rs.
|
||||
let typed_path = if *kind == "script" { "path" } else { "draft_path" };
|
||||
// filters on.
|
||||
let typed_path = draft_typed_path(kind);
|
||||
let effective_path =
|
||||
format!("COALESCE(NULLIF(d.value->>'{typed_path}', ''), d.path) as path");
|
||||
let mut w = vec![
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -138,37 +138,70 @@ struct PrivateMetadata {
|
||||
skin: ApprovalSkin,
|
||||
}
|
||||
|
||||
// Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify
|
||||
// the Slack request signature (which also defeats replay). It is NOT the primary defense:
|
||||
// the secret is unset in the default deployment, so authorization of the sensitive actions
|
||||
// is instead anchored on a per-workspace HMAC over the callback payload itself (see
|
||||
// `verify_slack_payload`), which holds even when this check is a no-op.
|
||||
#[cfg(feature = "oauth2")]
|
||||
fn verify_slack_callback_signature(headers: &HeaderMap, body: &str) -> Result<(), Error> {
|
||||
if let Some(sv) = crate::SLACK_SIGNING_SECRET.as_ref() {
|
||||
let sig = headers
|
||||
.get("X-Slack-Signature")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("");
|
||||
let ts = headers
|
||||
.get("X-Slack-Request-Timestamp")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("");
|
||||
sv.verify(ts, body, sig)
|
||||
.map_err(|_| Error::BadRequest("Slack signature verification failed".to_string()))?;
|
||||
/// Slack's signature over a callback, checked once the workspace it belongs to is known.
|
||||
#[cfg_attr(not(feature = "oauth2"), allow(dead_code))]
|
||||
struct SlackCallbackSignature {
|
||||
sig: String,
|
||||
ts: String,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl SlackCallbackSignature {
|
||||
fn new(headers: &HeaderMap, body: &str) -> Self {
|
||||
let header = |name: &str| {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
};
|
||||
Self {
|
||||
sig: header("X-Slack-Signature"),
|
||||
ts: header("X-Slack-Request-Timestamp"),
|
||||
body: body.to_string(),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Transport-level check, which also defeats replay. It is NOT the primary defense: authorization of
|
||||
// the sensitive actions is anchored on a per-workspace HMAC over the callback payload itself (see
|
||||
// `verify_slack_payload`). Only an approval sent through the workspace's Slack connection comes from
|
||||
// an app whose secret Windmill knows; any other resource may belong to any Slack app, so its
|
||||
// callbacks are left to the HMAC.
|
||||
#[cfg(feature = "oauth2")]
|
||||
async fn verify_slack_callback_signature(
|
||||
db: &DB,
|
||||
w_id: &str,
|
||||
resource_path: &str,
|
||||
request: &SlackCallbackSignature,
|
||||
) -> Result<(), Error> {
|
||||
if resource_path != windmill_common::oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH {
|
||||
return Ok(());
|
||||
}
|
||||
crate::slack_agents::verify_approval_slack_request(
|
||||
db,
|
||||
w_id,
|
||||
&request.sig,
|
||||
&request.ts,
|
||||
&request.body,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "oauth2"))]
|
||||
fn verify_slack_callback_signature(_headers: &HeaderMap, _body: &str) -> Result<(), Error> {
|
||||
async fn verify_slack_callback_signature(
|
||||
_db: &DB,
|
||||
_w_id: &str,
|
||||
_resource_path: &str,
|
||||
_request: &SlackCallbackSignature,
|
||||
) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// HMAC keyed on the per-workspace encryption key (the same trust anchor as resume-URL
|
||||
/// signatures). Used to authenticate the `/api/slack` callback payload itself so the
|
||||
/// unauthenticated route cannot be driven into decrypting arbitrary workspace variables,
|
||||
/// regardless of whether `SLACK_SIGNING_SECRET` is configured.
|
||||
/// whether or not a Slack signing secret is set.
|
||||
type SlackPayloadHmac = Hmac<Sha256>;
|
||||
|
||||
/// Domain-separation tag prepended to every Slack-payload MAC. The workspace key is also used
|
||||
@@ -232,8 +265,7 @@ pub async fn slack_app_callback_handler(
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Result<StatusCode, Error> {
|
||||
let body_str = String::from_utf8_lossy(&body);
|
||||
verify_slack_callback_signature(&headers, &body_str)?;
|
||||
let request = SlackCallbackSignature::new(&headers, &String::from_utf8_lossy(&body));
|
||||
|
||||
let form_data: SlackFormData = serde_urlencoded::from_bytes(&body)
|
||||
.map_err(|e| Error::BadRequest(format!("invalid form data: {}", e)))?;
|
||||
@@ -244,10 +276,10 @@ pub async fn slack_app_callback_handler(
|
||||
|
||||
match payload.r#type {
|
||||
PayloadType::ViewSubmission => {
|
||||
handle_submission(authed, opt_tokened, db, &payload, "resume").await?
|
||||
handle_submission(authed, opt_tokened, db, &payload, &request, "resume").await?
|
||||
}
|
||||
PayloadType::ViewClosed => {
|
||||
handle_submission(authed, opt_tokened, db, &payload, "cancel").await?
|
||||
handle_submission(authed, opt_tokened, db, &payload, &request, "cancel").await?
|
||||
}
|
||||
_ => {
|
||||
if let Some(actions) = &payload.actions {
|
||||
@@ -274,8 +306,8 @@ pub async fn slack_app_callback_handler(
|
||||
// payload was minted by `send_slack_message` with an HMAC over
|
||||
// (w_id, job_id, path) keyed on the workspace key. Without a valid
|
||||
// signature an unauthenticated caller cannot reach the decryption
|
||||
// below for an arbitrary variable, even when SLACK_SIGNING_SECRET
|
||||
// is unset.
|
||||
// below for an arbitrary variable, even when no Slack signing
|
||||
// secret is set.
|
||||
verify_slack_payload(
|
||||
&db,
|
||||
w_id,
|
||||
@@ -283,6 +315,7 @@ pub async fn slack_app_callback_handler(
|
||||
parsed_value.signature.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
verify_slack_callback_signature(&db, w_id, path, &request).await?;
|
||||
|
||||
// Map any lookup/decryption failure to a generic error: the
|
||||
// raw error echoes the probed `path`/`w_id` back, which would be
|
||||
@@ -411,6 +444,7 @@ async fn handle_submission(
|
||||
opt_tokened: OptTokened,
|
||||
db: DB,
|
||||
payload: &Payload,
|
||||
request: &SlackCallbackSignature,
|
||||
action: &str,
|
||||
) -> Result<(), Error> {
|
||||
let view = payload
|
||||
@@ -455,6 +489,7 @@ async fn handle_submission(
|
||||
// the resume/cancel and reach the decryption below with a swapped path. Require the
|
||||
// workspace-keyed HMAC minted when the modal was built.
|
||||
verify_slack_payload(&db, w_id, &[resource_path.as_bytes()], signature.as_deref()).await?;
|
||||
verify_slack_callback_signature(&db, w_id, &resource_path, request).await?;
|
||||
|
||||
// Use the common handler to process the resume/cancel action
|
||||
handle_resume_action(
|
||||
|
||||
@@ -105,6 +105,12 @@ fn build_standard_scope_domains() -> Vec<ScopeDomain> {
|
||||
"AI agent eval datasets and standalone runs",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"slack_agents",
|
||||
"Slack Agents",
|
||||
"AI agents answering in Slack channels",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"agent_workers",
|
||||
"Agent Workers",
|
||||
|
||||
@@ -34,10 +34,7 @@ use windmill_store::resources::try_get_resource_from_db_as;
|
||||
use windmill_trigger::TriggerMode;
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
use {
|
||||
crate::job_helpers_oss::get_workspace_s3_resource,
|
||||
windmill_object_store::build_object_store_client,
|
||||
};
|
||||
use crate::job_helpers_oss::get_workspace_s3_resource;
|
||||
|
||||
/// Which router a request's CORS decision must be looked up in.
|
||||
///
|
||||
@@ -624,7 +621,13 @@ async fn route_job(
|
||||
let s3_resource = s3_resource_opt.ok_or_else(|| {
|
||||
windmill_object_store::workspace_storage_misconfigured(config.storage.as_deref())
|
||||
})?;
|
||||
let s3_client = build_object_store_client(&s3_resource).await?;
|
||||
let s3_client = windmill_object_store::fork_store::workspace_object_store(
|
||||
&db,
|
||||
&trigger.workspace_id,
|
||||
config.storage.as_deref(),
|
||||
&s3_resource,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let path = if trigger.is_static_website {
|
||||
let subpath = params
|
||||
|
||||
@@ -194,6 +194,41 @@ async fn list_guests(
|
||||
Ok(Json(GuestList { usage, guests }))
|
||||
}
|
||||
|
||||
/// Shared while issuing an MCP OAuth grant, exclusive while revoking: no grant outlives a revoke.
|
||||
pub(crate) async fn lock_mcp_oauth_grants(
|
||||
conn: &mut sqlx::PgConnection,
|
||||
email: &str,
|
||||
exclusive: bool,
|
||||
) -> std::result::Result<(), sqlx::Error> {
|
||||
let q = if exclusive {
|
||||
"SELECT pg_advisory_xact_lock(hashtext('mcp_oauth_grant:' || $1))"
|
||||
} else {
|
||||
"SELECT pg_advisory_xact_lock_shared(hashtext('mcp_oauth_grant:' || $1))"
|
||||
};
|
||||
sqlx::query(q).bind(email).execute(conn).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn revoke_mcp_oauth_grants(tx: &mut sqlx::PgConnection, email: &str) -> Result<()> {
|
||||
lock_mcp_oauth_grants(&mut *tx, email, true).await?;
|
||||
// The token trigger only notifies for sessions, so evict deleted MCP access tokens explicitly.
|
||||
sqlx::query!(
|
||||
"WITH revoked AS (
|
||||
DELETE FROM token WHERE email = $1 AND label LIKE 'mcp-oauth-%' RETURNING token_prefix
|
||||
), refresh AS (
|
||||
DELETE FROM mcp_oauth_refresh_token WHERE user_email = $1
|
||||
), code AS (
|
||||
DELETE FROM mcp_oauth_server_code WHERE user_email = $1
|
||||
)
|
||||
INSERT INTO notify_event (channel, payload)
|
||||
SELECT 'notify_token_invalidation', token_prefix FROM revoked",
|
||||
email
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set_password(
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(argon2): Extension<Arc<Argon2<'_>>>,
|
||||
@@ -215,6 +250,9 @@ async fn set_password(
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let mut tx = db.begin().await?;
|
||||
revoke_mcp_oauth_grants(&mut *tx, &email).await?;
|
||||
tx.commit().await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
@@ -235,6 +273,9 @@ async fn set_password_of_user(
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let mut tx = db.begin().await?;
|
||||
revoke_mcp_oauth_grants(&mut *tx, &email).await?;
|
||||
tx.commit().await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
@@ -297,8 +338,21 @@ async fn rename_user(
|
||||
// keeps a like-named member of some other workspace out of it.
|
||||
if let Some(old_username) = old_instance_username.filter(|u| *u != ru.new_username) {
|
||||
let old_principal = windmill_common::users::username_to_permissioned_as(&old_username);
|
||||
let new_principal =
|
||||
windmill_common::users::username_to_permissioned_as(&ru.new_username);
|
||||
let new_principal = windmill_common::users::username_to_permissioned_as(&ru.new_username);
|
||||
// An agent records no address, so a workspace where the name is a member's own is
|
||||
// left to the per-workspace sweep: there the principal is that member.
|
||||
sqlx::query(
|
||||
"UPDATE resource SET value = jsonb_set(value, '{on_behalf_of}', to_jsonb($1::text))
|
||||
WHERE resource_type = 'ai_agent' AND value->>'on_behalf_of' = $2
|
||||
AND NOT EXISTS (SELECT 1 FROM usr
|
||||
WHERE usr.workspace_id = resource.workspace_id
|
||||
AND usr.username = $3)",
|
||||
)
|
||||
.bind(&new_principal)
|
||||
.bind(&old_principal)
|
||||
.bind(&old_username)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text))
|
||||
WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3",
|
||||
@@ -617,6 +671,15 @@ async fn update_username_in_workpsace<'c>(
|
||||
).execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
r#"UPDATE native_trigger SET connection_path = REGEXP_REPLACE(connection_path, 'u/' || $2 || '/(.*)', 'u/' || $1 || '/\1') WHERE connection_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
|
||||
new_username,
|
||||
old_username,
|
||||
w_id
|
||||
)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
r#"UPDATE workspace_integrations SET resource_path = REGEXP_REPLACE(resource_path, 'u/' || $2 || '/(.*)', 'u/' || $1 || '/\1') WHERE resource_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
|
||||
new_username,
|
||||
@@ -817,6 +880,17 @@ async fn update_username_in_workpsace<'c>(
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE resource SET value = jsonb_set(value, '{on_behalf_of}', to_jsonb('u/' || $1))
|
||||
WHERE resource_type = 'ai_agent' AND value->>'on_behalf_of' = ('u/' || $2)
|
||||
AND workspace_id = $3",
|
||||
)
|
||||
.bind(new_username)
|
||||
.bind(old_username)
|
||||
.bind(w_id)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
// An app draft carries a copy of the deployed policy, so the rename must reach it
|
||||
// there too — same reason as the script/flow draft sweep above.
|
||||
sqlx::query!(
|
||||
@@ -1063,6 +1137,7 @@ async fn reset_password(
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
revoke_mcp_oauth_grants(&mut *tx, &email).await?;
|
||||
|
||||
// Audit log
|
||||
let audit_author = AuditAuthor {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -61,6 +61,11 @@ pub fn workspaced_service() -> Router {
|
||||
)
|
||||
.route("/tarball", get(crate::workspaces_export::tarball_workspace))
|
||||
.route("/edit_copilot_config", post(edit_copilot_config))
|
||||
.route(
|
||||
"/settings_document",
|
||||
get(crate::workspace_settings_document::get_settings_document)
|
||||
.post(crate::workspace_settings_document::apply_settings_document),
|
||||
)
|
||||
.route("/get_copilot_info", get(get_copilot_info))
|
||||
.route("/critical_alerts", get(get_critical_alerts))
|
||||
.route(
|
||||
@@ -85,14 +90,13 @@ pub fn workspaced_service() -> Router {
|
||||
router
|
||||
}
|
||||
|
||||
async fn edit_copilot_config(
|
||||
pub(crate) async fn edit_copilot_config(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
ApiAuthed { is_admin, username, .. }: ApiAuthed,
|
||||
Json(ai_config): Json<AIConfig>,
|
||||
) -> JsonResult<EditCopilotConfigResponse> {
|
||||
require_admin(is_admin, &username)?;
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
|
||||
if let Some(ref custom_prompts) = ai_config.custom_prompts {
|
||||
for (mode, prompt) in custom_prompts.iter() {
|
||||
@@ -190,7 +194,7 @@ async fn edit_copilot_config(
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EditCopilotConfigResponse {
|
||||
pub(crate) struct EditCopilotConfigResponse {
|
||||
effective_ai_config: AIConfig,
|
||||
has_instance_ai_config: bool,
|
||||
uses_instance_ai_config: bool,
|
||||
@@ -321,7 +325,7 @@ pub struct MuteCriticalAlertRequest {
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn mute_critical_alerts(
|
||||
pub(crate) async fn mute_critical_alerts(
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
authed: ApiAuthed,
|
||||
|
||||
@@ -359,6 +359,9 @@ pub(crate) struct ArchiveQueryParams {
|
||||
include_users: Option<bool>,
|
||||
include_groups: Option<bool>,
|
||||
include_settings: Option<bool>,
|
||||
/// Opt-in, with `settings_version=v2`: state the data tables' roles in settings.json. Their
|
||||
/// tenants name this workspace's users and groups, and their roles this instance's catalog.
|
||||
include_datatable_permissions: Option<bool>,
|
||||
include_key: Option<bool>,
|
||||
include_workspace_dependencies: Option<bool>,
|
||||
/// Default `false`: data table migrations ship in the tarball unless the repo
|
||||
@@ -513,49 +516,6 @@ struct SimplifiedGroup {
|
||||
admins: Vec<String>,
|
||||
}
|
||||
|
||||
// V2 format: New grouped format
|
||||
#[derive(Serialize)]
|
||||
struct SimplifiedSettings {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
auto_invite: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
webhook: Option<String>,
|
||||
// Always serialize (including as `null`) so that `wmill sync pull` emits
|
||||
// these fields in settings.yaml unconditionally. Makes round-trip
|
||||
// bijective: YAML is the source of truth, absence/null = "clear remote",
|
||||
// mirroring every other workspace setting.
|
||||
error_handler: Option<Value>,
|
||||
success_handler: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
ai_config: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
large_file_storage: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
git_sync: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
default_app: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
default_scripts: Option<Value>,
|
||||
name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
mute_critical_alerts: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
color: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
operator_settings: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
datatable: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
slack_team_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
slack_name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
slack_command_script: Option<String>,
|
||||
// Always serialize (see note above on error_handler / success_handler).
|
||||
slack_oauth_client_id: Option<String>,
|
||||
slack_oauth_client_secret: Option<String>,
|
||||
}
|
||||
|
||||
// V1 format: Legacy flat format for backward compatibility (matches main branch exactly)
|
||||
#[derive(Serialize)]
|
||||
struct SimplifiedSettingsLegacy {
|
||||
@@ -602,7 +562,6 @@ struct SettingsRow {
|
||||
auto_invite: Option<Value>,
|
||||
webhook: Option<String>,
|
||||
error_handler: Option<Value>,
|
||||
success_handler: Option<Value>,
|
||||
ai_config: Option<serde_json::Value>,
|
||||
large_file_storage: Option<Value>,
|
||||
git_sync: Option<Value>,
|
||||
@@ -616,8 +575,6 @@ struct SettingsRow {
|
||||
slack_team_id: Option<String>,
|
||||
slack_name: Option<String>,
|
||||
slack_command_script: Option<String>,
|
||||
slack_oauth_client_id: Option<String>,
|
||||
slack_oauth_client_secret: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) async fn tarball_workspace(
|
||||
@@ -638,6 +595,7 @@ pub(crate) async fn tarball_workspace(
|
||||
include_users,
|
||||
include_groups,
|
||||
include_settings,
|
||||
include_datatable_permissions,
|
||||
include_key,
|
||||
include_workspace_dependencies,
|
||||
skip_datatable_migrations,
|
||||
@@ -936,7 +894,14 @@ pub(crate) async fn tarball_workspace(
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
for resource in resources {
|
||||
for mut resource in resources {
|
||||
// An agent's identity belongs to this workspace, and whoever writes it elsewhere gets
|
||||
// their own, as `to_string_without_metadata` leaves out a flow's.
|
||||
if resource.resource_type == "ai_agent" {
|
||||
if let Some(obj) = resource.value.as_mut().and_then(|v| v.as_object_mut()) {
|
||||
obj.remove("on_behalf_of");
|
||||
}
|
||||
}
|
||||
let resource_str =
|
||||
&to_string_without_metadata(&resource, ExtraPermsBehavior::Drop, None).unwrap();
|
||||
archive
|
||||
@@ -1003,11 +968,13 @@ pub(crate) async fn tarball_workspace(
|
||||
}
|
||||
|
||||
if !skip_variables.unwrap_or(false) {
|
||||
// `account` is left out: the variable endpoints ignore it, so a pushed spec carrying it
|
||||
// would never match the variable it created and every sync would report it changed.
|
||||
let variables =
|
||||
sqlx::query_as::<_, ExportableListableVariable>(if !skip_secrets.unwrap_or(false) {
|
||||
"SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND expires_at IS NULL"
|
||||
"SELECT workspace_id, path, value, is_secret, description, extra_perms, NULL::int AS account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND expires_at IS NULL"
|
||||
} else {
|
||||
"SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL"
|
||||
"SELECT workspace_id, path, value, is_secret, description, extra_perms, NULL::int AS account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL"
|
||||
})
|
||||
.bind(&w_id)
|
||||
.fetch_all(&mut *tx)
|
||||
@@ -1593,7 +1560,6 @@ pub(crate) async fn tarball_workspace(
|
||||
auto_invite,
|
||||
webhook,
|
||||
error_handler,
|
||||
success_handler,
|
||||
ai_config,
|
||||
large_file_storage,
|
||||
git_sync,
|
||||
@@ -1606,9 +1572,7 @@ pub(crate) async fn tarball_workspace(
|
||||
datatable,
|
||||
slack_team_id,
|
||||
slack_name,
|
||||
slack_command_script,
|
||||
slack_oauth_client_id,
|
||||
slack_oauth_client_secret
|
||||
slack_command_script
|
||||
FROM workspace_settings
|
||||
LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id
|
||||
WHERE workspace_id = $1"#,
|
||||
@@ -1617,73 +1581,23 @@ pub(crate) async fn tarball_workspace(
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
// Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility
|
||||
// Server-owned state (the HMAC webhook secret + hook id/error, the
|
||||
// synced-sha / last-pull status, the admin automatic pulls run as, and what
|
||||
// the credential check observed) must never leave the server: keep it out of
|
||||
// export archives and synced repos, and don't let a re-imported workspace
|
||||
// inherit another install's hook/sync state or pull identity.
|
||||
fn redact_git_sync_for_export(git_sync: Option<Value>) -> Option<Value> {
|
||||
let mut git_sync = git_sync?;
|
||||
if let Some(repos) = git_sync
|
||||
.get_mut("repositories")
|
||||
.and_then(|r| r.as_array_mut())
|
||||
{
|
||||
for repo in repos {
|
||||
if let Some(auto_pull) =
|
||||
repo.get_mut("auto_pull").and_then(|a| a.as_object_mut())
|
||||
{
|
||||
for field in [
|
||||
"webhook_secret",
|
||||
"webhook_id",
|
||||
"webhook_error",
|
||||
"last_synced_sha",
|
||||
"last_pull_status",
|
||||
"enabled_by",
|
||||
] {
|
||||
auto_pull.remove(field);
|
||||
}
|
||||
}
|
||||
// What this install observed about its own credential: a token
|
||||
// id and expiry, and a `checked_at` that moves on its own.
|
||||
// None of it describes the workspace, and in a git-synced
|
||||
// `wmill.yaml` it would churn the file for no reason.
|
||||
if let Some(repo) = repo.as_object_mut() {
|
||||
repo.remove("credential");
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(git_sync)
|
||||
}
|
||||
|
||||
let settings_str = if settings_version.as_deref() == Some("v2") {
|
||||
let settings = SimplifiedSettings {
|
||||
auto_invite: row.auto_invite,
|
||||
webhook: row.webhook,
|
||||
error_handler: row.error_handler,
|
||||
success_handler: row.success_handler,
|
||||
ai_config: row.ai_config,
|
||||
large_file_storage: row.large_file_storage,
|
||||
git_sync: redact_git_sync_for_export(row.git_sync),
|
||||
default_app: row.default_app,
|
||||
default_scripts: row.default_scripts,
|
||||
name: row.name.clone().unwrap_or_default(),
|
||||
mute_critical_alerts: row.mute_critical_alerts,
|
||||
color: row.color.clone(),
|
||||
operator_settings: row.operator_settings.clone(),
|
||||
datatable: windmill_common::workspaces::strip_datatable_permissions(
|
||||
row.datatable.clone(),
|
||||
),
|
||||
slack_team_id: row.slack_team_id.clone(),
|
||||
slack_name: row.slack_name.clone(),
|
||||
slack_command_script: row.slack_command_script.clone(),
|
||||
slack_oauth_client_id: row.slack_oauth_client_id.clone(),
|
||||
slack_oauth_client_secret: row.slack_oauth_client_secret.clone(),
|
||||
let secrets = if export_plain_secrets {
|
||||
crate::workspace_settings_document::SecretMode::Plain
|
||||
} else {
|
||||
crate::workspace_settings_document::SecretMode::Encrypted
|
||||
};
|
||||
serde_json::to_value(settings)
|
||||
.map(|v| serde_json::to_string_pretty(&v).ok())
|
||||
.ok()
|
||||
.flatten()
|
||||
let mut doc = crate::workspace_settings_document::load_settings_document(
|
||||
&mut *tx, &db, &w_id, secrets,
|
||||
)
|
||||
.await?;
|
||||
// A repo syncs its rules through protection-rules.yaml, so settings.yaml leaving
|
||||
// them out keeps one writer for them.
|
||||
doc.remove("protection_rules");
|
||||
if !include_datatable_permissions.unwrap_or(false) {
|
||||
doc.remove("datatable_permissions");
|
||||
}
|
||||
serde_json::to_string_pretty(&doc).ok()
|
||||
} else {
|
||||
// V1 (legacy) format: convert JSONB to flat fields (matches main branch exactly)
|
||||
let (auto_invite_enabled, auto_invite_as, auto_invite_mode) =
|
||||
@@ -1730,7 +1644,7 @@ pub(crate) async fn tarball_workspace(
|
||||
error_handler_muted_on_cancel,
|
||||
ai_config: row.ai_config,
|
||||
large_file_storage: row.large_file_storage,
|
||||
git_sync: redact_git_sync_for_export(row.git_sync),
|
||||
git_sync: crate::workspace_settings_document::redact_git_sync(row.git_sync),
|
||||
default_app: row.default_app,
|
||||
default_scripts: row.default_scripts,
|
||||
name: row.name.unwrap_or_default(),
|
||||
|
||||
Reference in New Issue
Block a user