mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
fix: parse flow step, ai tool and wac step paths on the worker (#12264)
This commit is contained in:
+3
-1
@@ -1760,11 +1760,13 @@ async fn process_notify_event(
|
||||
windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE
|
||||
.remove(&format!("{workspace_id}:{path}"));
|
||||
if *kind == "preprocessor" {
|
||||
// A stored flow can spell the path with the leading `/` the worker
|
||||
// tolerates, and must not keep the old format when it runs.
|
||||
match sqlx::query_scalar::<_, i64>(
|
||||
"SELECT fv.id
|
||||
FROM flow f
|
||||
INNER JOIN flow_version fv ON fv.id = f.versions[array_upper(f.versions, 1)]
|
||||
WHERE fv.value->'preprocessor_module'->'value'->>'path' = $1 AND f.workspace_id = $2",
|
||||
WHERE fv.value->'preprocessor_module'->'value'->>'path' IN ($1, '/' || $1) AND f.workspace_id = $2",
|
||||
)
|
||||
.bind(*path)
|
||||
.bind(*workspace_id)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
use serde_json::json;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use windmill_common::{flows::FlowValue, jobs::JobPayload};
|
||||
use windmill_test_utils::{ApiServer, RunJob};
|
||||
|
||||
async fn run_step(
|
||||
db: &Pool<Postgres>,
|
||||
port: u16,
|
||||
kind: &str,
|
||||
path: &str,
|
||||
) -> anyhow::Result<(bool, String, Vec<String>)> {
|
||||
run_step_value(
|
||||
db,
|
||||
port,
|
||||
json!({"type": kind, "path": path, "input_transforms": {}}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn run_step_value(
|
||||
db: &Pool<Postgres>,
|
||||
port: u16,
|
||||
step: serde_json::Value,
|
||||
) -> anyhow::Result<(bool, String, Vec<String>)> {
|
||||
let value: FlowValue =
|
||||
serde_json::from_value(json!({"modules": [{"id": "a", "value": step}]}))?;
|
||||
let job = RunJob::from(JobPayload::RawFlow { value, path: None, restarted_from: None })
|
||||
.run_until_complete(db, false, port)
|
||||
.await;
|
||||
let children: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT runnable_path FROM v2_job WHERE parent_job = $1 AND kind IN ('script', 'flow')",
|
||||
)
|
||||
.bind(job.id)
|
||||
.fetch_all(db)
|
||||
.await?;
|
||||
Ok((
|
||||
job.success,
|
||||
job.json_result().unwrap().to_string(),
|
||||
children,
|
||||
))
|
||||
}
|
||||
|
||||
/// A stored step path reaches the worker unvalidated from a raw preview or an older flow, and must
|
||||
/// mean what the run routes make of it.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn flow_steps_resolve_their_parsed_path(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock)
|
||||
VALUES ('test-workspace', 'test-user', 'echo ran', '{}', '', '', 'f/system/step', 2762, 'bash', '')",
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let sub_flow = json!({"modules": [{"id": "s", "value": {"type": "identity"}}]}).to_string();
|
||||
sqlx::query(
|
||||
"INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by)
|
||||
VALUES ('test-workspace', '', '', 'f/system/sub', '{}', '{}', $1::text::jsonb, 'test-user')",
|
||||
)
|
||||
.bind(&sub_flow)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"WITH v AS (
|
||||
INSERT INTO flow_version (workspace_id, path, value, schema, created_by)
|
||||
VALUES ('test-workspace', 'f/system/sub', $1::text::jsonb, '{}', 'test-user') RETURNING id
|
||||
)
|
||||
UPDATE flow SET versions = ARRAY[(SELECT id FROM v)]
|
||||
WHERE workspace_id = 'test-workspace' AND path = 'f/system/sub'",
|
||||
)
|
||||
.bind(&sub_flow)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
for (kind, path, canonical) in [
|
||||
("script", "/f/system/step", "f/system/step"),
|
||||
("flow", "/f/system/sub", "f/system/sub"),
|
||||
] {
|
||||
let (success, result, children) = run_step(&db, port, kind, path).await?;
|
||||
assert!(success, "{path}: {result}");
|
||||
assert_eq!(children, [canonical], "{path}");
|
||||
|
||||
let twice = format!("/{path}");
|
||||
let (success, result, _) = run_step(&db, port, kind, &twice).await?;
|
||||
assert!(!success, "{twice}: {result}");
|
||||
assert!(
|
||||
result.contains(&format!("invalid {kind} path")),
|
||||
"{twice}: {result}"
|
||||
);
|
||||
}
|
||||
|
||||
// A pinned step runs its hash whatever the path beside it says.
|
||||
for path in ["", "//f/system/step", "hub/abc/x"] {
|
||||
let step = json!({
|
||||
"type": "script", "path": path, "hash": format!("{:016x}", 2762), "input_transforms": {}
|
||||
});
|
||||
let (success, result, children) = run_step_value(&db, port, step).await?;
|
||||
assert!(success, "{path}: {result}");
|
||||
assert_eq!(children, [path], "{path}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -7928,8 +7928,9 @@ pub async fn run_workflow_as_code(
|
||||
JobKind::Script => {
|
||||
let userdb_authed =
|
||||
UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() };
|
||||
let script = RunnableRef::from_path(job.script_path(), false)?;
|
||||
script_path_to_payload(
|
||||
job.script_path(),
|
||||
script.path(),
|
||||
Some(userdb_authed),
|
||||
db.clone(),
|
||||
&w_id,
|
||||
|
||||
@@ -243,10 +243,11 @@ fn validate_retry(retry: &Retry, module_id: &str) -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
/// Script/sub-flow step references must be workspace paths (`u/`, `f/`, `g/`) or a hub
|
||||
/// reference (`hub/`). Empty is tolerated for intermediate/incomplete steps. This blocks
|
||||
/// reference (`hub/`). Empty is tolerated for intermediate/incomplete steps. This keeps
|
||||
/// absolute or local filesystem paths (e.g. `/tmp/.../ops/scripts/...` baked in by a
|
||||
/// `wmill sync push` from a feature-branch checkout) from being persisted into a flow,
|
||||
/// where they silently mis-resolve to an unrelated script at runtime (#9751).
|
||||
/// `wmill sync push` from a feature-branch checkout) out of stored flows (#9751). Refuses a
|
||||
/// leading `/`, which `RunnableRef` tolerates, so stored paths keep the canonical spelling; it
|
||||
/// is not a substitute for the parse, which refuses some paths this accepts.
|
||||
fn is_workspace_runnable_path(path: &str) -> bool {
|
||||
path.is_empty()
|
||||
|| path.starts_with("u/")
|
||||
|
||||
@@ -66,6 +66,7 @@ use windmill_common::{
|
||||
flows::{AgentTool, FlowModule, FlowModuleValue, InputTransform, ToolValue},
|
||||
get_latest_hash_for_path,
|
||||
jobs::JobKind,
|
||||
runnable_ref::RunnableRef,
|
||||
scripts::get_full_hub_script_by_path,
|
||||
utils::{StripPath, HTTP_CLIENT},
|
||||
worker::{to_raw_value, Connection},
|
||||
@@ -1062,9 +1063,10 @@ pub async fn handle_ai_agent_job(
|
||||
)
|
||||
}
|
||||
None => {
|
||||
if path.starts_with("hub/") {
|
||||
let tool = RunnableRef::from_path(path, false)?;
|
||||
if tool.path().starts_with("hub/") {
|
||||
let hub_script = get_full_hub_script_by_path(
|
||||
StripPath(path.to_string()),
|
||||
StripPath(tool.path().to_string()),
|
||||
&HTTP_CLIENT,
|
||||
None,
|
||||
)
|
||||
@@ -1080,7 +1082,7 @@ pub async fn handle_ai_agent_job(
|
||||
let hash = get_latest_hash_for_path(
|
||||
&mut *db.acquire().await?,
|
||||
&job.workspace_id,
|
||||
path.as_str(),
|
||||
tool.path(),
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
@@ -1088,7 +1090,7 @@ pub async fn handle_ai_agent_job(
|
||||
// update module definition to use a fixed hash so all tool calls match the same schema
|
||||
t.value = to_raw_value(&FlowModuleValue::Script {
|
||||
hash: Some(hash),
|
||||
path: path.clone(),
|
||||
path: tool.path().to_string(),
|
||||
tag_override: tag_override.clone(),
|
||||
input_transforms: input_transforms.clone(),
|
||||
is_trigger: *is_trigger,
|
||||
|
||||
@@ -3517,6 +3517,7 @@ pub async fn handle_wac_v2_output(
|
||||
use serde_json::Value;
|
||||
use windmill_common::get_latest_flow_version_info_for_path;
|
||||
use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode};
|
||||
use windmill_common::runnable_ref::RunnableRef;
|
||||
use windmill_common::runnable_settings::{
|
||||
ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings,
|
||||
};
|
||||
@@ -3792,9 +3793,10 @@ pub async fn handle_wac_v2_output(
|
||||
(ChildRunnable::Module(module), None)
|
||||
}
|
||||
"script" => {
|
||||
let script = RunnableRef::from_path(&step.script, false)?;
|
||||
// Resolve script path to job payload (handles hash, lang, etc.)
|
||||
let (payload, _, _, _, _, on_behalf_of) = script_path_to_payload(
|
||||
&step.script,
|
||||
script.path(),
|
||||
None, // no authed db for background workers
|
||||
db.clone(),
|
||||
&job.workspace_id,
|
||||
@@ -3804,16 +3806,17 @@ pub async fn handle_wac_v2_output(
|
||||
(ChildRunnable::Deployed(payload), on_behalf_of)
|
||||
}
|
||||
"flow" => {
|
||||
let flow = RunnableRef::from_path(&step.script, true)?;
|
||||
let flow_info = get_latest_flow_version_info_for_path(
|
||||
None,
|
||||
db,
|
||||
&job.workspace_id,
|
||||
&step.script,
|
||||
flow.path(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
let payload = JobPayload::Flow {
|
||||
path: step.script.clone(),
|
||||
path: flow.path().to_string(),
|
||||
dedicated_worker: flow_info.dedicated_worker,
|
||||
apply_preprocessor: false,
|
||||
version: flow_info.version,
|
||||
|
||||
@@ -48,6 +48,7 @@ use windmill_common::jobs::{
|
||||
script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, TriggerKindLabel,
|
||||
ENTRYPOINT_OVERRIDE,
|
||||
};
|
||||
use windmill_common::runnable_ref::RunnableRef;
|
||||
use windmill_common::runnable_settings::{
|
||||
ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettingsTrait,
|
||||
};
|
||||
@@ -6851,11 +6852,13 @@ async fn flow_to_payload(
|
||||
w_id: &str,
|
||||
db: &DB,
|
||||
) -> Result<JobPayloadWithTag, Error> {
|
||||
let flow_info = get_latest_flow_version_info_for_path(None, &db, w_id, &path, true).await?;
|
||||
let flow = RunnableRef::from_path(&path, true)?;
|
||||
let flow_info =
|
||||
get_latest_flow_version_info_for_path(None, &db, w_id, flow.path(), true).await?;
|
||||
let on_behalf_of = flow_info.on_behalf_of(w_id, &db).await?;
|
||||
let FlowVersionInfo { version, tag, job_token_scopes, .. } = flow_info;
|
||||
let payload = JobPayload::Flow {
|
||||
path,
|
||||
path: flow.path().to_string(),
|
||||
dedicated_worker: None,
|
||||
apply_preprocessor: false,
|
||||
version,
|
||||
@@ -6917,9 +6920,10 @@ pub async fn script_to_payload(
|
||||
};
|
||||
let (payload, tag, delete_after_use, delete_after_secs, script_timeout, on_behalf_of) =
|
||||
if script_hash.is_none() {
|
||||
let script = RunnableRef::from_path(&script_path, false)?;
|
||||
let (jp, tag, delete_after_use, delete_after_secs, script_timeout, on_behalf_of) =
|
||||
script_path_to_payload(
|
||||
&script_path,
|
||||
script.path(),
|
||||
None,
|
||||
db.clone(),
|
||||
&flow_job.workspace_id,
|
||||
@@ -6959,6 +6963,8 @@ pub async fn script_to_payload(
|
||||
(
|
||||
JobPayload::ScriptHash {
|
||||
hash,
|
||||
// Not parsed: the hash alone decides what runs, and flows pinned with a path
|
||||
// the parser refuses (deploy accepts an empty one) must keep running.
|
||||
path: script_path,
|
||||
concurrency_settings,
|
||||
debouncing_settings,
|
||||
|
||||
Reference in New Issue
Block a user