From 2a780ad87af69358f241536697f694612fe92d93 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 20 May 2026 12:44:29 +0000 Subject: [PATCH] feat: resolve relative imports from local content in script/flow preview (#9233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: thread temp_script_refs into preview jobs Co-Authored-By: Claude Opus 4.7 (1M context) * feat: resolve python preview relative imports from temp script refs Co-Authored-By: Claude Opus 4.7 (1M context) * feat: use local relative imports in wmill script preview Co-Authored-By: Claude Opus 4.7 (1M context) * feat: use local relative imports in wmill flow preview Co-Authored-By: Claude Opus 4.7 (1M context) * feat: add temp_script_refs to Preview and FlowPreview openapi schemas Co-Authored-By: Claude Opus 4.7 (1M context) * fix: pass temp_script_refs to bun lockfile gen for no-lock preview Co-Authored-By: Claude Opus 4.7 (1M context) * refactor: route script preview through shared buildPreviewTempScriptRefs Co-Authored-By: Claude Opus 4.7 (1M context) * feat: resolve local relative imports in wmill app dev inline scripts Co-Authored-By: Claude Opus 4.7 (1M context) * fix: address cubic review — bundle cache key, preview-mode gate, error masking Co-Authored-By: Claude Opus 4.7 (1M context) * perf: skip dep-tree build when previewed script has no relative imports; narrow old-backend classifier Co-Authored-By: Claude Opus 4.7 (1M context) * fix: address review issues (preview-only gate, bundle preview, app dev cwd) Three P1s flagged in repeated codex/pi reviews on PR #9233: - Gate _TEMP_SCRIPT_REFS extraction on JobKind::Preview (bun + python executors) and propagation in worker_flow on JobKind::FlowPreview. job.args includes caller-controlled request args, so honoring this key on deployed runs would let a caller swap import resolution to local content uploaded via /raw_temp. - run_bundle_preview_script now injects temp_script_refs into PushArgs.extra, mirroring run_preview_script — closes the silent data drop for the bundle preview path. - wmill app dev chdirs to the wmill.yaml root before buildPreviewTempScriptRefs and restores after, so the `cd __raw_app && wmill app dev` invocation (cwd is the raw_app folder, no app_folder arg) still walks sibling workspace scripts like f/lib.ts. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ee): bump ee-repo-ref to 5b347d6 (handle_python_deps arity fix) Picks up the EE arity fix so cargo_test + check_ee_full compile cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ee): bump ee-repo-ref to 52e273d (agent-workers bundle path arity fix) Picks up windmill-ee-private 52e273d which adds the missing &None arg to compute_bundle_local_and_remote_path in windmill-api-agent-workers/src/ee.rs. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ee): bump ee-repo-ref to 2d6ffd3 (EE main merged in) Previous bump pinned an older EE commit, missing the audit-log object-store export module (EE PR #579, commit ec3cd35) and other EE main updates. The CE backend's `crate::ee_oss::anchor_audit_logs_s3_checkpoint_env_var` and `export_audit_logs_to_object_store` references need the new EE definitions. Merged origin/main into the EE branch and pinned the merge commit. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: update ee-repo-ref to b0c87b1272c25dca4aa9148c87fb024a9d9ef322 This commit updates the EE repository reference after PR #583 was merged in windmill-ee-private. Previous ee-repo-ref: 2d6ffd32c99bd93e79cf78675cb89499a81b17e1 New ee-repo-ref: b0c87b1272c25dca4aa9148c87fb024a9d9ef322 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 18 ++++ backend/windmill-api/src/apps.rs | 24 ++++- backend/windmill-api/src/jobs.rs | 37 +++++++- backend/windmill-worker/loader.py | 8 ++ backend/windmill-worker/src/bun_executor.rs | 48 +++++++++- backend/windmill-worker/src/common.rs | 2 + .../windmill-worker/src/python_executor.rs | 44 ++++++++- backend/windmill-worker/src/worker_flow.rs | 18 +++- cli/src/commands/app/dev.ts | 50 +++++++++- cli/src/commands/flow/flow.ts | 17 ++++ .../generate-metadata/generate-metadata.ts | 94 +++++++++++++++++++ cli/src/commands/script/script.ts | 29 ++++++ 13 files changed, 376 insertions(+), 15 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index cd2c083feb..889151f3df 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -abeb405de36a3fe23382f19e762268f87b6679be +b0c87b1272c25dca4aa9148c87fb024a9d9ef322 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4b3da914f7..37d19b2bd7 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10712,6 +10712,12 @@ paths: run_query_params: type: object description: Runnable query parameters + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash. Only honored for inline-script (raw_code) execution so app dev resolves those imports from not-yet-deployed local content." + additionalProperties: + type: string required: - args - component @@ -23220,6 +23226,12 @@ components: description: "Additional script modules keyed by relative file path" additionalProperties: $ref: "#/components/schemas/ScriptModule" + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash so the preview job resolves those imports from not-yet-deployed local content instead of the deployed script" + additionalProperties: + type: string required: - args @@ -26566,6 +26578,12 @@ components: type: string restarted_from: $ref: "#/components/schemas/RestartedFrom" + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash, propagated to each flow step so inline-script relative imports resolve from not-yet-deployed local content instead of the deployed script" + additionalProperties: + type: string required: - value diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 4ecbae879a..ab36b8cb89 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -2039,6 +2039,10 @@ pub struct ExecuteApp { pub force_viewer_delete_after_secs: Option, /// Runnable query parameters (e.g., memory_id for chat-enabled flows) pub run_query_params: Option, + /// Map of relative-import script path -> temp storage hash. Only honored for + /// inline-script (raw_code, preview) execution so `wmill app dev` resolves + /// those imports from not-yet-deployed local content instead of deployed. + pub temp_script_refs: Option>, } fn digest(code: &str) -> String { @@ -2113,8 +2117,16 @@ async fn execute_component( Extension(db): Extension, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Json(payload): Json, + Json(mut payload): Json, ) -> Result { + // Only honor temp_script_refs for the inline-script preview path: + // preview/editor mode (force_viewer_static_fields set, == `is_preview`), + // raw_code present, and no deployed app_script id — i.e. `wmill app dev`. + let temp_script_refs = payload.temp_script_refs.take(); + let inject_temp_refs = temp_script_refs.is_some() + && payload.force_viewer_static_fields.is_some() + && payload.raw_code.is_some() + && payload.id.is_none(); match (payload.path.is_some(), payload.raw_code.is_some()) { (false, false) => { return Err(Error::BadRequest( @@ -2343,7 +2355,7 @@ async fn execute_component( let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); - let (args, job_id) = build_args( + let (mut args, job_id) = build_args( policy, policy_triggerables, payload.args, @@ -2354,6 +2366,14 @@ async fn execute_component( ) .await?; + if inject_temp_refs { + if let Some(refs) = temp_script_refs { + args.extra + .get_or_insert_with(HashMap::new) + .insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(&refs)); + } + } + let is_flow = payload .path .as_ref() diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4d61df606e..cb8d71022f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -3633,6 +3633,10 @@ struct Preview { format: Option, flow_path: Option, modules: Option>, + /// Map of relative-import script path -> temp storage hash. When set, the + /// preview job resolves those imports from not-yet-deployed local content + /// (uploaded to raw_script_temp) instead of the deployed script. + temp_script_refs: Option>, } #[cfg(feature = "run_inline")] @@ -3661,6 +3665,10 @@ struct PreviewFlow { args: Option>>, tag: Option, restarted_from: Option, + /// Map of relative-import script path -> temp storage hash. Propagated to + /// each flow step so inline-script relative imports resolve from + /// not-yet-deployed local content instead of the deployed script. + temp_script_refs: Option>, } #[derive(Debug, Deserialize)] @@ -5662,6 +5670,12 @@ async fn run_preview_script( if let Some(ref modules) = preview.modules { extra.insert("_MODULES".to_string(), to_raw_value(modules)); } + if let Some(ref temp_script_refs) = preview.temp_script_refs { + extra.insert( + "_TEMP_SCRIPT_REFS".to_string(), + to_raw_value(temp_script_refs), + ); + } let extra = if extra.is_empty() { None } else { Some(extra) }; let push_args = PushArgs { extra, args: &preview_args }; @@ -6010,6 +6024,17 @@ async fn run_bundle_preview_script( let args = preview.args.unwrap_or_default(); + // The bundle's runtime still resolves workspace-path imports + // (`/f/...`) via loader.bun.js, so pass through temp_script_refs the + // same way `run_preview_script` does — otherwise codebase previews + // silently fall back to deployed content for those imports. + let extra = preview.temp_script_refs.as_ref().map(|refs| { + let mut m = HashMap::new(); + m.insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(refs)); + m + }); + let push_args = PushArgs { extra, args: &args }; + is_tar = match preview.kind { Some(PreviewKind::Tarbundle) => true, _ => false, @@ -6038,7 +6063,7 @@ async fn run_bundle_preview_script( modules: None, tag: None, }), - PushArgs::from(&args), + push_args, authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -6668,6 +6693,14 @@ async fn run_preview_flow_job( .and_then(|args| args.get("user_message")) .cloned(); + let mut flow_args = raw_flow.args.unwrap_or_default(); + if let Some(ref temp_script_refs) = raw_flow.temp_script_refs { + flow_args.insert( + "_TEMP_SCRIPT_REFS".to_string(), + to_raw_value(temp_script_refs), + ); + } + let (uuid, mut tx) = push( &db, tx, @@ -6677,7 +6710,7 @@ async fn run_preview_flow_job( path: raw_flow.path, restarted_from: raw_flow.restarted_from, }, - PushArgs::from(&raw_flow.args.unwrap_or_default()), + PushArgs::from(&flow_args), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index c387e5e0fc..d3dc7b8a66 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -4,6 +4,11 @@ from importlib.abc import MetaPathFinder, Loader from importlib.machinery import ModuleSpec, SourceFileLoader import time +# Injected by backend: maps script path -> temp storage hash so preview jobs +# resolve relative imports from not-yet-deployed local content. Empty ({}) for +# deployed runs. +TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER + class WindmillLoader(Loader): def __init__(self, path): self.path = path @@ -48,6 +53,9 @@ class WindmillFinder(MetaPathFinder): runnable_id = os.environ.get('WM_RUNNABLE_ID') if runnable_id: query_params += f"&cache_key={runnable_id}" + temp_hash = TEMP_SCRIPT_REFS.get(script_path) if TEMP_SCRIPT_REFS else None + if temp_hash: + query_params += f"&temp_script_hash={temp_hash}" url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 163de9f7b3..7b04d47dce 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -28,6 +28,7 @@ use crate::{ }; use windmill_common::{ client::AuthedClient, + jobs::JobKind, scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, @@ -1160,8 +1161,15 @@ pub async fn prebundle_bun_script( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, temp_script_refs: &Option>, ) -> Result<()> { - let (local_path, remote_path) = - compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; + let (local_path, remote_path) = compute_bundle_local_and_remote_path( + inner_content, + lock, + script_path, + db, + w_id, + temp_script_refs, + ) + .await; if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } @@ -1252,6 +1260,7 @@ pub async fn compute_bundle_local_and_remote_path( script_path: &str, db: Option<&DB>, w_id: &str, + temp_script_refs: &Option>, ) -> (String, String) { let mut input_src = format!("{inner_content}{lock}",); @@ -1269,6 +1278,18 @@ pub async fn compute_bundle_local_and_remote_path( } }; + // Keep temp-script-ref (preview) bundles in a distinct cache slot: their + // imports come from not-yet-deployed local content, so they must neither + // reuse a deployed-content bundle nor be saved under the deployed key. + if let Some(refs) = temp_script_refs { + let mut entries: Vec<(&String, &String)> = refs.iter().collect(); + entries.sort(); + for (path, hash) in entries { + input_src.push_str(path); + input_src.push_str(hash); + } + } + let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await; input_src.push_str(&ws_suffix); let hash = windmill_common::utils::calculate_hash(&input_src); @@ -1333,6 +1354,22 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + // Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from + // not-yet-deployed local content uploaded to raw_script_temp. Extracted up + // front so it reaches both lockfile generation and the runtime loader. + // Gated on JobKind::Preview because job.args includes caller-controlled + // request args; honoring this key on deployed runs would let a caller swap + // import resolution targets in deployed code. + let temp_script_refs: Option> = if matches!(job.kind, JobKind::Preview) + { + job.args + .as_ref() + .and_then(|x| x.get("_TEMP_SCRIPT_REFS")) + .and_then(|v| serde_json::from_str(v.get()).ok()) + } else { + None + }; + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(error::Error::ExecutionErr( "Script has //sandbox annotation but nsjail is not available on this worker. \ @@ -1353,6 +1390,7 @@ pub async fn handle_bun_job( job.runnable_path(), Some(db), &job.workspace_id, + &temp_script_refs, ) .await } @@ -1512,7 +1550,7 @@ pub async fn handle_bun_job( workspace_dependencies, annotation.npm, &mut Some(occupancy_metrics), - &None, + &temp_script_refs, wac_replay_info.is_some(), ) .await?; @@ -1886,7 +1924,7 @@ try {{ } else { LoaderMode::BunBundle }, - &None, + &temp_script_refs, ) .await?; @@ -1903,7 +1941,7 @@ try {{ } else { LoaderMode::Bun }, - &None, + &temp_script_refs, ) .await } else { diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 75cd8bf029..8a3ee06452 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -114,6 +114,7 @@ pub async fn create_args_and_out_file( if let Some(args) = job.args.as_ref() { if let Some(mut x) = transform_json(client, &job.workspace_id, &args.0, job, conn).await? { x.remove("_MODULES"); + x.remove("_TEMP_SCRIPT_REFS"); write_file( job_dir, "args.json", @@ -122,6 +123,7 @@ pub async fn create_args_and_out_file( } else { let mut filtered = args.0.clone(); filtered.remove("_MODULES"); + filtered.remove("_TEMP_SCRIPT_REFS"); write_file( job_dir, "args.json", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d3d3de62d6..7f9c94d844 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -31,6 +31,7 @@ use windmill_common::{ self, Error::{self}, }, + jobs::JobKind, scripts::ScriptLang, utils::calculate_hash, worker::{ @@ -119,6 +120,18 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); +/// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a +/// Python dict literal. Preview jobs pass a path -> temp-hash map so relative +/// imports resolve from not-yet-deployed local content; deployed runs pass +/// `None` which renders an empty dict (deployed resolution unchanged). +fn render_relative_python_loader(temp_script_refs: &Option>) -> String { + let temp_refs_py = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "{}".to_string()); + RELATIVE_PYTHON_LOADER.replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_py) +} + #[cfg(any(feature = "private", test))] pub fn has_relative_imports(content: &str) -> bool { RELATIVE_IMPORT_REGEX.is_match(content) @@ -643,6 +656,21 @@ pub async fn handle_python_job( )); } + // Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from + // not-yet-deployed local content uploaded to raw_script_temp. Gated on + // JobKind::Preview because job.args includes caller-controlled request + // args; honoring this key on deployed runs would let a caller swap import + // resolution targets in deployed code. + let temp_script_refs: Option> = if matches!(job.kind, JobKind::Preview) + { + job.args + .as_ref() + .and_then(|x| x.get("_TEMP_SCRIPT_REFS")) + .and_then(|v| serde_json::from_str(v.get()).ok()) + } else { + None + }; + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -658,6 +686,7 @@ pub async fn handle_python_job( &mut Some(occupancy_metrics), precomputed_agent_info, annotations.clone(), + &temp_script_refs, ) .await?; @@ -709,6 +738,7 @@ pub async fn handle_python_job( job.script_entrypoint_override.as_deref(), inner_content, &script_path, + &temp_script_refs, ) .await?; @@ -1532,6 +1562,7 @@ async fn prepare_wrapper( job_script_entrypoint_override: Option<&str>, inner_content: &str, script_path: &str, + temp_script_refs: &Option>, ) -> error::Result<( &'static str, &'static str, @@ -1571,7 +1602,11 @@ async fn prepare_wrapper( let _ = write_file(&module_dir, &format!("{last}.py"), inner_content)?; if relative_imports { - let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; + let _ = write_file( + job_dir, + "loader.py", + &render_relative_python_loader(temp_script_refs), + )?; } let sig = windmill_parser_py::parse_python_signature( @@ -1768,6 +1803,7 @@ pub(crate) async fn handle_python_deps( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, precomputed_agent_info: Option, annotations: PythonAnnotations, + temp_script_refs: &Option>, ) -> error::Result<(PyV, Vec)> { create_dependencies_dir(job_dir).await; @@ -1796,7 +1832,7 @@ pub(crate) async fn handle_python_deps( &mut version_specifiers, &mut locked_v, &None, - &None, // temp_script_refs: only used during CLI lock generation + temp_script_refs, )) .await?; @@ -3042,7 +3078,8 @@ pub async fn start_worker( let any_relative_imports = RELATIVE_IMPORT_REGEX.is_match(inner_content); if any_relative_imports { - let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; + // Dedicated worker runs deployed scripts only — no temp refs. + let _ = write_file(job_dir, "loader.py", &render_relative_python_loader(&None))?; } let mut mem_peak: i32 = 0; @@ -3087,6 +3124,7 @@ pub async fn start_worker( &mut None, None, annotations, + &None, // dedicated worker runs deployed scripts only ) .await?; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 2256513b5c..6e2deb7446 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -4061,7 +4061,7 @@ async fn push_next_flow_job( _ => nargs, }; - let push_args; + let mut push_args; let err; let ov; @@ -4077,6 +4077,22 @@ async fn push_next_flow_job( } }; + // Propagate temp script refs from the flow preview job to each step so + // relative imports in inline scripts resolve from not-yet-deployed local + // content (uploaded to raw_script_temp) instead of the deployed script. + // Gated on JobKind::FlowPreview because flow_job.args includes + // caller-controlled request args; honoring this key on deployed flow + // runs would let a caller swap import resolution targets in deployed + // step code. + if matches!(flow_job.kind, JobKind::FlowPreview) { + if let Some(temp_script_refs) = arc_flow_job_args.as_ref().get("_TEMP_SCRIPT_REFS") { + push_args + .extra + .get_or_insert_with(HashMap::new) + .insert("_TEMP_SCRIPT_REFS".to_string(), temp_script_refs.clone()); + } + } + tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed args for job {i} of {len}"); let value_with_parallel = module.get_value_with_parallel()?; diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 7edb793bea..c73a8685c8 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -28,7 +28,12 @@ import * as wmill from "../../../gen/services.gen.ts"; import { logQueueStatus } from "../../utils/job_polling.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; -import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts"; +import { + getWmillYamlPath, + GLOBAL_CONFIG_OPT, + mergeConfigWithConfigFile, +} from "../../core/conf.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; import { replaceInlineScripts, repopulateFields } from "./app.ts"; import { Runnable } from "./metadata.ts"; import { @@ -386,6 +391,43 @@ async function dev(opts: DevOptions, appFolder?: string) { await requireLogin(opts); const workspaceId = workspace.workspaceId; + // Resolve relative imports in app inline scripts from local (not-yet-deployed) + // content so previews use locally-edited workspace libs instead of deployed. + // Computed here as a startup snapshot; re-run `wmill app dev` to pick up + // later edits to imported workspace scripts. Degrades gracefully (undefined) + // on older backends. The walk must run from the wmill.yaml root so that the + // supported `cd __raw_app && wmill app dev` invocation (cwd is the + // raw_app folder) still sees sibling workspace scripts like `f/lib.ts`. + let appTempRefs: Record | undefined = undefined; + { + const wmillYamlPath = getWmillYamlPath(); + const workspaceRoot = wmillYamlPath + ? path.dirname(wmillYamlPath) + : originalCwd; + const relAppFolder = path.relative(workspaceRoot, targetDir) || "."; + const mergedOpts = await mergeConfigWithConfigFile(opts); + const codebases = await listSyncCodebases(mergedOpts); + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + const savedCwd = process.cwd(); + if (workspaceRoot !== savedCwd) { + process.chdir(workspaceRoot); + } + try { + appTempRefs = await buildPreviewTempScriptRefs( + workspace, + mergedOpts as any, + codebases, + { kind: "app", folder: relAppFolder, rawApp: true }, + ); + } finally { + if (workspaceRoot !== savedCwd) { + process.chdir(savedCwd); + } + } + } + // Change to target directory for the rest of the command if (appFolder) { process.chdir(targetDir); @@ -941,6 +983,7 @@ async function dev(opts: DevOptions, appFolder?: string) { appPath, runnableId, args, + appTempRefs, ); log.info(colors.gray(`[backend] Job started: ${uuid}`)); @@ -987,6 +1030,7 @@ async function dev(opts: DevOptions, appFolder?: string) { appPath, runnable_id, v, + appTempRefs, ); log.info(colors.gray(`[backendAsync] Job started: ${uuid}`)); @@ -1563,6 +1607,7 @@ async function executeRunnable( appPath: string, runnableId: string, args: any, + tempScriptRefs?: Record, ): Promise { const requestBody: any = { component: runnableId, @@ -1602,6 +1647,9 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; + if (inlineScript.id === undefined && tempScriptRefs) { + requestBody.temp_script_refs = tempScriptRefs; + } } else if ( (runnable.type === "path" || runnable.type === "runnableByPath") && runnable.runType && diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7f437b8005..03a8d14d51 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -618,6 +618,22 @@ async function preview( await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log); } + // Resolve relative imports in inline scripts from local (not-yet-deployed) + // content so previewing a flow uses locally-edited dependency scripts. + let tempScriptRefs: Record | undefined = undefined; + if (useLocalPathScripts) { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + const resolvedCodebases = (await Promise.resolve(codebases)) as SyncCodebase[]; + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + opts, + resolvedCodebases, + { kind: "flow", folder: flowPath } + ); + } + const input = opts.data ? await resolve(opts.data) : {}; if (!opts.silent) { @@ -633,6 +649,7 @@ async function preview( value: localFlow.value, path: flowPath.substring(0, flowPath.indexOf(".flow")).replaceAll(SEP, "/"), args: input, + temp_script_refs: tempScriptRefs, }, }); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 1de173e582..76c82ed39f 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -7,6 +7,7 @@ import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; import * as log from "../../core/log.ts"; +import { Workspace } from "../workspace/workspace.ts"; import { beginLockfileBatch, flushLockfileBatch, @@ -91,6 +92,99 @@ async function walkLocalAppItems( })); } +/** + * Build the path -> temp-storage-hash map for a preview target (script, flow, + * or app), so a preview run resolves relative imports from not-yet-deployed + * local content instead of the deployed scripts. Walks all local scripts so + * transitive relative-import targets can be uploaded, then for flow/app adds + * that item's node. Degrades gracefully (returns undefined) on older backends + * without the /raw_temp endpoints. + */ +export async function buildPreviewTempScriptRefs( + workspace: Workspace, + opts: GlobalOptions & SyncOptions & { defaultTs?: "bun" | "deno" }, + codebases: SyncCodebase[], + target: + | { kind: "script"; path: string } + | { kind: "flow"; folder: string } + | { kind: "app"; folder: string; rawApp: boolean }, +): Promise | undefined> { + try { + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); + const ignore = await ignoreF(opts); + + for (const e of await walkLocalScripts(codebases, ignore)) { + await generateScriptMetadataInternal( + e, + workspace, + opts, + true, // dryRun: only populate the tree + true, // noStaleMessage + rawWorkspaceDependencies, + codebases, + false, + tree, + ); + } + + let nodePath: string; + if (target.kind === "script") { + nodePath = scriptPathToRemotePath(target.path); + } else if (target.kind === "flow") { + const folder = target.folder.endsWith(SEP) + ? target.folder.slice(0, -1) + : target.folder; + await generateFlowLockInternal(folder, true, workspace, opts, false, true, tree); + nodePath = folder.replaceAll(SEP, "/"); + } else { + const folder = target.folder.endsWith(SEP) + ? target.folder.slice(0, -1) + : target.folder; + await generateAppLocksInternal( + folder, + target.rawApp, + true, + workspace, + opts, + false, + true, + tree, + ); + nodePath = folder.replaceAll(SEP, "/"); + } + + tree.propagateStaleness(); + await uploadScripts(tree, workspace); + const refs = tree.getTempScriptRefs(nodePath); + return refs && Object.keys(refs).length > 0 ? refs : undefined; + } catch (e) { + // Degrade gracefully (preview still runs against deployed versions) but do + // NOT mask the real error: only the missing-/raw_temp-endpoint case is an + // expected old-backend incompatibility — anything else is surfaced verbatim. + const msg = e instanceof Error ? e.message : String(e); + // Narrow: only the missing raw_temp endpoint is the expected old-backend + // signal. A bare 404/"not found" matches far too much (module/command/ + // ENOENT "not found", "Script X not found", …) and would mislabel real + // bugs as a backend-too-old issue. + const isOldBackend = /raw_temp|raw_script_temp/i.test(msg); + if (!(opts as { silent?: boolean }).silent) { + log.warn( + colors.yellow( + isOldBackend + ? `Backend does not support local-import resolution for preview ` + + `(requires the /raw_temp endpoints); relative imports will use ` + + `deployed script versions.` + : `Failed to resolve local relative imports for preview: ${msg}. ` + + `Falling back to deployed script versions.`, + ), + ); + } + return undefined; + } +} + /** * Categorize a flat list of file paths into scripts / flow folders / app * file paths. Used to derive item lists from a precomputed FS map (e.g. diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index b9d9c6ee60..808456f151 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1521,6 +1521,33 @@ async function preview( const codebase = language == "bun" ? findCodebase(filePath, codebases) : undefined; + // Resolve relative imports from local (not-yet-deployed) content so previewing + // a script that imports other locally-edited scripts uses the local versions + // instead of the deployed ones. Shared with `wmill flow preview` so both + // entry points behave identically; degrades gracefully on older backends. + // Short-circuit when the script has no relative imports: the full-workspace + // dependency walk + diff round-trip is pure overhead in that (common) case. + let tempScriptRefs: Record | undefined = undefined; + const { extractRelativeImports } = await import( + "../../utils/relative_imports.ts" + ); + const relImports = await extractRelativeImports( + content, + scriptPathToRemotePath(filePath), + language + ); + if (relImports.length > 0) { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + opts, + codebases, + { kind: "script", path: filePath } + ); + } + let bundledContent: string | Blob | undefined = undefined; let isTar = false; @@ -1616,6 +1643,7 @@ async function preview( language: language, kind: isTar ? "tarbundle" : "bundle", format: codebase?.format ?? "cjs", + temp_script_refs: tempScriptRefs, }; form.append("preview", JSON.stringify(previewPayload)); form.append( @@ -1683,6 +1711,7 @@ async function preview( args: input, language: language as any, modules: modules ?? undefined, + temp_script_refs: tempScriptRefs, }, });