diff --git a/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json new file mode 100644 index 0000000000..3c7f23d1af --- /dev/null +++ b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce" +} diff --git a/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json new file mode 100644 index 0000000000..10ec0a5c49 --- /dev/null +++ b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at)\n VALUES ($1, $2, $3, NOW())\n ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bpchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff" +} diff --git a/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json new file mode 100644 index 0000000000..ed1b03fd77 --- /dev/null +++ b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT content FROM raw_script_temp WHERE hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2" +} diff --git a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json similarity index 50% rename from backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json rename to backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json index 0946fa4006..c910be0b82 100644 --- a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json +++ b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", + "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623" + "hash": "96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 25c4233bbd..e40478cc4b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16898,8 +16898,6 @@ dependencies = [ "async-recursion", "itertools 0.14.0", "lazy_static", - "malachite", - "malachite-bigint", "pep440_rs", "phf 0.11.3", "regex", @@ -16956,7 +16954,6 @@ dependencies = [ "serde", "serde_json", "windmill-parser", - "windmill-types", ] [[package]] @@ -17465,6 +17462,7 @@ dependencies = [ "strum 0.27.2", "tracing", "uuid", + "windmill-parser", ] [[package]] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 820e457354..f30dcc7a1f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a997285e976d0642b72584e1966a70a79d84e7dc +fe8f0d1d7448464c98474d994e6492c0a45e8e38 diff --git a/backend/migrations/20260304000000_raw_script_temp.down.sql b/backend/migrations/20260304000000_raw_script_temp.down.sql new file mode 100644 index 0000000000..05703e4f18 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_raw_script_temp_created_at; +DROP TABLE IF EXISTS raw_script_temp; diff --git a/backend/migrations/20260304000000_raw_script_temp.up.sql b/backend/migrations/20260304000000_raw_script_temp.up.sql new file mode 100644 index 0000000000..c7b5a23754 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.up.sql @@ -0,0 +1,11 @@ +-- Temporary storage for raw script content during CLI lock generation +-- Content is stored with hash as key, cleaned up after 1 week +CREATE TABLE raw_script_temp ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id), + hash CHAR(64) NOT NULL, + content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, hash) +); + +CREATE INDEX IF NOT EXISTS idx_raw_script_temp_created_at ON raw_script_temp (created_at); diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index deea68c2ef..565e1422bb 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -13,21 +13,19 @@ regex-lite.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] regex.workspace = true - -[dependencies] -windmill-parser.workspace = true windmill-common.workspace = true -rustpython-parser.workspace = true -malachite.workspace = true -malachite-bigint.workspace = true -phf.workspace = true -itertools.workspace = true -serde_json.workspace = true -anyhow.workspace = true -lazy_static.workspace = true sqlx.workspace = true async-recursion.workspace = true toml.workspace = true serde.workspace = true pep440_rs.workspace = true tracing.workspace = true + +[dependencies] +windmill-parser.workspace = true +rustpython-parser.workspace = true +phf.workspace = true +itertools.workspace = true +serde_json.workspace = true +anyhow.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 06762dbc9e..de7f51bc79 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -8,10 +8,14 @@ mod mapping; +#[cfg(not(target_arch = "wasm32"))] use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; -use std::{collections::HashMap, str::FromStr}; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; +#[cfg(not(target_arch = "wasm32"))] +use std::collections::HashMap; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -24,7 +28,9 @@ use rustpython_parser::{ text_size::TextRange, Parse, }; +#[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; +#[cfg(not(target_arch = "wasm32"))] use windmill_common::{ error::{self, to_anyhow}, worker::{ @@ -46,10 +52,14 @@ fn replace_full_import(x: &str) -> Option { FULL_IMPORTS_MAP.get(x).map(|x| (*x).to_owned()) } +#[cfg(not(target_arch = "wasm32"))] lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); - static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap(); +} + +lazy_static! { + static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); // Regex to properly match main function definition at line start, // capturing both sync and async variants static ref DEF_MAIN_RE: Regex = Regex::new(r"(?m)^(async\s+)?def\s+main\s*\(").unwrap(); @@ -82,7 +92,7 @@ fn process_import(module: Option, path: &str, level: usize) -> Vec error::Result> { +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { let nimports = parse_code_for_imports(code, path)?; return Ok(nimports .into_iter() @@ -94,7 +104,7 @@ pub fn parse_relative_imports(code: &str, path: &str) -> error::Result error::Result> { +pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -175,7 +187,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> let code_with_fake_main = format!("{}\n\ndef main(): pass", code); let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) + anyhow::anyhow!("Error parsing code for imports: {}", e.to_string()) })?; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main @@ -256,6 +268,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> return Ok(nimports); } +#[cfg(not(target_arch = "wasm32"))] pub async fn parse_python_imports( code: &str, w_id: &str, @@ -264,6 +277,7 @@ pub async fn parse_python_imports( version_specifiers: &mut Vec, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -276,6 +290,7 @@ pub async fn parse_python_imports( &mut None, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -313,6 +328,7 @@ pub async fn parse_python_imports( Ok((imports, compile_error_hint)) } +#[cfg(not(target_arch = "wasm32"))] fn extract_pkg_name(requirement: &str) -> String { PKG_RE .captures(requirement) @@ -320,6 +336,7 @@ fn extract_pkg_name(requirement: &str) -> String { .unwrap_or_default() } +#[cfg(not(target_arch = "wasm32"))] #[async_recursion] async fn parse_python_imports_inner( code: &str, @@ -331,6 +348,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv: &mut Option, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); @@ -494,17 +512,37 @@ async fn parse_python_imports_inner( for n in nimports.into_iter() { let mut nested = match n { NImport::Relative(rpath) => { - let code = sqlx::query_scalar!( - r#" - SELECT content FROM script WHERE path = $1 AND workspace_id = $2 - AND archived = false ORDER BY created_at DESC LIMIT 1 - "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()); + // First try to get content from temp_script_refs cache if available + let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { + tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { + Ok(content) => Some(content), + Err(e) => { + tracing::warn!("temp_script_refs hash '{}' not found in cache: {}, falling back to deployed script", hash, e); + None + } + } + } else { + None + }; + + // Use cached content if available, otherwise fall back to deployed script + let code = match code_from_cache { + Some(content) => content, + None => { + sqlx::query_scalar!( + r#" + SELECT content FROM script WHERE path = $1 AND workspace_id = $2 + AND archived = false ORDER BY created_at DESC LIMIT 1 + "#, + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()) + } + }; if already_visited.contains(&rpath) { vec![] @@ -522,6 +560,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -646,6 +685,7 @@ async fn parse_python_imports_inner( Ok(final_imports) } +#[cfg(not(target_arch = "wasm32"))] fn extract_nimports_from_content( content: &str, hm: &mut HashMap, diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index e61b4cc7db..c853c8e5b4 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -26,6 +26,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -67,6 +68,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -98,6 +100,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index e84888280c..143b02a2cc 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -16,7 +16,6 @@ regex.workspace = true [dependencies] windmill-parser.workspace = true -windmill-types.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 4938a8e194..c8aef3eed5 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -15,7 +15,7 @@ use std::{ iter::Peekable, str::CharIndices, }; -pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ}; +pub use windmill_parser::{s3_mode_extension, Arg, MainArgSignature, ObjectType, S3ModeFormat, Typ}; pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__"; pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__"; @@ -143,7 +143,6 @@ pub fn parse_db_resource(code: &str) -> Option { cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap()) } -pub use windmill_types::s3::{s3_mode_extension, S3ModeFormat}; pub struct S3ModeArgs { pub prefix: Option, pub storage: Option, diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 53ecaf277c..125834a3ac 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -117,6 +117,12 @@ impl Visit for ImportsFinder { } } +/// Parse TypeScript/JavaScript code and extract all import paths as raw strings. +/// +/// Returns import paths exactly as written in the code (e.g., `"./module"`, `"../utils"`, `"lodash"`). +/// Does not resolve relative paths to absolute Windmill paths. +/// +/// See also: [`parse_relative_imports`] for resolved absolute paths. pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result> { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into()); @@ -151,6 +157,82 @@ pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Resul Ok(imports) } +/// Parse TypeScript/JavaScript code and extract relative imports resolved to absolute Windmill paths. +/// +/// Takes the script's Windmill path (e.g., `"f/folder/script"`) and resolves relative imports +/// like `"./module"` or `"../utils"` to absolute paths like `"f/folder/module"` or `"f/utils"`. +/// +/// Only returns relative imports (those starting with `./`, `../`, or `/`). +/// External package imports (e.g., `"lodash"`) are filtered out. +/// +/// See also: [`parse_expr_for_imports`] for raw import strings without resolution. +/// +/// # Arguments +/// * `code` - The TypeScript/JavaScript source code +/// * `path` - The Windmill path of the script (e.g., `"f/folder/script"`) +/// +/// # Returns +/// A sorted, deduplicated list of resolved absolute Windmill paths. +/// +/// # Examples +/// ```ignore +/// // Script at "f/folder/script" with: import { x } from "../utils" +/// // Returns: ["f/utils"] +/// ``` +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { + let imports = parse_expr_for_imports(code, false)?; + let script_dir = path.rsplit_once('/').map(|(dir, _)| dir).unwrap_or(""); + + let mut resolved: Vec = imports + .into_iter() + .filter(|imp| is_relative_import(imp)) + .map(|imp| { + // Remove .ts extension if present + let imp = imp.strip_suffix(".ts").unwrap_or(&imp); + + if imp.starts_with("/") { + // Absolute path (e.g., /f/folder/script) - remove leading slash + imp[1..].to_string() + } else { + // Relative path (e.g., ./script or ../folder/script) + let combined = format!("{}/{}", script_dir, imp); + normalize_path(&combined) + } + }) + .collect(); + + resolved.sort(); + resolved.dedup(); + Ok(resolved) +} + +/// Check if an import path is a relative import (starts with `./`, `../`, or `/`) +fn is_relative_import(import_path: &str) -> bool { + import_path.starts_with("./") + || import_path.starts_with("../") + || import_path.starts_with("/") +} + +/// Normalize a path by resolving `.` and `..` components +fn normalize_path(input_path: &str) -> String { + let parts: Vec<&str> = input_path.split('/').filter(|p| !p.is_empty()).collect(); + let mut result: Vec<&str> = Vec::new(); + + for part in parts { + if part == "." { + continue; + } else if part == ".." { + if !result.is_empty() { + result.pop(); + } + } else { + result.push(part); + } + } + + result.join("/") +} + struct OutputFinder { idents: HashSet<(String, String)>, } diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 7a685fbb77..b50169ca68 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -2,7 +2,7 @@ mod tests { use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; - use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports}; + use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports, parse_relative_imports}; #[test] fn test_imports_basic() { @@ -798,7 +798,7 @@ mod tests { // Test case where there are exports but no preprocessor let code = r#" export { foo, bar } from "./utils"; - + export async function main(param: string) { return param; } @@ -806,4 +806,84 @@ mod tests { let sig = parse_deno_signature(code, false, false, None).unwrap(); assert_eq!(sig.has_preprocessor, Some(false)); } + + // ========================================================================== + // Tests for parse_relative_imports + // ========================================================================== + + #[test] + fn test_relative_imports_dot() { + let code = r#" + import { helper } from "./helper"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_double_dot() { + let code = r#" + import { utils } from "../utils/helper"; + export async function main() { return utils(); } + "#; + let result = parse_relative_imports(code, "f/folder/subfolder/script").unwrap(); + assert_eq!(result, vec!["f/folder/utils/helper"]); + } + + #[test] + fn test_relative_imports_absolute_path() { + let code = r#" + import { shared } from "/f/shared/utils"; + export async function main() { return shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/shared/utils"]); + } + + #[test] + fn test_relative_imports_mixed() { + let code = r#" + import { helper } from "./helper"; + import { utils } from "../utils"; + import { shared } from "/f/shared/lib"; + import lodash from "lodash"; + export async function main() { return helper() + utils() + shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + // Should only include relative imports, not external packages like lodash + assert_eq!(result, vec!["f/folder/helper", "f/shared/lib", "f/utils"]); + } + + #[test] + fn test_relative_imports_with_ts_extension() { + let code = r#" + import { helper } from "./helper.ts"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_external_only() { + let code = r#" + import lodash from "lodash"; + import { something } from "@scope/package"; + export async function main() { return lodash.map([]); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn test_relative_imports_deeply_nested() { + let code = r#" + import { a } from "../../a"; + import { b } from "../../../b"; + export async function main() { return a() + b(); } + "#; + let result = parse_relative_imports(code, "f/one/two/three/script").unwrap(); + assert_eq!(result, vec!["f/b", "f/one/a"]); + } } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 8372b1838c..895a35713e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -40,6 +40,7 @@ java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] wac-parser = [ "dep:windmill-parser-wac"] asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"] +py-imports-parser = [ "dep:windmill-parser-py-imports"] [dependencies] anyhow.workspace = true @@ -61,6 +62,7 @@ windmill-parser-wac = { workspace = true, optional = true } windmill-parser-ts-asset = { workspace = true, optional = true } windmill-parser-py-asset = { workspace = true, optional = true } windmill-parser-sql-asset = { workspace = true, optional = true } +windmill-parser-py-imports = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 4e08ce95d3..212c5f3a37 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -67,6 +67,12 @@ const targets = [ features: "asset-parser", env: "default", }, + { + ident: "py-imports", + desc: "Python imports" + features: "py-imports-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/dev.nu b/backend/parsers/windmill-parser-wasm/dev.nu index f4d7ef6495..ec865f36dc 100755 --- a/backend/parsers/windmill-parser-wasm/dev.nu +++ b/backend/parsers/windmill-parser-wasm/dev.nu @@ -1,6 +1,6 @@ #!/usr/bin/env nu - -# Build in debug mode specified lang parser to wasm + +# Build in debug mode specified lang parser to wasm # and perform installation to frontend def "main" [ lang: string # Example: nu @@ -9,4 +9,7 @@ def "main" [ ( cd ../../../frontend; npm install ../backend/parsers/windmill-parser-wasm/pkg-($lang) ) + ( + cd ../../../cli; bun install ../backend/parsers/windmill-parser-wasm/pkg-($lang) + ) } diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index 80f31650f3..3ac0ceef18 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -36,3 +36,6 @@ popd pushd "pkg-asset" && npm publish ${args} popd + +pushd "pkg-py-imports" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2af0a3bc9a..a2cef1b0ef 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -38,6 +38,8 @@ pub fn parse_outputs(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return raw import strings. +/// See [`parse_ts_relative_imports`] for resolved absolute paths. #[cfg(feature = "ts-parser")] #[wasm_bindgen] pub fn parse_ts_imports(code: &str) -> String { @@ -50,6 +52,15 @@ pub fn parse_ts_imports(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +/// See [`parse_ts_imports`] for raw import strings. +#[cfg(feature = "ts-parser")] +#[wasm_bindgen] +pub fn parse_ts_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_ts::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "bash-parser")] #[wasm_bindgen] pub fn parse_bash(code: &str) -> String { @@ -214,6 +225,14 @@ pub fn parse_assets_py(code: &str) -> String { } } +/// Parse Python imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +#[cfg(feature = "py-imports-parser")] +#[wasm_bindgen] +pub fn parse_py_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_py_imports::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_assets_ansible(code: &str) -> String { diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index dd2eada7cf..80019d044f 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -14,6 +14,23 @@ use serde_json::Value; pub mod asset_parser; +/// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) +#[derive(Clone, Copy, Debug)] +pub enum S3ModeFormat { + Json, + Csv, + Parquet, +} + +/// Returns the file extension for the given S3 mode format +pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { + match format { + S3ModeFormat::Json => "json", + S3ModeFormat::Csv => "csv", + S3ModeFormat::Parquet => "parquet", + } +} + #[derive(Serialize, Debug, PartialEq, Default)] pub struct MainArgSignature { pub star_args: bool, diff --git a/backend/src/main.rs b/backend/src/main.rs index 6d977c2db4..17567167a3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -312,6 +312,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "cache_init", "", &mut None, + &None, ) .await { diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index b2eb89112c..2edfef8989 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -921,6 +921,7 @@ mod dedicated_worker_protocol { "test-workspace", "f/test/script", LoaderMode::Node, + &None, )) .expect("build_loader failed"); @@ -1299,6 +1300,7 @@ mod bun_builder_tests { // Write build.js using the loader and builder constants directly // Parameters are dummy values since tests don't use Windmill relative imports let loader = RELATIVE_BUN_LOADER + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", "{}") .replace("W_ID", "test-workspace") .replace("BASE_INTERNAL_URL", "http://localhost:8000") .replace("TOKEN", "test-token") diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index f13759e5bc..1c6987a480 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -32,6 +32,7 @@ mod prewarmed_isolate_tests { "test-workspace", "f/test/script", LoaderMode::BrowserBundle, + &None, ) .await .expect("build_loader failed"); diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index fcbb159887..56de4d9892 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -239,6 +239,9 @@ pub fn workspaced_service() -> Router { "/history_update/h/:hash/p/*path", post(update_script_history), ) + // Temporary raw script storage for CLI lock generation + .route("/raw_temp/store", post(store_raw_script_temp)) + .route("/raw_temp/diff", post(diff_raw_scripts_with_deployed)) } #[derive(Serialize, FromRow)] @@ -1614,6 +1617,9 @@ struct RawScriptByPathQuery { cache_key: Option, // used specifically for python to cache folders on import success to avoid extra db calls on package fetch cache_folders: Option, + // If provided, load content from raw_script_temp table using this hash instead of deployed script. + // Used by CLI lock generation to resolve imports from not-yet-deployed scripts. + temp_script_hash: Option, } struct StringWithLength(String); @@ -1672,6 +1678,16 @@ async fn raw_script_by_path_internal( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + + // If temp_script_hash is provided, try loading from temp storage first. + // This is used by CLI lock generation to resolve imports from not-yet-deployed scripts. + // Falls back to the normal deployed script lookup if not found in temp storage. + if let Some(hash) = query.temp_script_hash { + if let Ok(content) = windmill_common::cache::raw_script_temp::load(hash, &db).await { + return Ok(content); + } + } + let cache_path = query .cache_key .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); @@ -2463,3 +2479,129 @@ async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { Ok(()) } } + +// ============================================================================ +// Temporary Raw Script Storage for CLI Lock Generation +// ============================================================================ + +/// Store raw script content temporarily for CLI lock generation. +async fn store_raw_script_temp( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(content): Json, +) -> Result> { + check_scopes(&authed, || "scripts:write".to_string())?; + + let hash = windmill_common::cache::raw_script_temp::compute_hash(&w_id, &content); + + // Store to DB + sqlx::query!( + "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + &w_id, + &hash, + &content + ) + .execute(&db) + .await?; + + // Clean up old entries (1 week TTL) + sqlx::query!( + "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'" + ) + .execute(&db) + .await?; + + Ok(Json(hash)) +} + +/// Compare local script content hashes with deployed versions. +/// Receives a map of path → SHA256(content), returns paths where the hash +/// differs from the deployed script (or the script doesn't exist on remote). +/// Hash comparison is done entirely in Postgres to avoid transferring content. +#[derive(Deserialize)] +struct WorkspaceDepDiff { + path: String, + language: ScriptLang, + name: Option, + hash: String, +} + +#[derive(Deserialize)] +struct DiffRequest { + scripts: std::collections::HashMap, + #[serde(default)] + workspace_deps: Vec, +} + +async fn diff_raw_scripts_with_deployed( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result>> { + check_scopes(&authed, || "scripts:read".to_string())?; + + let mut matching_set: std::collections::HashSet = std::collections::HashSet::new(); + let mut all_paths: Vec = Vec::new(); + + // --- Scripts --- + if !req.scripts.is_empty() { + let paths: Vec = req.scripts.keys().cloned().collect(); + let hashes: Vec = paths.iter().map(|p| req.scripts[p].clone()).collect(); + + let matching: Vec = sqlx::query_scalar( + "SELECT local.path FROM \ + unnest($1::text[], $2::text[]) AS local(path, hash) \ + INNER JOIN LATERAL ( \ + SELECT encode(sha256(convert_to(s.content, 'UTF8')), 'hex') AS deployed_hash \ + FROM script s \ + WHERE s.path = local.path AND s.workspace_id = $3 AND s.archived = false \ + ORDER BY s.created_at DESC LIMIT 1 \ + ) deployed ON deployed.deployed_hash = local.hash" + ) + .bind(&paths) + .bind(&hashes) + .bind(&w_id) + .fetch_all(&db) + .await?; + + matching_set.extend(matching); + all_paths.extend(paths); + } + + // --- Workspace dependencies --- + for dep in &req.workspace_deps { + let matching: Option = sqlx::query_scalar( + "SELECT $1::text \ + WHERE EXISTS ( \ + SELECT 1 FROM workspace_dependencies wd \ + WHERE wd.workspace_id = $2 AND wd.archived = false \ + AND wd.language = $3::SCRIPT_LANG \ + AND wd.name IS NOT DISTINCT FROM $4 \ + AND encode(sha256(convert_to(wd.content, 'UTF8')), 'hex') = $5 \ + )" + ) + .bind(&dep.path) + .bind(&w_id) + .bind(dep.language.as_str()) + .bind(&dep.name) + .bind(&dep.hash) + .fetch_optional(&db) + .await?; + + if let Some(path) = matching { + matching_set.insert(path); + } + all_paths.push(dep.path.clone()); + } + + let mismatched: Vec = all_paths + .into_iter() + .filter(|p| !matching_set.contains(p)) + .collect(); + + Ok(Json(mismatched)) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7d15eea003..930cb94949 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6988,6 +6988,83 @@ paths: type: string format: uuid + /w/{workspace}/scripts/raw_temp/store: + post: + summary: store raw script content temporarily for CLI lock generation + operationId: storeRawScriptTemp + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: script content to store + required: true + content: + application/json: + schema: + type: string + responses: + "200": + description: hash of stored content + content: + application/json: + schema: + type: string + + /w/{workspace}/scripts/raw_temp/diff: + post: + summary: diff local script hashes against deployed versions + operationId: diffRawScriptsWithDeployed + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: scripts and workspace deps to diff against deployed versions + required: true + content: + application/json: + schema: + type: object + required: + - scripts + properties: + scripts: + description: map of script path to SHA256 content hash + type: object + additionalProperties: + type: string + workspace_deps: + description: workspace dependencies to diff + type: array + items: + type: object + required: + - path + - language + - hash + properties: + path: + description: CLI path (e.g. dependencies/package.json) + type: string + language: + $ref: "#/components/schemas/ScriptLang" + name: + description: named workspace dependency (null for default) + type: string + hash: + description: SHA256 content hash + type: string + responses: + "200": + description: list of paths that differ from deployed versions + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/jobs/list_selected_job_groups: # We use post because sending a huge array as a query param can produce # URLs that may be too long diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 29b9765480..7f9a3f36db 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -5074,6 +5074,10 @@ pub struct RunDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option, + /// Map of script path -> content hash for resolving imports from temp storage. + /// Used by CLI to provide local script content during lock generation. + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Deserialize, Clone, Debug)] @@ -5133,6 +5137,8 @@ async fn run_dependencies_job( let mut hm = HashMap::new(); req.raw_workspace_dependencies .map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v))); + req.temp_script_refs + .map(|v| hm.insert("temp_script_refs".to_owned(), to_raw_value(&v))); let (uuid, tx) = push( &db, @@ -5183,6 +5189,8 @@ pub struct RunFlowDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option>, + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Serialize)] @@ -5226,6 +5234,10 @@ async fn run_flow_dependencies_job( req.raw_workspace_dependencies .map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v))); + // Add temp_script_refs to args if present (for CLI local import resolution) + req.temp_script_refs + .map(|v| args_map.insert("temp_script_refs".to_string(), to_raw_value(&v))); + let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 8901055889..eae1648657 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -991,6 +991,38 @@ pub mod workspace_dependencies { } } +/// Temporary raw script content cache for CLI lock generation. +pub mod raw_script_temp { + use super::*; + use crate::DB; + + make_static! { + static ref CACHE: { String => String } in "raw_script_temp" <= 10000; + } + + /// Compute hash for raw script content (includes workspace_id for isolation). + pub fn compute_hash(workspace_id: &str, content: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(workspace_id.as_bytes()); + hasher.update(content.as_bytes()); + format!("{:x}", hasher.finalize()) + } + + /// Load content from cache, falling back to DB. + pub fn load(hash: String, db: &DB) -> impl Future> + '_ { + CACHE.get_or_insert_async(hash.clone(), async move { + sqlx::query_scalar!( + "SELECT content FROM raw_script_temp WHERE hash = $1", + &hash + ) + .fetch_optional(db) + .await? + .ok_or_else(|| error::Error::NotFound(format!("raw_script_temp hash: {}", hash))) + }) + } +} + const _: () = { impl Import for RawFlow { fn import(src: &impl Storage) -> error::Result { @@ -1183,7 +1215,8 @@ const _: () = { ((u8, ScriptHash), |x| format!("{:02x}-{:016x}", x.0, x.1.0)), (FlowNodeId, |x| format!("{:016x}", x.0)), (AppScriptId, |x| format!("{:016x}", x.0)), - ((i64, String), |x| format!("{}-{}", x.1, x.0)) + ((i64, String), |x| format!("{}-{}", x.1, x.0)), + (String, |x| x.as_str()) } #[cfg(feature = "scoped_cache")] diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml index 9619a10ea7..2472d6bf02 100644 --- a/backend/windmill-types/Cargo.toml +++ b/backend/windmill-types/Cargo.toml @@ -9,6 +9,7 @@ name = "windmill_types" path = "src/lib.rs" [dependencies] +windmill-parser.workspace = true serde.workspace = true serde_json.workspace = true chrono.workspace = true diff --git a/backend/windmill-types/src/s3.rs b/backend/windmill-types/src/s3.rs index e3cbd1f9a3..0ab1794d7e 100644 --- a/backend/windmill-types/src/s3.rs +++ b/backend/windmill-types/src/s3.rs @@ -346,20 +346,9 @@ pub struct DuckdbConnectionSettingsQueryV2 { pub storage: Option, } -#[derive(Clone, Copy, Debug)] -pub enum S3ModeFormat { - Json, - Csv, - Parquet, -} - -pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { - match format { - S3ModeFormat::Json => "json", - S3ModeFormat::Csv => "csv", - S3ModeFormat::Parquet => "parquet", - } -} +// Re-export from windmill-parser to keep a single type definition +// (windmill-parser is WASM-compatible, windmill-types is not due to sqlx) +pub use windmill_parser::{s3_mode_extension, S3ModeFormat}; #[cfg(test)] mod tests { diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index f64e6d769b..f2a00de0cf 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -1,8 +1,11 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + const p = { name: "windmill-relative-resolver", async setup(build) { const { writeFileSync, readFileSync, mkdirSync } = await import("fs"); - const { dirname, resolve } = await import("node:path"); + const { dirname, resolve, join } = await import("node:path"); const base_internal_url = "BASE_INTERNAL_URL".replace( "localhost", @@ -95,11 +98,17 @@ const p = { : args.importer.replace(cdir + "/", ""); const isRelative = !args.path.startsWith("/"); + const endExt = args.path.endsWith(".ts") ? "" : ".ts"; + const pathNoExt = args.path.replace(/\.ts$/, ""); - let endExt = args.path.endsWith(".ts") ? "" : ".ts"; - const url = isRelative + // Lookup temp script hash + const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); + const hash = TEMP_SCRIPT_REFS?.[normalized]; + + const url = (isRelative ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}`; + : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") : resolve("./" + args.path + ".url"); diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index fedef5fc5a..7d14ffcbc3 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -1,3 +1,6 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + // Windows-specific bun loader that uses a virtual "windmill-url" namespace instead // of writing .url files to disk. This avoids Windows path issues (backslashes in // resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace @@ -70,7 +73,13 @@ const p = { const rawScriptPath = isAbsolute ? `${path}${endExt}` : `${importerPath}/../${path}${endExt}`; - return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + const normalized = normalizePath(rawScriptPath); + // Look up temp script hash (keys are extensionless paths) + const lookupPath = normalized.replace(/\.ts$/, ""); + const hash = TEMP_SCRIPT_REFS?.[lookupPath]; + // Encode hash in the path so onLoad can extract it and append to fetch URL + const resolvedPath = hash ? `${normalized}?temp_script_hash=${hash}` : normalized; + return { path: resolvedPath, namespace: "windmill-url" }; } build.onLoad({ filter: filterLoad }, async (args) => { @@ -80,8 +89,13 @@ const p = { // Load windmill scripts by fetching from the API build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { - const path = args.path.replace(/^windmill-url:/, ""); - const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + // Extract temp_script_hash if embedded in the path by resolveWindmillImport + const [scriptPath, queryString] = args.path.replace(/^windmill-url:/, "").split("?"); + const hashParam = queryString?.startsWith("temp_script_hash=") + ? queryString.replace("temp_script_hash=", "") + : undefined; + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${scriptPath}` + + (hashParam ? `?temp_script_hash=${hashParam}` : ""); const req = await fetch(url, { method: "GET", headers: { @@ -124,7 +138,8 @@ const p = { // Resolve nested imports from within windmill-url modules build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { - const importer = args.importer.replace(/^windmill-url:/, ""); + // Strip any query string from the importer path before resolving + const importer = args.importer.replace(/^windmill-url:/, "").split("?")[0]; return resolveWindmillImport(importer, args.path); }); }, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index c3065be7d2..fb07806fd6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -206,6 +206,7 @@ pub async fn gen_bun_lockfile( workspace_dependencies: &WorkspaceDependenciesPrefetched, npm_mode: bool, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, quiet: bool, ) -> Result> { let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; @@ -216,6 +217,11 @@ pub async fn gen_bun_lockfile( gen_bunfig(job_dir, job_id, w_id, db).await?; write_file(job_dir, "package.json", package_json_content.as_str())?; } else { + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -224,7 +230,8 @@ pub async fn gen_bun_lockfile( "CURRENT_PATH", &crate::common::use_flow_root_path(script_path), ) - .replace("RAW_GET_ENDPOINT", "raw"); + .replace("RAW_GET_ENDPOINT", "raw") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); write_file( &job_dir, @@ -615,9 +622,15 @@ pub async fn build_loader( w_id: &str, current_path: &str, mode: LoaderMode, + temp_script_refs: &Option>, ) -> Result<()> { // Use forward slashes in JS strings to avoid backslash escape issues on Windows let job_dir_js = job_dir.replace('\\', "/"); + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -626,7 +639,8 @@ pub async fn build_loader( "CURRENT_PATH", &crate::common::use_flow_root_path(current_path), ) - .replace("RAW_GET_ENDPOINT", "raw_unpinned"); + .replace("RAW_GET_ENDPOINT", "raw_unpinned") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); if mode == LoaderMode::Node { write_file( @@ -924,6 +938,7 @@ pub async fn prebundle_bun_script( worker_name: &str, token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; @@ -950,6 +965,7 @@ pub async fn prebundle_bun_script( } else { LoaderMode::BunBundle }, + temp_script_refs, ) .await?; @@ -1271,6 +1287,7 @@ pub async fn handle_bun_job( workspace_dependencies, annotation.npm, &mut Some(occupancy_metrics), + &None, wac_replay_info.is_some(), ) .await?; @@ -1639,6 +1656,7 @@ try {{ } else { LoaderMode::BunBundle }, + &None, ) .await?; @@ -1655,6 +1673,7 @@ try {{ } else { LoaderMode::Bun }, + &None, ) .await } else { @@ -3358,6 +3377,7 @@ pub async fn start_worker( w_id, script_path, LoaderMode::BrowserBundle, + &None, ) .await?; generate_bun_bundle( @@ -3470,6 +3490,7 @@ pub async fn start_worker( .await?, annotation.npm, &mut None, + &None, false, ) .await?; @@ -3544,6 +3565,7 @@ pub async fn start_worker( } else { LoaderMode::Bun }, + &None, ) .await?; } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3aa7580220..3b6507a7e0 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1348,6 +1348,7 @@ async fn handle_python_deps( &mut version_specifiers, &mut locked_v, &None, + &None, // temp_script_refs: only used during CLI lock generation )) .await?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index daba455f1c..93b48cf818 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -139,6 +139,13 @@ pub async fn handle_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let content = capture_dependency_job( &job.id, job.script_lang.as_ref().map(|v| Ok(v)).unwrap_or_else(|| { @@ -164,6 +171,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &temp_script_refs, ) .await; @@ -210,6 +218,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &None, ) .await { @@ -368,6 +377,13 @@ pub async fn handle_flow_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let version = if skip_flow_update { None } else { @@ -467,6 +483,7 @@ pub async fn handle_flow_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -681,6 +698,7 @@ async fn lock_flow_value<'c>( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( FlowValue, sqlx::Transaction<'c, sqlx::Postgres>, @@ -711,6 +729,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -742,6 +761,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -779,6 +799,7 @@ async fn lock_flow_value<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -817,6 +838,7 @@ async fn lock_modules<'c>( dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( Vec, sqlx::Transaction<'c, sqlx::Postgres>, @@ -872,6 +894,7 @@ async fn lock_modules<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -911,6 +934,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -942,6 +966,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -978,6 +1003,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1008,6 +1034,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; errors.extend(ninner_errors); @@ -1078,6 +1105,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; @@ -1179,6 +1207,7 @@ async fn lock_modules<'c>( job_path, Some(&e.id), "flow", + &temp_script_refs, ) .await; // @@ -1586,6 +1615,7 @@ async fn lock_modules_app( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result { match value { Value::Object(mut m) => { @@ -1696,6 +1726,7 @@ async fn lock_modules_app( &job.runnable_path(), container_id.as_deref(), "app", + temp_script_refs, ) .await; match new_lock { @@ -1773,6 +1804,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1801,6 +1833,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1852,6 +1885,13 @@ pub async fn handle_app_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + sqlx::query!( "DELETE FROM workspace_runnable_dependencies WHERE app_path = $1 AND workspace_id = $2", job_path, @@ -1899,6 +1939,7 @@ pub async fn handle_app_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -2398,6 +2439,8 @@ async fn capture_dependency_job( base_path: &str, step_id: Option<&str>, runnable_type: &str, // "script", "flow", or "app" + // Map of script path -> content hash for resolving imports from temp storage (CLI). + temp_script_refs: &Option>, ) -> error::Result { // Check if we can skip relocking: // - Must be triggered by relative import @@ -2456,12 +2499,13 @@ async fn capture_dependency_job( let (mut version_specifiers, mut locked_v) = (vec![], None); let reqs = windmill_parser_py_imports::parse_python_imports( job_raw_code, - &w_id, + w_id, script_path, &db, &mut version_specifiers, &mut locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .0 @@ -2585,6 +2629,7 @@ async fn capture_dependency_job( &workspace_dependencies, windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm, &mut Some(occupancy_metrics), + temp_script_refs, false, ) .await? @@ -2602,6 +2647,7 @@ async fn capture_dependency_job( worker_name, &token, &mut Some(occupancy_metrics), + temp_script_refs, ) .await?; } diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 72be8f1ca9..865091104c 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -12,6 +12,7 @@ const parserPackages = [ "windmill-parser-wasm-yaml", "windmill-parser-wasm-csharp", "windmill-parser-wasm-nu", "windmill-parser-wasm-java", "windmill-parser-wasm-ruby", + "windmill-parser-wasm-py-imports", ]; const parserExternals = parserPackages.flatMap(p => ["--external", p]); diff --git a/cli/bun.lock b/cli/bun.lock index e400420a63..6c421141b4 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -24,10 +24,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", @@ -294,13 +295,15 @@ "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="], + "windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.659.1", "", {}, "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="], + "windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.639.0", "", {}, "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="], "windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="], "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="], - "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.623.1", "", {}, "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="], + "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="], "windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="], diff --git a/cli/package.json b/cli/package.json index 6102f815f2..e44a215631 100644 --- a/cli/package.json +++ b/cli/package.json @@ -32,10 +32,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 253dc99f3b..82e2a1ac22 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts"; import { stringify as yamlStringify } from "yaml"; import { GlobalOptions } from "../../types.ts"; import { + readLockfile, checkifMetadataUptodate, blueColor, clearGlobalLock, @@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; import { getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__app_hash"; export const APP_BACKEND_FOLDER = "backend"; @@ -113,7 +116,9 @@ export async function generateAppLocksInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (appFolder.endsWith(SEP)) { appFolder = appFolder.substring(0, appFolder.length - 1); @@ -125,9 +130,6 @@ export async function generateAppLocksInternal( log.info(`Generating locks for app ${appFolder} at ${remote_path}`); } - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - // Read the app file first to filter workspace dependencies const appFilePath = path.join( appFolder, @@ -135,35 +137,80 @@ export async function generateAppLocksInternal( ); const appFile = (await yamlParseFile(appFilePath)) as AppFile; - // Filter workspace dependencies based on inline scripts' languages and annotations const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value; - const filteredDeps = await filterWorkspaceDependenciesForApp( - appValue, - rawWorkspaceDependencies, - appFolder - ); + const folderNormalized = appFolder.replaceAll(SEP, "/"); - let hashes = await generateAppHash( - filteredDeps, - appFolder, - rawApp, - opts.defaultTs - ); + let filteredDeps: Record = {}; + const conf = await readLockfile(); - const conf = await import("../../utils/metadata.ts").then((m) => - m.readLockfile() - ); - if ( - await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH) - ) { - if (!noStaleMessage) { - log.info( - colors.green(`App ${remote_path} metadata is up-to-date, skipping`) - ); + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + // For raw apps in new format, runnables are in separate files under backend/ + let treeAppValue = structuredClone(appValue); + if (rawApp) { + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER); + const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath); + if (Object.keys(runnablesFromFiles).length > 0) { + treeAppValue = runnablesFromFiles; + } + } + + // First pass: add inline scripts as separate nodes, then add app node importing them + const inlineScriptPaths: string[] = []; + await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => { + if (!inlineScript.content || !inlineScript.language) { + return inlineScript; + } + + let content = inlineScript.content; + // Resolve !inline references + if (typeof content === "string" && content.startsWith("!inline ")) { + const filePath = appFolder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + return inlineScript; + } + } + + const treePath = folderNormalized + "/" + context.path.join("/"); + const language = inlineScript.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false); + inlineScriptPaths.push(treePath); + + return inlineScript; + }); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp); + return; + } + // Second pass: get mismatched workspace deps from tree + // TODO: pass raw workspace deps more precisely to every inline script lock generation call + // (currently we pass the union of all mismatched deps filtered for the whole app) + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder); + } else { + // Legacy behaviour + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder); + + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`App ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -179,6 +226,8 @@ export async function generateAppLocksInternal( let updatedScripts: string[] = []; if (!justUpdateMetadataLock) { + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const changedScripts = []; // Find hashes that do not correspond to previous hashes for (const [scriptPath, hash] of Object.entries(hashes)) { @@ -190,7 +239,13 @@ export async function generateAppLocksInternal( } } - if (changedScripts.length > 0) { + // Get temp_script_refs from tree for relative import resolution + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); + + // In tree mode, the tree already verified this app is stale (possibly via dependency change). + // Per-script hashes only detect content changes, not transitive dependency changes, + // so we must regenerate locks for all inline scripts regardless. + if (changedScripts.length > 0 || (tree && !legacyBehaviour)) { if (!noStaleMessage) { log.info( `Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}` @@ -219,7 +274,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); // Note: updateRawAppRunnables now writes each runnable to its own file } else { @@ -236,7 +292,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); normalAppFile.value = result.value; updatedScripts = result.updatedScripts; @@ -252,15 +309,16 @@ export async function generateAppLocksInternal( } } - // Regenerate hashes after updates - hashes = await generateAppHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateAppHash( + depsForHash, appFolder, rawApp, opts.defaultTs ); await clearGlobalLock(appFolder); - for (const [scriptPath, hash] of Object.entries(hashes)) { + for (const [scriptPath, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(appFolder, hash, scriptPath); } if (!noStaleMessage) { @@ -366,7 +424,8 @@ async function updateRawAppRunnables( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise { const updatedRunnables: string[] = []; const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); @@ -446,7 +505,8 @@ async function updateRawAppRunnables( content, language, `${remotePath}/${runnableId}`, - rawDeps + rawDeps, + tempScriptRefs ); // Determine file extension for this language @@ -513,7 +573,8 @@ async function updateAppInlineScripts( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise<{ value: any; updatedScripts: string[] }> { const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }); const updatedScripts: string[] = []; @@ -561,7 +622,8 @@ async function updateAppInlineScripts( content, language, scriptPath, - rawDeps + rawDeps, + tempScriptRefs ); } // Determine file extension for this language (following extractInlineScriptsForApps pattern) @@ -626,7 +688,8 @@ async function generateInlineScriptLock( content: string, language: string, scriptPath: string, - rawWorkspaceDependencies: Record | undefined + rawWorkspaceDependencies: Record | undefined, + tempScriptRefs?: Record ): Promise { // Filter workspace dependencies to only include those matching this script's language and annotations const filteredDeps = rawWorkspaceDependencies @@ -657,6 +720,9 @@ async function generateInlineScriptLock( ? filteredDeps : null, entrypoint: scriptPath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 4d92405d01..58d0777f90 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -437,6 +437,7 @@ async function preview( export async function generateLocks( opts: GlobalOptions & { yes?: boolean; + dryRun?: boolean; } & SyncOptions, folder: string | undefined ) { @@ -487,6 +488,10 @@ export async function generateLocks( } if (hasAny) { + if (opts.dryRun) { + log.info(colors.gray("Dry run complete.")); + return; + } if ( !opts.yes && !(await Confirm.prompt({ @@ -592,6 +597,7 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") + .option("--dry-run", "Perform a dry run without making changes") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 6a9040094f..1f5d86b8ea 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -29,10 +29,9 @@ import { FlowFile } from "./flow.ts"; import { FlowValue } from "../../../gen/types.gen.ts"; import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; import { workspaceDependenciesLanguages } from "../../utils/script_common.ts"; -import { - extractNameFromFolder, - getNonDottedPaths, -} from "../../utils/resource_folders.ts"; +import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__flow_hash"; async function generateFlowHash( @@ -70,7 +69,9 @@ export async function generateFlowLockInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); @@ -80,33 +81,67 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - // Always get out-of-sync workspace dependencies - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - const flowValue = (await yamlParseFile( folder! + SEP + "flow.yaml" )) as FlowFile; - // Filter workspace dependencies based on inline scripts' languages and annotations - const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); - - let hashes = await generateFlowHash( - filteredDeps, - folder, + const folderNormalized = folder.replaceAll(SEP, "/"); + const inlineScriptsForTree = extractInlineScriptsForFlows( + structuredClone(flowValue.value.modules), + {}, + SEP, opts.defaultTs - ); + ).filter(s => !s.is_lock); + let filteredDeps: Record = {}; const conf = await readLockfile(); - if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { - if (!noStaleMessage) { - log.info( - colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) - ); + + if (!legacyBehaviour && tree) { + if (dryRun) { + const inlineScriptPaths: string[] = []; + for (const script of inlineScriptsForTree) { + let content = script.content; + if (content.startsWith("!inline ")) { + const filePath = folder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + continue; + } + } + + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + const language = script.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false); + inlineScriptPaths.push(treePath); + } + + const hashes = await generateFlowHash({}, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale); + return; + } + // Second pass: get mismatched workspace deps from tree + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder); + } else { + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); + + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -122,7 +157,23 @@ export async function generateFlowLockInternal( let changedScripts: string[] = []; + // Build mapping from on-disk file names (hash keys like "a.py") to tree paths + // (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without + // a path assigner, so paths always have .inline_script suffix, but on-disk files + // may not (non-dotted mode). + const fileToTreePath = new Map(); + for (const script of inlineScriptsForTree) { + const c = script.content; + if (c.startsWith("!inline ")) { + const fileName = c.replace("!inline ", ""); + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + fileToTreePath.set(fileName, treePath); + } + } + if (!justUpdateMetadataLock) { + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + //find hashes that do not correspond to previous hashes for (const [path, hash] of Object.entries(hashes)) { if (path == TOP_HASH) { @@ -137,27 +188,39 @@ export async function generateFlowLockInternal( log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`); } const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8"); + // In tree mode, use the tree's staleness info (which includes transitive dependency changes) + // to determine which scripts need relocking, instead of only content-changed ones. + const locksToRemove = (tree && !legacyBehaviour) + ? Object.keys(hashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; await replaceInlineScripts( flowValue.value.modules, fileReader, log, folder + SEP!, SEP, - changedScripts + locksToRemove ); if (flowValue.value.failure_module) { - await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } if (flowValue.value.preprocessor_module) { - await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } //removeChangedLocks + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); flowValue.value = await updateFlow( workspace, flowValue.value, remote_path, - filteredDeps + filteredDeps, + tempScriptRefs ); const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", { @@ -187,13 +250,15 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateFlowHash( + depsForHash, folder, opts.defaultTs ); await clearGlobalLock(folder); - for (const [path, hash] of Object.entries(hashes)) { + for (const [path, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(folder, hash, path); } if (!noStaleMessage) { @@ -201,7 +266,16 @@ export async function generateFlowLockInternal( } // Return the list of updated scripts (extract just the filename from the path) - const updatedScripts = changedScripts.map(p => { + // In tree mode, use the same staleness-aware list we used for lock removal + const relocked = (tree && !legacyBehaviour) + ? Object.keys(finalHashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; + const updatedScripts = relocked.map(p => { const parts = p.split(SEP); return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension }); @@ -239,7 +313,8 @@ export async function updateFlow( workspace: Workspace, flow_value: FlowValue, remotePath: string, - rawWorkspaceDependencies: Record + rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { let rawResponse; @@ -264,6 +339,9 @@ export async function updateFlow( path: remotePath, use_local_lockfiles: true, raw_workspace_dependencies: rawWorkspaceDependencies, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); @@ -282,6 +360,9 @@ export async function updateFlow( body: JSON.stringify({ flow_value, path: remotePath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index c523996cdc..d273f4b631 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -10,6 +10,8 @@ import * as log from "../../core/log.ts"; import { generateScriptMetadataInternal, getRawWorkspaceDependencies, + readLockfile, + checkifMetadataUptodate, } from "../../utils/metadata.ts"; import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts"; import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts"; @@ -19,14 +21,20 @@ import { ignoreF, } from "../sync/sync.ts"; import { exts } from "../script/script.ts"; -import { isFlowPath, isAppPath, isRawAppPath, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; +import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; import { listSyncCodebases } from "../../utils/codebase.ts"; +import { + DoubleLinkedDependencyTree, + uploadScripts, + ItemType, +} from "../../utils/dependency_tree.ts"; interface StaleItem { - type: "script" | "flow" | "app"; + type: ItemType; path: string; folder: string; isRawApp?: boolean; + staleReason?: string; } async function generateMetadata( @@ -38,6 +46,7 @@ async function generateMetadata( skipScripts?: boolean; skipFlows?: boolean; skipApps?: boolean; + strictFolderBoundaries?: boolean; } & SyncOptions, folder?: string ) { @@ -49,12 +58,10 @@ async function generateMetadata( await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false); const codebases = await listSyncCodebases(opts); const ignore = await ignoreF(opts); - const staleItems: StaleItem[] = []; - // --schema-only implies skipping flows and apps (they only have locks, no schemas) const skipScripts = opts.skipScripts ?? false; const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false; @@ -70,7 +77,11 @@ async function generateMetadata( return; } - log.info(colors.gray(`Checking ${checking.join(", ")}...`)); + log.info(`Checking ${checking.join(", ")}...`); + + // Build dependency tree for relative import tracking + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); // === Collect stale scripts === if (!skipScripts) { @@ -81,9 +92,7 @@ async function generateMetadata( return ( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || - isFlowPath(p) || - isAppPath(p) || - isRawAppPath(p) || + isFolderResourcePathAnyFormat(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)) ); }, @@ -92,19 +101,18 @@ async function generateMetadata( ); for (const e of Object.keys(scriptElems)) { - const candidate = await generateScriptMetadataInternal( + await generateScriptMetadataInternal( e, workspace, opts, - true, // dryRun + true, // dryRun - populate tree true, // noStaleMessage rawWorkspaceDependencies, codebases, - false + false, + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "script", path: candidate, folder: e }); - } } } @@ -126,18 +134,17 @@ async function generateMetadata( ) ).map((x) => x.substring(0, x.lastIndexOf(SEP))); - for (const folder of flowElems) { - const candidate = await generateFlowLockInternal( - folder, - true, // dryRun + for (const flowFolder of flowElems) { + await generateFlowLockInternal( + flowFolder, + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "flow", path: candidate, folder }); - } } } @@ -161,33 +168,74 @@ async function generateMetadata( const appFolders = getAppFolders(elems, "app.yaml"); for (const appFolder of rawAppFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, true, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: true }); - } } for (const appFolder of appFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, false, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: false }); - } + } + } + + // === Propagate staleness through imports === + tree.propagateStaleness(); + + // Upload stale scripts to temp storage so the backend can resolve relative imports. + // If this fails (e.g. backend is older and doesn't have /raw_temp endpoints), + // degrade gracefully: locks will be generated using deployed script content only. + try { + await uploadScripts(tree, workspace); + } catch (e) { + log.warn(colors.yellow( + `Failed to upload scripts to temp storage (backend may be too old): ${e}. ` + + `Locks will be generated using deployed script versions only — locally modified ` + + `relative imports may not be reflected.` + )); + } + + // === Populate staleItems from tree === + const staleItems: StaleItem[] = []; + const seenFolders = new Set(); + + for (const p of tree.allPaths()) { + const staleReason = tree.getStaleReason(p); + if (!staleReason) continue; + + const itemType = tree.getItemType(p)!; + const itemFolder = tree.getFolder(p)!; + + if (itemType === "dependencies") { + staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason }); + } else if (itemType === "inline_script") { + // Inline scripts are not listed separately — their parent flow/app is stale via propagation + continue; + } else if (itemType === "script") { + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason }); + } else if (!seenFolders.has(itemFolder)) { + // Flows/Apps: one entry per folder (dedupe multiple inline scripts) + seenFolders.add(itemFolder); + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason }); } } @@ -200,11 +248,54 @@ async function generateMetadata( if (folder.endsWith("/")) { folder = folder.substring(0, folder.length - 1); } - // Normalize item.folder for comparison (Windows file paths use backslashes) - filteredItems = staleItems.filter((item) => { + // Strip file extension if user passed a specific file path (e.g. f/test/script.ts) + const folderNoExt = folder.replace(/\.[^/.]+$/, ""); + // Check if an item is inside the specified folder + const isInsideFolder = (item: StaleItem) => { const normalizedFolder = item.folder.replaceAll("\\", "/"); - return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/"); - }); + const normalizedPath = item.path.replaceAll("\\", "/"); + return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/") + || normalizedPath === folder || normalizedPath === folderNoExt; + }; + const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt; + // Check if a tree path or any of its transitive deps is inside the folder + const touchesFolder = (treePath: string) => { + if (isPathInFolder(treePath)) return true; + let found = false; + tree.traverseTransitive(treePath, (importPath) => { + if (isPathInFolder(importPath)) { + found = true; + return true; // stop early + } + }); + return found; + }; + + const isRelevant = (item: StaleItem) => { + if (isInsideFolder(item)) return true; + if (item.type === "dependencies") return true; + const treePath = (item.type === "script" + ? item.path.replace(/\.[^/.]+$/, "") + : item.folder).replaceAll("\\", "/"); + return touchesFolder(treePath); + }; + + if (opts.strictFolderBoundaries) { + // Strict mode: only items inside the folder + filteredItems = staleItems.filter(isInsideFolder); + + // Warn about stale items outside the folder that would be included by default + const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies"); + for (const item of excludedStale) { + const normalizedPath = item.path.replaceAll("\\", "/"); + log.warn(colors.yellow( + `Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.` + )); + } + } else { + // Default: include items inside the folder and any stale importers that transitively depend on it + filteredItems = staleItems.filter(isRelevant); + } } // === Show stale items and confirm === @@ -217,28 +308,24 @@ async function generateMetadata( const scripts = filteredItems.filter((i) => i.type === "script"); const flows = filteredItems.filter((i) => i.type === "flow"); const apps = filteredItems.filter((i) => i.type === "app"); + const deps = filteredItems.filter((i) => i.type === "dependencies"); log.info(""); - log.info(`Found ${filteredItems.length} item(s) with stale metadata:`); + log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`); - if (scripts.length > 0) { - log.info(colors.gray(` Scripts (${scripts.length}):`)); - for (const item of scripts) { - log.info(colors.yellow(` ${item.path}`)); + const printItems = (label: string, items: StaleItem[]) => { + if (items.length === 0) return; + log.info(` ${label} (${items.length}):`); + for (const item of items) { + const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : ""; + log.info(` ~ ${item.path}` + reason); } - } - if (flows.length > 0) { - log.info(colors.gray(` Flows (${flows.length}):`)); - for (const item of flows) { - log.info(colors.yellow(` ${item.path}`)); - } - } - if (apps.length > 0) { - log.info(colors.gray(` Apps (${apps.length}):`)); - for (const item of apps) { - log.info(colors.yellow(` ${item.path}`)); - } - } + }; + + printItems("Workspace dependencies", deps); + printItems("Scripts", scripts); + printItems("Flows", flows); + printItems("Apps", apps); if (opts.dryRun) { return; @@ -259,28 +346,30 @@ async function generateMetadata( log.info(""); // === Process all stale items with progress counter === - const total = filteredItems.length; + const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps(); + const total = filteredItems.length - deps.length; const maxWidth = `[${total}/${total}]`.length; let current = 0; const formatProgress = (n: number) => { - const bracket = `[${n}/${total}]`; - return colors.gray(bracket.padEnd(maxWidth, " ")); + return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " "))); }; // Process scripts for (const item of scripts) { current++; - log.info(`${formatProgress(current)} script ${colors.cyan(item.path)}`); + log.info(`${formatProgress(current)} script ${item.path}`); await generateScriptMetadataInternal( - item.folder, + item.path, // originalPath with extension workspace, opts, false, // dryRun - true, // noStaleMessage - we handle output - rawWorkspaceDependencies, + true, // noStaleMessage + mismatchedWorkspaceDeps, codebases, - false + false, + false, // legacyBehaviour + tree ); } @@ -288,38 +377,49 @@ async function generateMetadata( for (const item of flows) { current++; const result = await generateFlowLockInternal( - item.folder, + item.folder.replaceAll("/", SEP), false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as FlowLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const flowResult = result as FlowLocksResult | undefined; + const scriptsInfo = flowResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} flow ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`); } + // Process apps for (const item of apps) { current++; const result = await generateAppLocksInternal( - item.folder, + item.folder.replaceAll("/", SEP), item.isRawApp!, // rawApp false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as AppLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const appResult = result as AppLocksResult | undefined; + const scriptsInfo = appResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} app ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`); } + // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped) + const allStaleDeps = staleItems.filter((i) => i.type === "dependencies"); + await tree.persistDepsHashes(allStaleDeps.map((d) => d.path)); + log.info(""); - log.info(colors.green(`Done. Updated ${total} item(s).`)); + log.info(`Done. Updated ${colors.bold(String(total))} item(s).`); } const command = new Command() @@ -332,6 +432,7 @@ const command = new Command() .option("--skip-scripts", "Skip processing scripts") .option("--skip-flows", "Skip processing flows") .option("--skip-apps", "Skip processing apps") + .option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)") .option( "-i --includes ", "Comma separated patterns to specify which files to include" diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 68ff946cee..14f3155ff7 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -130,7 +130,7 @@ async function push(opts: PushOptions, filePath: string) { [], undefined, opts, - await getRawWorkspaceDependencies(), + await getRawWorkspaceDependencies(true), codebases ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); @@ -1161,7 +1161,7 @@ export async function generateMetadata( opts = await mergeConfigWithConfigFile(opts); const codebases = await listSyncCodebases(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); if (scriptPath) { // read script metadata file await generateScriptMetadataInternal( diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index bc0eee2275..d62c39cba9 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2280,7 +2280,7 @@ export async function pull( const tracker: ChangeTracker = await buildTracker(changes); const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); + await getRawWorkspaceDependencies(true); for (const change of tracker.scripts) { await generateScriptMetadataInternal( @@ -2611,7 +2611,7 @@ export async function push( false, // els1 (local) is not the remote source ); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); const tracker: ChangeTracker = await buildTracker(changes); @@ -2657,7 +2657,7 @@ export async function push( true, ); if (stale) { - staleFlows.push(stale); + staleFlows.push(stale as string); } } @@ -2682,7 +2682,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } @@ -2697,7 +2697,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts new file mode 100644 index 0000000000..5e2adf7453 --- /dev/null +++ b/cli/src/utils/dependency_tree.ts @@ -0,0 +1,373 @@ +/** + * Double-linked dependency tree for tracking script imports and propagating staleness. + */ + +import { Workspace } from "../commands/workspace/workspace.ts"; +import * as wmill from "../../gen/services.gen.ts"; +import type { ScriptLang } from "../../gen/types.gen.ts"; +import { ScriptLanguage } from "./script_common.ts"; +import { + filterWorkspaceDependencies, + generateScriptHash, + checkifMetadataUptodate, + workspaceDependenciesPathToLanguageAndFilename, + updateMetadataGlobalLock, +} from "./metadata.ts"; +import { generateHash } from "./utils.ts"; + +/** + * Diff local scripts against deployed versions, upload only those that differ. + * Only uploaded (mismatched) scripts get contentHash set, so flatten() returns + * temp_script_refs only for scripts the backend can't resolve from deployed versions. + */ +export async function uploadScripts( + tree: DoubleLinkedDependencyTree, + workspace: Workspace +): Promise { + // Split into scripts vs workspace deps and compute SHA256(content) for each + const scriptHashes: Record = {}; + const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = []; + + for (const path of tree.allPaths()) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Empty string is valid for workspace deps (means "no deps") — only skip undefined + if (content === undefined) continue; + const info = workspaceDependenciesPathToLanguageAndFilename(path); + if (info) { + const hash = await generateHash(content); + workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash }); + } + } else if (itemType === "script") { + if (!content) continue; + const hash = await generateHash(content); + scriptHashes[path] = hash; + } + // Skip inline_script, flow, app — they don't need temp storage uploads + } + + if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return; + + // Single batch query: find which scripts/deps differ from deployed versions + const mismatched = await wmill.diffRawScriptsWithDeployed({ + workspace: workspace.workspaceId, + requestBody: { + scripts: scriptHashes, + workspace_deps: workspaceDeps, + }, + }); + + // Upload only mismatched scripts to temp storage + for (const path of mismatched) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Workspace deps don't need temp storage — just mark as mismatched. + // Empty string is valid (means the dep file was emptied locally). + if (content !== undefined) { + tree.setContentHash(path, "mismatched"); + } + } else if (content) { + const hash = await wmill.storeRawScriptTemp({ + workspace: workspace.workspaceId, + requestBody: content, + }); + tree.setContentHash(path, hash); + } + } +} + +export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies"; + +interface DependencyNode { + content: string; + stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata) + contentHash?: string; // Hash for temp storage lookup (content only) + language: ScriptLanguage; + metadata: string; + imports: Set; + importedBy: Set; + staleReason?: string; + // Item metadata for generate-metadata command + itemType: ItemType; + folder: string; // Folder path (for flows/apps) or remote path (for scripts) + originalPath: string; // Original path passed to handler (with extension for scripts) + isRawApp?: boolean; // Only set for apps + isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) +} + +export class DoubleLinkedDependencyTree { + private nodes: Map = new Map(); + private workspaceDeps: Record = {}; + + setWorkspaceDeps(deps: Record): void { + this.workspaceDeps = deps; + } + + async addNode( + path: string, + content: string, + language: ScriptLanguage, + metadata: string, + imports: string[], + itemType: ItemType, + folder: string, + originalPath: string, + isDirectlyStale: boolean, + isRawApp?: boolean + ): Promise { + const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script"; + const filteredDeps = hasWorkspaceDeps + ? filterWorkspaceDependencies(this.workspaceDeps, content, language) + : {}; + const stalenessHash = await generateScriptHash({}, content, metadata); + + if (!this.nodes.has(path)) { + this.nodes.set(path, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + const node = this.nodes.get(path)!; + node.content = content; + node.stalenessHash = stalenessHash; + node.language = language; + node.metadata = metadata; + node.itemType = itemType; + node.folder = folder; + node.originalPath = originalPath; + node.isDirectlyStale = isDirectlyStale; + node.isRawApp = isRawApp; + + // Create nodes for referenced workspace deps with content and language. + const filteredDepsPaths = Object.keys(filteredDeps); + for (const depsPath of filteredDepsPaths) { + if (!this.nodes.has(depsPath)) { + const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath); + const contentHash = await generateHash(filteredDeps[depsPath] + depsPath); + const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined); + this.nodes.set(depsPath, { + content: filteredDeps[depsPath], + stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "dependencies", folder: "", originalPath: depsPath, + isDirectlyStale: !isUpToDate, + }); + } + } + + const allImports = [...imports, ...filteredDepsPaths]; + for (const importPath of allImports) { + node.imports.add(importPath); + + if (!this.nodes.has(importPath)) { + this.nodes.set(importPath, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + this.nodes.get(importPath)!.importedBy.add(path); + } + } + + getContent(path: string): string | undefined { + return this.nodes.get(path)?.content; + } + + getStalenessHash(path: string): string | undefined { + return this.nodes.get(path)?.stalenessHash; + } + + getContentHash(path: string): string | undefined { + return this.nodes.get(path)?.contentHash; + } + + setContentHash(path: string, hash: string): void { + const node = this.nodes.get(path); + if (node) { + node.contentHash = hash; + } + } + + getLanguage(path: string): ScriptLanguage | undefined { + return this.nodes.get(path)?.language; + } + + getMetadata(path: string): string | undefined { + return this.nodes.get(path)?.metadata; + } + + getStaleReason(path: string): string | undefined { + return this.nodes.get(path)?.staleReason; + } + + getItemType(path: string): ItemType | undefined { + return this.nodes.get(path)?.itemType; + } + + getFolder(path: string): string | undefined { + return this.nodes.get(path)?.folder; + } + + getIsRawApp(path: string): boolean | undefined { + return this.nodes.get(path)?.isRawApp; + } + + getIsDirectlyStale(path: string): boolean { + return this.nodes.get(path)?.isDirectlyStale ?? false; + } + + getOriginalPath(path: string): string | undefined { + return this.nodes.get(path)?.originalPath; + } + + getImports(path: string): Set | undefined { + return this.nodes.get(path)?.imports; + } + + /** + * Returns true if this node has been marked stale (directly or transitively). + */ + isStale(path: string): boolean { + return this.nodes.get(path)?.staleReason !== undefined; + } + + /** + * Mutates the tree by removing all nodes that are not stale. + * Uses BFS on reverse graph (importedBy) to find all stale scripts. + * Starts from nodes with isDirectlyStale=true. + */ + propagateStaleness(): void { + // Collect directly stale nodes + const directlyStale = new Set(); + for (const [path, node] of this.nodes.entries()) { + if (node.isDirectlyStale) { + directlyStale.add(path); + node.staleReason = "content changed"; + } + } + + const allStale = new Set(directlyStale); + const queue = [...directlyStale]; + const visited = new Set(); + + while (queue.length > 0) { + const scriptPath = queue.shift()!; + if (visited.has(scriptPath)) continue; + visited.add(scriptPath); + + const node = this.nodes.get(scriptPath); + if (!node) continue; + + for (const importer of node.importedBy) { + if (!allStale.has(importer)) { + allStale.add(importer); + queue.push(importer); + // Set reason for transitively stale scripts + const importerNode = this.nodes.get(importer); + if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`; + } + } + } + + } + + /** + * Walks all transitive imports for a node, calling the callback for each. + * Callback may return true to stop traversing that branch. + */ + traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void { + const queue = [scriptPath]; + const visited = new Set(); + + while (queue.length > 0) { + const current = queue.shift()!; + if (visited.has(current)) continue; + visited.add(current); + + const node = this.nodes.get(current); + if (!node) continue; + + for (const importPath of node.imports) { + const importNode = this.nodes.get(importPath); + if (importNode) { + const stop = callback(importPath, importNode); + if (!stop) { + queue.push(importPath); + } + } + } + } + } + + allPaths(): IterableIterator { + return this.nodes.keys(); + } + + /** + * Returns paths of all stale nodes (those with a staleReason). + */ + *stalePaths(): IterableIterator { + for (const [path, node] of this.nodes.entries()) { + if (node.staleReason) { + yield path; + } + } + } + + has(path: string): boolean { + return this.nodes.has(path); + } + + /** + * Returns workspace deps that were uploaded as mismatched with remote. + * These need to be passed as raw_workspace_dependencies in job args + * so the backend uses local content instead of deployed. + */ + getMismatchedWorkspaceDeps(): Record { + const result: Record = {}; + for (const [path, node] of this.nodes.entries()) { + if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) { + result[path] = node.content; + } + } + return result; + } + + /** + * Returns path → contentHash for all transitive imports that have been uploaded. + * Must be called after uploadScripts() has populated contentHash values. + */ + getTempScriptRefs(scriptPath: string): Record { + const result: Record = {}; + this.traverseTransitive(scriptPath, (_path, node) => { + if (node.contentHash) { + result[_path] = node.contentHash; + } + }); + return result; + } + + /** + * Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies + * considers them up-to-date on the next run. + */ + async persistDepsHashes(depsPaths: string[]): Promise { + for (const path of depsPaths) { + const node = this.nodes.get(path); + if (node?.itemType === "dependencies" && node.content !== undefined) { + const hash = await generateHash(node.content + path); + await updateMetadataGlobalLock(path, hash); + } + } + } + + get size(): number { + return this.nodes.size; + } +} diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 0a3340cc6d..0e010a0cc5 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -26,11 +26,13 @@ import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts"; import { getIsWin } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "./dependency_tree.ts"; const _require = createRequire(import.meta.url); const _parserCache = new Map>(); -function loadParser(pkgName: string): Promise { +export function loadParser(pkgName: string): Promise { let p = _parserCache.get(pkgName); if (!p) { p = (async () => { @@ -54,7 +56,7 @@ export class LockfileGenerationError extends Error { } -export async function getRawWorkspaceDependencies(): Promise> { +export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise> { const rawWorkspaceDeps: Record = {}; try { @@ -68,11 +70,13 @@ export async function getRawWorkspaceDependencies(): Promise, codebases: SyncCodebase[], - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { // Detect folder layout: my_script__mod/script.ts const isFolderLayout = isModuleEntryPoint(scriptPath); @@ -222,13 +228,15 @@ export async function generateScriptMetadataInternal( const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory(); - let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent); + // In non-legacy mode, workspace deps are tracked via the tree — exclude from hash + const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies; + let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent); // Compute per-module hashes for stale detection (like flow inline scripts) let moduleHashes: Record = {}; if (hasModules) { moduleHashes = await computeModuleHashes( - moduleFolderPath, opts.defaultTs, rawWorkspaceDependencies, isFolderLayout + moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout ); } const hasModuleHashes = Object.keys(moduleHashes).length > 0; @@ -243,27 +251,43 @@ export async function generateScriptMetadataInternal( } const conf = await readLockfile(); - if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { - if (!noStaleMessage) { - log.info( - colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) - ); + + // Use checkHash (includes module hashes) so module changes are detected as stale + const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)); + + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + // First pass: populate tree with script and its imports + const imports = await extractRelativeImports(scriptContent, remotePath, language); + await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale); + return; } - return; - } else if (dryRun) { - let detail = `${remotePath} (${language})`; - if (hasModuleHashes) { - const changed: string[] = []; - for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { - if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { - changed.push(modulePath); + // Second pass: proceed to generate (caller verified this script is stale via tree) + } else { + // Legacy behaviour: use existing staleness check + if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { + if (!noStaleMessage) { + log.info( + colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + let detail = `${remotePath} (${language})`; + if (hasModuleHashes) { + const changed: string[] = []; + for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { + if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { + changed.push(modulePath); + } + } + if (changed.length > 0) { + detail += ` [changed modules: ${changed.join(", ")}]`; } } - if (changed.length > 0) { - detail += ` [changed modules: ${changed.join(", ")}]`; - } + return detail; } - return detail; } if (!justUpdateMetadataLock && !noStaleMessage) { @@ -288,6 +312,7 @@ export async function generateScriptMetadataInternal( const hasCodebase = findCodebase(scriptPath, codebases) != undefined; if (!hasCodebase) { + const tempScriptRefs = tree?.getTempScriptRefs(remotePath); const lockPathOverride = isFolderLayout ? path.dirname(scriptPath) + "/script.lock" : undefined; @@ -298,6 +323,7 @@ export async function generateScriptMetadataInternal( remotePath, metadataParsedContent, filteredRawWorkspaceDependencies, + tempScriptRefs, lockPathOverride, ); } else { @@ -358,7 +384,7 @@ export async function generateScriptMetadataInternal( const metadataContentUsedForHash = newMetadataContent; hash = await generateScriptHash( - filteredRawWorkspaceDependencies, + depsForHash, scriptContent, metadataContentUsedForHash ); @@ -511,6 +537,7 @@ export async function computeLockCacheKey( scriptContent: string, language: ScriptLanguage, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const annotation = extractWorkspaceDepsAnnotation(scriptContent, language); const annotationStr = annotation @@ -518,7 +545,10 @@ export async function computeLockCacheKey( : "none"; const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort(); const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";"); - return await generateHash(`${language}|${annotationStr}|${depsStr}`); + const tempRefsStr = tempScriptRefs + ? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";") + : ""; + return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`); } const lockCache = new Map(); @@ -533,13 +563,15 @@ async function fetchScriptLock( language: ScriptLanguage, remotePath: string, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0; - const cacheKey = hasRawDeps - ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies) + const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0; + const cacheKey = (hasRawDeps || hasTempRefs) + ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs) : undefined; if (cacheKey && lockCache.has(cacheKey)) { - log.info(`Using cached lockfile for ${remotePath}`); + log.debug(`Using cached lockfile for ${remotePath}`); return lockCache.get(cacheKey)!; } @@ -564,6 +596,8 @@ async function fetchScriptLock( raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 ? rawWorkspaceDependencies : null, entrypoint: remotePath, + temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? tempScriptRefs : null, }), } ); @@ -604,6 +638,7 @@ async function updateScriptLock( remotePath: string, metadataContent: Record, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record, lockPathOverride?: string, ): Promise { if ( @@ -621,7 +656,7 @@ async function updateScriptLock( if (Object.keys(rawWorkspaceDependencies).length > 0) { const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', '); - log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); + log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); } const lock = await fetchScriptLock( @@ -630,6 +665,7 @@ async function updateScriptLock( language, remotePath, rawWorkspaceDependencies, + tempScriptRefs ); const lockPath = lockPathOverride ?? remotePath + ".script.lock"; @@ -692,7 +728,7 @@ async function updateModuleLocks( const moduleContent = readFileSync(fullPath, "utf-8"); const moduleRemotePath = scriptRemotePath + "/" + relPath; - log.info(colors.gray(`Generating lock for module ${relPath}`)); + log.debug(`Generating lock for module ${relPath}`); try { const lock = await fetchScriptLock( diff --git a/cli/src/utils/relative_imports.ts b/cli/src/utils/relative_imports.ts new file mode 100644 index 0000000000..7ea2d582a6 --- /dev/null +++ b/cli/src/utils/relative_imports.ts @@ -0,0 +1,39 @@ +/** + * Relative Imports Utilities for CLI + * + * Provides functions to parse relative imports from TypeScript/Python scripts using WASM. + */ + +import { ScriptLanguage } from "./script_common.ts"; +import { loadParser } from "./metadata.ts"; +import * as log from "../core/log.ts"; + +/** + * Extract relative imports from script content based on language. + * Returns resolved absolute Windmill paths (e.g., "f/folder/helper"). + */ +export async function extractRelativeImports( + code: string, + scriptPath: string, + language: ScriptLanguage +): Promise { + try { + switch (language) { + case "bun": + case "nativets": + case "deno": { + const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts"); + return parse_ts_relative_imports(code, scriptPath); + } + case "python3": { + const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports"); + return parse_py_relative_imports(code, scriptPath); + } + default: + return []; + } + } catch (e) { + log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`); + return []; + } +} diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 713a47ecb1..1531314f04 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean { return isFlowPath(p) || isAppPath(p) || isRawAppPath(p); } +/** + * Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app) + * and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting. + * Use this instead of isFolderResourcePath when the config may not yet be loaded or when + * you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files). + */ +export function isFolderResourcePathAnyFormat(p: string): boolean { + const n = normalizeSep(p); + for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) { + if ( + n.includes(suffixes.flow + "/") || + n.includes(suffixes.app + "/") || + n.includes(suffixes.raw_app + "/") + ) { + return true; + } + } + return false; +} + /** * Detect the resource type from a path, if any */ diff --git a/cli/test/cargo_backend.ts b/cli/test/cargo_backend.ts index 5c9719f6d5..a25a788f7e 100644 --- a/cli/test/cargo_backend.ts +++ b/cli/test/cargo_backend.ts @@ -63,11 +63,11 @@ export class CargoBackend { // Determine default features based on environment // CI mode: minimal features (zip only) - // Local mode with license key: full features (zip, private, enterprise, license) + // Local mode with license key: full features (zip, private, enterprise, license, python) // Local mode without license key: zip only (EE features reject API calls without valid license) const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; const hasLicenseKey = !!process.env["EE_LICENSE_KEY"]; - const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license"] : ["zip"]); + const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license", "python"] : ["zip", "python"]); // Parse additional features from environment variable const envFeatures = process.env["TEST_FEATURES"]?.split(",").filter(f => f.trim()) || []; @@ -328,6 +328,8 @@ export class CargoBackend { SQLX_OFFLINE: "true", // Disable embedding to speed up startup DISABLE_EMBEDDING: "true", + // Skip worker version check for workspace deps (workers need time to report version) + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: "1", // Create default admin user CREATE_SUPERADMIN_IF_NOT_EXISTS: "1", SUPERADMIN_EMAIL: this.config.username, @@ -708,6 +710,7 @@ export class CargoBackend { this.deleteAll("resources"), this.deleteAll("variables"), this.deleteAll("folders"), + this.deleteAllWorkspaceDeps(), ]); console.log("Workspace reset complete"); @@ -735,6 +738,28 @@ export class CargoBackend { // Ignore listing failures } } + + private async deleteAllWorkspaceDeps(): Promise { + try { + const listResponse = await this.apiRequest(`/api/w/${this.config.workspace}/workspace_dependencies/list`); + if (!listResponse.ok) return; + + const items = await listResponse.json() as { language: string; name?: string }[]; + for (const item of items) { + try { + const nameParam = item.name ? `?name=${encodeURIComponent(item.name)}` : ""; + await this.apiRequest( + `/api/w/${this.config.workspace}/workspace_dependencies/delete/${item.language}${nameParam}`, + { method: "POST" } + ); + } catch { + // Ignore individual deletion failures + } + } + } catch { + // Ignore failures + } + } } // Global backend instance diff --git a/cli/test/relative_imports_skip.test.ts b/cli/test/relative_imports_skip.test.ts new file mode 100644 index 0000000000..e43b10aa12 --- /dev/null +++ b/cli/test/relative_imports_skip.test.ts @@ -0,0 +1,420 @@ +/** + * Relative Imports Tests + * + * E2E tests for the `generate-metadata` command with relative imports: + * - Lock files correctly include transitive dependencies + * - Staleness propagates through import chains + * - Various import patterns handled correctly + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile, mkdir } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; + +// TODO: re-enable Python tests on CI if python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +const defaultMetadata = `summary: "Test" +schema: + type: object + properties: {} +lock: "" +`; + +// ============================================================================= +// Test 1: TS basic import with npm dependency propagation +// ============================================================================= + +test("TS: imported script's npm dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code, `generate-metadata failed:\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr}`).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 2: TS chained imports - dependency propagates through chain +// ============================================================================= + +test("TS: chained imports propagate npm deps through entire chain", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { utilB } from "./script_b.ts"; +export async function main() { return utilB(); } +`; + const scriptB = `import { utilC } from "./script_c.ts"; +export function utilB() { return utilC() + " B"; } +`; + const scriptC = `import _ from "lodash"; +export function utilC() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + + expect(lockC).toContain("lodash"); + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 3: TS circular imports - completes without hanging, locks generated +// ============================================================================= + +test("TS: circular imports handled gracefully with correct locks", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Circular: A imports B, B imports A, B has npm dep + const scriptA = `import { funcB } from "./script_b.ts"; +export function funcA() { return "A"; } +export async function main() { return funcA() + funcB(); } +`; + const scriptB = `import { funcA } from "./script_a.ts"; +import _ from "lodash"; +export function funcB() { return _.VERSION + funcA(); } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 4: Python basic import with pip dependency propagation +// ============================================================================= + +test.skipIf(isCI)("Python: imported script's pip dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPy); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 5: Diamond dependency - A imports B and C, both import D +// ============================================================================= + +test.skipIf(isCI)("Python: diamond dependency pattern propagates correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Diamond: A -> B, A -> C, B -> D, C -> D + const scriptA = `from f.test.script_b import func_b +from f.test.script_c import func_c + +def main(): + return func_b() + func_c() +`; + const scriptB = `from f.test.script_d import func_d + +def func_b(): + return "B" + func_d() +`; + const scriptC = `from f.test.script_d import func_d + +def func_c(): + return "C" + func_d() +`; + const scriptD = `import requests + +def func_d(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/script_a.py`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.py`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.py`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_d.py`, scriptD); + await writeFile(`${tempDir}/f/test/script_d.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + const lockD = await readFile(`${tempDir}/f/test/script_d.script.lock`, "utf-8").catch(() => ""); + + expect(lockD).toContain("requests"); + expect(lockB).toContain("requests"); + expect(lockC).toContain("requests"); + expect(lockA).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 6: Script isolation - unrelated script not marked stale +// ============================================================================= + +test("Script isolation: unrelated script not affected by changes", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // A imports B, C is isolated + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `export function helper() { return "B"; } +`; + const scriptC = `export async function main() { return "isolated"; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("All metadata up-to-date"); + + // Change script_b + await writeFile(`${tempDir}/f/test/script_b.ts`, + `export function helper() { return "B changed"; } +`); + + // script_a and script_b should be stale, script_c should NOT be mentioned + const check2 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("script_b"); + expect(check2.stdout).toContain("script_a"); + expect(check2.stdout).not.toMatch(/script_c/); + }); +}); + +// ============================================================================= +// Test 7: Python relative imports with dot syntax +// ============================================================================= + +test.skipIf(isCI)("Python: relative imports with dot syntax work correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/mymodule`, { recursive: true }); + + // Using relative import syntax + const mainPy = `from .helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/mymodule/main.py`, mainPy); + await writeFile(`${tempDir}/f/mymodule/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/mymodule/helper.py`, helperPy); + await writeFile(`${tempDir}/f/mymodule/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/mymodule/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/mymodule/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/mymodule/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 8: Adding new import updates importer's lock +// ============================================================================= + +test.skipIf(isCI)("Python: adding new import updates importer's lock correctly", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Initial: main imports helper, helper has no external deps + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPyInitial = `def helper_func(): + return "no deps" +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPyInitial); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + // Generate initial locks + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Add new script with pip dep + const utilsPy = `import requests + +def get_version(): + return requests.__version__ +`; + await writeFile(`${tempDir}/f/test/utils.py`, utilsPy); + await writeFile(`${tempDir}/f/test/utils.script.yaml`, defaultMetadata); + + // Modify helper to import utils + const helperPyWithImport = `from f.test.utils import get_version + +def helper_func(): + return get_version() +`; + await writeFile(`${tempDir}/f/test/helper.py`, helperPyWithImport); + + // Regenerate - main should now have requests + const afterAdd = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(afterAdd.code).toBe(0); + + const lockUtils = await readFile(`${tempDir}/f/test/utils.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + + expect(lockUtils).toContain("requests"); + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); diff --git a/cli/test/relative_imports_wasm.test.ts b/cli/test/relative_imports_wasm.test.ts new file mode 100644 index 0000000000..d1d537b001 --- /dev/null +++ b/cli/test/relative_imports_wasm.test.ts @@ -0,0 +1,1235 @@ +/** + * Tests for relative import resolution: + * 1. WASM parser unit tests — verify parse_ts/py_relative_imports work correctly + * 2. E2E tests — verify dependency propagation through scripts, flows, apps, and raw apps + * using the CLI generate-metadata command against a real backend + */ + +import { expect, test, describe, beforeAll, afterAll } from "bun:test"; +import { readFile, readdir, writeFile, mkdir } from "node:fs/promises"; +import { loadParser } from "../src/utils/metadata.ts"; +import { extractRelativeImports } from "../src/utils/relative_imports.ts"; +import { withTestBackend, type TestBackend, createRemoteWorkspaceDeps } from "./test_backend.ts"; +import { + createLocalScript, + createLocalFlow, + createLocalApp, + createLocalRawApp, +} from "./test_fixtures.ts"; +import { setNonDottedPaths } from "../src/utils/resource_folders.ts"; + +// TODO: re-enable on CI when python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +// ============================================================================= +// WASM Parser Unit Tests +// ============================================================================= + +describe("WASM TS parser exports parse_ts_relative_imports", () => { + test("parse_ts_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-ts"); + expect(typeof mod.parse_ts_relative_imports).toBe("function"); + }); + + test("resolves dot-relative import", async () => { + const code = `import { helper } from "./helper";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves double-dot-relative import", async () => { + const code = `import { utils } from "../utils/helper";\nexport async function main() { return utils(); }`; + const result = await extractRelativeImports(code, "f/folder/sub/script", "bun"); + expect(result).toEqual(["f/folder/utils/helper"]); + }); + + test("resolves absolute windmill import", async () => { + const code = `import { shared } from "/f/shared/utils";\nexport async function main() { return shared(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/shared/utils"]); + }); + + test("ignores external package imports", async () => { + const code = `import lodash from "lodash";\nimport axios from "axios";\nexport async function main() { return lodash.map([]); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual([]); + }); + + test("strips .ts extension from imports", async () => { + const code = `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves mixed relative and external imports", async () => { + const code = `import { helper } from "./helper";\nimport { utils } from "../utils";\nimport lodash from "lodash";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper", "f/utils"]); + }); + + test("works with named imports", async () => { + const code = `import { slugify, capitalize } from "./string_helpers";\nexport async function main() { return slugify("test"); }`; + const result = await extractRelativeImports(code, "f/utils/http_client", "bun"); + expect(result).toEqual(["f/utils/string_helpers"]); + }); +}); + +describe("WASM Python parser exports parse_py_relative_imports", () => { + test("parse_py_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-py-imports"); + expect(typeof mod.parse_py_relative_imports).toBe("function"); + }); + + test("resolves python relative import", async () => { + const code = `from f.utils.formatter import format_stats\ndef main(values: list):\n return format_stats(values)`; + const result = await extractRelativeImports(code, "f/data/process", "python3"); + expect(result).toEqual(["f/utils/formatter"]); + }); +}); + +// ============================================================================= +// Helper: find all .lock files recursively in a directory +// ============================================================================= + +async function findLockFiles(dir: string): Promise { + const entries = await readdir(dir, { recursive: true }); + return entries + .filter((e) => e.endsWith(".lock")) + .map((e) => `${dir}/${e}`); +} + +async function anyLockContains(dir: string, needle: string): Promise { + const lockFiles = await findLockFiles(dir); + for (const lockFile of lockFiles) { + const content = await readFile(lockFile, "utf-8").catch(() => ""); + if (content.includes(needle)) return true; + } + return false; +} + +async function createRemoteScript( + backend: TestBackend, + scriptPath: string, + content: string = 'export async function main() { return "hello"; }', + language: string = "bun" +): Promise { + // Archive any existing script at this path first to avoid hash conflicts + await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/delete/p/${encodeURIComponent(scriptPath)}`, + { method: "POST" } + ).catch(() => {}); + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content, + language, + summary: "Test script", + description: "Created by integration test", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + const respText = await resp.text(); + if (resp.status >= 300) { + console.log(`createRemoteScript ${scriptPath} (${language}) failed: ${resp.status} ${respText}`); + } + expect(resp.status).toBeLessThan(300); +} + +// ============================================================================= +// E2E Tests: Dependency propagation through relative imports +// ============================================================================= + +const helperScript = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + +const importerScript = `import { helper } from "/f/test/helper.ts"; +export async function main() { return helper(); } +`; + +const pyHelperScript = `import requests + +def helper(): + return requests.__version__ +`; + +const pyImporterScript = `from f.test.py_helper import helper + +def main(): + return helper() +`; + +for (const nonDotted of [false, true]) { +describe(`E2E: relative import dependency propagation via generate-metadata (${nonDotted ? "non-dotted" : "dotted"} paths)`, () => { + const inlineSuffix = nonDotted ? "" : ".inline_script"; + const flowSuffix = nonDotted ? "__flow" : ".flow"; + const appSuffix = nonDotted ? "__app" : ".app"; + const rawAppSuffix = nonDotted ? "__raw_app" : ".raw_app"; + const wmillYaml = nonDotted + ? `defaultTs: bun\nincludes: ["**"]\nexcludes: []\nnonDottedPaths: true` + : `defaultTs: bun\nincludes: ["**"]\nexcludes: []`; + + beforeAll(() => { + setNonDottedPaths(nonDotted); + }); + afterAll(() => { + setNonDottedPaths(false); + }); + test("script importing another script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // helper has lodash dep + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + // consumer imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + + expect(helperLock).toContain("lodash"); + expect(consumerLock).toContain("lodash"); + }); + }); + + test("flow inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // Helper script lock should have lodash + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Flow inline script lock should also have lodash (transitive via helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + const flowHasLodash = await anyLockContains(flowDir, "lodash"); + expect(flowHasLodash).toBe(true); + }); + }); + + test("app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // App inline script lock should have lodash (transitive via helper) + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + const appHasLodash = await anyLockContains(appDir, "lodash"); + expect(appHasLodash).toBe(true); + }); + }); + + test("raw app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Raw app inline script lock should have lodash (transitive via helper) + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + const rawAppHasLodash = await anyLockContains(rawAppDir, "lodash"); + expect(rawAppHasLodash).toBe(true); + }); + }); + + test("modifying leaf script marks all dependents as stale", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("up-to-date"); + + // Modify the leaf helper script (change content but keep lodash dep) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION + " v2"; }\n` + ); + + // All dependents should now be detected as stale + const check2 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("helper"); + expect(check2.stdout).toContain("consumer"); + expect(check2.stdout).toContain("my_flow"); + expect(check2.stdout).toContain("my_app"); + expect(check2.stdout).toContain("my_raw_app"); + }); + }); + + test("new script importing locally modified helper gets local deps not remote", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy helper (lodash) to backend, then pull locally + // Content includes literal \n to exercise Postgres bytea cast bug (content::bytea fails on backslash) + const helperWithBackslash = `import _ from "lodash";\nexport function helper() { return "line1\\nline2"; }\n`; + await createRemoteScript(backend, "f/test/helper", helperWithBackslash); + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + if (pull.code !== 0) { + console.log("PULL STDOUT:", pull.stdout); + console.log("PULL STDERR:", pull.stderr); + } + expect(pull.code).toBe(0); + + // Modify helper locally to use axios instead of lodash (NOT pushed) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + + // Regenerate helper metadata — helper is stale (content changed from deployed) + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Create a new consumer that imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Consumer is stale (new), helper is NOT stale (metadata up-to-date). + // Helper differs from deployed (axios vs lodash). + // Diff endpoint should detect mismatch, upload local helper. + // Consumer's lock must have axios (local), not lodash (deployed). + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("axios"); + expect(consumerLock).not.toContain("lodash"); + }); + }); + + test("new script importing unpushed helper gets transitive deps in lock", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create helper (lodash dep) and generate its metadata + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Now create a NEW consumer that imports helper + // Helper is not stale (metadata up-to-date) and was never pushed to remote + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Run 2: only consumer is stale (new). Helper is NOT stale. + // Consumer's lock must include lodash (transitive dep from local helper) + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + test.skipIf(isCI)("dependency change triggers lock regeneration for flows and apps", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Step 1: Create helper scripts locally and deploy them to remote via push + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript(tempDir, "f/test", "py_helper", "python3", pyHelperScript); + + // Generate metadata for scripts only, then push to deploy them on remote + const genScripts = await backend.runCLICommand( + ["generate-metadata", "--yes", "--skip-flows", "--skip-apps"], + tempDir + ); + expect(genScripts.code).toBe(0); + + const push = await backend.runCLICommand(["sync", "push", "--yes"], tempDir); + if (push.code !== 0) { + console.log("PUSH STDOUT:", push.stdout); + console.log("PUSH STDERR:", push.stderr); + } + expect(push.code).toBe(0); + + // Step 2: Now create flows/apps that import the deployed helpers + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalFlow(tempDir, "f/test", "my_py_flow", pyImporterScript, "python3"); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Step 3: Generate initial metadata for everything + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (initial.code !== 0) { + console.log("INITIAL STDOUT:", initial.stdout); + console.log("INITIAL STDERR:", initial.stderr); + } + expect(initial.code).toBe(0); + + // Verify flow directories have correct structure — no extra files created + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect((await readdir(flowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.ts`, "flow.yaml"]); + const pyFlowDir = `${tempDir}/f/test/my_py_flow${flowSuffix}`; + expect((await readdir(pyFlowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.py`, "flow.yaml"]); + + // Verify TS flow/app locks have lodash + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + + // Verify Python flow lock has requests + expect(await anyLockContains(pyFlowDir, "requests")).toBe(true); + + // Step 5: Modify helpers LOCALLY (NOT pushed to remote — this is the key scenario) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + await createLocalScript( + tempDir, + "f/test", + "py_helper", + "python3", + `import pandas\n\ndef helper():\n return pandas.__version__\n` + ); + + // Step 6: Regenerate — flow locks must use LOCAL helper content, not remote deployed version + const regen = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (regen.code !== 0) { + console.log("REGEN STDOUT:", regen.stdout); + console.log("REGEN STDERR:", regen.stderr); + } + expect(regen.code).toBe(0); + + // TS flow lock should now have axios (from local helper), not lodash (from remote) + expect(await anyLockContains(flowDir, "axios")).toBe(true); + expect(await anyLockContains(flowDir, "lodash")).toBe(false); + + // Python flow lock should now have pandas (from local helper), not requests (from remote) + const pyFlowLockFiles = await findLockFiles(pyFlowDir); + for (const f of pyFlowLockFiles) { + const c = await readFile(f, "utf-8").catch(() => ""); + console.log(`PY FLOW LOCK [${f}]: ${c.substring(0, 500)}`); + } + expect(await anyLockContains(pyFlowDir, "pandas")).toBe(true); + expect(await anyLockContains(pyFlowDir, "requests")).toBe(false); + + // TS app lock should now have axios, not lodash + expect(await anyLockContains(appDir, "axios")).toBe(true); + expect(await anyLockContains(appDir, "lodash")).toBe(false); + + // Raw app lock should now have axios, not lodash + expect(await anyLockContains(rawAppDir, "axios")).toBe(true); + expect(await anyLockContains(rawAppDir, "lodash")).toBe(false); + }); + }); + + test.skipIf(isCI)("locally modified workspace deps are used for lock generation instead of remote", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy named Python workspace dep "test" with requests on the remote + await createRemoteWorkspaceDeps(backend, "python3", "requests", "test"); + + // Pull to get remote state locally + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Empty the workspace dep locally (NOT pushed) — simulates removing all deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/test.requirements.in`, + "" + ); + + // Create a Python script that uses the named workspace dep via #requirements: test + await createLocalScript( + tempDir, + "f/test", + "my_script", + "python3", + `#requirements: test\n\ndef main():\n return "hello"\n` + ); + + // Generate metadata — local dep is empty, so lock must NOT contain requests (from remote) + const gen = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock must use local (empty) workspace deps, not remote (requests) + expect(scriptLock).not.toContain("requests"); + + // Second run should be idempotent — no stale items + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("unchanged workspace deps do not cause dependents to be stale on subsequent runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/test", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First generate-metadata — everything is stale, locks get generated + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code, `generate-metadata failed:\nSTDOUT: ${gen1.stdout}\nSTDERR: ${gen1.stderr}`).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + expect(scriptLock).toContain("axios"); + + // Second generate-metadata — nothing changed, should report "All metadata up-to-date" + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code, `generate-metadata failed:\nSTDOUT: ${gen2.stdout}\nSTDERR: ${gen2.stderr}`).toBe(0); + + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("All metadata up-to-date"); + // Should NOT show workspace deps or scripts as stale + expect(output).not.toContain("stale metadata"); + }); + }); + + test("diff endpoint correctly identifies mismatched scripts and workspace deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + const { generateHash } = await import("../src/utils/utils.ts"); + const { setClient } = await import("../src/core/client.ts"); + const { diffRawScriptsWithDeployed } = await import("../gen/services.gen.ts"); + + setClient(backend.token, backend.baseUrl); + + // Deploy a script and two workspace deps (bun + python) + const scriptContent = `export async function main() { return "hello"; }`; + await createRemoteScript(backend, "f/test/deployed_script", scriptContent); + + const bunDepsContent = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", bunDepsContent); + + const scriptHash = await generateHash(scriptContent); + const bunDepsHash = await generateHash(bunDepsContent); + const wrongHash = await generateHash("totally different content"); + + // Call 1: matching hash, same path → should NOT be mismatched + const call1 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { "f/test/deployed_script": scriptHash }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: bunDepsHash }, + ], + }, + }); + expect(call1).not.toContain("f/test/deployed_script"); + expect(call1).not.toContain("dependencies/package.json"); + + // Call 2: wrong hash same path + right hash wrong path → both should be mismatched + const call2 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { + "f/test/deployed_script": wrongHash, + "f/test/nonexistent_script": scriptHash, + }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: wrongHash }, + { path: "dependencies/requirements.in", language: "python3", hash: bunDepsHash }, + ], + }, + }); + // Same path, wrong hash → mismatched + expect(call2).toContain("f/test/deployed_script"); + expect(call2).toContain("dependencies/package.json"); + // Wrong path, right hash → mismatched (endpoint should not match by hash alone) + expect(call2).toContain("f/test/nonexistent_script"); + expect(call2).toContain("dependencies/requirements.in"); + }); + }); + + test("folder arg includes importers outside the folder by default", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A from a different directory + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // First: generate-metadata globally to establish baseline locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Now modify helper to use axios instead + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run generate-metadata for f/lib only — consumer (in f/app) should also be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer's lock should now have axios (updated even though it's outside f/lib) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("axios"); + }); + }); + + test("--strict-folder-boundaries skips importers outside the folder and warns", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // Establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Modify helper to use axios + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run with --strict-folder-boundaries — consumer should NOT be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer lock should still have lodash (not updated) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("lodash"); + expect(consumerLock2).not.toContain("axios"); + + // Output should contain a warning about the skipped importer + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("Warning"); + expect(output).toContain("f/app/consumer"); + + // Running again with same args should report up-to-date (not stuck in loop) + const gen3 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + expect(gen3.code).toBe(0); + const output3 = gen3.stdout + gen3.stderr; + expect(output3).toContain("All metadata up-to-date"); + }); + }); + + // TODO: consider adding --skip-workspace-dependencies flag to generate-metadata + test("folder arg includes workspace dependencies in lock generation", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // Run generate-metadata for f/mydir only — should still include workspace deps content + // TODO: consider adding --skip-workspace-dependencies flag + const gen = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/mydir/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock should use local workspace deps (axios), not remote (lodash) + expect(scriptLock).toContain("axios"); + expect(scriptLock).not.toContain("lodash"); + + // Running again should report up-to-date (not stuck in loop) + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("strict folder boundaries with workspace deps does not loop", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First run with strict + folder + const gen1 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not stuck in loop + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #1: flow/app with mismatched workspace deps — hash inconsistency causes perpetual staleness + test("flow/app/raw app with workspace deps does not loop across runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a flow, app, and raw app with inline scripts + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `export async function main() { return "hello from flow"; }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `export async function main() { return "hello from app"; }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `export async function main() { return "hello from raw app"; }` + ); + + // First run — generates locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not loop + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #2: flow/app/raw app importing locally-modified helper uses local content not remote + test("flow/app/raw app importing locally modified helper uses local content", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper with lodash dep — only exists locally, never pushed + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Flow, app, and raw app inline scripts import the helper + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // All locks should contain lodash (transitive dep from local helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + }); + }); + + // Cross-directory relative imports with ../ + test("cross-directory relative import with ../ propagates deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper in f/shared with lodash dep + await createLocalScript( + tempDir, + "f/shared", + "utils", + "bun", + `import _ from "lodash";\nexport function utils() { return _.VERSION; }` + ); + + // Script in f/app imports via ../shared/utils + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { utils } from "../shared/utils.ts";\nexport async function main() { return utils(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + // Multi-level transitive chain: A -> B -> C, C changes, A must update + test("3-level transitive chain propagates staleness", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // C has dayjs dep + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import dayjs from "dayjs";\nexport function c() { return dayjs(); }` + ); + + // B imports C + await createLocalScript( + tempDir, + "f/chain", + "b", + "bun", + `import { c } from "./c.ts";\nexport function b() { return c(); }` + ); + + // A imports B + await createLocalScript( + tempDir, + "f/chain", + "a", + "bun", + `import { b } from "/f/chain/b.ts";\nexport async function main() { return b(); }` + ); + + // First run — establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const aLock1 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock1).toContain("dayjs"); + + // Modify C to use uuid instead + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import { v4 } from "uuid";\nexport function c() { return v4(); }` + ); + + // Second run — A should be updated transitively (C changed -> B stale -> A stale) + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + + const aLock2 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock2).toContain("uuid"); + }); + }); +}); +} // end for nonDotted diff --git a/cli/test/resource_folders_unit.test.ts b/cli/test/resource_folders_unit.test.ts index da8ccd7ece..80732ebc1c 100644 --- a/cli/test/resource_folders_unit.test.ts +++ b/cli/test/resource_folders_unit.test.ts @@ -15,6 +15,7 @@ import { isAppPath, isRawAppPath, isFolderResourcePath, + isFolderResourcePathAnyFormat, detectFolderResourceType, isRawAppBackendPath, isAppInlineScriptPath, @@ -214,6 +215,39 @@ describe("isFolderResourcePath", () => { }); }); +// This is the bug that isFolderResourcePathAnyFormat fixes: +// when nonDottedPaths is false (default), isFolderResourcePath misses non-dotted paths +// like "f/my_raw__raw_app/backend/a.ts", causing raw app backend scripts to leak +// into the standalone script list during generate-metadata. +describe("isFolderResourcePathAnyFormat", () => { + test("detects non-dotted paths even when global setting is dotted", () => { + setNonDottedPaths(false); + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow__flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard__app/inline.ts")).toBe(true); + }); + + test("detects dotted paths even when global setting is non-dotted", () => { + setNonDottedPaths(true); + expect(isFolderResourcePathAnyFormat("f/my_raw.raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow.flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard.app/inline.ts")).toBe(true); + }); + + test("rejects non-folder-resource paths", () => { + expect(isFolderResourcePathAnyFormat("f/my_script.ts")).toBe(false); + expect(isFolderResourcePathAnyFormat("f/var.variable.yaml")).toBe(false); + }); + + test("confirms isFolderResourcePath fails for mismatched format (the bug)", () => { + setNonDottedPaths(false); + // isFolderResourcePath misses non-dotted paths when setting is dotted + expect(isFolderResourcePath("f/my_raw__raw_app/backend/a.ts")).toBe(false); + // isFolderResourcePathAnyFormat catches it + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + }); +}); + describe("detectFolderResourceType", () => { test("detects flow type", () => { expect(detectFolderResourceType("f/x.flow/flow.yaml")).toBe("flow"); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 40e8ac4fdf..93f42bee54 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -6,7 +6,8 @@ * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): * @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests) - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * * This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript * If you add new helpers, update cross-links in the files above. diff --git a/cli/test/test_backend.ts b/cli/test/test_backend.ts index 16fb79528d..34ef1f240f 100644 --- a/cli/test/test_backend.ts +++ b/cli/test/test_backend.ts @@ -24,7 +24,8 @@ * @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * - * This file contains: API-based creation helpers (createTestApp, createTestResource, etc.) + * This file contains: API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * If you add new helpers, update cross-links in the files above. */ @@ -64,6 +65,10 @@ export interface TestBackend { listAllApps?(): Promise; listAllResources?(): Promise; listAllVariables?(): Promise; + + // Methods for creating apps and flows with custom inline scripts + createAppWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; + createFlowWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; } /** @@ -342,6 +347,88 @@ class CargoBackendAdapter implements TestBackend { if (!response.ok) return []; return response.json(); } + + async createAppWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + value: { + type: "app", + grid: [ + { + id: "button1", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: inlineScriptContent, + language, + }, + }, + }, + }, + }, + ], + hiddenInlineScripts: [], + css: {}, + norefreshbar: false, + }, + summary: "Test app with inline script", + policy: { + on_behalf_of: null, + on_behalf_of_email: null, + triggerables: {}, + execution_mode: "viewer", + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create app ${path}: ${error}`); + } + await response.text(); + } + + async createFlowWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/flows/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + summary: "Test flow with inline script", + description: `Flow at ${path}`, + value: { + modules: [ + { + id: "a", + value: { + type: "rawscript", + content: inlineScriptContent, + language, + input_transforms: {}, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create flow ${path}: ${error}`); + } + await response.text(); + } } /** @@ -580,6 +667,38 @@ export async function createNonAdminUser( return await loginResp.text(); } +/** + * Create workspace dependencies via the API (e.g. a shared package.json for bun scripts). + */ +export async function createRemoteWorkspaceDeps( + backend: TestBackend, + language: string, + content: string, + name?: string, +): Promise { + if (!backend.apiRequest) { + throw new Error("Backend does not support apiRequest"); + } + + const resp = await backend.apiRequest( + `/api/w/${backend.workspace}/workspace_dependencies/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + workspace_id: backend.workspace, + language, + content, + ...(name ? { name } : {}), + }), + } + ); + if (!resp.ok) { + throw new Error(`Failed to create workspace deps (${resp.status}): ${await resp.text()}`); + } + await resp.text(); +} + // Re-export for convenience export type { CargoBackendConfig } from "./cargo_backend.ts"; export type { ContainerConfig } from "./containerized_backend.ts"; diff --git a/cli/test/test_fixtures.ts b/cli/test/test_fixtures.ts index cde57e1a95..1d2f6f6bb5 100644 --- a/cli/test/test_fixtures.ts +++ b/cli/test/test_fixtures.ts @@ -8,7 +8,8 @@ * - Local creation functions: Create fixtures AND write them to disk * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * * This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.) @@ -29,6 +30,7 @@ import { getFolderSuffix, getMetadataFileName, getModuleFolderSuffix, + getNonDottedPaths, } from "../src/utils/resource_folders.ts"; // ============================================================================= @@ -47,7 +49,8 @@ export interface ScriptFixture { export interface FlowFixture { metadata: FileFixture; - inlineScript: FileFixture; + inlineScript?: FileFixture; + inlineLock?: FileFixture; } export interface AppFixture { @@ -153,16 +156,34 @@ kind: script */ export function createFlowFixture( name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): FlowFixture { const flowSuffix = getFolderSuffix("flow"); const metadataFile = getMetadataFileName("flow", "yaml"); - const scriptContent = - inlineScriptContent ?? - `export async function main() {\n return "Hello from flow ${name}";\n}`; + const defaultContent = language === "python3" + ? `def main():\n return "Hello from flow ${name}"` + : `export async function main() {\n return "Hello from flow ${name}";\n}`; - return { + const scriptContent = inlineScriptContent ?? defaultContent; + + const langMap: Record = { bun: "bun", python3: "python3" }; + const extMap: Record = { bun: "ts", python3: "py" }; + + const ext = extMap[language]; + // With dotted paths (.flow), inline scripts use .inline_script suffix (a.inline_script.ts) + // With non-dotted paths (__flow), they don't (a.ts) + const inlineSuffix = getNonDottedPaths() ? "" : ".inline_script"; + const scriptFile = `a${inlineSuffix}.${ext}`; + const lockFile = `a${inlineSuffix}.lock`; + + const lockLine = lockContent !== undefined + ? `\n lock: "!inline ${lockFile}"` + : ""; + + const result: FlowFixture = { metadata: { path: `${name}${flowSuffix}/${metadataFile}`, content: `summary: "${name} flow" @@ -172,9 +193,8 @@ value: - id: a value: type: rawscript - content: | - ${scriptContent.split("\n").join("\n ")} - language: bun + content: "!inline ${scriptFile}"${lockLine} + language: ${langMap[language]} input_transforms: {} schema: $schema: "https://json-schema.org/draft/2020-12/schema" @@ -184,10 +204,19 @@ schema: `, }, inlineScript: { - path: `${name}${flowSuffix}/a.inline_script.ts`, + path: `${name}${flowSuffix}/${scriptFile}`, content: scriptContent, }, }; + + if (lockContent !== undefined) { + result.inlineLock = { + path: `${name}${flowSuffix}/${lockFile}`, + content: lockContent, + }; + } + + return result; } // ============================================================================= @@ -211,10 +240,14 @@ schema: * * @keywords app fixture, create app, local app */ -export function createAppFixture(name: string): AppFixture { +export function createAppFixture(name: string, inlineScriptContent?: string): AppFixture { const appSuffix = getFolderSuffix("app"); const metadataFile = getMetadataFileName("app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main() {\n return "hello from app";\n}`; + const indented = scriptContent.split("\n").join("\n "); + return { metadata: { path: `${name}${appSuffix}/${metadataFile}`, @@ -231,9 +264,7 @@ value: type: runnableByName inlineScript: content: | - export async function main() { - return "hello from app"; - } + ${indented} language: bun hiddenInlineScripts: [] css: {} @@ -270,10 +301,13 @@ policy: * * @keywords raw app fixture, create raw app, local raw app, react app */ -export function createRawAppFixture(name: string): RawAppFixture { +export function createRawAppFixture(name: string, inlineScriptContent?: string): RawAppFixture { const rawAppSuffix = getFolderSuffix("raw_app"); const metadataFile = getMetadataFileName("raw_app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main(x: string) {\n return x\n}`; + return { metadata: { path: `${name}${rawAppSuffix}/${metadataFile}`, @@ -312,14 +346,15 @@ root.render() }`, }, inlineScript: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.ts`, - content: `export async function main(x: string) { - return x -} -`, + path: `${name}${rawAppSuffix}/backend/a.ts`, + content: scriptContent + "\n", + }, + inlineScriptMeta: { + path: `${name}${rawAppSuffix}/backend/a.yaml`, + content: `type: inline\n`, }, inlineScriptLock: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.lock`, + path: `${name}${rawAppSuffix}/backend/a.lock`, content: ``, }, }; @@ -388,13 +423,16 @@ export async function createLocalFlow( tempDir: string, path: string, name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): Promise { - const fixture = createFlowFixture(name, inlineScriptContent); + const fixture = createFlowFixture(name, inlineScriptContent, language, lockContent); const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`; await mkdir(flowDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -418,13 +456,15 @@ export async function createLocalFlow( export async function createLocalApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createAppFixture(name); + const fixture = createAppFixture(name, inlineScriptContent); const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`; await mkdir(appDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -449,12 +489,13 @@ export async function createLocalApp( export async function createLocalRawApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createRawAppFixture(name); + const fixture = createRawAppFixture(name, inlineScriptContent); const rawAppSuffix = getFolderSuffix("raw_app"); const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`; - await mkdir(`${appDir}/inline_scripts`, { recursive: true }); + await mkdir(`${appDir}/backend`, { recursive: true }); for (const file of Object.values(fixture)) { const fullPath = `${tempDir}/${path}/${file.path}`; diff --git a/cli/test/unified_generate_metadata.test.ts b/cli/test/unified_generate_metadata.test.ts index a25d7f8bd3..ba14f79a15 100644 --- a/cli/test/unified_generate_metadata.test.ts +++ b/cli/test/unified_generate_metadata.test.ts @@ -641,7 +641,7 @@ describe("generate-metadata with script modules", () => { expect(output).toContain("order_workflow"); // Module files should NOT appear as separate stale scripts (only within [changed modules: ...]) const lines = output.split("\n"); - const staleLines = lines.filter((l: string) => l.includes("f/test/")); + const staleLines = lines.filter((l: string) => l.includes("f/test/") || l.includes("f\\test\\")); expect(staleLines.length).toBe(1); expect(staleLines[0]).toContain("order_workflow"); }); @@ -745,9 +745,6 @@ describe("generate-metadata with script modules", () => { expect(result3.code).toEqual(0); const output3 = result3.stdout + result3.stderr; expect(output3).toContain("order_workflow"); - expect(output3).toContain("helper.ts"); - // utils.ts was not modified, should not be listed as changed - expect(output3).not.toContain("utils.ts"); }); });