diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts index b360c8b802..abee64cbf3 100644 --- a/cli/src/commands/pipeline/docs.ts +++ b/cli/src/commands/pipeline/docs.ts @@ -43,7 +43,7 @@ async function fetchDeployedGraph( } // Render the pipeline graph as a markdown document. -function generatePipelineMarkdown( +export function generatePipelineMarkdown( folder: string, graph: AssetGraph, datatableSchemas: any[], @@ -73,7 +73,26 @@ function generatePipelineMarkdown( } } - const scripts = graph.runnables.filter((r) => r.usage_kind === "script").map((r) => r.path).sort(); + // `// macros` libraries are definition-only nodes (not runnable pipeline + // steps), so they get their own section below and are excluded from the + // per-script listing + the script count. + const macroLibs = graph.runnables + .filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0) + .sort((a, b) => a.path.localeCompare(b.path)); + const macroLibPaths = new Set(macroLibs.map((r) => r.path)); + const macroConsumersByLib = new Map(); + for (const me of graph.macro_edges ?? []) { + (macroConsumersByLib.get(me.lib_path) ?? macroConsumersByLib.set(me.lib_path, []).get(me.lib_path)!).push({ + consumer: me.consumer_path, + names: me.macro_names, + viaUse: me.via_use, + }); + } + + const scripts = graph.runnables + .filter((r) => r.usage_kind === "script" && !macroLibPaths.has(r.path)) + .map((r) => r.path) + .sort(); let md = `# Pipeline \`f/${folder}\` @@ -85,7 +104,7 @@ produces data by reading/writing assets in its body (\`datatable://\`, \`ducklake://\`, \`s3://\`, \`volume://\`). The cascade runs a producer, then every downstream subscriber, in topological order. -- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"} +- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"}${macroLibs.length > 0 ? ` · **${macroLibs.length}** macro librar${macroLibs.length === 1 ? "y" : "ies"}` : ""} ## Scripts @@ -107,6 +126,30 @@ downstream subscriber, in topological order. md += `\n`; } + // Macro libraries: `// macros` scripts whose macros are injected into consuming + // DuckDB scripts at run time. List each library's signatures and its callers so + // an agent discovers the reuse layer instead of re-inlining the logic. + if (macroLibs.length > 0) { + md += `## Macro libraries\n\n`; + md += `\`// macros\` DuckDB libraries. Their \`CREATE MACRO\` definitions are injected as\nTEMP macros into consuming scripts at run time — call a macro by name, or force the\nwhole library in with \`// use \` (needed for macros only reached via dynamic SQL).\n\n`; + for (const lib of macroLibs) { + md += `### \`${lib.path}\`\n\n`; + for (const m of lib.macros ?? []) { + md += `- \`${m.name}(${m.params ?? ""})\`${m.is_table ? " → TABLE" : ""}\n`; + } + const consumers = [...(macroConsumersByLib.get(lib.path) ?? [])].sort((a, b) => + a.consumer.localeCompare(b.consumer), + ); + if (consumers.length > 0) { + md += `- **Used by:**\n`; + for (const c of consumers) { + md += ` - \`${c.consumer}\` (${c.viaUse ? "via \`// use\`" : `calls ${c.names.map((n) => `\`${n}\``).join(", ")}`})\n`; + } + } + md += `\n`; + } + } + // Datatable schemas, restricted to datatables this pipeline references. const referencedDatatables = new Set( graph.assets.filter((a) => a.kind === "datatable").map((a) => a.path.split("/")[0]), diff --git a/cli/src/commands/pipeline/duckdbMacros.ts b/cli/src/commands/pipeline/duckdbMacros.ts new file mode 100644 index 0000000000..be505874d7 --- /dev/null +++ b/cli/src/commands/pipeline/duckdbMacros.ts @@ -0,0 +1,261 @@ +// Faithful TS port of the lexical rules in +// `backend/parsers/windmill-parser/src/duckdb_macros.rs` — keep the two in +// lockstep. Deliberately lexical, not an AST parse: over-matching a call only +// draws a spurious edge, never a wrong run. A macro reached only through dynamic +// SQL (`query('…')`) is invisible to call detection here as on the server — +// `// use ` forces the whole-library edge instead. + +export type ParsedMacro = { + // Lowercased bare identifier (DuckDB identifiers are case-insensitive unquoted; + // qualified / quoted names are rejected, matching the backend). + name: string; + // Verbatim text inside the parameter parens (may be empty). + params: string; + isTable: boolean; +}; + +// Case-insensitive whole-word prefix strip (whitespace-bounded). Returns the +// remainder with leading whitespace trimmed, or null when `s` does not start with +// `kw` as a whole word. `kw` must be lowercase. Mirrors `strip_kw`. +function stripKw(s: string, kw: string): string | null { + if (s.length < kw.length) return null; + if (s.slice(0, kw.length).toLowerCase() !== kw) return null; + const after = s.slice(kw.length); + if (after.length === 0 || /^\s/.test(after)) return after.replace(/^\s+/, ""); + return null; +} + +function isIdent(s: string): boolean { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(s); +} + +// Split a script body into `;`-terminated statements with line comments +// (`--`, `//`), block comments (`/* */`), and quoted spans stripped/preserved as +// in the SQL executor. Char-wise (not byte-wise) so multi-byte text survives. +// Mirrors `split_statements`. +export function splitStatements(sql: string): string[] { + const out: string[] = []; + let cur = ""; + const chars = [...sql]; + let i = 0; + const n = chars.length; + while (i < n) { + const c = chars[i]; + // line comment: `--` (SQL) or `//` (Windmill annotation prefix) + if ( + (c === "-" && i + 1 < n && chars[i + 1] === "-") || + (c === "/" && i + 1 < n && chars[i + 1] === "/") + ) { + while (i < n && chars[i] !== "\n") i += 1; + continue; + } + // block comment + if (c === "/" && i + 1 < n && chars[i + 1] === "*") { + i += 2; + while (i + 1 < n && !(chars[i] === "*" && chars[i + 1] === "/")) i += 1; + i += 2; + continue; + } + // single-quoted string ('' escapes an embedded quote) + if (c === "'") { + cur += c; + i += 1; + while (i < n) { + cur += chars[i]; + if (chars[i] === "'") { + if (i + 1 < n && chars[i + 1] === "'") { + cur += "'"; + i += 2; + continue; + } + i += 1; + break; + } + i += 1; + } + continue; + } + // double-quoted identifier + if (c === '"') { + cur += c; + i += 1; + while (i < n) { + cur += chars[i]; + if (chars[i] === '"') { + i += 1; + break; + } + i += 1; + } + continue; + } + if (c === ";") { + const t = cur.trim(); + if (t !== "") out.push(t); + cur = ""; + i += 1; + continue; + } + cur += c; + i += 1; + } + const t = cur.trim(); + if (t !== "") out.push(t); + return out; +} + +// Parse one comment-free, `;`-less statement as a CREATE MACRO. Returns null when +// the statement is not a well-formed macro definition (unlike the backend, which +// distinguishes "not macro-shaped" from "malformed" to surface deploy errors — +// the local graph is lenient and simply skips anything it can't read). Mirrors +// `parse_create_macro` for the shape it accepts. +export function parseCreateMacro(stmt: string): ParsedMacro | null { + let rest = stripKw(stmt.trim(), "create"); + if (rest === null) return null; + const afterOr = stripKw(rest, "or"); + if (afterOr !== null) { + const afterReplace = stripKw(afterOr, "replace"); + if (afterReplace === null) return null; // CREATE OR + rest = afterReplace; + } + const afterTemp = stripKw(rest, "temp") ?? stripKw(rest, "temporary"); + if (afterTemp !== null) rest = afterTemp; + // `FUNCTION` is DuckDB's alias for `MACRO`. + const afterMacro = stripKw(rest, "macro") ?? stripKw(rest, "function"); + if (afterMacro === null) return null; + rest = afterMacro; + + const nameEndMatch = rest.match(/[\s(]/); + const nameEnd = nameEndMatch ? nameEndMatch.index! : rest.length; + const rawName = rest.slice(0, nameEnd); + if (rawName === "" || rawName.includes(".") || rawName.includes('"') || !isIdent(rawName)) { + return null; + } + const name = rawName.toLowerCase(); + + rest = rest.slice(nameEnd).replace(/^\s+/, ""); + if (!rest.startsWith("(")) return null; + // Balanced-paren scan for the verbatim param list; skip quoted spans (a default + // value may contain a paren). + const pchars = [...rest]; + let depth = 0; + let j = 0; + let close = -1; + while (j < pchars.length) { + const ch = pchars[j]; + if (ch === "(") depth += 1; + else if (ch === ")") { + depth -= 1; + if (depth === 0) { + close = j; + break; + } + } else if (ch === "'" || ch === '"') { + const q = ch; + j += 1; + while (j < pchars.length && pchars[j] !== q) j += 1; + } + j += 1; + } + if (close === -1) return null; // unbalanced + const params = pchars.slice(1, close).join("").trim(); + + const afterParams = pchars.slice(close + 1).join("").replace(/^\s+/, ""); + let body = stripKw(afterParams, "as"); + if (body === null) return null; + let isTable = false; + const afterTable = stripKw(body, "table"); + if (afterTable !== null) { + body = afterTable; + isTable = true; + } + const trimmedBody = body.replace(/;+\s*$/, "").trim(); + if (trimmedBody === "") return null; // empty body + return { name, params, isTable }; +} + +// All well-formed macro definitions in a `// macros` library body, in source +// order. Non-macro statements (setup: ATTACH/INSTALL/… ) are skipped. +export function parseMacroLibrary(sql: string): ParsedMacro[] { + const out: ParsedMacro[] = []; + for (const stmt of splitStatements(sql)) { + const m = parseCreateMacro(stmt); + if (m) out.push(m); + } + return out; +} + +// Names from `names` that `sql` calls: an identifier token immediately followed +// by `(` (after optional whitespace), not `.`-qualified, outside strings and +// comments. Lexical (over-matching only adds an unused edge). Mirrors +// `detect_macro_calls`. +export function detectMacroCalls(sql: string, names: Set): Set { + const found = new Set(); + if (names.size === 0) return found; + for (const stmt of splitStatements(sql)) { + const chars = [...stmt]; + const n = chars.length; + let i = 0; + let prev: string | undefined = undefined; + while (i < n) { + const c = chars[i]; + if (c === "'" || c === '"') { + const q = c; + i += 1; + while (i < n && chars[i] !== q) i += 1; + i += 1; + prev = q; + continue; + } + const isIdentStart = /[A-Za-z_]/.test(c); + const prevBlocks = prev !== undefined && /[.A-Za-z0-9_]/.test(prev); + if (isIdentStart && !prevBlocks) { + const start = i; + while (i < n && /[A-Za-z0-9_]/.test(chars[i])) i += 1; + const word = chars.slice(start, i).join("").toLowerCase(); + let k = i; + while (k < n && /\s/.test(chars[k])) k += 1; + if (k < n && chars[k] === "(" && names.has(word)) found.add(word); + prev = chars[i - 1]; + continue; + } + prev = c; + i += 1; + } + } + return found; +} + +// Scan the LEADING comment header for the `// macros` marker and `// use ` +// annotations, independent of the wasm parser (which drops both). Faithful mirror +// of the canonical `parse_pipeline_annotations` (asset_parser.rs): each line's +// prefix is stripped as `//`, `--`, or `#` REGARDLESS of language (a DuckDB +// library may legitimately head its annotations with `// macros`), scanning stops +// at the first non-comment line, the marker must stand alone on its line, and a +// `// use` target is a single whitespace-free token containing `/`. +export function parseMacroAnnotations( + content: string, +): { macros: boolean; useLibs: string[] } { + let macros = false; + const useLibs: string[] = []; + for (const raw of content.split("\n")) { + const line = raw.replace(/^\s+/, ""); + if (line === "") continue; + let rest: string; + if (line.startsWith("//") || line.startsWith("--")) rest = line.slice(2); + else if (line.startsWith("#")) rest = line.slice(1); + else break; // first line of actual code ends the annotation header + rest = rest.replace(/^\s+/, ""); + // marker: `macros` as the only content on the line (rejects `macros_v2`, + // `macros are below`) + if (/^macros\s*$/.test(rest)) { + macros = true; + continue; + } + // `use `: a single whitespace-free path token (must contain `/`, so + // prose like `use this to …` is dropped) + const u = rest.match(/^use\s+(\S+)\s*$/); + if (u && u[1].includes("/") && !useLibs.includes(u[1])) useLibs.push(u[1]); + } + return { macros, useLibs }; +} diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index df1018ff0d..c1a5882849 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -19,6 +19,12 @@ import * as log from "../../core/log.ts"; import { exts, removeExtensionToPath } from "../script/script.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { getWmillYamlPath } from "../../core/conf.ts"; +import { + detectMacroCalls, + parseMacroAnnotations, + parseMacroLibrary, + type ParsedMacro, +} from "./duckdbMacros.ts"; // Resolve the workspace root (the directory containing wmill.yaml) for reading // local files, falling back to the current directory. @@ -53,8 +59,9 @@ export type GraphRunnable = { // schema-contract warnings; only present when set (default `warn` is absent), // mirroring the deployed graph node. materialize_on_schema_change?: string; - // `// macros` library: the macros it defines (deployed graph only — the wasm - // asset parser does not emit them). Non-empty ⇒ definition-only node. + // `// macros` library: the macros it defines. Derived locally by + // `buildMacroEdges` (the wasm asset parser emits neither the marker nor the + // registry). Non-empty ⇒ definition-only node. macros?: { name: string; params?: string; is_table?: boolean }[]; }; export type GraphEdge = { @@ -89,7 +96,7 @@ export type AssetGraph = { edges: GraphEdge[]; triggers: GraphTrigger[]; // ƒ edges from a `// macros` library to the scripts calling its macros - // (deployed graph only). + // (present on both the deployed graph and the locally-derived one). macro_edges?: { lib_path: string; consumer_path: string; @@ -456,6 +463,10 @@ export async function buildLocalPipelineGraph(args: { const folderDir = path.join(args.root, "f", folderClean); const all = await collectScripts(folderDir, args.root, args.defaultTs); + // `// macros` DuckDB libraries across the whole workspace (a pipeline may use a + // shared library outside its folder). + const libMacros = collectMacroLibraries(args.root); + const runnables: GraphRunnable[] = []; const edges: GraphEdge[] = []; const triggers: GraphTrigger[] = []; @@ -471,6 +482,26 @@ export async function buildLocalPipelineGraph(args: { for (const s of all) { const out = await inferScriptAssets(s.content, s.language); + // A `// macros` library is a pipeline-member node carrying its macro + // signatures — whether or not any consumer uses it — matching the deployed + // graph, which marks every macro library `auto_kind='pipeline'`. It is + // definition-only (no assets/triggers, never run), so we add just the node; + // it is NOT a previewable script (excluded from `pipelineScripts`). + const macroLibDefs = libMacros.get(s.path); + if (macroLibDefs) { + runnables.push({ + path: s.path, + usage_kind: "script", + in_pipeline: true, + ...(out.tag ? { tag: out.tag } : {}), + macros: macroLibDefs.map((m) => ({ + name: m.name, + params: m.params, + is_table: m.isTable, + })), + }); + continue; + } if (!out.in_pipeline) continue; // not a pipeline member const retry = normalizeRetry(out.retry); const nativeTriggers = recoverHeaderNativeTriggers(s.content, s.language); @@ -598,6 +629,8 @@ export async function buildLocalPipelineGraph(args: { } } + const macroEdges = buildMacroEdges(all, libMacros, runnables); + const assets = [...assetSet.entries()].map(([key, a]) => { const derived_from = a.derived_from ?? derivedFromByKey.get(key); return derived_from ? { ...a, derived_from } : a; @@ -609,7 +642,162 @@ export async function buildLocalPipelineGraph(args: { assets, edges, triggers, + ...(macroEdges.length > 0 ? { macro_edges: macroEdges } : {}), }, scripts: pipelineScripts, }; } + +// Every `// macros` DuckDB library in the workspace, keyed by its windmill path, +// mapped to its parsed macro definitions. Walks the whole `f/` tree (not just the +// pipeline folder) because the deployed graph resolves consumers against the +// workspace-wide macro registry — a pipeline may `// use` / call a shared library +// living outside its own folder. Only `*.duckdb.sql` files are read (macros are +// DuckDB-only), so the extra walk stays cheap. +function collectMacroLibraries(root: string): Map { + const out = new Map(); + const walk = (dir: string) => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + if (e.name.startsWith(".") || e.name === "node_modules") continue; + const abs = path.join(dir, e.name); + if (e.isDirectory()) { + walk(abs); + continue; + } + if (!e.isFile() || !e.name.endsWith(".duckdb.sql")) continue; + let content: string; + try { + content = fs.readFileSync(abs, "utf-8"); + } catch { + continue; + } + if (!parseMacroAnnotations(content).macros) continue; + const relFromRoot = path.relative(root, abs).replaceAll("\\", "/"); + out.set(removeExtensionToPath(relFromRoot), parseMacroLibrary(content)); + } + }; + walk(path.join(root, "f")); + return out; +} + +// Derive lib→consumer edges (lexical calls + `// use`). In-folder libraries are +// already member nodes; this adds any out-of-folder provider referenced by an +// in-folder consumer. Mirrors the deployed `asset_graph`: libraries resolve +// workspace-wide, consumers are folder-scoped. +function buildMacroEdges( + all: LocalScript[], + libMacros: Map, + runnables: GraphRunnable[], +): NonNullable { + if (libMacros.size === 0) return []; + const useLibsByScript = new Map(); + for (const s of all) { + if (s.language !== "duckdb") continue; + const { useLibs } = parseMacroAnnotations(s.content); + if (useLibs.length > 0) useLibsByScript.set(s.path, useLibs); + } + + // Macro name → providing library. Names are workspace-unique (the deploy path + // enforces this); on a local collision, last-writer-wins, harmlessly. + const providerByName = new Map(); + for (const [lib, macros] of libMacros) { + for (const m of macros) providerByName.set(m.name, lib); + } + const allMacroNames = new Set(providerByName.keys()); + + // Aggregate per (lib, consumer): the set of called macro names and whether the + // edge came (also) from a whole-library `// use`. Consumers are folder-scoped + // (`all` is this folder's scripts). + const pipelinePaths = new Set(runnables.map((r) => r.path)); + type EdgeAgg = { names: Set; viaUse: boolean }; + // lib_path → consumer_path → aggregate. Nested (not a packed single-string key) + // so no separator can ever collide with a path. + const edgeMap = new Map>(); + const aggFor = (lib: string, consumer: string): EdgeAgg => { + let byConsumer = edgeMap.get(lib); + if (!byConsumer) { + byConsumer = new Map(); + edgeMap.set(lib, byConsumer); + } + let agg = byConsumer.get(consumer); + if (!agg) { + agg = { names: new Set(), viaUse: false }; + byConsumer.set(consumer, agg); + } + return agg; + }; + for (const s of all) { + if (s.language !== "duckdb") continue; + // Lexical call edges: the deploy path records `macro_usage` for EVERY DuckDB + // script in the folder (not only pipeline members) — including a macro + // library that calls another library's macros — so a lib→lib edge and its + // upstream provider node survive. Mirror that: any folder DuckDB script is a + // candidate consumer here. + for (const name of detectMacroCalls(s.content, allMacroNames)) { + const lib = providerByName.get(name)!; + if (lib === s.path) continue; // a library calling its own macro is not an edge + aggFor(lib, s.path).names.add(name); + } + // `// use` whole-library edges: the deployed graph re-parses these only from + // pipeline members (`annotations_by_path`), so scope them the same way. + if (!pipelinePaths.has(s.path)) continue; + for (const lib of useLibsByScript.get(s.path) ?? []) { + const macros = libMacros.get(lib); + // An out-of-tree / unknown `// use` target can't be resolved locally. + if (!macros || lib === s.path) continue; + const agg = aggFor(lib, s.path); + agg.viaUse = true; + for (const m of macros) agg.names.add(m.name); + } + } + + const edges = [...edgeMap.entries()] + .flatMap(([lib_path, byConsumer]) => + [...byConsumer.entries()].map(([consumer_path, agg]) => ({ + lib_path, + consumer_path, + macro_names: [...agg.names].sort(), + // `via_use` is always present (the deployed `MacroEdge` serializes it + // unconditionally) so `--json` matches byte-for-byte. + via_use: agg.viaUse, + })), + ) + .sort((a, b) => + a.lib_path.localeCompare(b.lib_path) || + a.consumer_path.localeCompare(b.consumer_path), + ); + + // In-folder libraries are already member nodes (added above with `in_pipeline` + // + macros). An OUT-OF-folder library referenced by an in-folder consumer is + // added here as a non-member provider node (no `in_pipeline`), matching the + // deployed graph, which folder-scopes membership but pulls the out-of-folder + // provider in as the edge's endpoint. + const libPaths = new Set(edges.map((e) => e.lib_path)); + for (const lib of libPaths) { + if (runnables.some((r) => r.path === lib)) continue; + const macros = (libMacros.get(lib) ?? []).map((m) => ({ + name: m.name, + params: m.params, + is_table: m.isTable, + })); + runnables.push({ path: lib, usage_kind: "script", macros }); + } + // Force every edge's CONSUMER endpoint into the node set too, like the deployed + // builder, so no edge dangles at a missing runnable. Members and libraries are + // already nodes; only a non-member DuckDB helper (calls a macro but isn't + // `// pipeline` and isn't itself a library) needs a bare node here. + for (const consumer of new Set(edges.map((e) => e.consumer_path))) { + if (!runnables.some((r) => r.path === consumer)) { + runnables.push({ path: consumer, usage_kind: "script" }); + } + } + runnables.sort((a, b) => a.path.localeCompare(b.path)); + + return edges; +} diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts index 2d207444eb..0738283fba 100644 --- a/cli/src/commands/pipeline/pipeline.ts +++ b/cli/src/commands/pipeline/pipeline.ts @@ -304,8 +304,9 @@ async function renderGraph( } // `// macros` libraries: badge the node with the macros it defines and list - // its consumers as ƒ edges. Deployed graphs only — the local wasm parse does - // not emit `macros`/`macro_edges`, so local mode renders them as plain nodes. + // its consumers as ƒ edges. Populated on both the deployed graph and the local + // graph (localGraph.ts derives `macros`/`macro_edges` from the working tree, + // since the wasm asset parser emits neither). const macrosByLib = new Map( graph.runnables .filter((r) => (r.macros?.length ?? 0) > 0) @@ -590,19 +591,25 @@ async function run( // Resolve relative imports from local (not-yet-deployed) content so a script // that imports a sibling workspace lib previews against local edits. Same // trick as `wmill dev` / `wmill app dev`; degrades to undefined gracefully. - try { - const codebases = await listSyncCodebases(merged); - const { buildPreviewTempScriptRefs } = await import( - "../generate-metadata/generate-metadata.ts" - ); - tempScriptRefs = await buildPreviewTempScriptRefs( - workspace, - merged, - codebases, - { kind: "all" }, - ); - } catch { - // best-effort: relative imports fall back to deployed versions + // Skipped for `--dry-run`: it locks/uploads scripts as a side effect (it + // regenerates `*.script.yaml` / `*.script.lock` / `wmill-lock.yaml`), and a + // plan preview must stay READ-ONLY — the refs are only consumed when a + // preview job actually runs below, which dry-run never reaches. + if (!opts.dryRun) { + try { + const codebases = await listSyncCodebases(merged); + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + merged, + codebases, + { kind: "all" }, + ); + } catch { + // best-effort: relative imports fall back to deployed versions + } } } else { graph = await apiGet( @@ -671,6 +678,20 @@ async function run( .filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0) .map((r) => r.path), ); + // Non-runnable graph nodes: the macro libraries above, plus (under `--local`) + // any script node with no local file. The local graph surfaces macro-consumer + // nodes for lineage display — a DuckDB script that calls a macro but isn't a + // `// pipeline` member (so has no previewable content). They must never enter + // the run: as a manual root they'd be scheduled, and a preview would fail with + // no local content to send. + const notRunnablePaths = new Set(macroLibPaths); + if (opts.local && localScripts) { + for (const r of graph.runnables) { + if (r.usage_kind === "script" && !localScripts.has(r.path)) { + notRunnablePaths.add(r.path); + } + } + } // Resolve the start. Two eligibility sets: // `starts` — schedule/manual roots (+ `--upload`-bound handlers). The @@ -684,12 +705,12 @@ async function run( // unless `--upload`-bound. const starts = new Set( [...validStarts(graph), ...boundNodeIds].filter( - (id) => !macroLibPaths.has(scriptPathOf(id)), + (id) => !notRunnablePaths.has(scriptPathOf(id)), ), ); const fromEligible = new Set( [...validFromStarts(graph), ...boundNodeIds].filter( - (id) => !macroLibPaths.has(scriptPathOf(id)), + (id) => !notRunnablePaths.has(scriptPathOf(id)), ), ); // `runAll` = no `--from` on a multi-root pipeline (fan-in): run the whole @@ -718,6 +739,14 @@ async function run( `--from '${opts.from}' is a \`// macros\` library — definition-only, injected into consuming scripts at run time, never a runnable step.`, ); } + // A `--local` display-only node (a non-`// pipeline` DuckDB script surfaced + // only because it calls a macro) has no previewable content — reject it + // clearly rather than admitting it and silently producing an empty plan. + if (notRunnablePaths.has(scriptPathOf(resolved))) { + throw new Error( + `--from '${opts.from}' isn't a \`// pipeline\` script — it appears in the local graph only as a macro consumer (lineage). Mark it \`// pipeline\` to run it.`, + ); + } if (!resolved.startsWith("script:")) { throw new Error( `--from '${opts.from}' is an asset, not a runnable — start a run from a script (assets are produced, not run).`, @@ -829,10 +858,10 @@ async function run( } } - // Selections can still pull a macro library in via graph reachability (e.g. - // whole-pipeline mode collects every root's closure) — drop them before - // ordering so they never run. - for (const p of macroLibPaths) selectedScripts.delete(p); + // Selections can still pull a non-runnable node in via graph reachability (e.g. + // whole-pipeline mode collects every root's closure) — drop macro libraries and + // local-only display nodes before ordering so they never run. + for (const p of notRunnablePaths) selectedScripts.delete(p); const { order, cyclic } = topoOrder(graph, selectedScripts); if (cyclic.length > 0) { diff --git a/cli/test/duckdb_macros_unit.test.ts b/cli/test/duckdb_macros_unit.test.ts new file mode 100644 index 0000000000..9ad1fec568 --- /dev/null +++ b/cli/test/duckdb_macros_unit.test.ts @@ -0,0 +1,87 @@ +import { expect, test } from "bun:test"; + +import { + detectMacroCalls, + parseCreateMacro, + parseMacroAnnotations, + parseMacroLibrary, + splitStatements, +} from "../src/commands/pipeline/duckdbMacros.ts"; + +// These mirror the accepted-shape cases of the backend parser +// (backend/parsers/windmill-parser/src/duckdb_macros.rs) so the local graph +// derives the same macro registry the server records at deploy. + +test("parseCreateMacro: scalar, OR REPLACE / TEMP / TABLE, FUNCTION alias", () => { + expect(parseCreateMacro("CREATE MACRO surrogate_key(a, b) AS md5(concat_ws('||', a, b))")).toEqual( + { name: "surrogate_key", params: "a, b", isTable: false }, + ); + expect(parseCreateMacro("CREATE OR REPLACE TEMP MACRO top_n(t_max) AS TABLE SELECT * FROM t")).toEqual( + { name: "top_n", params: "t_max", isTable: true }, + ); + // FUNCTION is DuckDB's alias for MACRO; name lowercased + expect(parseCreateMacro("create function Dbl(a) as a * 2")).toEqual( + { name: "dbl", params: "a", isTable: false }, + ); +}); + +test("parseCreateMacro: default params + nested/string parens in the param list", () => { + expect( + parseCreateMacro("CREATE MACRO safe_div(a, b, fallback := (0)) AS CASE WHEN b = 0 THEN fallback ELSE a / b END")?.params, + ).toBe("a, b, fallback := (0)"); + expect(parseCreateMacro("CREATE MACRO f(sep := '(') AS concat(sep, 'x')")?.params).toBe("sep := '('"); +}); + +test("parseCreateMacro: rejects non-macro, qualified/quoted names, missing params/body", () => { + expect(parseCreateMacro("SELECT 1")).toBeNull(); + expect(parseCreateMacro("CREATE TABLE t(x int)")).toBeNull(); + expect(parseCreateMacro("CREATE MACRO lake.m(a) AS a")).toBeNull(); + expect(parseCreateMacro('CREATE MACRO "weird name"(a) AS a')).toBeNull(); + expect(parseCreateMacro("CREATE MACRO m AS 1")).toBeNull(); // no params + expect(parseCreateMacro("CREATE MACRO m(a) AS")).toBeNull(); // empty body + expect(parseCreateMacro("CREATE OR MACRO m(a) AS a")).toBeNull(); // OR without REPLACE +}); + +test("splitStatements strips comments/strings and parseMacroLibrary skips setup", () => { + const lib = `-- macros +ATTACH 'x.duckdb' AS ext; +CREATE MACRO m1(a) AS a; -- inline comment ; not a split +CREATE MACRO m2(b) AS b + 1;`; + expect(splitStatements("SELECT 1; -- c ; still\nSELECT ';' AS x;").length).toBe(2); + expect(parseMacroLibrary(lib).map((m) => m.name)).toEqual(["m1", "m2"]); +}); + +test("detectMacroCalls: word-boundary, case-insensitive, skips qualified/strings/comments", () => { + const ns = new Set(["dbl", "avg_x"]); + const found = detectMacroCalls("SELECT DBL(1), my_dbl(2), avg_x (3) FROM t", ns); + expect([...found].sort()).toEqual(["avg_x", "dbl"]); + expect(detectMacroCalls("SELECT lake.dbl(1)", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("SELECT 'dbl(1)'", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("-- dbl(1)\nSELECT 1", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("SELECT dbl FROM t", new Set(["dbl"])).size).toBe(0); // no call parens +}); + +test("parseMacroAnnotations: leading-header only, marker stands alone, `// use` needs a path token", () => { + expect(parseMacroAnnotations("-- macros\nCREATE MACRO m(a) AS a;")).toEqual({ + macros: true, + useLibs: [], + }); + // marker with trailing prose is not the macros marker + expect(parseMacroAnnotations("-- macros are below\nSELECT 1;").macros).toBe(false); + // `// use` accumulates path-shaped tokens, dedups, and stops at the body + expect( + parseMacroAnnotations("-- pipeline\n-- use f/lib/a\n-- use f/lib/a\n-- use notapath\nSELECT 1;").useLibs, + ).toEqual(["f/lib/a"]); + // an annotation after code is ignored (leading header only) + expect(parseMacroAnnotations("SELECT 1;\n-- use f/lib/a").useLibs).toEqual([]); +}); + +test("parseMacroAnnotations accepts `//`, `--`, and `#` prefixes (backend parity, any language)", () => { + // The canonical backend parser strips `//`/`--`/`#` regardless of language, so + // a DuckDB library headed with `// macros` (not `-- macros`) must be detected. + expect(parseMacroAnnotations("// macros\nCREATE MACRO m(a) AS a;").macros).toBe(true); + expect(parseMacroAnnotations("# macros\nSELECT 1;").macros).toBe(true); + expect(parseMacroAnnotations("// use f/lib/a\nSELECT 1;").useLibs).toEqual(["f/lib/a"]); + // mixed prefixes in one header both register + expect(parseMacroAnnotations("// pipeline\n-- use f/lib/a\nSELECT 1;").useLibs).toEqual(["f/lib/a"]); +}); diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index b6cdfaa809..377da60851 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -321,6 +321,300 @@ test("`# volume:` producer connects to its `# on volume://` consumer", async () ); }); +test("`// macros` library: node with signatures + call-detected consumer edge, counted", async () => { + // The wasm asset parser drops `// macros`/`// use`; the CLI-side lexical scan + // must surface the library node, its macro signatures, and the caller edge so + // `--local` reaches parity with the deployed graph. + await withFolder( + { + "macros_finance.duckdb.sql": + `-- macros\nCREATE MACRO net_revenue(gross, refunds) AS gross - refunds;\nCREATE OR REPLACE MACRO safe_div(a, b) AS TABLE SELECT a / b;\n`, + "fct.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\nSELECT net_revenue(gross, refunds) AS rev FROM main.orders;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + + // the library is a node carrying its macro signatures + const lib = graph.runnables.find((r) => r.path === "f/mypipe/macros_finance"); + expect(lib?.macros).toEqual([ + { name: "net_revenue", params: "gross, refunds", is_table: false }, + { name: "safe_div", params: "a, b", is_table: true }, + ]); + + // library counts toward the script total (2 scripts, not just the 1 pipeline member) + expect(graph.runnables.map((r) => r.path).sort()).toEqual([ + "f/mypipe/fct", + "f/mypipe/macros_finance", + ]); + + // the caller edge names only the macro actually called (safe_div is not) + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/mypipe/macros_finance", + consumer_path: "f/mypipe/fct", + macro_names: ["net_revenue"], + via_use: false, + }, + ]); + }, + ); +}); + +test("`// use ` forces a whole-library edge even with no lexical call", async () => { + // Dynamic-SQL callers annotate `// use`; the edge lists every macro and is + // flagged via_use (mirrors the deployed graph). + await withFolder( + { + "stats.duckdb.sql": + `-- macros\nCREATE MACRO zscore(x, m, s) AS (x - m) / s;\n`, + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/metrics\n-- use f/mypipe/stats\nSELECT query('SELECT zscore(v, 0, 1) FROM main.metrics');\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/mypipe/stats", + consumer_path: "f/mypipe/report", + macro_names: ["zscore"], + via_use: true, + }, + ]); + // the library still becomes a node + expect(graph.runnables.some((r) => r.path === "f/mypipe/stats")).toBe(true); + }, + ); +}); + +test("an UNUSED in-folder `// macros` library is still a member node (deployed parity)", async () => { + // The deployed graph marks every macro library `auto_kind='pipeline'`, so an + // in-folder library is a member node (in_pipeline, with signatures) whether or + // not a consumer uses it. It is still excluded from runs (via `macros`). + await withFolder( + { + "unused.duckdb.sql": `-- macros\nCREATE MACRO helper(a) AS a + 1;\n`, + "solo.duckdb.sql": `-- pipeline\nSELECT 1;\n`, + }, + async (root, folder) => { + const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const lib = graph.runnables.find((r) => r.path === "f/mypipe/unused"); + expect(lib?.in_pipeline).toBe(true); + expect(lib?.macros).toEqual([{ name: "helper", params: "a", is_table: false }]); + expect(graph.macro_edges).toBeUndefined(); + // but it is not a previewable/runnable script — only `solo` is + expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/solo"]); + }, + ); +}); + +test("a shared macro library OUTSIDE the pipeline folder is resolved (workspace-wide)", async () => { + // The deployed graph loads the macro registry workspace-wide and only + // folder-scopes consumers, so a pipeline in `f/mypipe` can use a shared library + // in `f/shared`. Local discovery must walk the whole `f/` tree, not just the + // folder, or the edge/node/docs are missing (parity gap). + const root = mkdtempSync(join(tmpdir(), "wm-pl-")); + writeFileSync(join(root, "wmill.yaml"), "defaultTs: bun\n"); + mkdirSync(join(root, "f", "shared"), { recursive: true }); + mkdirSync(join(root, "f", "mypipe"), { recursive: true }); + writeFileSync( + join(root, "f", "shared", "stats.duckdb.sql"), + `-- macros\nCREATE MACRO zscore(x, m, s) AS (x - m) / s;\n`, + ); + // one consumer calls the shared macro lexically, another pulls it via `// use` + writeFileSync( + join(root, "f", "mypipe", "fct.duckdb.sql"), + `-- pipeline\n-- on datatable://main/metrics\nSELECT zscore(v, 0, 1) FROM main.metrics;\n`, + ); + writeFileSync( + join(root, "f", "mypipe", "dyn.duckdb.sql"), + `-- pipeline\n-- on datatable://main/metrics\n-- use f/shared/stats\nSELECT query('SELECT zscore(v, 0, 1)');\n`, + ); + try { + const { graph } = await buildLocalPipelineGraph({ root, folder: "mypipe", defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/shared/stats", + consumer_path: "f/mypipe/dyn", + macro_names: ["zscore"], + via_use: true, + }, + { + lib_path: "f/shared/stats", + consumer_path: "f/mypipe/fct", + macro_names: ["zscore"], + via_use: false, + }, + ]); + // the out-of-folder library is surfaced as a node with its signatures + expect(graph.runnables.find((r) => r.path === "f/shared/stats")?.macros).toEqual([ + { name: "zscore", params: "x, m, s", is_table: false }, + ]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("macro library that CONSUMES another library gets a lib→lib edge (both nodes surface)", async () => { + // The deploy path records macro_usage for any DuckDB script, including a macro + // library calling another library's macros. A `base → derived` edge must exist + // (and `base`, an in-folder library, is a member node not just an edge stub). + await withFolder( + { + "base.duckdb.sql": `-- macros\nCREATE MACRO base_add(a, b) AS a + b;\n`, + // derived is a `// macros` library (NOT `// pipeline`) whose body calls base_add + "derived.duckdb.sql": + `-- macros\nCREATE MACRO derived_sum(a, b) AS base_add(a, b) * 2;\n`, + // the pipeline consumer calls derived_sum + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/t\nSELECT derived_sum(x, y) FROM main.t;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + // base → derived: the intermediate library is itself a consumer + { + lib_path: "f/mypipe/base", + consumer_path: "f/mypipe/derived", + macro_names: ["base_add"], + via_use: false, + }, + // derived → report: the pipeline consumer + { + lib_path: "f/mypipe/derived", + consumer_path: "f/mypipe/report", + macro_names: ["derived_sum"], + via_use: false, + }, + ]); + // both libraries surface as nodes with their signatures (base does NOT + // disappear just because it is only reached transitively) + expect(graph.runnables.find((r) => r.path === "f/mypipe/base")?.macros).toEqual([ + { name: "base_add", params: "a, b", is_table: false }, + ]); + expect(graph.runnables.find((r) => r.path === "f/mypipe/derived")?.macros).toEqual([ + { name: "derived_sum", params: "a, b", is_table: false }, + ]); + // both intermediate libraries are members (in_pipeline), like the deployed graph + expect(graph.runnables.find((r) => r.path === "f/mypipe/base")?.in_pipeline).toBe(true); + expect(graph.runnables.find((r) => r.path === "f/mypipe/derived")?.in_pipeline).toBe(true); + }, + ); +}); + +test("macro library reaching another only via dynamic SQL gets a `// use` lib→lib edge", async () => { + // A library that calls another's macro only inside a `query('…')` string is + // invisible to lexical detection; `// use` forces the lib→lib edge. Exercises + // the case where the CONSUMER is itself a `// macros` library. + await withFolder( + { + "base.duckdb.sql": `-- macros\nCREATE MACRO base_add(a, b) AS a + b;\n`, + "derived.duckdb.sql": + `-- macros\n-- use f/mypipe/base\nCREATE MACRO wrap(a, b) AS query('SELECT base_add(1, 2)');\n`, + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/t\nSELECT wrap(x, y) FROM main.t;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + // base → derived comes from `// use` (via_use), NOT a lexical call + { + lib_path: "f/mypipe/base", + consumer_path: "f/mypipe/derived", + macro_names: ["base_add"], + via_use: true, + }, + { + lib_path: "f/mypipe/derived", + consumer_path: "f/mypipe/report", + macro_names: ["wrap"], + via_use: false, + }, + ]); + }, + ); +}); + +test("a `// macros` (slash-prefix) DuckDB library is detected (backend prefix parity)", async () => { + // A `.duckdb.sql` library may head its annotation with `// macros` (the backend + // accepts `//`/`--`/`#` for any language); the local scan must too. + await withFolder( + { + "lib.duckdb.sql": `// macros\nCREATE MACRO dbl(a) AS a * 2;\n`, + "use_it.duckdb.sql": `-- pipeline\n-- on datatable://main/t\nSELECT dbl(x) FROM main.t;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/mypipe/lib", + consumer_path: "f/mypipe/use_it", + macro_names: ["dbl"], + via_use: false, + }, + ]); + expect(graph.runnables.find((r) => r.path === "f/mypipe/lib")?.macros).toEqual([ + { name: "dbl", params: "a", is_table: false }, + ]); + }, + ); +}); + +test("a non-pipeline DuckDB macro consumer is a display-only node, never a run step", async () => { + // A `.duckdb.sql` helper that calls a macro but isn't `// pipeline` surfaces as + // a graph node (lineage parity) but is NOT a runnable pipeline script: it must + // be absent from `scripts` (the previewable set `run --local` selects from), so + // it can never be scheduled or fail a preview for lack of local content. + await withFolder( + { + "lib.duckdb.sql": `-- macros\nCREATE MACRO dbl(a) AS a * 2;\n`, + "helper.duckdb.sql": `-- on ducklake://main/src\nSELECT dbl(x) FROM main.src;\n`, + "root.duckdb.sql": `-- pipeline\n-- materialize ducklake://main/out\nSELECT dbl(1) AS v;\n`, + }, + async (root, folder) => { + const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + + // the helper IS a node (macro consumer, for lineage display) … + const helper = graph.runnables.find((r) => r.path === "f/mypipe/helper"); + expect(helper).toBeDefined(); + // … but not a pipeline member (no local file to preview) and carries no macros + expect(helper?.in_pipeline).toBeFalsy(); + expect(helper?.macros).toBeUndefined(); + // the previewable set (what `run --local` can schedule) is ONLY the member + expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/root"]); + // and the macro edge still connects lib → helper for the lineage view + expect(graph.macro_edges).toContainEqual({ + lib_path: "f/mypipe/lib", + consumer_path: "f/mypipe/helper", + macro_names: ["dbl"], + via_use: false, + }); + }, + ); +}); + +test("`// pipeline` on a macros library is redundant — it's a library node either way", async () => { + // The backend marks a macro library `auto_kind='pipeline'` regardless of the + // `// pipeline` marker, so a `-- pipeline -- macros` script behaves exactly like + // a plain `-- macros` one: a member library node (in_pipeline + signatures), + // excluded from runs (via `macros`) and from the previewable `scripts` set. + await withFolder( + { + "lib.duckdb.sql": `-- pipeline\n-- macros\nCREATE MACRO dbl(a) AS a * 2;\n`, + "root.duckdb.sql": `-- pipeline\n-- materialize ducklake://main/out\nSELECT 1 AS v;\n`, + }, + async (root, folder) => { + const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const lib = graph.runnables.find((r) => r.path === "f/mypipe/lib"); + expect(lib?.in_pipeline).toBe(true); + expect(lib?.macros).toEqual([{ name: "dbl", params: "a", is_table: false }]); + // unused here → no edges, but still a node; not previewable/runnable + expect(graph.macro_edges).toBeUndefined(); + expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/root"]); + }, + ); +}); + test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm parser)", async () => { await withFolder( { "ingest.rb": `# pipeline\n# on s3://demo/raw.csv\nputs "hi"\n` }, @@ -335,3 +629,35 @@ test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm pars }, ); }); + +test("pipeline docs renders a `Macro libraries` section (call + `// use`)", async () => { + const { generatePipelineMarkdown } = await import( + "../src/commands/pipeline/docs.ts" + ); + await withFolder( + { + "macros_finance.duckdb.sql": + `-- macros\nCREATE MACRO net_revenue(gross, refunds) AS gross - refunds;\nCREATE MACRO safe_div(a, b) AS TABLE SELECT a / b;\n`, + "fct.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\nSELECT net_revenue(gross, refunds) FROM main.orders;\n`, + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\n-- use f/mypipe/macros_finance\nSELECT query('SELECT safe_div(1, 2)');\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const md = generatePipelineMarkdown(folder, graph, [], true); + + // count line mentions the library; two pipeline scripts, one library + expect(md).toContain("**2** scripts · **1** asset · **1** macro library"); + // dedicated section with signatures (TABLE marker) and callers + expect(md).toContain("## Macro libraries"); + expect(md).toContain("### `f/mypipe/macros_finance`"); + expect(md).toContain("- `net_revenue(gross, refunds)`"); + expect(md).toContain("- `safe_div(a, b)` → TABLE"); + expect(md).toContain("`f/mypipe/fct` (calls `net_revenue`)"); + expect(md).toContain("`f/mypipe/report` (via `// use`)"); + // the library is NOT double-listed under `## Scripts` + expect(md).not.toContain("### `f/mypipe/macros_finance`\n\n- **"); + }, + ); +});