diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts index b360c8b802..abee64cbf3 100644 --- a/cli/src/commands/pipeline/docs.ts +++ b/cli/src/commands/pipeline/docs.ts @@ -43,7 +43,7 @@ async function fetchDeployedGraph( } // Render the pipeline graph as a markdown document. -function generatePipelineMarkdown( +export function generatePipelineMarkdown( folder: string, graph: AssetGraph, datatableSchemas: any[], @@ -73,7 +73,26 @@ function generatePipelineMarkdown( } } - const scripts = graph.runnables.filter((r) => r.usage_kind === "script").map((r) => r.path).sort(); + // `// macros` libraries are definition-only nodes (not runnable pipeline + // steps), so they get their own section below and are excluded from the + // per-script listing + the script count. + const macroLibs = graph.runnables + .filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0) + .sort((a, b) => a.path.localeCompare(b.path)); + const macroLibPaths = new Set(macroLibs.map((r) => r.path)); + const macroConsumersByLib = new Map(); + for (const me of graph.macro_edges ?? []) { + (macroConsumersByLib.get(me.lib_path) ?? macroConsumersByLib.set(me.lib_path, []).get(me.lib_path)!).push({ + consumer: me.consumer_path, + names: me.macro_names, + viaUse: me.via_use, + }); + } + + const scripts = graph.runnables + .filter((r) => r.usage_kind === "script" && !macroLibPaths.has(r.path)) + .map((r) => r.path) + .sort(); let md = `# Pipeline \`f/${folder}\` @@ -85,7 +104,7 @@ produces data by reading/writing assets in its body (\`datatable://\`, \`ducklake://\`, \`s3://\`, \`volume://\`). The cascade runs a producer, then every downstream subscriber, in topological order. -- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"} +- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"}${macroLibs.length > 0 ? ` · **${macroLibs.length}** macro librar${macroLibs.length === 1 ? "y" : "ies"}` : ""} ## Scripts @@ -107,6 +126,30 @@ downstream subscriber, in topological order. md += `\n`; } + // Macro libraries: `// macros` scripts whose macros are injected into consuming + // DuckDB scripts at run time. List each library's signatures and its callers so + // an agent discovers the reuse layer instead of re-inlining the logic. + if (macroLibs.length > 0) { + md += `## Macro libraries\n\n`; + md += `\`// macros\` DuckDB libraries. Their \`CREATE MACRO\` definitions are injected as\nTEMP macros into consuming scripts at run time — call a macro by name, or force the\nwhole library in with \`// use \` (needed for macros only reached via dynamic SQL).\n\n`; + for (const lib of macroLibs) { + md += `### \`${lib.path}\`\n\n`; + for (const m of lib.macros ?? []) { + md += `- \`${m.name}(${m.params ?? ""})\`${m.is_table ? " → TABLE" : ""}\n`; + } + const consumers = [...(macroConsumersByLib.get(lib.path) ?? [])].sort((a, b) => + a.consumer.localeCompare(b.consumer), + ); + if (consumers.length > 0) { + md += `- **Used by:**\n`; + for (const c of consumers) { + md += ` - \`${c.consumer}\` (${c.viaUse ? "via \`// use\`" : `calls ${c.names.map((n) => `\`${n}\``).join(", ")}`})\n`; + } + } + md += `\n`; + } + } + // Datatable schemas, restricted to datatables this pipeline references. const referencedDatatables = new Set( graph.assets.filter((a) => a.kind === "datatable").map((a) => a.path.split("/")[0]), diff --git a/cli/src/commands/pipeline/duckdbMacros.ts b/cli/src/commands/pipeline/duckdbMacros.ts new file mode 100644 index 0000000000..cd1e7c3543 --- /dev/null +++ b/cli/src/commands/pipeline/duckdbMacros.ts @@ -0,0 +1,272 @@ +// Local, lexical DuckDB `// macros`-library parsing for the pipeline graph. +// +// The deployed graph records a workspace macro registry (`macro_definition`) and +// call edges (`macro_usage`) at deploy time and surfaces them on `/assets/graph`. +// The wasm asset parser used locally does NOT emit any of that (it predates the +// `// macros` / `// use` annotations — it drops both), so to reach local/deployed +// parity (`pipeline show --local`, `pipeline dev`, `pipeline docs`) the CLI must +// derive the same data from the working tree itself. +// +// This is a faithful TS port of the lexical rules in +// `backend/parsers/windmill-parser/src/duckdb_macros.rs` (`split_statements`, +// `parse_create_macro`, `detect_macro_calls`): top-level `CREATE [OR REPLACE] +// [TEMP] MACRO|FUNCTION () AS [TABLE] `, with string/comment +// spans skipped. It is deliberately lexical, not an AST parse — over-matching a +// call only draws a spurious edge, never a wrong run. +// +// LIMITATION (same as the server's static detection): a macro invoked only from +// dynamic SQL inside a `query('…')` string is invisible to call detection — +// annotate the consumer with `// use ` to force the whole-library edge. + +export type ParsedMacro = { + // Lowercased bare identifier (DuckDB identifiers are case-insensitive unquoted; + // qualified / quoted names are rejected, matching the backend). + name: string; + // Verbatim text inside the parameter parens (may be empty). + params: string; + isTable: boolean; +}; + +// Case-insensitive whole-word prefix strip (whitespace-bounded). Returns the +// remainder with leading whitespace trimmed, or null when `s` does not start with +// `kw` as a whole word. `kw` must be lowercase. Mirrors `strip_kw`. +function stripKw(s: string, kw: string): string | null { + if (s.length < kw.length) return null; + if (s.slice(0, kw.length).toLowerCase() !== kw) return null; + const after = s.slice(kw.length); + if (after.length === 0 || /^\s/.test(after)) return after.replace(/^\s+/, ""); + return null; +} + +function isIdent(s: string): boolean { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(s); +} + +// Split a script body into `;`-terminated statements with line comments +// (`--`, `//`), block comments (`/* */`), and quoted spans stripped/preserved as +// in the SQL executor. Char-wise (not byte-wise) so multi-byte text survives. +// Mirrors `split_statements`. +export function splitStatements(sql: string): string[] { + const out: string[] = []; + let cur = ""; + const chars = [...sql]; + let i = 0; + const n = chars.length; + while (i < n) { + const c = chars[i]; + // line comment: `--` (SQL) or `//` (Windmill annotation prefix) + if ( + (c === "-" && i + 1 < n && chars[i + 1] === "-") || + (c === "/" && i + 1 < n && chars[i + 1] === "/") + ) { + while (i < n && chars[i] !== "\n") i += 1; + continue; + } + // block comment + if (c === "/" && i + 1 < n && chars[i + 1] === "*") { + i += 2; + while (i + 1 < n && !(chars[i] === "*" && chars[i + 1] === "/")) i += 1; + i += 2; + continue; + } + // single-quoted string ('' escapes an embedded quote) + if (c === "'") { + cur += c; + i += 1; + while (i < n) { + cur += chars[i]; + if (chars[i] === "'") { + if (i + 1 < n && chars[i + 1] === "'") { + cur += "'"; + i += 2; + continue; + } + i += 1; + break; + } + i += 1; + } + continue; + } + // double-quoted identifier + if (c === '"') { + cur += c; + i += 1; + while (i < n) { + cur += chars[i]; + if (chars[i] === '"') { + i += 1; + break; + } + i += 1; + } + continue; + } + if (c === ";") { + const t = cur.trim(); + if (t !== "") out.push(t); + cur = ""; + i += 1; + continue; + } + cur += c; + i += 1; + } + const t = cur.trim(); + if (t !== "") out.push(t); + return out; +} + +// Parse one comment-free, `;`-less statement as a CREATE MACRO. Returns null when +// the statement is not a well-formed macro definition (unlike the backend, which +// distinguishes "not macro-shaped" from "malformed" to surface deploy errors — +// the local graph is lenient and simply skips anything it can't read). Mirrors +// `parse_create_macro` for the shape it accepts. +export function parseCreateMacro(stmt: string): ParsedMacro | null { + let rest = stripKw(stmt.trim(), "create"); + if (rest === null) return null; + const afterOr = stripKw(rest, "or"); + if (afterOr !== null) { + const afterReplace = stripKw(afterOr, "replace"); + if (afterReplace === null) return null; // CREATE OR + rest = afterReplace; + } + const afterTemp = stripKw(rest, "temp") ?? stripKw(rest, "temporary"); + if (afterTemp !== null) rest = afterTemp; + // `FUNCTION` is DuckDB's alias for `MACRO`. + const afterMacro = stripKw(rest, "macro") ?? stripKw(rest, "function"); + if (afterMacro === null) return null; + rest = afterMacro; + + const nameEndMatch = rest.match(/[\s(]/); + const nameEnd = nameEndMatch ? nameEndMatch.index! : rest.length; + const rawName = rest.slice(0, nameEnd); + if (rawName === "" || rawName.includes(".") || rawName.includes('"') || !isIdent(rawName)) { + return null; + } + const name = rawName.toLowerCase(); + + rest = rest.slice(nameEnd).replace(/^\s+/, ""); + if (!rest.startsWith("(")) return null; + // Balanced-paren scan for the verbatim param list; skip quoted spans (a default + // value may contain a paren). + const pchars = [...rest]; + let depth = 0; + let j = 0; + let close = -1; + while (j < pchars.length) { + const ch = pchars[j]; + if (ch === "(") depth += 1; + else if (ch === ")") { + depth -= 1; + if (depth === 0) { + close = j; + break; + } + } else if (ch === "'" || ch === '"') { + const q = ch; + j += 1; + while (j < pchars.length && pchars[j] !== q) j += 1; + } + j += 1; + } + if (close === -1) return null; // unbalanced + const params = pchars.slice(1, close).join("").trim(); + + const afterParams = pchars.slice(close + 1).join("").replace(/^\s+/, ""); + let body = stripKw(afterParams, "as"); + if (body === null) return null; + let isTable = false; + const afterTable = stripKw(body, "table"); + if (afterTable !== null) { + body = afterTable; + isTable = true; + } + const trimmedBody = body.replace(/;+\s*$/, "").trim(); + if (trimmedBody === "") return null; // empty body + return { name, params, isTable }; +} + +// All well-formed macro definitions in a `// macros` library body, in source +// order. Non-macro statements (setup: ATTACH/INSTALL/… ) are skipped. +export function parseMacroLibrary(sql: string): ParsedMacro[] { + const out: ParsedMacro[] = []; + for (const stmt of splitStatements(sql)) { + const m = parseCreateMacro(stmt); + if (m) out.push(m); + } + return out; +} + +// Names from `names` that `sql` calls: an identifier token immediately followed +// by `(` (after optional whitespace), not `.`-qualified, outside strings and +// comments. Lexical (over-matching only adds an unused edge). Mirrors +// `detect_macro_calls`. +export function detectMacroCalls(sql: string, names: Set): Set { + const found = new Set(); + if (names.size === 0) return found; + for (const stmt of splitStatements(sql)) { + const chars = [...stmt]; + const n = chars.length; + let i = 0; + let prev: string | undefined = undefined; + while (i < n) { + const c = chars[i]; + if (c === "'" || c === '"') { + const q = c; + i += 1; + while (i < n && chars[i] !== q) i += 1; + i += 1; + prev = q; + continue; + } + const isIdentStart = /[A-Za-z_]/.test(c); + const prevBlocks = prev !== undefined && /[.A-Za-z0-9_]/.test(prev); + if (isIdentStart && !prevBlocks) { + const start = i; + while (i < n && /[A-Za-z0-9_]/.test(chars[i])) i += 1; + const word = chars.slice(start, i).join("").toLowerCase(); + let k = i; + while (k < n && /\s/.test(chars[k])) k += 1; + if (k < n && chars[k] === "(" && names.has(word)) found.add(word); + prev = chars[i - 1]; + continue; + } + prev = c; + i += 1; + } + } + return found; +} + +// Scan the LEADING comment header for the `// macros` marker and `// use ` +// annotations, independent of the wasm parser (which drops both). Mirrors the +// canonical `parse_pipeline_annotations` handling: the marker must stand alone on +// its line; a `// use` target is a single whitespace-free token containing `/`. +// `prefix` is the language comment prefix (`--` for SQL). Scanning stops at the +// first non-comment line so a body comment can't inject a phantom annotation. +export function parseMacroAnnotations( + content: string, + prefix: string, +): { macros: boolean; useLibs: string[] } { + const p = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const macrosRe = new RegExp(`^\\s*${p}\\s*macros\\s*$`); + const useRe = new RegExp(`^\\s*${p}\\s*use\\s+(\\S+)\\s*$`); + let macros = false; + const useLibs: string[] = []; + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (trimmed === "") continue; + if (!trimmed.startsWith(prefix)) break; + if (macrosRe.test(line)) { + macros = true; + continue; + } + const u = line.match(useRe); + if (u) { + const path = u[1]; + if (path.includes("/") && !useLibs.includes(path)) useLibs.push(path); + } + } + return { macros, useLibs }; +} diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 805f30c710..42dfcd4b6c 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -19,6 +19,12 @@ import * as log from "../../core/log.ts"; import { exts, removeExtensionToPath } from "../script/script.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { getWmillYamlPath } from "../../core/conf.ts"; +import { + detectMacroCalls, + parseMacroAnnotations, + parseMacroLibrary, + type ParsedMacro, +} from "./duckdbMacros.ts"; // Resolve the workspace root (the directory containing wmill.yaml) for reading // local files, falling back to the current directory. @@ -564,13 +570,118 @@ export async function buildLocalPipelineGraph(args: { } } + // `// macros` libraries + lib→consumer edges. The wasm asset parser drops the + // `// macros` / `// use` annotations and never emits a macro registry, so we + // derive both from the working tree here (see ./duckdbMacros.ts) to match the + // deployed graph, which records them at deploy. Macros are DuckDB-only. + const macroEdges = buildMacroEdges(all, runnables); + return { graph: { runnables, assets: [...assetSet.values()], edges, triggers, + ...(macroEdges.length > 0 ? { macro_edges: macroEdges } : {}), }, scripts: pipelineScripts, }; } + +// Detect `// macros` libraries, resolve which pipeline scripts call their macros +// (by lexical call detection + `// use` annotations), and (a) mutate `runnables` +// to add each referenced library as a node carrying its macro signatures, and +// (b) return the lib→consumer edges. Mirrors the deployed graph builder +// (`asset_graph` in windmill-api-assets): a library node appears only when it is +// an endpoint of at least one edge, so an unused library is not surfaced. +function buildMacroEdges( + all: LocalScript[], + runnables: GraphRunnable[], +): NonNullable { + // Parse every `// macros` library body into its macro definitions. Only DuckDB + // scripts can be libraries. + const libMacros = new Map(); + const useLibsByScript = new Map(); + for (const s of all) { + if (s.language !== "duckdb") continue; + const { macros, useLibs } = parseMacroAnnotations(s.content, "--"); + if (useLibs.length > 0) useLibsByScript.set(s.path, useLibs); + if (macros) libMacros.set(s.path, parseMacroLibrary(s.content)); + } + if (libMacros.size === 0) return []; + + // Macro name → providing library. Names are workspace-unique (the deploy path + // enforces this); on a local collision, last-writer-wins, harmlessly. + const providerByName = new Map(); + for (const [lib, macros] of libMacros) { + for (const m of macros) providerByName.set(m.name, lib); + } + const allMacroNames = new Set(providerByName.keys()); + + // Only `// pipeline` scripts are graph nodes, so only they are candidate + // consumers. Aggregate per (lib, consumer): the set of called macro names and + // whether the edge came (also) from a whole-library `// use`. + const pipelinePaths = new Set(runnables.map((r) => r.path)); + type EdgeAgg = { names: Set; viaUse: boolean }; + const edgeMap = new Map(); + const edgeKey = (lib: string, consumer: string) => `${lib}${consumer}`; + const aggFor = (lib: string, consumer: string): EdgeAgg => { + const key = edgeKey(lib, consumer); + let agg = edgeMap.get(key); + if (!agg) { + agg = { names: new Set(), viaUse: false }; + edgeMap.set(key, agg); + } + return agg; + }; + for (const s of all) { + if (s.language !== "duckdb" || !pipelinePaths.has(s.path)) continue; + for (const name of detectMacroCalls(s.content, allMacroNames)) { + const lib = providerByName.get(name)!; + if (lib === s.path) continue; // a library calling its own macro is not an edge + aggFor(lib, s.path).names.add(name); + } + for (const lib of useLibsByScript.get(s.path) ?? []) { + const macros = libMacros.get(lib); + // An out-of-tree / unknown `// use` target can't be resolved locally. + if (!macros || lib === s.path) continue; + const agg = aggFor(lib, s.path); + agg.viaUse = true; + for (const m of macros) agg.names.add(m.name); + } + } + + const edges = [...edgeMap.entries()] + .map(([key, agg]) => { + const [lib_path, consumer_path] = key.split(""); + // `via_use` is always present (the deployed `MacroEdge` serializes it + // unconditionally) so `--json` matches byte-for-byte. + return { + lib_path, + consumer_path, + macro_names: [...agg.names].sort(), + via_use: agg.viaUse, + }; + }) + .sort((a, b) => + a.lib_path.localeCompare(b.lib_path) || + a.consumer_path.localeCompare(b.consumer_path), + ); + + // Surface each referenced library as a node with its macro signatures. If the + // library is also a `// pipeline` member it already has a runnable — enrich it + // in place rather than duplicating. + for (const lib of new Set(edges.map((e) => e.lib_path))) { + const macros = (libMacros.get(lib) ?? []).map((m) => ({ + name: m.name, + params: m.params, + is_table: m.isTable, + })); + const existing = runnables.find((r) => r.path === lib); + if (existing) existing.macros = macros; + else runnables.push({ path: lib, usage_kind: "script", macros }); + } + runnables.sort((a, b) => a.path.localeCompare(b.path)); + + return edges; +} diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts index 5e686fcf57..63ae3213ef 100644 --- a/cli/src/commands/pipeline/pipeline.ts +++ b/cli/src/commands/pipeline/pipeline.ts @@ -587,19 +587,25 @@ async function run( // Resolve relative imports from local (not-yet-deployed) content so a script // that imports a sibling workspace lib previews against local edits. Same // trick as `wmill dev` / `wmill app dev`; degrades to undefined gracefully. - try { - const codebases = await listSyncCodebases(merged); - const { buildPreviewTempScriptRefs } = await import( - "../generate-metadata/generate-metadata.ts" - ); - tempScriptRefs = await buildPreviewTempScriptRefs( - workspace, - merged, - codebases, - { kind: "all" }, - ); - } catch { - // best-effort: relative imports fall back to deployed versions + // Skipped for `--dry-run`: it locks/uploads scripts as a side effect (it + // regenerates `*.script.yaml` / `*.script.lock` / `wmill-lock.yaml`), and a + // plan preview must stay READ-ONLY — the refs are only consumed when a + // preview job actually runs below, which dry-run never reaches. + if (!opts.dryRun) { + try { + const codebases = await listSyncCodebases(merged); + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + merged, + codebases, + { kind: "all" }, + ); + } catch { + // best-effort: relative imports fall back to deployed versions + } } } else { graph = await apiGet( diff --git a/cli/test/duckdb_macros_unit.test.ts b/cli/test/duckdb_macros_unit.test.ts new file mode 100644 index 0000000000..3ccd7afbb3 --- /dev/null +++ b/cli/test/duckdb_macros_unit.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from "bun:test"; + +import { + detectMacroCalls, + parseCreateMacro, + parseMacroAnnotations, + parseMacroLibrary, + splitStatements, +} from "../src/commands/pipeline/duckdbMacros.ts"; + +// These mirror the accepted-shape cases of the backend parser +// (backend/parsers/windmill-parser/src/duckdb_macros.rs) so the local graph +// derives the same macro registry the server records at deploy. + +test("parseCreateMacro: scalar, OR REPLACE / TEMP / TABLE, FUNCTION alias", () => { + expect(parseCreateMacro("CREATE MACRO surrogate_key(a, b) AS md5(concat_ws('||', a, b))")).toEqual( + { name: "surrogate_key", params: "a, b", isTable: false }, + ); + expect(parseCreateMacro("CREATE OR REPLACE TEMP MACRO top_n(t_max) AS TABLE SELECT * FROM t")).toEqual( + { name: "top_n", params: "t_max", isTable: true }, + ); + // FUNCTION is DuckDB's alias for MACRO; name lowercased + expect(parseCreateMacro("create function Dbl(a) as a * 2")).toEqual( + { name: "dbl", params: "a", isTable: false }, + ); +}); + +test("parseCreateMacro: default params + nested/string parens in the param list", () => { + expect( + parseCreateMacro("CREATE MACRO safe_div(a, b, fallback := (0)) AS CASE WHEN b = 0 THEN fallback ELSE a / b END")?.params, + ).toBe("a, b, fallback := (0)"); + expect(parseCreateMacro("CREATE MACRO f(sep := '(') AS concat(sep, 'x')")?.params).toBe("sep := '('"); +}); + +test("parseCreateMacro: rejects non-macro, qualified/quoted names, missing params/body", () => { + expect(parseCreateMacro("SELECT 1")).toBeNull(); + expect(parseCreateMacro("CREATE TABLE t(x int)")).toBeNull(); + expect(parseCreateMacro("CREATE MACRO lake.m(a) AS a")).toBeNull(); + expect(parseCreateMacro('CREATE MACRO "weird name"(a) AS a')).toBeNull(); + expect(parseCreateMacro("CREATE MACRO m AS 1")).toBeNull(); // no params + expect(parseCreateMacro("CREATE MACRO m(a) AS")).toBeNull(); // empty body + expect(parseCreateMacro("CREATE OR MACRO m(a) AS a")).toBeNull(); // OR without REPLACE +}); + +test("splitStatements strips comments/strings and parseMacroLibrary skips setup", () => { + const lib = `-- macros +ATTACH 'x.duckdb' AS ext; +CREATE MACRO m1(a) AS a; -- inline comment ; not a split +CREATE MACRO m2(b) AS b + 1;`; + expect(splitStatements("SELECT 1; -- c ; still\nSELECT ';' AS x;").length).toBe(2); + expect(parseMacroLibrary(lib).map((m) => m.name)).toEqual(["m1", "m2"]); +}); + +test("detectMacroCalls: word-boundary, case-insensitive, skips qualified/strings/comments", () => { + const ns = new Set(["dbl", "avg_x"]); + const found = detectMacroCalls("SELECT DBL(1), my_dbl(2), avg_x (3) FROM t", ns); + expect([...found].sort()).toEqual(["avg_x", "dbl"]); + expect(detectMacroCalls("SELECT lake.dbl(1)", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("SELECT 'dbl(1)'", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("-- dbl(1)\nSELECT 1", new Set(["dbl"])).size).toBe(0); + expect(detectMacroCalls("SELECT dbl FROM t", new Set(["dbl"])).size).toBe(0); // no call parens +}); + +test("parseMacroAnnotations: leading-header only, marker stands alone, `// use` needs a path token", () => { + expect(parseMacroAnnotations("-- macros\nCREATE MACRO m(a) AS a;", "--")).toEqual({ + macros: true, + useLibs: [], + }); + // marker with trailing prose is not the macros marker + expect(parseMacroAnnotations("-- macros are below\nSELECT 1;", "--").macros).toBe(false); + // `// use` accumulates path-shaped tokens, dedups, and stops at the body + expect( + parseMacroAnnotations("-- pipeline\n-- use f/lib/a\n-- use f/lib/a\n-- use notapath\nSELECT 1;", "--").useLibs, + ).toEqual(["f/lib/a"]); + // an annotation after code is ignored (leading header only) + expect(parseMacroAnnotations("SELECT 1;\n-- use f/lib/a", "--").useLibs).toEqual([]); +}); diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index 1368e14504..86897c13f7 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -286,6 +286,88 @@ test("`# volume:` producer connects to its `# on volume://` consumer", async () ); }); +test("`// macros` library: node with signatures + call-detected consumer edge, counted", async () => { + // The wasm asset parser drops `// macros`/`// use`; the CLI-side lexical scan + // must surface the library node, its macro signatures, and the caller edge so + // `--local` reaches parity with the deployed graph. + await withFolder( + { + "macros_finance.duckdb.sql": + `-- macros\nCREATE MACRO net_revenue(gross, refunds) AS gross - refunds;\nCREATE OR REPLACE MACRO safe_div(a, b) AS TABLE SELECT a / b;\n`, + "fct.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\nSELECT net_revenue(gross, refunds) AS rev FROM main.orders;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + + // the library is a node carrying its macro signatures + const lib = graph.runnables.find((r) => r.path === "f/mypipe/macros_finance"); + expect(lib?.macros).toEqual([ + { name: "net_revenue", params: "gross, refunds", is_table: false }, + { name: "safe_div", params: "a, b", is_table: true }, + ]); + + // library counts toward the script total (2 scripts, not just the 1 pipeline member) + expect(graph.runnables.map((r) => r.path).sort()).toEqual([ + "f/mypipe/fct", + "f/mypipe/macros_finance", + ]); + + // the caller edge names only the macro actually called (safe_div is not) + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/mypipe/macros_finance", + consumer_path: "f/mypipe/fct", + macro_names: ["net_revenue"], + via_use: false, + }, + ]); + }, + ); +}); + +test("`// use ` forces a whole-library edge even with no lexical call", async () => { + // Dynamic-SQL callers annotate `// use`; the edge lists every macro and is + // flagged via_use (mirrors the deployed graph). + await withFolder( + { + "stats.duckdb.sql": + `-- macros\nCREATE MACRO zscore(x, m, s) AS (x - m) / s;\n`, + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/metrics\n-- use f/mypipe/stats\nSELECT query('SELECT zscore(v, 0, 1) FROM main.metrics');\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.macro_edges).toEqual([ + { + lib_path: "f/mypipe/stats", + consumer_path: "f/mypipe/report", + macro_names: ["zscore"], + via_use: true, + }, + ]); + // the library still becomes a node + expect(graph.runnables.some((r) => r.path === "f/mypipe/stats")).toBe(true); + }, + ); +}); + +test("an unused `// macros` library is not surfaced as a node", async () => { + // Parity with the deployed graph: a library appears only when it is an edge + // endpoint (a consumer actually uses it). + await withFolder( + { + "unused.duckdb.sql": `-- macros\nCREATE MACRO helper(a) AS a + 1;\n`, + "solo.duckdb.sql": `-- pipeline\nSELECT 1;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.runnables.map((r) => r.path)).toEqual(["f/mypipe/solo"]); + expect(graph.macro_edges).toBeUndefined(); + }, + ); +}); + test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm parser)", async () => { await withFolder( { "ingest.rb": `# pipeline\n# on s3://demo/raw.csv\nputs "hi"\n` }, @@ -300,3 +382,35 @@ test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm pars }, ); }); + +test("pipeline docs renders a `Macro libraries` section (call + `// use`)", async () => { + const { generatePipelineMarkdown } = await import( + "../src/commands/pipeline/docs.ts" + ); + await withFolder( + { + "macros_finance.duckdb.sql": + `-- macros\nCREATE MACRO net_revenue(gross, refunds) AS gross - refunds;\nCREATE MACRO safe_div(a, b) AS TABLE SELECT a / b;\n`, + "fct.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\nSELECT net_revenue(gross, refunds) FROM main.orders;\n`, + "report.duckdb.sql": + `-- pipeline\n-- on datatable://main/orders\n-- use f/mypipe/macros_finance\nSELECT query('SELECT safe_div(1, 2)');\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const md = generatePipelineMarkdown(folder, graph, [], true); + + // count line mentions the library; two pipeline scripts, one library + expect(md).toContain("**2** scripts · **1** asset · **1** macro library"); + // dedicated section with signatures (TABLE marker) and callers + expect(md).toContain("## Macro libraries"); + expect(md).toContain("### `f/mypipe/macros_finance`"); + expect(md).toContain("- `net_revenue(gross, refunds)`"); + expect(md).toContain("- `safe_div(a, b)` → TABLE"); + expect(md).toContain("`f/mypipe/fct` (calls `net_revenue`)"); + expect(md).toContain("`f/mypipe/report` (via `// use`)"); + // the library is NOT double-listed under `## Scripts` + expect(md).not.toContain("### `f/mypipe/macros_finance`\n\n- **"); + }, + ); +});