diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts index 63ae3213ef..7aba9659b8 100644 --- a/cli/src/commands/pipeline/pipeline.ts +++ b/cli/src/commands/pipeline/pipeline.ts @@ -674,12 +674,26 @@ async function run( .filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0) .map((r) => r.path), ); + // Non-runnable graph nodes: the macro libraries above, plus (under `--local`) + // any script node with no local file. The local graph surfaces macro-consumer + // nodes for lineage display — a DuckDB script that calls a macro but isn't a + // `// pipeline` member (so has no previewable content). They must never enter + // the run: as a manual root they'd be scheduled, and a preview would fail with + // no local content to send. + const notRunnablePaths = new Set(macroLibPaths); + if (opts.local && localScripts) { + for (const r of graph.runnables) { + if (r.usage_kind === "script" && !localScripts.has(r.path)) { + notRunnablePaths.add(r.path); + } + } + } // Resolve the start: explicit --from (must be a valid start) or the folder's // sole valid start. `--upload`-bound scripts join the valid starts. const starts = new Set( [...validStarts(graph), ...boundNodeIds].filter( - (id) => !macroLibPaths.has(scriptPathOf(id)), + (id) => !notRunnablePaths.has(scriptPathOf(id)), ), ); // `runAll` = no `--from` on a multi-root pipeline (fan-in): run the whole @@ -810,10 +824,10 @@ async function run( } } - // Selections can still pull a macro library in via graph reachability (e.g. - // whole-pipeline mode collects every root's closure) — drop them before - // ordering so they never run. - for (const p of macroLibPaths) selectedScripts.delete(p); + // Selections can still pull a non-runnable node in via graph reachability (e.g. + // whole-pipeline mode collects every root's closure) — drop macro libraries and + // local-only display nodes before ordering so they never run. + for (const p of notRunnablePaths) selectedScripts.delete(p); const { order, cyclic } = topoOrder(graph, selectedScripts); if (cyclic.length > 0) { diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index b7cb86ceb6..6b0c79fee4 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -493,6 +493,39 @@ test("a `// macros` (slash-prefix) DuckDB library is detected (backend prefix pa ); }); +test("a non-pipeline DuckDB macro consumer is a display-only node, never a run step", async () => { + // A `.duckdb.sql` helper that calls a macro but isn't `// pipeline` surfaces as + // a graph node (lineage parity) but is NOT a runnable pipeline script: it must + // be absent from `scripts` (the previewable set `run --local` selects from), so + // it can never be scheduled or fail a preview for lack of local content. + await withFolder( + { + "lib.duckdb.sql": `-- macros\nCREATE MACRO dbl(a) AS a * 2;\n`, + "helper.duckdb.sql": `-- on ducklake://main/src\nSELECT dbl(x) FROM main.src;\n`, + "root.duckdb.sql": `-- pipeline\n-- materialize ducklake://main/out\nSELECT dbl(1) AS v;\n`, + }, + async (root, folder) => { + const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + + // the helper IS a node (macro consumer, for lineage display) … + const helper = graph.runnables.find((r) => r.path === "f/mypipe/helper"); + expect(helper).toBeDefined(); + // … but not a pipeline member (no local file to preview) and carries no macros + expect(helper?.in_pipeline).toBeFalsy(); + expect(helper?.macros).toBeUndefined(); + // the previewable set (what `run --local` can schedule) is ONLY the member + expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/root"]); + // and the macro edge still connects lib → helper for the lineage view + expect(graph.macro_edges).toContainEqual({ + lib_path: "f/mypipe/lib", + consumer_path: "f/mypipe/helper", + macro_names: ["dbl"], + via_use: false, + }); + }, + ); +}); + test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm parser)", async () => { await withFolder( { "ingest.rb": `# pipeline\n# on s3://demo/raw.csv\nputs "hi"\n` },