fix(cli): exclude non-pipeline macro-consumer nodes from --local run selection

Address Codex P1: buildMacroEdges surfaces macro-consumer nodes (a DuckDB script
calling a macro but not marked // pipeline) for lineage display. Those have no
local file, so pipeline run --local must not treat them as manual roots — a
dry-run listed them and a real run failed resolving local content. Exclude any
--local graph node absent from localScripts (the previewable set) from starts and
selection, alongside the existing macro-library exclusion.
This commit is contained in:
Ruben Fiszel
2026-07-05 22:19:30 +00:00
parent 99f0d14ed1
commit 87604add9c
2 changed files with 52 additions and 5 deletions
+19 -5
View File
@@ -674,12 +674,26 @@ async function run(
.filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0)
.map((r) => r.path),
);
// Non-runnable graph nodes: the macro libraries above, plus (under `--local`)
// any script node with no local file. The local graph surfaces macro-consumer
// nodes for lineage display — a DuckDB script that calls a macro but isn't a
// `// pipeline` member (so has no previewable content). They must never enter
// the run: as a manual root they'd be scheduled, and a preview would fail with
// no local content to send.
const notRunnablePaths = new Set<string>(macroLibPaths);
if (opts.local && localScripts) {
for (const r of graph.runnables) {
if (r.usage_kind === "script" && !localScripts.has(r.path)) {
notRunnablePaths.add(r.path);
}
}
}
// Resolve the start: explicit --from (must be a valid start) or the folder's
// sole valid start. `--upload`-bound scripts join the valid starts.
const starts = new Set(
[...validStarts(graph), ...boundNodeIds].filter(
(id) => !macroLibPaths.has(scriptPathOf(id)),
(id) => !notRunnablePaths.has(scriptPathOf(id)),
),
);
// `runAll` = no `--from` on a multi-root pipeline (fan-in): run the whole
@@ -810,10 +824,10 @@ async function run(
}
}
// Selections can still pull a macro library in via graph reachability (e.g.
// whole-pipeline mode collects every root's closure) — drop them before
// ordering so they never run.
for (const p of macroLibPaths) selectedScripts.delete(p);
// Selections can still pull a non-runnable node in via graph reachability (e.g.
// whole-pipeline mode collects every root's closure) — drop macro libraries and
// local-only display nodes before ordering so they never run.
for (const p of notRunnablePaths) selectedScripts.delete(p);
const { order, cyclic } = topoOrder(graph, selectedScripts);
if (cyclic.length > 0) {
@@ -493,6 +493,39 @@ test("a `// macros` (slash-prefix) DuckDB library is detected (backend prefix pa
);
});
test("a non-pipeline DuckDB macro consumer is a display-only node, never a run step", async () => {
// A `.duckdb.sql` helper that calls a macro but isn't `// pipeline` surfaces as
// a graph node (lineage parity) but is NOT a runnable pipeline script: it must
// be absent from `scripts` (the previewable set `run --local` selects from), so
// it can never be scheduled or fail a preview for lack of local content.
await withFolder(
{
"lib.duckdb.sql": `-- macros\nCREATE MACRO dbl(a) AS a * 2;\n`,
"helper.duckdb.sql": `-- on ducklake://main/src\nSELECT dbl(x) FROM main.src;\n`,
"root.duckdb.sql": `-- pipeline\n-- materialize ducklake://main/out\nSELECT dbl(1) AS v;\n`,
},
async (root, folder) => {
const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" });
// the helper IS a node (macro consumer, for lineage display) …
const helper = graph.runnables.find((r) => r.path === "f/mypipe/helper");
expect(helper).toBeDefined();
// … but not a pipeline member (no local file to preview) and carries no macros
expect(helper?.in_pipeline).toBeFalsy();
expect(helper?.macros).toBeUndefined();
// the previewable set (what `run --local` can schedule) is ONLY the member
expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/root"]);
// and the macro edge still connects lib → helper for the lineage view
expect(graph.macro_edges).toContainEqual({
lib_path: "f/mypipe/lib",
consumer_path: "f/mypipe/helper",
macro_names: ["dbl"],
via_use: false,
});
},
);
});
test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm parser)", async () => {
await withFolder(
{ "ingest.rb": `# pipeline\n# on s3://demo/raw.csv\nputs "hi"\n` },