mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-10 16:05:58 +00:00
fix(cli): exclude non-pipeline macro-consumer nodes from --local run selection
Address Codex P1: buildMacroEdges surfaces macro-consumer nodes (a DuckDB script calling a macro but not marked // pipeline) for lineage display. Those have no local file, so pipeline run --local must not treat them as manual roots — a dry-run listed them and a real run failed resolving local content. Exclude any --local graph node absent from localScripts (the previewable set) from starts and selection, alongside the existing macro-library exclusion.
This commit is contained in:
@@ -674,12 +674,26 @@ async function run(
|
||||
.filter((r) => r.usage_kind === "script" && (r.macros?.length ?? 0) > 0)
|
||||
.map((r) => r.path),
|
||||
);
|
||||
// Non-runnable graph nodes: the macro libraries above, plus (under `--local`)
|
||||
// any script node with no local file. The local graph surfaces macro-consumer
|
||||
// nodes for lineage display — a DuckDB script that calls a macro but isn't a
|
||||
// `// pipeline` member (so has no previewable content). They must never enter
|
||||
// the run: as a manual root they'd be scheduled, and a preview would fail with
|
||||
// no local content to send.
|
||||
const notRunnablePaths = new Set<string>(macroLibPaths);
|
||||
if (opts.local && localScripts) {
|
||||
for (const r of graph.runnables) {
|
||||
if (r.usage_kind === "script" && !localScripts.has(r.path)) {
|
||||
notRunnablePaths.add(r.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the start: explicit --from (must be a valid start) or the folder's
|
||||
// sole valid start. `--upload`-bound scripts join the valid starts.
|
||||
const starts = new Set(
|
||||
[...validStarts(graph), ...boundNodeIds].filter(
|
||||
(id) => !macroLibPaths.has(scriptPathOf(id)),
|
||||
(id) => !notRunnablePaths.has(scriptPathOf(id)),
|
||||
),
|
||||
);
|
||||
// `runAll` = no `--from` on a multi-root pipeline (fan-in): run the whole
|
||||
@@ -810,10 +824,10 @@ async function run(
|
||||
}
|
||||
}
|
||||
|
||||
// Selections can still pull a macro library in via graph reachability (e.g.
|
||||
// whole-pipeline mode collects every root's closure) — drop them before
|
||||
// ordering so they never run.
|
||||
for (const p of macroLibPaths) selectedScripts.delete(p);
|
||||
// Selections can still pull a non-runnable node in via graph reachability (e.g.
|
||||
// whole-pipeline mode collects every root's closure) — drop macro libraries and
|
||||
// local-only display nodes before ordering so they never run.
|
||||
for (const p of notRunnablePaths) selectedScripts.delete(p);
|
||||
|
||||
const { order, cyclic } = topoOrder(graph, selectedScripts);
|
||||
if (cyclic.length > 0) {
|
||||
|
||||
@@ -493,6 +493,39 @@ test("a `// macros` (slash-prefix) DuckDB library is detected (backend prefix pa
|
||||
);
|
||||
});
|
||||
|
||||
test("a non-pipeline DuckDB macro consumer is a display-only node, never a run step", async () => {
|
||||
// A `.duckdb.sql` helper that calls a macro but isn't `// pipeline` surfaces as
|
||||
// a graph node (lineage parity) but is NOT a runnable pipeline script: it must
|
||||
// be absent from `scripts` (the previewable set `run --local` selects from), so
|
||||
// it can never be scheduled or fail a preview for lack of local content.
|
||||
await withFolder(
|
||||
{
|
||||
"lib.duckdb.sql": `-- macros\nCREATE MACRO dbl(a) AS a * 2;\n`,
|
||||
"helper.duckdb.sql": `-- on ducklake://main/src\nSELECT dbl(x) FROM main.src;\n`,
|
||||
"root.duckdb.sql": `-- pipeline\n-- materialize ducklake://main/out\nSELECT dbl(1) AS v;\n`,
|
||||
},
|
||||
async (root, folder) => {
|
||||
const { graph, scripts } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" });
|
||||
|
||||
// the helper IS a node (macro consumer, for lineage display) …
|
||||
const helper = graph.runnables.find((r) => r.path === "f/mypipe/helper");
|
||||
expect(helper).toBeDefined();
|
||||
// … but not a pipeline member (no local file to preview) and carries no macros
|
||||
expect(helper?.in_pipeline).toBeFalsy();
|
||||
expect(helper?.macros).toBeUndefined();
|
||||
// the previewable set (what `run --local` can schedule) is ONLY the member
|
||||
expect(scripts.map((s) => s.path)).toEqual(["f/mypipe/root"]);
|
||||
// and the macro edge still connects lib → helper for the lineage view
|
||||
expect(graph.macro_edges).toContainEqual({
|
||||
lib_path: "f/mypipe/lib",
|
||||
consumer_path: "f/mypipe/helper",
|
||||
macro_names: ["dbl"],
|
||||
via_use: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("`#`-comment languages (ruby) use the `#` annotation fallback (no wasm parser)", async () => {
|
||||
await withFolder(
|
||||
{ "ingest.rb": `# pipeline\n# on s3://demo/raw.csv\nputs "hi"\n` },
|
||||
|
||||
Reference in New Issue
Block a user