feat(run): report Zig test runs in the test runner (#3355)

Zig test blocks had run markers but their runs never reached the test runner: no parser read the runners' output. A parser now reads zig test's per-test progress lines, error-return traces and summary, and zig build test's failure blocks and --summary all counts (added to those runs). Results are rooted at the module stem so gutter icons turn green/red per test; Rerun Failed and Run Selected build --test-filter configs for standalone zig test runs. test names are unescaped in run targets so the marker label, --test-filter and the runner's names agree.

Co-authored-by: akbar1214 <43771058+akbar1214@users.noreply.github.com>
This commit is contained in:
Akbar Husain
2026-11-14 22:16:35 +08:00
committed by GitHub
parent 4ab85d5f49
commit 07bd8ded5e
10 changed files with 409 additions and 6 deletions
+1 -1
View File
@@ -1132,7 +1132,7 @@ function elementProduction(input: ContextInput, env: ToolEnv, target: RunTarget)
}
if (/\.zig$/.test(input.path)) {
// `--test-filter` matches a substring of the test's full name.
return zigFile(input, env, s, { name: target.name, filter: target.name.replace(/\\(.)/g, "$1") });
return zigFile(input, env, s, { name: target.name, filter: target.name });
}
const rust = /\.rs$/.test(input.path);
if (rust && !(s.inProject && cargoOwner(rel, input.configs).length)) {
+2 -2
View File
@@ -131,8 +131,8 @@ function showTargetMenu(view: EditorView, target: RunTarget, path: string, at: {
showContextMenu(at, targetMenuItems(target, path, view.state.doc.toString()));
}
/** Results of frameworks that root their test tree at the file (jest/vitest, pytest) also match by file. */
const fileRootedResults = (path: string) => /\.(py|[cm]?[jt]sx?)$/.test(path);
/** Results of frameworks that root their test tree at the file (jest/vitest, pytest, Zig's module) also match by file. */
const fileRootedResults = (path: string) => /\.(py|[cm]?[jt]sx?|zig)$/.test(path);
/** Last test result of a gutter target (IDEA: green / red run icons after a test run). */
export function gutterTestState(target: RunTarget, path: string): TestState | null {
+2 -1
View File
@@ -361,7 +361,8 @@ export function zigTargets(content: string): RunTarget[] {
if (/^pub\s+fn\s+main\s*\(/.test(line)) out.push({ line: i, kind: "main", name: "main", path: [] });
const test = /^\s*test\s+(?:"((?:[^"\\]|\\.)*)"|([A-Za-z_]\w*))\s*\{/.exec(line);
if (test) {
const name = test[1] ?? test[2]!;
// Unescape for display and for `--test-filter`: the runner reports the raw name.
const name = test[1]?.replace(/\\(.)/g, "$1") ?? test[2]!;
out.push({ line: i, kind: "test", name, path: [name] });
}
});
+32 -2
View File
@@ -8,12 +8,23 @@ import { phpunitParser, withTeamcity } from "./phpunit";
import { pytestParser } from "./pytest";
import { rubyParser } from "./ruby";
import { unittestParser } from "./unittest";
import { zigParser } from "./zig";
import type { TestEvent, TestOutputParser } from "./types";
export type { TestEvent, TestOutputParser, TestStatus } from "./types";
export { parseDuration } from "./types";
export { TestResults, nodeOutput, nodeStatus, type TestNode } from "./results";
export { cargoParser, goParser, jsParser, jvmParser, phpunitParser, pytestParser, rubyParser, unittestParser };
export {
cargoParser,
goParser,
jsParser,
jvmParser,
phpunitParser,
pytestParser,
rubyParser,
unittestParser,
zigParser,
};
/** Runs several parsers side by side (unknown tool: e.g. `make test`). */
export function combinedParser(parsers: TestOutputParser[]): TestOutputParser {
@@ -23,7 +34,18 @@ export function combinedParser(parsers: TestOutputParser[]): TestOutputParser {
};
}
export type Framework = "cargo" | "js" | "pytest" | "unittest" | "phpunit" | "go" | "jvm" | "ruby" | "dotnet" | "auto";
export type Framework =
| "cargo"
| "js"
| "pytest"
| "unittest"
| "phpunit"
| "go"
| "jvm"
| "ruby"
| "dotnet"
| "zig"
| "auto";
export function frameworkFor(config: Pick<RunConfig, "kind" | "command" | "args">): Framework {
const cmd = [config.command, ...config.args].join(" ");
@@ -54,6 +76,8 @@ export function frameworkFor(config: Pick<RunConfig, "kind" | "command" | "args"
case "rake":
case "rspec":
return "ruby";
case "zig":
return "zig";
}
if (/(^|[/\\])dotnet(?:\.exe)?$/i.test(config.command) && config.args[0] === "test") return "dotnet";
if (/\bcargo\b/.test(cmd)) return "cargo";
@@ -98,6 +122,10 @@ export function withTestReporting<T extends Pick<RunConfig, "kind" | "command" |
};
}
if (frameworkFor(config) === "phpunit") return { ...config, args: withTeamcity(config.args) };
if (frameworkFor(config) === "zig" && config.args[0] === "build") {
// `zig build test` prints nothing for a passing (or cached) run; the summary reports the counts.
return config.args.includes("--summary") ? config : { ...config, args: [...config.args, "--summary", "all"] };
}
if (frameworkFor(config) === "unittest") {
const chosen = config.args.some((a) => /^(-v|--verbosity)(=?\d)?$/.test(a));
return chosen ? config : { ...config, args: [...config.args, "-v", "2"] };
@@ -131,6 +159,8 @@ export function createTestParser(config: Pick<RunConfig, "kind" | "command" | "a
return jvmParser();
case "ruby":
return rubyParser();
case "zig":
return zigParser();
default:
return combinedParser([
cargoParser(),
+17
View File
@@ -180,6 +180,23 @@ describe("rerunFailedConfig", () => {
).toEqual(["test", "-Dtest=com.a.FooTest#bar+baz,com.a.BarTest"]);
});
it("zig: --test-filter per failed test for `zig test`, null for build-based runs", async () => {
expect(
(await rerunFailedConfig(cfg("zig", "zig", ["test", "src/calc.zig"]), [
["calc", "adds"],
["calc", "subs"],
]))!.args,
).toEqual(["test", "src/calc.zig", "--test-filter", "adds", "--test-filter", "subs"]);
// Existing filters are replaced, not stacked.
expect(
(await rerunFailedConfig(cfg("zig", "zig", ["test", "src/calc.zig", "--test-filter", "adds"]), [
["calc", "subs"],
]))!.args,
).toEqual(["test", "src/calc.zig", "--test-filter", "subs"]);
// `zig build` forwards no args to the test binary, so a build-based run can't narrow.
expect(await rerunFailedConfig(cfg("zig", "zig", ["build", "test"]), [["tally", "subs"]])).toBeNull();
});
it("returns null when nothing failed or the framework is unknown", async () => {
expect(await rerunFailedConfig(cfg("cargo", "cargo", ["test"]), [])).toBeNull();
expect(await rerunFailedConfig(cfg("make", "make", ["test"]), [["x"]])).toBeNull();
+15
View File
@@ -249,6 +249,21 @@ function builtinTestsConfig(config: RunConfig, paths: string[][], name: string):
case "go":
args = goArgs(config.args, paths, config.params?.scope === "file" ? config.params.target : undefined);
break;
case "zig": {
// Only a standalone `zig test <file>` can filter: `zig build` forwards no args to the test binary.
const file = config.args[0] === "test" ? config.args[1] : undefined;
const names = unique(paths.map((p) => p[p.length - 1]!).filter(Boolean));
args =
file && names.length
? [
config.args[0]!,
file,
...dropFlag(config.args.slice(2), ["--test-filter"]),
...names.flatMap((n) => ["--test-filter", n]),
]
: null;
break;
}
case "jvm":
args = jvmArgs(config, paths);
break;
+13
View File
@@ -58,6 +58,19 @@ describe("test states for the run gutter", () => {
expect(testState({ path: ["QTests", "test_a"], file: "/work/app/città/tests.py" })).toBe("passed");
});
it("matches Zig results rooted at the module stem of the file", () => {
run([
{ type: "result", path: ["tally", "adds"], status: "passed" },
{ type: "result", path: ["tally", "subs"], status: "failed" },
{ type: "result", path: ["other", "adds"], status: "failed" },
]);
expect(testState({ path: ["adds"], file: "src/tally.zig" })).toBe("passed");
expect(testState({ path: ["subs"], file: "/work/app/src/tally.zig" })).toBe("failed");
// Same-named tests of another module don't bleed over.
expect(testState({ path: ["adds"], file: "src/other.zig" })).toBe("failed");
expect(testState({ path: ["adds"], file: "src/tally/main.zig" })).toBeNull();
});
it("newer runs win and listeners hear about changes", () => {
const listener = vi.fn();
const off = onTestStatesChanged(listener);
+4
View File
@@ -69,6 +69,8 @@ function segmentMatches(result: string, target: string): boolean {
const cleanPath = (p: string) => p.replace(/\\/g, "/").replace(/^\.\//, "");
const stemOf = (p: string) => p.replace(/.*\//, "").replace(/\.[^.]+$/, "");
function sameFile(resultRoot: string, relPath: string): boolean {
const b = cleanPath(relPath);
// unittest / Django results are rooted at the dotted module (`polls.tests`).
@@ -80,6 +82,8 @@ function sameFile(resultRoot: string, relPath: string): boolean {
const module = resultRoot.replace(/\./g, "/");
return [`${module}.py`, `${module}/__init__.py`].some((f) => b === f || b.endsWith(`/${f}`));
}
// Zig results are rooted at the module stem (`tally` for `src/tally.zig`).
if (/^[\p{L}_][\p{L}\p{N}_]*$/u.test(resultRoot) && stemOf(b) === resultRoot) return true;
const a = cleanPath(resultRoot);
return a === b || b.endsWith(`/${a}`) || a.endsWith(`/${b}`);
}
+174
View File
@@ -0,0 +1,174 @@
import { describe, expect, it } from "vitest";
import { createTestParser, frameworkFor, parseAll, withTestReporting } from "./index";
import { TestResults, type TestOutputParser } from "./index";
import { zigParser } from "./zig";
function results(parser: TestOutputParser, text: string) {
const r = new TestResults();
r.apply(parseAll(parser, text));
return r;
}
function leaf(r: TestResults, ...path: string[]) {
return r.get(path.join("\u0000"));
}
const FAILING_RUN = `1/3 calc.test.adds two numbers...OK
2/3 calc.test.fails on purpose...expected 7, found 5
FAIL (TestExpectedEqual)
/Users/me/.zvm/0.16.0/lib/std/testing.zig:118:17: 0x10262da43 in expectEqualInner__anon_36302 (test)
return error.TestExpectedEqual;
^
/Users/me/.zvm/0.16.0/lib/std/testing.zig:83:5: 0x10262dae3 in expectEqual (test)
return expectEqualInner(T, expected, actual);
^
/work/app/src/calc.zig:12:5: 0x10262db23 in test.fails on purpose (test)
try std.testing.expectEqual(@as(u32, 7), add(2, 3));
^
3/3 calc.test.printing helper...boom
OK
2 passed; 0 skipped; 1 failed.
error: the following test command failed with exit code 1:
/Users/me/.cache/zig/o/91f9863b4e219205507fbe58005bc1c5/test --seed=0xebfccadb
`;
describe("zig test", () => {
it("builds the module's tests with failure output and the error's location", () => {
const r = results(zigParser(), FAILING_RUN);
expect(r.roots.map((n) => n.name)).toEqual(["calc"]);
expect(leaf(r, "calc", "adds two numbers")?.own).toBe("passed");
expect(leaf(r, "calc", "printing helper")?.own).toBe("passed");
expect(leaf(r, "calc", "printing helper")?.output).toEqual(["boom"]);
const failed = leaf(r, "calc", "fails on purpose")!;
expect(failed.own).toBe("failed");
expect(failed.output[0]).toBe("expected 7, found 5");
expect(failed.output).toContain("try std.testing.expectEqual(@as(u32, 7), add(2, 3));");
expect(failed.location).toEqual({
path: "/work/app/src/calc.zig",
line: 12,
col: 5,
resolve: "path",
});
expect(r.summary).toEqual({ passed: 2, skipped: 0, failed: 1 });
expect(r.counts()).toMatchObject({ passed: 2, failed: 1 });
});
it("parses a quiet passing run and a filtered single test", () => {
const r = results(
zigParser(),
`1/3 tally.test.total adds the prices...OK
2/3 tally.test.total of nothing is zero...OK
3/3 tally.test.format pads the cents...OK
All 3 tests passed.
`,
);
expect(leaf(r, "tally", "total adds the prices")?.own).toBe("passed");
expect(r.counts()).toMatchObject({ passed: 3, failed: 0 });
const filtered = results(
zigParser(),
`1/1 calc.test.adds two numbers...OK
All 1 tests passed.
`,
);
expect(filtered.counts()).toMatchObject({ passed: 1 });
// A filter matching nothing leaves no nodes but reports the counts.
expect(results(zigParser(), "All 0 tests passed.\n").counts()).toMatchObject({ passed: 0 });
});
it("reports skips, and status printed after output without a newline", () => {
const r = results(
zigParser(),
`1/2 calc.test.will skip...SKIP
2/2 calc.test.greets...helloOK
1 passed; 1 skipped; 0 failed.
`,
);
expect(leaf(r, "calc", "will skip")?.own).toBe("skipped");
expect(leaf(r, "calc", "greets")?.own).toBe("passed");
expect(leaf(r, "calc", "greets")?.output).toEqual(["hello"]);
expect(r.summary).toEqual({ passed: 1, skipped: 1, failed: 0 });
});
it("parses zig build test failure blocks (the build runner's captured output)", () => {
const r = results(
zigParser(),
`test
+- run test 2 pass, 1 fail (3 total)
error: 'tally.test.total of nothing is zero' failed:
expected 1, found 0
/Users/me/.zvm/0.16.0/lib/std/testing.zig:94:17: 0x102d48893 in expectEqualInner__func_1083 (test)
return error.TestExpectedEqual;
^
/Users/me/.zvm/0.16.0/lib/std/testing.zig:58:5: 0x102d48937 in expectEqual (test)
return expectEqualInner(T, expected, actual);
^
/work/app/src/tally.zig:22:5: 0x102d48977 in test.total of nothing is zero (test)
try std.testing.expectEqual(@as(u32, 1), total(&.{}));
^
failed command: ./.zig-cache/o/b84ba4b8338e114a2e03f2758f6ca155/test --cache-dir=./.zig-cache --seed=0x7689fac0 --listen=-
Build Summary: 1/3 steps succeeded (1 failed); 2/3 tests passed (1 failed)
test transitive failure
+- run test 2 pass, 1 fail (3 total)
+- compile test debug native success 790ms MaxRSS:249M
`,
);
// The build runner reports passing tests only as counts; failures get nodes.
const failed = leaf(r, "tally", "total of nothing is zero")!;
expect(failed.own).toBe("failed");
expect(failed.output[0]).toBe("expected 1, found 0");
expect(failed.output).toContain("try std.testing.expectEqual(@as(u32, 1), total(&.{}));");
expect(failed.location).toEqual({ path: "/work/app/src/tally.zig", line: 22, col: 5, resolve: "path" });
expect(r.counts()).toMatchObject({ passed: 2, failed: 1 });
});
it("parses a passing zig build test --summary all (counts only, no nodes)", () => {
const r = results(
zigParser(),
`Build Summary: 3/3 steps succeeded; 3/3 tests passed
test success
+- run test 3 pass (3 total) 2ms MaxRSS:2M
+- compile test Debug native cached 36ms MaxRSS:32M
`,
);
expect(r.roots).toEqual([]);
expect(r.counts()).toMatchObject({ passed: 3, failed: 0 });
});
it("a failure block left open at the end of the output still gets its location", () => {
const r = results(
zigParser(),
`error: 'calc.test.breaks' failed:
/work/app/src/calc.zig:4:5: 0x102 in test.breaks (test)
`,
);
const failed = leaf(r, "calc", "breaks")!;
expect(failed.own).toBe("failed");
expect(failed.location).toEqual({ path: "/work/app/src/calc.zig", line: 4, col: 5, resolve: "path" });
});
});
describe("zig framework selection", () => {
const test = (args: string[]) => ({ kind: "zig", command: "zig", args }) as const;
it("selects the zig parser for zig test configurations", () => {
expect(frameworkFor(test(["build", "test"]))).toBe("zig");
expect(frameworkFor(test(["test", "src/calc.zig", "--test-filter", "adds"]))).toBe("zig");
const events = createTestParser(test(["test", "calc.zig"])).feed("1/1 calc.test.adds two numbers...OK");
expect(events).toEqual([
{ type: "start", path: ["calc", "adds two numbers"] },
{ type: "result", path: ["calc", "adds two numbers"], status: "passed" },
]);
});
it("adds --summary all to zig build test runs so passing runs report counts", () => {
expect(withTestReporting(test(["build", "test"]))).toMatchObject({ args: ["build", "test", "--summary", "all"] });
expect(withTestReporting(test(["build", "test", "--summary", "none"]))).toMatchObject({
args: ["build", "test", "--summary", "none"],
});
// A standalone `zig test` always prints per-test lines.
expect(withTestReporting(test(["test", "calc.zig"]))).toMatchObject({ args: ["test", "calc.zig"] });
});
});
+149
View File
@@ -0,0 +1,149 @@
/** `zig test` (the runner's progress lines) and `zig build test` (failure blocks from the build runner). */
import type { TestEvent, TestOutputParser } from "./types";
// `1/3 tally.test.total adds the prices...OK` — the status lands here when the test printed nothing.
const PROGRESS_RE = /^(\d+)\/(\d+)\s+(.+?)\.\.\.(.*)$/;
// `…/src/tally.zig:22:5: 0x… in test.total of nothing is zero (test)` — error-return trace frames.
const FRAME_RE = /^\s*(.+\.zig):(\d+):(\d+): 0x[0-9a-fA-F]+ in /;
const SUMMARY_RE = /^(\d+) passed(?:; (\d+) skipped)?(?:; (\d+) failed)?\.?$/;
const BUILD_SUMMARY_RE = /^Build Summary: .*?(\d+)\/\d+ tests passed(?: \((\d+) failed\))?/;
const STEP_RE = /^\s*\+- run [^()]*?(\d+) pass(?:, (\d+) skip)?(?:, (\d+) fail)? \(\d+ total\)/;
const FAILED_RE = /^error: '(.+)' failed:$/;
// Frames inside the toolchain's standard library (`<zig>/lib/std/…`; stage3 builds `/lib/zig/std/…`).
const STD_LIB_RE = /\/lib\/(?:zig\/)?std\//;
interface Frame {
path: string;
line: number;
col: number;
}
interface Awaiting {
path: string[];
frames: Frame[];
}
/** `tally.test.total adds the prices` → module `tally`, name `total adds the prices` (the last `.test.` wins). */
function splitName(full: string): { stem: string | null; name: string } {
const at = full.lastIndexOf(".test.");
return at < 0 ? { stem: null, name: full } : { stem: full.slice(0, at), name: full.slice(at + 6) };
}
/**
* The runner writes the status right after whatever the test printed, on the same line
* (`…boomOK`) or, when the print ended in a newline, alone on the next one.
*/
function statusOf(text: string): { status: "passed" | "skipped" | "failed"; output: string } | null {
const trimmed = text.replace(/\s+$/, "");
if (trimmed.endsWith("OK")) return { status: "passed", output: trimmed.slice(0, -2) };
if (trimmed.endsWith("SKIP")) return { status: "skipped", output: trimmed.slice(0, -4) };
const failed = /^(.*)FAIL(?: \([^)]*\))?$/.exec(trimmed);
return failed ? { status: "failed", output: failed[1]! } : null;
}
export function zigParser(): TestOutputParser {
let pending: Awaiting | null = null;
let block: Awaiting | null = null;
let trace: Awaiting | null = null;
const pathOf = (stem: string | null, name: string) => (stem ? [stem, name] : [name]);
const location = (events: TestEvent[], path: string[], frames: Frame[]) => {
// The error originates in the first non-library frame; the outermost is the fallback.
const frame = frames.find((f) => !STD_LIB_RE.test(f.path)) ?? frames[frames.length - 1];
if (frame)
events.push({
type: "location",
path,
location: { path: frame.path, line: frame.line, col: frame.col, resolve: "path" },
});
};
/** A failed test's error-return trace: indented source lines and frames, up to the next unindented line. */
const traced = (events: TestEvent[], text: Awaiting, line: string): boolean => {
const m = FRAME_RE.exec(line);
if (m) {
text.frames.push({ path: m[1]!, line: Number(m[2]), col: Number(m[3]) });
return true;
}
if (line === "" || /^\s/.test(line)) {
if (line.trim()) events.push({ type: "output", path: text.path, line: line.trim() });
return true;
}
location(events, text.path, text.frames);
return false;
};
return {
feed(line) {
const events: TestEvent[] = [];
let m: RegExpExecArray | null;
// Inside a `zig build` failure block: indented (or blank) lines are the test's captured output.
if (block) {
if (traced(events, block, line)) return events;
block = null;
}
// After a `zig test` FAIL: the error-return trace follows on the next lines.
if (trace) {
if (traced(events, trace, line)) return events;
trace = null;
}
if (pending && (m = /^(OK|SKIP|FAIL(?: \([^)]*\))?)$/.exec(line))) {
const status = m[1] === "OK" ? "passed" : m[1] === "SKIP" ? "skipped" : "failed";
events.push({ type: "result", path: pending.path, status });
// A failure's trace follows on the next lines; the summary must not become its output.
trace = status === "failed" ? pending : null;
pending = null;
return events;
}
if (pending && (m = FRAME_RE.exec(line))) {
pending.frames.push({ path: m[1]!, line: Number(m[2]), col: Number(m[3]) });
return events;
}
if ((m = PROGRESS_RE.exec(line))) {
const { stem, name } = splitName(m[3]!);
pending = { path: pathOf(stem, name), frames: [] };
events.push({ type: "start", path: pending.path });
const status = statusOf(m[4]!);
if (status) {
if (status.output) events.push({ type: "output", path: pending.path, line: status.output });
events.push({ type: "result", path: pending.path, status: status.status });
trace = status.status === "failed" ? pending : null;
pending = null;
} else if (m[4]) {
events.push({ type: "output", path: pending.path, line: m[4] });
}
return events;
}
if ((m = FAILED_RE.exec(line))) {
const { stem, name } = splitName(m[1]!);
block = { path: pathOf(stem, name), frames: [] };
events.push({ type: "result", path: block.path, status: "failed" });
return events;
}
if (pending) {
// Anything else before the status is the running test's output (prints, panic text).
events.push({ type: "output", path: pending.path, line });
return events;
}
if ((m = SUMMARY_RE.exec(line))) {
events.push({ type: "summary", passed: Number(m[1]), skipped: Number(m[2] ?? 0), failed: Number(m[3] ?? 0) });
} else if ((m = BUILD_SUMMARY_RE.exec(line))) {
events.push({ type: "summary", passed: Number(m[1]), failed: Number(m[2] ?? 0) });
} else if ((m = STEP_RE.exec(line))) {
events.push({ type: "summary", passed: Number(m[1]), skipped: Number(m[2] ?? 0), failed: Number(m[3] ?? 0) });
}
return events;
},
end() {
const events: TestEvent[] = [];
if (trace) {
location(events, trace.path, trace.frames);
trace = null;
}
if (block) {
location(events, block.path, block.frames);
block = null;
}
return events;
},
};
}