mirror of
https://github.com/l0ng-ai/openidea.git
synced 2026-11-15 00:08:15 +00:00
l0ng-ai
b2d506a929
pnpm eval drives the production workbench in Chromium against a release openidea-server and real jdtls, judging each scenario (navigation, usages, implementations, gutter markers, lenses, diagnostics, completion, docs, rename) on time to feedback, first and complete result, and checking results against evals/java/golden/. Supporting changes: data-* hooks in the usages popup, hints and rows; a shared in-page Tauri shim for smoke tests and evals; a shared perf bundle builder; language-server versions in .github/tool-versions.env shared with Language smoke; the eval agent skill.
36 lines
1.3 KiB
TypeScript
36 lines
1.3 KiB
TypeScript
/** What an eval scenario is: a project, a step the user takes, and what they should see. */
|
|
import type { ResultSet } from "./golden.ts";
|
|
import type { Tracked } from "./measure.ts";
|
|
import type { Session } from "./session.ts";
|
|
import type { Condition } from "./verdict.ts";
|
|
|
|
export interface Measured {
|
|
timing: Tracked;
|
|
results?: ResultSet;
|
|
/** Why the answer is weaker than a semantic one, when the UI says so. */
|
|
degraded?: string | null;
|
|
}
|
|
|
|
export interface EvalScenario {
|
|
/** Stable id: `<language>.<group>.<case>`, used by `--only`, goldens and budgets. */
|
|
id: string;
|
|
title: string;
|
|
/** `fixtures/<fixture>`. */
|
|
fixture: string;
|
|
/** The language server whose readiness the warm runs wait for. */
|
|
server: string;
|
|
conditions: Condition[];
|
|
/** Brings the idle workbench to where the step starts (open a file, place the caret). */
|
|
setup(s: Session): Promise<void>;
|
|
/** The measured step. */
|
|
step(s: Session): Promise<Measured>;
|
|
}
|
|
|
|
/** Project-relative path, or a library class's URI without its machine-specific query. */
|
|
export function normalizePath(root: string, path: string): string {
|
|
if (path.startsWith(root + "/")) return path.slice(root.length + 1);
|
|
const uri = /^[a-z]+:\/\//.exec(path);
|
|
if (uri) return path.replace(/[?#].*$/, "");
|
|
return path;
|
|
}
|