From b2d506a929036c60411944c9cfd4613d6dc33e14 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:17:07 +0800 Subject: [PATCH] test(evals): measure Java code intelligence as the user sees it (#2758) pnpm eval drives the production workbench in Chromium against a release openidea-server and real jdtls, judging each scenario (navigation, usages, implementations, gutter markers, lenses, diagnostics, completion, docs, rename) on time to feedback, first and complete result, and checking results against evals/java/golden/. Supporting changes: data-* hooks in the usages popup, hints and rows; a shared in-page Tauri shim for smoke tests and evals; a shared perf bundle builder; language-server versions in .github/tool-versions.env shared with Language smoke; the eval agent skill. --- .agents/skills/eval/SKILL.md | 83 ++ .claude/skills/eval | 1 + .github/tool-versions.env | 14 + .github/workflows/language-smoke.yml | 15 +- .gitignore | 1 + .oxfmtrc.json | 1 + AGENTS.md | 14 +- evals/budgets.json | 11 + .../golden/java.app.completion.jdkMember.json | 70 ++ .../java.app.completion.projectMember.json | 43 + .../java.app.declaration.dependency.json | 11 + .../java.app.declaration.inJdkSource.json | 11 + .../java/golden/java.app.declaration.jdk.json | 11 + .../java.app.declaration.jdkMethod.json | 11 + .../golden/java.app.declaration.project.json | 11 + .../java.app.diagnostics.afterError.json | 11 + .../golden/java.app.diagnostics.afterFix.json | 11 + evals/java/golden/java.app.doc.jdk.json | 11 + evals/java/golden/java.app.doc.project.json | 11 + .../golden/java.app.edit.declaration.json | 11 + .../golden/java.app.edit.usagesOtherFile.json | 14 + .../golden/java.app.edit.usagesSameFile.json | 13 + .../golden/java.app.gutter.interface.json | 12 + .../java/golden/java.app.gutter.jdkClass.json | 11 + .../java/golden/java.app.gutter.subclass.json | 11 + ...va.app.implementation.interfaceMethod.json | 12 + .../java.app.implementation.jdkInterface.json | 81 ++ ...a.app.implementation.projectInterface.json | 12 + evals/java/golden/java.app.lens.jdkClass.json | 13 + evals/java/golden/java.app.lens.project.json | 13 + .../golden/java.app.rename.acrossFiles.json | 14 + evals/java/golden/java.app.rename.local.json | 13 + .../golden/java.app.usages.inJdkSource.json | 13 + .../java.app.usages.jdkMethodLibraries.json | 16 + evals/java/golden/java.app.usages.method.json | 13 + evals/java/golden/java.app.usages.record.json | 21 + ...ava.petclinic.completion.libraryClass.json | 18 + ...etclinic.declaration.inheritedLibrary.json | 11 + ...va.petclinic.declaration.libraryClass.json | 11 + .../java.petclinic.declaration.project.json | 11 + .../java.petclinic.diagnostics.cleanFile.json | 9 + .../golden/java.petclinic.doc.library.json | 11 + .../java.petclinic.gutter.baseClass.json | 11 + .../java.petclinic.gutter.midClass.json | 11 + ...va.petclinic.implementation.baseClass.json | 20 + .../golden/java.petclinic.lens.entity.json | 16 + .../java.petclinic.rename.acrossFiles.json | 13 + .../golden/java.petclinic.usages.entity.json | 83 ++ ...java.petclinic.usages.inheritedMethod.json | 24 + evals/java/scenarios.ts | 841 ++++++++++++++++++ evals/lib/golden.test.ts | 40 + evals/lib/golden.ts | 62 ++ evals/lib/machine.ts | 86 ++ evals/lib/measure.ts | 154 ++++ evals/lib/projects.ts | 70 ++ evals/lib/scenario.ts | 35 + evals/lib/session.ts | 425 +++++++++ evals/lib/verdict.test.ts | 86 ++ evals/lib/verdict.ts | 196 ++++ evals/projects.json | 16 + evals/run.ts | 365 ++++++++ evals/tsconfig.json | 4 + .../java/com/example/orders/BulkDiscount.java | 18 + .../java/com/example/orders/ListPrice.java | 9 + .../java/com/example/orders/PricingRule.java | 6 + knip.ts | 1 + package.json | 3 +- perf/lib/bundle.ts | 47 + perf/run.ts | 49 +- smoke/desktop.ts | 76 +- smoke/server.ts | 1 + smoke/tauriShim.ts | 111 +++ src/editor/lsp/Popups.tsx | 14 +- src/editor/lsp/actions.ts | 4 +- src/editor/lsp/hint.test.ts | 11 + src/editor/lsp/hint.ts | 14 +- src/editor/lsp/navigation.ts | 4 + src/editor/lsp/popupStore.ts | 4 + src/features/code-insight/UsagesPopup.tsx | 19 +- src/features/code-insight/ipc.ts | 6 +- src/features/code-insight/usages.ts | 24 +- src/workbench/perfProbe.ts | 74 +- vite.config.ts | 2 +- 83 files changed, 3622 insertions(+), 149 deletions(-) create mode 100644 .agents/skills/eval/SKILL.md create mode 120000 .claude/skills/eval create mode 100644 .github/tool-versions.env create mode 100644 evals/budgets.json create mode 100644 evals/java/golden/java.app.completion.jdkMember.json create mode 100644 evals/java/golden/java.app.completion.projectMember.json create mode 100644 evals/java/golden/java.app.declaration.dependency.json create mode 100644 evals/java/golden/java.app.declaration.inJdkSource.json create mode 100644 evals/java/golden/java.app.declaration.jdk.json create mode 100644 evals/java/golden/java.app.declaration.jdkMethod.json create mode 100644 evals/java/golden/java.app.declaration.project.json create mode 100644 evals/java/golden/java.app.diagnostics.afterError.json create mode 100644 evals/java/golden/java.app.diagnostics.afterFix.json create mode 100644 evals/java/golden/java.app.doc.jdk.json create mode 100644 evals/java/golden/java.app.doc.project.json create mode 100644 evals/java/golden/java.app.edit.declaration.json create mode 100644 evals/java/golden/java.app.edit.usagesOtherFile.json create mode 100644 evals/java/golden/java.app.edit.usagesSameFile.json create mode 100644 evals/java/golden/java.app.gutter.interface.json create mode 100644 evals/java/golden/java.app.gutter.jdkClass.json create mode 100644 evals/java/golden/java.app.gutter.subclass.json create mode 100644 evals/java/golden/java.app.implementation.interfaceMethod.json create mode 100644 evals/java/golden/java.app.implementation.jdkInterface.json create mode 100644 evals/java/golden/java.app.implementation.projectInterface.json create mode 100644 evals/java/golden/java.app.lens.jdkClass.json create mode 100644 evals/java/golden/java.app.lens.project.json create mode 100644 evals/java/golden/java.app.rename.acrossFiles.json create mode 100644 evals/java/golden/java.app.rename.local.json create mode 100644 evals/java/golden/java.app.usages.inJdkSource.json create mode 100644 evals/java/golden/java.app.usages.jdkMethodLibraries.json create mode 100644 evals/java/golden/java.app.usages.method.json create mode 100644 evals/java/golden/java.app.usages.record.json create mode 100644 evals/java/golden/java.petclinic.completion.libraryClass.json create mode 100644 evals/java/golden/java.petclinic.declaration.inheritedLibrary.json create mode 100644 evals/java/golden/java.petclinic.declaration.libraryClass.json create mode 100644 evals/java/golden/java.petclinic.declaration.project.json create mode 100644 evals/java/golden/java.petclinic.diagnostics.cleanFile.json create mode 100644 evals/java/golden/java.petclinic.doc.library.json create mode 100644 evals/java/golden/java.petclinic.gutter.baseClass.json create mode 100644 evals/java/golden/java.petclinic.gutter.midClass.json create mode 100644 evals/java/golden/java.petclinic.implementation.baseClass.json create mode 100644 evals/java/golden/java.petclinic.lens.entity.json create mode 100644 evals/java/golden/java.petclinic.rename.acrossFiles.json create mode 100644 evals/java/golden/java.petclinic.usages.entity.json create mode 100644 evals/java/golden/java.petclinic.usages.inheritedMethod.json create mode 100644 evals/java/scenarios.ts create mode 100644 evals/lib/golden.test.ts create mode 100644 evals/lib/golden.ts create mode 100644 evals/lib/machine.ts create mode 100644 evals/lib/measure.ts create mode 100644 evals/lib/projects.ts create mode 100644 evals/lib/scenario.ts create mode 100644 evals/lib/session.ts create mode 100644 evals/lib/verdict.test.ts create mode 100644 evals/lib/verdict.ts create mode 100644 evals/projects.json create mode 100644 evals/run.ts create mode 100644 evals/tsconfig.json create mode 100644 fixtures/java-app/src/main/java/com/example/orders/BulkDiscount.java create mode 100644 fixtures/java-app/src/main/java/com/example/orders/ListPrice.java create mode 100644 fixtures/java-app/src/main/java/com/example/orders/PricingRule.java create mode 100644 perf/lib/bundle.ts create mode 100644 smoke/tauriShim.ts diff --git a/.agents/skills/eval/SKILL.md b/.agents/skills/eval/SKILL.md new file mode 100644 index 000000000..be5709212 --- /dev/null +++ b/.agents/skills/eval/SKILL.md @@ -0,0 +1,83 @@ +--- +name: eval +description: Run the code-intelligence evals (`pnpm eval`), pick the worst failing scenario, diagnose it from the attached timeline, fix it, prove the fix with before / after numbers measured on this machine without regressing other scenarios, and open one PR. Use for "跑 eval", "优化 Java 代码智能", "eval 失败了", an issue labelled `eval`, or any change to Java navigation / usages / gutter markers that needs numbers. +--- + +# Eval loop: measure → fix → prove + +`pnpm eval` drives the production workbench in Chromium against a release `openidea-server` +with real language servers (Java / jdtls for now) and judges every scenario on what the user +sees: time to feedback, to the first result, to the final one, and the results themselves vs +`evals//golden/`. One machine, before and after: numbers from different machines or +from the nightly host don't judge a fix. + +## 0. Preconditions + +- `jdtls` on PATH and `JAVA_HOME` pointing at a JDK of the pinned `JAVA` version (projects + compile against it, and the JDK scenarios' results are in its sources; jdtls itself runs on + a newer installed JDK when it needs one). Pinned versions: `.github/tool-versions.env`. A + mismatch prints a warning and the report records both: say so in the PR when it applies. +- First run builds `cargo build --release -p ide-host --bin openidea-server` and the perf + bundle; later iterations of a frontend-only change can use `--no-build` after a + `pnpm exec vite build --mode perf --outDir target/eval/dist`, a backend-only one after the + cargo build alone. +- Run evals one at a time on this machine, with nothing heavy (builds, other evals) beside them. + +## 1. Baseline on the base commit + +```sh +git switch --detach origin/main # or the fix branch's merge base +pnpm eval --save-baseline # target/eval/baseline.json, this machine only +``` + +Read `target/eval/report.json`. Each verdict has `status`, `failures` (`kind`: `error`, +`timeout`, `degraded`, `correctness`, `budget`, `golden`), `regressions`, `repro` (the command +that reruns it) and, for failures, `diagnostics`: every command sent to the server in the +slowest run's window (`cmd`, `start`, `ms`, `ok`, `bytes`, `items`), background tasks, server +status changes and the server's log tail. + +## 2. Pick one problem + +Worst first: wrong or missing results and silent degradation, then timeouts, then the +feedback budget (100 ms), then the slowest `completeMs`. One problem per PR. Say which +scenario and metric, with its value vs budget. + +## 3. Diagnose from the timeline + +- Commands in the window ordered by `start`: what ran, what waited on what, what's serial + that could overlap, which request returned many `items`/`bytes`, which failed. +- `feedbackMs` far above 16 ms with a fast `firstResultMs`: the UI shows nothing while it + waits (a hint or popup that only appears after a delay). +- A task still running across the window: background work competing with the request. +- For the frontend side rerun with `--trace` and open `target/eval/traces/@.json` + in Chrome's Performance panel. +- Name the cause at class level (as in `issue-fix`): it usually affects sibling scenarios. + +## 4. Fix, then rerun that scenario until it passes + +```sh +pnpm eval --only --condition # the verdict's `repro` +``` + +Correctness is enforced, so a faster answer that drops, truncates or serves stale results +fails. Don't trade a budget miss for a degraded answer. + +## 5. Full suite, then PR + +```sh +pnpm eval # vs the baseline from step 1: no new failures or regressions +``` + +PR per `issue-fix` §6, plus in **How it was verified** the before / after table from +`target/eval/report.md` (both on this machine, commits named) and the tool versions line. + +## Guard rails + +- Goldens (`evals/*/golden/`) and budgets (`evals/budgets.json`) are human-owned. Don't edit + them, and don't run `--update-golden`, to make a scenario pass. When a golden looks wrong + (a tool bump, a deliberate behaviour change), stop and say so: a PR that touches them gets + `needs-decision`. +- Don't weaken a scenario's checks or timeouts in `evals//scenarios.ts` to pass. + Adding a scenario (for a case the sweep or a user found) is fine; record its golden with + `--update-golden --only ` and ask a person to review the file. +- A flaky timing (passes on rerun) is not a regression: rerun once before acting on one. diff --git a/.claude/skills/eval b/.claude/skills/eval new file mode 120000 index 000000000..da9d362b1 --- /dev/null +++ b/.claude/skills/eval @@ -0,0 +1 @@ +../../.agents/skills/eval \ No newline at end of file diff --git a/.github/tool-versions.env b/.github/tool-versions.env new file mode 100644 index 000000000..50ea7d560 --- /dev/null +++ b/.github/tool-versions.env @@ -0,0 +1,14 @@ +# Language server and tool versions for Language smoke (.github/workflows/language-smoke.yml) +# and the code-intelligence evals (pnpm eval). Pinned so a release can't fail an unrelated +# PR or move a golden result; bump them deliberately (and re-record goldens: pnpm eval --update-golden). +VTSLS=0.3.0 +TYPESCRIPT=6.0.3 +BASEDPYRIGHT=1.40.2 +RUFF=0.17.3 +GOPLS=v0.24.0 +JDTLS=1.63.0-202610191322 +KOTLIN_LSP=263.7084.0 +GRADLE=8.14.3 +INTELEPHENSE=1.18.5 +# The JDK projects compile against (JAVA_HOME): library navigation opens its sources. +JAVA=21 diff --git a/.github/workflows/language-smoke.yml b/.github/workflows/language-smoke.yml index 875011f98..0bc8cfe74 100644 --- a/.github/workflows/language-smoke.yml +++ b/.github/workflows/language-smoke.yml @@ -24,6 +24,7 @@ on: - "package.json" - "pnpm-lock.yaml" - ".github/workflows/language-smoke.yml" + - ".github/tool-versions.env" pull_request: branches: [main] types: [opened, synchronize, reopened] @@ -38,16 +39,6 @@ concurrency: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 - # Pinned so a server release can't fail an unrelated PR; bump them deliberately. - VTSLS: 0.3.0 - TYPESCRIPT: 6.0.3 - BASEDPYRIGHT: 1.40.2 - RUFF: 0.17.3 - GOPLS: v0.24.0 - JDTLS: 1.63.0-202610191322 - KOTLIN_LSP: 263.7084.0 - GRADLE: 8.14.3 - INTELEPHENSE: 1.18.5 jobs: smoke: @@ -61,6 +52,8 @@ jobs: timeout-minutes: 40 steps: - uses: actions/checkout@v4 + # Tool versions, shared with the evals (pnpm eval). + - run: grep -E '^[A-Z0-9_]+=' .github/tool-versions.env >> "$GITHUB_ENV" - uses: dtolnay/rust-toolchain@stable with: { components: "rust-analyzer, rustfmt" } # One build cache for every language; main saves it. @@ -96,7 +89,7 @@ jobs: - if: matrix.language == 'java' || matrix.language == 'kotlin' uses: actions/setup-java@v4 - with: { distribution: temurin, java-version: "21" } + with: { distribution: temurin, java-version: "${{ env.JAVA }}" } - if: matrix.language == 'java' run: | mkdir -p "$RUNNER_TEMP/jdtls" diff --git a/.gitignore b/.gitignore index 75785cf03..9a07d5fd7 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ CLAUDE.local.md .claude/skills/* !.claude/skills/verify !.claude/skills/issue-fix +!.claude/skills/eval # Local reference screenshots pycharm-screenshots/ diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 49531abcb..cde809749 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -9,6 +9,7 @@ "dist", "target", "src/bindings.ts", + "evals/*/golden", "src/editor/core/cm/angular/parser*.ts", "src/editor/core/cm/jsp/parser*.ts", "src/editor/core/cm/astro/parser*.ts", diff --git a/AGENTS.md b/AGENTS.md index d5336d37f..5443cb2b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,8 +24,8 @@ Visual style: dark, low-contrast greys, one cool accent Plugin docs: `plugins/README.md`, `plugins/themes.md`. - `java/` — JDT LS extension bundles (`java-usages`, `kotlin-bridge`) and the patched Lombok agent, embedded into the app by `ide-sdk` / `ide-tools`. -- `e2e/` — Playwright UI tests (`pnpm e2e`). `fixtures/` — sample projects per language - for manual and automated testing. +- `e2e/` — Playwright UI tests (`pnpm e2e`). `evals/` — code-intelligence evals (`pnpm eval`). + `fixtures/` — sample projects per language for manual and automated testing. - `docs/` — the Mintlify documentation site (`docs.json` navigation, `en/` and `zh/` pages); preview with `npx mint dev` in `docs/`. @@ -59,9 +59,12 @@ Unclear product decisions → `needs-decision` and stop. ## Skills +Do not use Impeccable or its related skills in this repository, including globally installed copies. + Shared agent skills live in `.agents/skills/` (`.claude/skills/` links to them): `verify` drives the running app through Tauri MCP and captures screenshot evidence; -`issue-fix` takes an issue from reproduction to one PR. +`issue-fix` takes an issue from reproduction to one PR; +`eval` takes a failing code-intelligence eval to a fix with before / after numbers. ## Commands @@ -81,6 +84,11 @@ Shared agent skills live in `.agents/skills/` (`.claude/skills/` links to them): `pnpm gen:bindings`. - `pnpm perf` (UI interaction latency / frames / leaks), `pnpm perf:rust` (criterion benches vs budgets). Budgets: `perf/budgets.json`. +- `pnpm eval [--only ] [--runs n] [--save-baseline]` — Java code intelligence (navigation, + usages, gutter markers) in the production workbench against a release `openidea-server` and + jdtls: time to feedback / first result / complete, results vs `evals//golden/`. + Report: `target/eval/report.json`. Goldens and `evals/budgets.json` are changed by people only. + Tool versions: `.github/tool-versions.env` (shared with Language smoke). - `pnpm release:prepare ` — the release commit's changes: app version everywhere, `Cargo.lock`, and the plugin API snapshot when `wit/` moved to a new version. diff --git a/evals/budgets.json b/evals/budgets.json new file mode 100644 index 000000000..c30d1f428 --- /dev/null +++ b/evals/budgets.json @@ -0,0 +1,11 @@ +{ + "$comment": "Human-owned: agents don't change this file or evals/*/golden/ (a PR that does is labelled needs-decision). `*Ms` are upper bounds on medians (the user's view: input → first painted frame), `recall` / `precision` lower bounds vs the golden. Keys: `` or `@`. Regressions vs target/eval/baseline.json need both the relative tolerance and the absolute floor.", + "regressionTolerance": 0.2, + "regressionFloorMs": 200, + "defaults": { + "feedbackMs": 100, + "recall": 1, + "precision": 1 + }, + "scenarios": {} +} diff --git a/evals/java/golden/java.app.completion.jdkMember.json b/evals/java/golden/java.app.completion.jdkMember.json new file mode 100644 index 000000000..c7027d487 --- /dev/null +++ b/evals/java/golden/java.app.completion.jdkMember.json @@ -0,0 +1,70 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "completions": [ + "add", + "addAll", + "addFirst", + "addLast", + "cast", + "clear", + "contains", + "containsAll", + "else", + "equals", + "for", + "forEach", + "fori", + "get", + "getClass", + "getFirst", + "getLast", + "hashCode", + "if", + "indexOf", + "isEmpty", + "iterator", + "lastIndexOf", + "listIterator", + "nn", + "not", + "notify", + "notifyAll", + "null", + "orders.nnull", + "orders.opt", + "orders.syserr", + "orders.sysouf", + "orders.sysout", + "orders.sysoutv", + "par", + "parallelStream", + "remove", + "removeAll", + "removeFirst", + "removeIf", + "removeLast", + "replaceAll", + "retainAll", + "return", + "reversed", + "set", + "size", + "sort", + "sout", + "spliterator", + "stream", + "subList", + "throw", + "toArray", + "toString", + "try", + "var", + "wait", + "while" + ] + } +} diff --git a/evals/java/golden/java.app.completion.projectMember.json b/evals/java/golden/java.app.completion.projectMember.json new file mode 100644 index 000000000..a719335d4 --- /dev/null +++ b/evals/java/golden/java.app.completion.projectMember.json @@ -0,0 +1,43 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "completions": [ + "book.nnull", + "book.opt", + "book.syserr", + "book.sysouf", + "book.sysout", + "book.sysoutv", + "byCustomer", + "cast", + "else", + "equals", + "for", + "fori", + "getClass", + "hashCode", + "if", + "largest", + "nn", + "not", + "notify", + "notifyAll", + "null", + "par", + "place", + "return", + "revenueCents", + "size", + "sout", + "throw", + "toString", + "try", + "var", + "wait", + "while" + ] + } +} diff --git a/evals/java/golden/java.app.declaration.dependency.json b/evals/java/golden/java.app.declaration.dependency.json new file mode 100644 index 000000000..c35a12752 --- /dev/null +++ b/evals/java/golden/java.app.declaration.dependency.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/junit-jupiter-api-5.11.3.jar/org.junit.jupiter.api/Assertions.java#assertEquals" + ] + } +} diff --git a/evals/java/golden/java.app.declaration.inJdkSource.json b/evals/java/golden/java.app.declaration.inJdkSource.json new file mode 100644 index 000000000..08631f08c --- /dev/null +++ b/evals/java/golden/java.app.declaration.inJdkSource.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/java.base/java.util/AbstractList.java#AbstractList" + ] + } +} diff --git a/evals/java/golden/java.app.declaration.jdk.json b/evals/java/golden/java.app.declaration.jdk.json new file mode 100644 index 000000000..11fc3b3a3 --- /dev/null +++ b/evals/java/golden/java.app.declaration.jdk.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/java.base/java.util/ArrayList.java#ArrayList" + ] + } +} diff --git a/evals/java/golden/java.app.declaration.jdkMethod.json b/evals/java/golden/java.app.declaration.jdkMethod.json new file mode 100644 index 000000000..35ee6b581 --- /dev/null +++ b/evals/java/golden/java.app.declaration.jdkMethod.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/java.base/java.util/Comparator.java#comparingLong" + ] + } +} diff --git a/evals/java/golden/java.app.declaration.project.json b/evals/java/golden/java.app.declaration.project.json new file mode 100644 index 000000000..315487018 --- /dev/null +++ b/evals/java/golden/java.app.declaration.project.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "src/main/java/com/example/orders/OrderBook.java:28" + ] + } +} diff --git a/evals/java/golden/java.app.diagnostics.afterError.json b/evals/java/golden/java.app.diagnostics.afterError.json new file mode 100644 index 000000000..ac376c67e --- /dev/null +++ b/evals/java/golden/java.app.diagnostics.afterError.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "problems": [ + "L33:error:Type mismatch: cannot convert from String to int" + ] + } +} diff --git a/evals/java/golden/java.app.diagnostics.afterFix.json b/evals/java/golden/java.app.diagnostics.afterFix.json new file mode 100644 index 000000000..9a898fb0b --- /dev/null +++ b/evals/java/golden/java.app.diagnostics.afterFix.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "problems": [ + "L34:warning:The value of the local variable used is not used" + ] + } +} diff --git a/evals/java/golden/java.app.doc.jdk.json b/evals/java/golden/java.app.doc.jdk.json new file mode 100644 index 000000000..a710be18b --- /dev/null +++ b/evals/java/golden/java.app.doc.jdk.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "documentation": [ + "java.util.ArrayList.ArrayList() Constructs an empty list with an initial capacity of ten. Source: Java 21.0.13.0.2 (module: java.base)" + ] + } +} diff --git a/evals/java/golden/java.app.doc.project.json b/evals/java/golden/java.app.doc.project.json new file mode 100644 index 000000000..13849c69a --- /dev/null +++ b/evals/java/golden/java.app.doc.project.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "documentation": [ + "Order com.example.orders.OrderBook.place(String customer, String product, int quantity, long unitPriceCents) Places an order and returns it. Parameters: customer product quantity unitPriceCents Source: openidea-fixture-orders" + ] + } +} diff --git a/evals/java/golden/java.app.edit.declaration.json b/evals/java/golden/java.app.edit.declaration.json new file mode 100644 index 000000000..5fe530267 --- /dev/null +++ b/evals/java/golden/java.app.edit.declaration.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "src/main/java/com/example/orders/OrderBook.java:32" + ] + } +} diff --git a/evals/java/golden/java.app.edit.usagesOtherFile.json b/evals/java/golden/java.app.edit.usagesOtherFile.json new file mode 100644 index 000000000..e20fd75ec --- /dev/null +++ b/evals/java/golden/java.app.edit.usagesOtherFile.json @@ -0,0 +1,14 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/com/example/orders/ListPrice.java:7", + "src/main/java/com/example/orders/OrderBook.java:11", + "src/main/java/com/example/orders/OrderBook.java:28", + "src/main/java/com/example/orders/OrderBook.java:33" + ] + } +} diff --git a/evals/java/golden/java.app.edit.usagesSameFile.json b/evals/java/golden/java.app.edit.usagesSameFile.json new file mode 100644 index 000000000..a7096c120 --- /dev/null +++ b/evals/java/golden/java.app.edit.usagesSameFile.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/com/example/orders/App.java:12", + "src/main/java/com/example/orders/OrderBook.java:11", + "src/test/java/com/example/orders/OrderBookTest.java:15" + ] + } +} diff --git a/evals/java/golden/java.app.gutter.interface.json b/evals/java/golden/java.app.gutter.interface.json new file mode 100644 index 000000000..c3355be0c --- /dev/null +++ b/evals/java/golden/java.app.gutter.interface.json @@ -0,0 +1,12 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "markers": [ + "L4:implemented", + "L5:implemented" + ] + } +} diff --git a/evals/java/golden/java.app.gutter.jdkClass.json b/evals/java/golden/java.app.gutter.jdkClass.json new file mode 100644 index 000000000..175880c85 --- /dev/null +++ b/evals/java/golden/java.app.gutter.jdkClass.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "markers": [ + "L109:overridden" + ] + } +} diff --git a/evals/java/golden/java.app.gutter.subclass.json b/evals/java/golden/java.app.gutter.subclass.json new file mode 100644 index 000000000..7dbe639b4 --- /dev/null +++ b/evals/java/golden/java.app.gutter.subclass.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "markers": [ + "L14:overriding" + ] + } +} diff --git a/evals/java/golden/java.app.implementation.interfaceMethod.json b/evals/java/golden/java.app.implementation.interfaceMethod.json new file mode 100644 index 000000000..39156183b --- /dev/null +++ b/evals/java/golden/java.app.implementation.interfaceMethod.json @@ -0,0 +1,12 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "implementations": [ + "src/main/java/com/example/orders/BulkDiscount.java:14", + "src/main/java/com/example/orders/ListPrice.java:6" + ] + } +} diff --git a/evals/java/golden/java.app.implementation.jdkInterface.json b/evals/java/golden/java.app.implementation.jdkInterface.json new file mode 100644 index 000000000..4aee99e50 --- /dev/null +++ b/evals/java/golden/java.app.implementation.jdkInterface.json @@ -0,0 +1,81 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "implementations": [ + "jdt://contents/java.base/java.lang.invoke/AbstractConstantGroup.java:175", + "jdt://contents/java.base/java.util.concurrent/CopyOnWriteArrayList.java:1269", + "jdt://contents/java.base/java.util.concurrent/CopyOnWriteArrayList.java:1742", + "jdt://contents/java.base/java.util.concurrent/CopyOnWriteArrayList.java:99", + "jdt://contents/java.base/java.util/AbstractList.java:75", + "jdt://contents/java.base/java.util/AbstractList.java:750", + "jdt://contents/java.base/java.util/AbstractList.java:919", + "jdt://contents/java.base/java.util/AbstractSequentialList.java:71", + "jdt://contents/java.base/java.util/ArrayList.java:109", + "jdt://contents/java.base/java.util/ArrayList.java:1194", + "jdt://contents/java.base/java.util/Arrays.java:4229", + "jdt://contents/java.base/java.util/Collections.java:1486", + "jdt://contents/java.base/java.util/Collections.java:1585", + "jdt://contents/java.base/java.util/Collections.java:2691", + "jdt://contents/java.base/java.util/Collections.java:2789", + "jdt://contents/java.base/java.util/Collections.java:3748", + "jdt://contents/java.base/java.util/Collections.java:3836", + "jdt://contents/java.base/java.util/Collections.java:4779", + "jdt://contents/java.base/java.util/Collections.java:5158", + "jdt://contents/java.base/java.util/Collections.java:5368", + "jdt://contents/java.base/java.util/ImmutableCollections.java:254", + "jdt://contents/java.base/java.util/ImmutableCollections.java:434", + "jdt://contents/java.base/java.util/ImmutableCollections.java:553", + "jdt://contents/java.base/java.util/ImmutableCollections.java:660", + "jdt://contents/java.base/java.util/LinkedList.java:1292", + "jdt://contents/java.base/java.util/LinkedList.java:90", + "jdt://contents/java.base/java.util/ReverseOrderListView.java:40", + "jdt://contents/java.base/java.util/ReverseOrderListView.java:55", + "jdt://contents/java.base/java.util/Stack.java:50", + "jdt://contents/java.base/java.util/Vector.java:92", + "jdt://contents/java.base/jdk.internal.classfile.components/ClassPrinter.java:151", + "jdt://contents/java.base/jdk.internal.classfile.impl/ClassPrinterImpl.java:78", + "jdt://contents/java.base/jdk.internal.classfile.impl/Util.java:161", + "jdt://contents/java.base/jdk.internal.org.objectweb.asm.tree/MethodNode.java:281", + "jdt://contents/java.base/sun.net.www.http/KeepAliveStreamCleaner.java:49", + "jdt://contents/java.base/sun.security.jca/ProviderList.java:153", + "jdt://contents/java.base/sun.security.jca/ProviderList.java:431", + "jdt://contents/java.desktop/javax.swing/JEditorPane.java:2010", + "jdt://contents/java.desktop/sun.awt.util/IdentityArrayList.java:95", + "jdt://contents/java.desktop/sun.swing/BakedArrayList.java:47", + "jdt://contents/java.management/com.sun.jmx.remote.internal/ArrayQueue.java:31", + "jdt://contents/java.management/javax.management.relation/RoleList.java:56", + "jdt://contents/java.management/javax.management.relation/RoleUnresolvedList.java:55", + "jdt://contents/java.management/javax.management/AttributeList.java:65", + "jdt://contents/java.xml/com.sun.org.apache.xalan.internal.xsltc.compiler.util/StringStack.java:30", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.util/ByteListImpl.java:36", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/Base64BinaryDV.java:60", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/HexBinaryDV.java:57", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/ListDV.java:58", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/XSSimpleTypeDecl.java:2263", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/XSSimpleTypeDecl.java:2298", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.dv.xs/XSSimpleTypeDecl.java:3447", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs.util/LSInputListImpl.java:38", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs.util/ObjectListImpl.java:35", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs.util/ShortListImpl.java:36", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs.util/StringListImpl.java:38", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs.util/XSObjectListImpl.java:41", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs/PSVIErrorList.java:35", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.impl.xs/XSModelImpl.java:63", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs.datatypes/ByteList.java:34", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs.datatypes/ObjectList.java:33", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs/LSInputList.java:33", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs/ShortList.java:31", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs/StringList.java:31", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs/XSNamespaceItemList.java:32", + "jdt://contents/java.xml/com.sun.org.apache.xerces.internal.xs/XSObjectList.java:32", + "jdt://contents/jdk.compiler/com.sun.tools.javac.model/FilteredMemberList.java:46", + "jdt://contents/jdk.compiler/com.sun.tools.javac.util/List.java:53", + "jdt://contents/jdk.compiler/com.sun.tools.javac.util/List.java:79", + "jdt://contents/jdk.jdi/com.sun.tools.jdi/EventSetImpl.java:86", + "jdt://contents/jdk.jshell/jdk.jshell/DiagList.java:38" + ] + } +} diff --git a/evals/java/golden/java.app.implementation.projectInterface.json b/evals/java/golden/java.app.implementation.projectInterface.json new file mode 100644 index 000000000..e5dbcf44d --- /dev/null +++ b/evals/java/golden/java.app.implementation.projectInterface.json @@ -0,0 +1,12 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "implementations": [ + "src/main/java/com/example/orders/BulkDiscount.java:4", + "src/main/java/com/example/orders/ListPrice.java:4" + ] + } +} diff --git a/evals/java/golden/java.app.lens.jdkClass.json b/evals/java/golden/java.app.lens.jdkClass.json new file mode 100644 index 000000000..582e6fdf3 --- /dev/null +++ b/evals/java/golden/java.app.lens.jdkClass.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "lenses": [ + "L33:2 usages", + "L73:no usages", + "L92:no usages" + ] + } +} diff --git a/evals/java/golden/java.app.lens.project.json b/evals/java/golden/java.app.lens.project.json new file mode 100644 index 000000000..59b41e54c --- /dev/null +++ b/evals/java/golden/java.app.lens.project.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "lenses": [ + "L10:3 usages", + "L3:12 usages", + "L6:2 usages" + ] + } +} diff --git a/evals/java/golden/java.app.rename.acrossFiles.json b/evals/java/golden/java.app.rename.acrossFiles.json new file mode 100644 index 000000000..897c1ca3c --- /dev/null +++ b/evals/java/golden/java.app.rename.acrossFiles.json @@ -0,0 +1,14 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "changes": [ + "src/main/java/com/example/orders/ListPrice.java:7", + "src/main/java/com/example/orders/Order.java:11", + "src/main/java/com/example/orders/OrderBook.java:24", + "src/main/java/com/example/orders/OrderBook.java:29" + ] + } +} diff --git a/evals/java/golden/java.app.rename.local.json b/evals/java/golden/java.app.rename.local.json new file mode 100644 index 000000000..89ed23dbd --- /dev/null +++ b/evals/java/golden/java.app.rename.local.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "changes": [ + "src/main/java/com/example/orders/OrderBook.java:14", + "src/main/java/com/example/orders/OrderBook.java:15", + "src/main/java/com/example/orders/OrderBook.java:16" + ] + } +} diff --git a/evals/java/golden/java.app.usages.inJdkSource.json b/evals/java/golden/java.app.usages.inJdkSource.json new file mode 100644 index 000000000..a71c1c092 --- /dev/null +++ b/evals/java/golden/java.app.usages.inJdkSource.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/com/example/orders/OrderBook.java:20", + "src/main/java/com/example/orders/OrderBook.java:24", + "src/main/java/com/example/orders/OrderBook.java:29" + ] + } +} diff --git a/evals/java/golden/java.app.usages.jdkMethodLibraries.json b/evals/java/golden/java.app.usages.jdkMethodLibraries.json new file mode 100644 index 000000000..c581c493d --- /dev/null +++ b/evals/java/golden/java.app.usages.jdkMethodLibraries.json @@ -0,0 +1,16 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "jdt://contents/java.base/java.util/Comparator.java:302", + "jdt://contents/java.base/java.util/Comparator.java:307", + "jdt://contents/jdk.jfr/jdk.jfr.internal.dcmd/AbstractDCmd.java:174", + "jdt://contents/jdk.jfr/jdk.jfr.internal.periodic/BatchManager.java:84", + "jdt://contents/jdk.jfr/jdk.jfr.internal/TypeLibrary.java:109", + "src/main/java/com/example/orders/OrderBook.java:24" + ] + } +} diff --git a/evals/java/golden/java.app.usages.method.json b/evals/java/golden/java.app.usages.method.json new file mode 100644 index 000000000..7f989f4be --- /dev/null +++ b/evals/java/golden/java.app.usages.method.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/com/example/orders/ListPrice.java:7", + "src/main/java/com/example/orders/OrderBook.java:24", + "src/main/java/com/example/orders/OrderBook.java:29" + ] + } +} diff --git a/evals/java/golden/java.app.usages.record.json b/evals/java/golden/java.app.usages.record.json new file mode 100644 index 000000000..1401190df --- /dev/null +++ b/evals/java/golden/java.app.usages.record.json @@ -0,0 +1,21 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/com/example/orders/BulkDiscount.java:14", + "src/main/java/com/example/orders/ListPrice.java:6", + "src/main/java/com/example/orders/OrderBook.java:10", + "src/main/java/com/example/orders/OrderBook.java:13", + "src/main/java/com/example/orders/OrderBook.java:14", + "src/main/java/com/example/orders/OrderBook.java:19", + "src/main/java/com/example/orders/OrderBook.java:23", + "src/main/java/com/example/orders/OrderBook.java:24", + "src/main/java/com/example/orders/OrderBook.java:29", + "src/main/java/com/example/orders/PricingRule.java:5", + "src/test/java/com/example/orders/OrderBookTest.java:21" + ] + } +} diff --git a/evals/java/golden/java.petclinic.completion.libraryClass.json b/evals/java/golden/java.petclinic.completion.libraryClass.json new file mode 100644 index 000000000..5446dfab9 --- /dev/null +++ b/evals/java/golden/java.petclinic.completion.libraryClass.json @@ -0,0 +1,18 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "completions": [ + "Pageable", + "PageableArgumentResolver", + "PageableDefault", + "PageableExecutionUtils", + "PageableHandlerMethodArgumentResolver", + "PageableHandlerMethodArgumentResolverCustomizer", + "PageableHandlerMethodArgumentResolverSupport", + "PageableUtils" + ] + } +} diff --git a/evals/java/golden/java.petclinic.declaration.inheritedLibrary.json b/evals/java/golden/java.petclinic.declaration.inheritedLibrary.json new file mode 100644 index 000000000..7e291c10e --- /dev/null +++ b/evals/java/golden/java.petclinic.declaration.inheritedLibrary.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/spring-data-commons-4.1.0.jar/org.springframework.data.repository/CrudRepository.java#save" + ] + } +} diff --git a/evals/java/golden/java.petclinic.declaration.libraryClass.json b/evals/java/golden/java.petclinic.declaration.libraryClass.json new file mode 100644 index 000000000..4645f4f3d --- /dev/null +++ b/evals/java/golden/java.petclinic.declaration.libraryClass.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "jdt://contents/spring-data-commons-4.1.0.jar/org.springframework.data.domain/PageRequest.java#PageRequest" + ] + } +} diff --git a/evals/java/golden/java.petclinic.declaration.project.json b/evals/java/golden/java.petclinic.declaration.project.json new file mode 100644 index 000000000..32bfb8e41 --- /dev/null +++ b/evals/java/golden/java.petclinic.declaration.project.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "declaration": [ + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:45" + ] + } +} diff --git a/evals/java/golden/java.petclinic.diagnostics.cleanFile.json b/evals/java/golden/java.petclinic.diagnostics.cleanFile.json new file mode 100644 index 000000000..2fcb87f0b --- /dev/null +++ b/evals/java/golden/java.petclinic.diagnostics.cleanFile.json @@ -0,0 +1,9 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "problems": [] + } +} diff --git a/evals/java/golden/java.petclinic.doc.library.json b/evals/java/golden/java.petclinic.doc.library.json new file mode 100644 index 000000000..991e5f363 --- /dev/null +++ b/evals/java/golden/java.petclinic.doc.library.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "documentation": [ + "org.springframework.data.domain.PageRequest Basic Java Bean implementation of Pageable. Author: Oliver Gierke Thomas Darimont Anastasiia Smirnova Mark Paluch Thach Le Source: spring-data-commons-4.1.0.jar" + ] + } +} diff --git a/evals/java/golden/java.petclinic.gutter.baseClass.json b/evals/java/golden/java.petclinic.gutter.baseClass.json new file mode 100644 index 000000000..b6d6dd9dc --- /dev/null +++ b/evals/java/golden/java.petclinic.gutter.baseClass.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "markers": [ + "L33:overridden" + ] + } +} diff --git a/evals/java/golden/java.petclinic.gutter.midClass.json b/evals/java/golden/java.petclinic.gutter.midClass.json new file mode 100644 index 000000000..4280f4ea1 --- /dev/null +++ b/evals/java/golden/java.petclinic.gutter.midClass.json @@ -0,0 +1,11 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "markers": [ + "L29:overridden" + ] + } +} diff --git a/evals/java/golden/java.petclinic.implementation.baseClass.json b/evals/java/golden/java.petclinic.implementation.baseClass.json new file mode 100644 index 000000000..001ccdccf --- /dev/null +++ b/evals/java/golden/java.petclinic.implementation.baseClass.json @@ -0,0 +1,20 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "implementations": [ + "src/main/java/org/springframework/samples/petclinic/model/NamedEntity.java:31", + "src/main/java/org/springframework/samples/petclinic/model/Person.java:29", + "src/main/java/org/springframework/samples/petclinic/owner/Owner.java:50", + "src/main/java/org/springframework/samples/petclinic/owner/Pet.java:46", + "src/main/java/org/springframework/samples/petclinic/owner/PetType.java:28", + "src/main/java/org/springframework/samples/petclinic/owner/Visit.java:37", + "src/main/java/org/springframework/samples/petclinic/vet/Specialty.java:30", + "src/main/java/org/springframework/samples/petclinic/vet/Vet.java:45", + "src/test/java/org/springframework/samples/petclinic/owner/PetTypeFormatterTests.java:84", + "src/test/java/org/springframework/samples/petclinic/owner/PetTypeFormatterTests.java:89" + ] + } +} diff --git a/evals/java/golden/java.petclinic.lens.entity.json b/evals/java/golden/java.petclinic.lens.entity.json new file mode 100644 index 000000000..744d27b5f --- /dev/null +++ b/evals/java/golden/java.petclinic.lens.entity.json @@ -0,0 +1,16 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "lenses": [ + "L38:82 usages", + "L72:no usages", + "L76:3 usages", + "L80:no usages", + "L84:3 usages", + "L88:no usages" + ] + } +} diff --git a/evals/java/golden/java.petclinic.rename.acrossFiles.json b/evals/java/golden/java.petclinic.rename.acrossFiles.json new file mode 100644 index 000000000..356f29e08 --- /dev/null +++ b/evals/java/golden/java.petclinic.rename.acrossFiles.json @@ -0,0 +1,13 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "changes": [ + "src/main/java/org/springframework/samples/petclinic/owner/Owner.java:176", + "src/main/java/org/springframework/samples/petclinic/owner/VisitController.java:108", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:229" + ] + } +} diff --git a/evals/java/golden/java.petclinic.usages.entity.json b/evals/java/golden/java.petclinic.usages.entity.json new file mode 100644 index 000000000..28f90ec45 --- /dev/null +++ b/evals/java/golden/java.petclinic.usages.entity.json @@ -0,0 +1,83 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:107", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:132", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:133", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:141", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:154", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:181", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:182", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:65", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:66", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:78", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:95", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:36", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:39", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:42", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:45", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:48", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:50", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:51", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:55", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerRepository.java:60", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:101", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:108", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:145", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:186", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:68", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:69", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:70", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:83", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:84", + "src/main/java/org/springframework/samples/petclinic/owner/VisitController.java:66", + "src/main/java/org/springframework/samples/petclinic/owner/VisitController.java:67", + "src/main/java/org/springframework/samples/petclinic/owner/VisitController.java:98", + "src/test/java/org/springframework/samples/petclinic/PetClinicConcurrencyTests.java:114", + "src/test/java/org/springframework/samples/petclinic/PetClinicConcurrencyTests.java:22", + "src/test/java/org/springframework/samples/petclinic/PetClinicConcurrencyTests.java:43", + "src/test/java/org/springframework/samples/petclinic/PetClinicConcurrencyTests.java:45", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:176", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:192", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:201", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:215", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:227", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:302", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:73", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:74", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerControllerTests.java:95", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerTests.java:27", + "src/test/java/org/springframework/samples/petclinic/owner/OwnerTests.java:40", + "src/test/java/org/springframework/samples/petclinic/owner/PetControllerTests.java:167", + "src/test/java/org/springframework/samples/petclinic/owner/PetControllerTests.java:258", + "src/test/java/org/springframework/samples/petclinic/owner/PetControllerTests.java:77", + "src/test/java/org/springframework/samples/petclinic/owner/VisitControllerTests.java:61", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:100", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:102", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:112", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:115", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:131", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:133", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:160", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:162", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:188", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:190", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:220", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:222", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:239", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:241", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:256", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:258", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:285", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:287", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:296", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:298", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:34", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:91" + ] + } +} diff --git a/evals/java/golden/java.petclinic.usages.inheritedMethod.json b/evals/java/golden/java.petclinic.usages.inheritedMethod.json new file mode 100644 index 000000000..8344fe3e1 --- /dev/null +++ b/evals/java/golden/java.petclinic.usages.inheritedMethod.json @@ -0,0 +1,24 @@ +{ + "tools": { + "jdtls": "1.63.0-202610191322", + "java": "21.0.13.0.2 (Homebrew)" + }, + "groups": { + "usages": [ + "src/main/java/org/springframework/samples/petclinic/owner/Owner.java:108", + "src/main/java/org/springframework/samples/petclinic/owner/Owner.java:132", + "src/main/java/org/springframework/samples/petclinic/owner/Owner.java:161", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:125", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:161", + "src/main/java/org/springframework/samples/petclinic/owner/OwnerController.java:86", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:153", + "src/main/java/org/springframework/samples/petclinic/owner/PetController.java:187", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:122", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:182", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:229", + "src/test/java/org/springframework/samples/petclinic/service/ClinicServiceTests.java:234", + "src/test/java/org/springframework/samples/petclinic/service/EntityUtils.java:46", + "src/test/java/org/springframework/samples/petclinic/vet/VetTests.java:38" + ] + } +} diff --git a/evals/java/scenarios.ts b/evals/java/scenarios.ts new file mode 100644 index 000000000..58ef38080 --- /dev/null +++ b/evals/java/scenarios.ts @@ -0,0 +1,841 @@ +/** + * Java code intelligence with jdtls: the steps users take most, timed as they see them and + * checked against `golden/.json`. On `fixtures/java-app` (small, every case visible) and + * on Spring PetClinic (`evals/projects.json`: a real Spring Boot / Maven project). + */ +import { join } from "node:path"; +import { track } from "../lib/measure.ts"; +import { normalizePath, type EvalScenario, type Measured } from "../lib/scenario.ts"; +import type { Session } from "../lib/session.ts"; + +const SRC = "src/main/java/com/example/orders"; +const TEST = "src/test/java/com/example/orders"; +const PET = "src/main/java/org/springframework/samples/petclinic"; +const TIMEOUT = 30_000; +/** Results a whole file's worth of server requests make up. */ +const LONG_TIMEOUT = 60_000; + +interface ProbeWindow { + __openideaPerf: { + editorState(): { path: string; head: number; line: number; word: string } | null; + editorText(): string | null; + setCursor(offset: number): void; + insertAfter(after: string, text: string): number; + completions(): string[]; + completionPending(): boolean; + diagnostics(): Array<{ line: number; severity: string; message: string }>; + lineAtY(y: number): number; + }; +} + +/** The visible editor hint, unless it only says a request is pending. */ +async function finalHint(s: Session): Promise { + return s.page.evaluate(() => { + const el = document.querySelector(".cm-lsp-hint:not([data-pending])") as HTMLElement | null; + return el && el.getClientRects().length ? el.textContent : null; + }); +} + +/** Where the caret is: `path:line` in the project, `uri#word` in a library class (its line depends on the JDK build). */ +async function caretLocation(s: Session): Promise { + const st = await s.page.evaluate(() => (window as unknown as ProbeWindow).__openideaPerf.editorState()); + if (!st) return null; + const path = normalizePath(s.root, st.path); + return path.includes("://") ? `${path}#${st.word}` : `${path}:${st.line + 1}`; +} + +/** + * Where a step starts: inside `word` on the first line containing `line` of `file`, or of the + * library class reached from `file` by going to the declaration of each `via` in turn. `edit` + * changes `file` first; `typed` changes the file reached right before the step, with no time + * for the server to catch up. + */ +interface Place { + file: string; + edit?: Insertion; + via?: Array<[line: string, word: string]>; + typed?: Insertion; + line: string; + word: string; +} + +interface Insertion { + after: string; + text: string; +} + +async function reach(s: Session, at: Place) { + await s.openFile(at.file); + if (at.edit) await s.edit(at.edit.after, at.edit.text); + for (const [line, word] of at.via ?? []) await s.declarationOf(line, word); + if (!at.typed) await s.caretAt(at.line, at.word); +} + +/** What a step does first when its place is `typed`. */ +async function type(s: Session, at: Place) { + if (!at.typed) return; + await s.edit(at.typed.after, at.typed.text); + await s.caretAt(at.line, at.word); +} + +/** Text that marks an answer from syntax instead of the language server. */ +const SYNTAX = /based on syntax|language server not ready/i; + +function declaration( + project: string, + id: string, + title: string, + at: Place, + conditions: EvalScenario["conditions"] = ["warm"], +): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions, + setup: (s) => reach(s, at), + async step(s): Promise { + await type(s, at); + // Moved to the declaration, or a hint saying why not (not one saying it is still looking). + const answered = (_: null, before: { path: string; head: number } | null) => { + const st = (window as unknown as ProbeWindow).__openideaPerf.editorState(); + if (st && before && (st.path !== before.path || st.head !== before.head)) return true; + return !!document.querySelector(".cm-lsp-hint:not([data-pending])"); + }; + const timing = await track(s.page, { + action: "navigate.declaration", + arg: null, + first: answered, + done: answered, + timeoutMs: TIMEOUT, + }); + // A note that the answer came from syntax follows the navigation. + await s.page.waitForTimeout(300); + const hint = await finalHint(s); + const landed = await caretLocation(s); + return { + timing, + results: { declaration: landed ? [landed] : [] }, + degraded: hint && SYNTAX.test(hint) ? hint : null, + }; + }, + }; +} + +/** Every usage row of the popup, scrolling its virtual list through. */ +async function usageRows(s: Session): Promise { + const rows = await s.page.evaluate(async () => { + const popup = document.querySelector('[data-testid="usages-popup"]'); + if (!popup) return []; + const found = new Set(); + const collect = () => { + for (const r of popup.querySelectorAll('[data-testid="usage-row"]')) + found.add(`${r.dataset.path}\u0000${Number(r.dataset.line) + 1}`); + }; + const scroller = [...popup.querySelectorAll("*")].find((el) => el.scrollHeight > el.clientHeight + 1); + collect(); + if (scroller) { + for (let top = 0; top < scroller.scrollHeight; top += scroller.clientHeight / 2) { + scroller.scrollTop = top; + await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))); + collect(); + } + } + return [...found]; + }); + return rows.map((r) => { + const [path, line] = r.split("\u0000") as [string, string]; + return `${normalizePath(s.root, path)}:${line}`; + }); +} + +/** + * Show Usages at `at`. `widen`: in Project and Libraries, as invoking it again while the popup + * or its hint offers to widen a Project Files search does (the step is that second invocation). + */ +function showUsages(project: string, id: string, title: string, at: Place, widen = false): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup(s) { + await reach(s, at); + if (!widen) return; + await s.runAction("find.showUsages"); + await s.page.waitForFunction( + () => + document.querySelector('[data-testid="usages-popup"][data-loading="false"][data-scope="project"]') || + document.querySelector(".cm-lsp-hint[data-usages]"), + undefined, + { timeout: TIMEOUT }, + ); + }, + async step(s): Promise { + await type(s, at); + const timing = await track(s.page, { + action: "find.showUsages", + arg: widen, + // A single usage is jumped to; none says so in a hint. + first: (widen, before) => { + const popup = document.querySelector('[data-testid="usages-popup"]'); + if (popup) + return ( + (!widen || popup.dataset.scope === "libraries") && !!popup.querySelector('[data-testid="usage-row"]') + ); + const st = (window as unknown as ProbeWindow).__openideaPerf.editorState(); + if (st && before && (st.path !== before.path || st.head !== before.head)) return true; + return !!document.querySelector(widen ? '.cm-lsp-hint[data-scope="libraries"]' : ".cm-lsp-hint[data-usages]"); + }, + done: (widen, before) => { + const popup = document.querySelector('[data-testid="usages-popup"]'); + if (popup) return (!widen || popup.dataset.scope === "libraries") && popup.dataset.loading === "false"; + const st = (window as unknown as ProbeWindow).__openideaPerf.editorState(); + if (st && before && (st.path !== before.path || st.head !== before.head)) return true; + return !!document.querySelector(widen ? '.cm-lsp-hint[data-scope="libraries"]' : ".cm-lsp-hint[data-usages]"); + }, + timeoutMs: TIMEOUT, + }); + const popup = await s.page.$('[data-testid="usages-popup"]'); + const rows = popup ? await usageRows(s) : []; + // Without a popup the hint gives the count: the only usage was jumped to (or the caret is on it). + const count = await s.page.evaluate( + () => (document.querySelector(".cm-lsp-hint[data-usages]") as HTMLElement | null)?.dataset.usages ?? null, + ); + const single = !popup && count === "1" ? await caretLocation(s) : null; + const note = await s.page.evaluate( + () => document.querySelector('[data-testid="usages-note"]')?.textContent ?? null, + ); + const hint = await finalHint(s); + return { + timing, + results: { usages: single ? [single] : rows }, + degraded: note ?? (hint && SYNTAX.test(hint) ? hint : null), + }; + }, + }; +} + +/** Every row of the targets chooser (`path:line`), scrolling its virtual list through. */ +async function chooserRows(s: Session): Promise { + const rows = await s.page.evaluate(async () => { + const popup = document.querySelector('[data-testid="list-popup"]'); + if (!popup) return []; + const found = new Set(); + const collect = () => { + for (const r of popup.querySelectorAll('[data-testid="list-popup-item"]')) + if (r.dataset.path) found.add(`${r.dataset.path}\u0000${Number(r.dataset.line) + 1}`); + }; + const scroller = [...popup.querySelectorAll("*")].find((el) => el.scrollHeight > el.clientHeight + 1); + collect(); + if (scroller) { + for (let top = 0; top < scroller.scrollHeight; top += scroller.clientHeight / 2) { + scroller.scrollTop = top; + await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))); + collect(); + } + } + return [...found]; + }); + return rows.map((r) => { + const [path, line] = r.split("\u0000") as [string, string]; + return `${normalizePath(s.root, path)}:${line}`; + }); +} + +/** Go to Implementation: the chooser's rows once it stops searching, or the only target. */ +function implementation(project: string, id: string, title: string, at: Place): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + setup: (s) => reach(s, at), + async step(s): Promise { + const before = await caretLocation(s); + // The chooser (busy while the server looks on), the only target, or a hint. + const answered = (searching: boolean, before: { path: string; head: number } | null) => { + const popup = document.querySelector('[data-testid="list-popup"]'); + if (popup) return searching || popup.getAttribute("aria-busy") !== "true"; + const st = (window as unknown as ProbeWindow).__openideaPerf.editorState(); + if (st && before && (st.path !== before.path || st.head !== before.head)) return true; + return !!document.querySelector(".cm-lsp-hint:not([data-pending])"); + }; + // Checks run in the page from their source, so the shared one goes in as the argument's source. + const timing = await track(s.page, { + action: "navigate.implementation", + arg: answered.toString(), + first: (src, before) => (new Function(`return ${src}`)() as typeof answered)(true, before), + done: (src, before) => (new Function(`return ${src}`)() as typeof answered)(false, before), + timeoutMs: TIMEOUT, + }); + const popup = await s.page.$('[data-testid="list-popup"]'); + const rows = popup ? await chooserRows(s) : []; + const at = await caretLocation(s); + const hint = await finalHint(s); + return { + timing, + results: { implementations: popup ? rows : at && at !== before ? [at] : [] }, + degraded: hint && SYNTAX.test(hint) ? hint : null, + }; + }, + }; +} + +/** + * What opening `file`, or the library class reached from it through `via` (see `Place`), brings + * up beside its text: every `selector` with its line, as `L:`. + */ +function onOpen( + project: string, + id: string, + title: string, + file: string, + via: NonNullable, + group: string, + selector: string, +): EvalScenario { + const last = via[via.length - 1]; + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup(s) { + if (!last) return; + await s.openFile(file); + for (const [line, word] of via.slice(0, -1)) await s.declarationOf(line, word); + await s.caretAt(last[0], last[1]); + }, + async step(s): Promise { + const items = (selector: string) => { + const probe = (window as unknown as ProbeWindow).__openideaPerf; + const out: string[] = []; + for (const m of document.querySelectorAll(selector)) { + // A lens is a row of its own above the line it is about. + const r = (m.closest(".cm-openidea-lens")?.nextElementSibling ?? m).getBoundingClientRect(); + out.push(`L${probe.lineAtY(r.top + r.height / 2) + 1}:${m.dataset.kind ?? m.textContent}`); + } + return out.sort(); + }; + const timing = await track(s.page, { + // A library class has no path to open: it is reached the way users do, which also shows its declaration. + ...(last ? { action: "navigate.declaration" } : { openFile: join(s.root, file) }), + arg: selector, + first: (selector, before) => + (window as unknown as ProbeWindow).__openideaPerf.editorState()?.path !== before?.path && + !!document.querySelector(selector), + done: () => true, + signature: items, + settleMs: 2000, + timeoutMs: LONG_TIMEOUT, + }); + return { timing, results: { [group]: await s.page.evaluate(items, selector) } }; + }, + }; +} + +const GUTTER = '.cm-editor [class*="cm-openidea-lineMarker-"]'; + +function gutterOnOpen(project: string, id: string, title: string, file: string, via: Place["via"] = []): EvalScenario { + return onOpen(project, id, title, file, via, "markers", GUTTER); +} + +function lensOnOpen(project: string, id: string, title: string, file: string, via: Place["via"] = []): EvalScenario { + return onOpen(project, id, title, file, via, "lenses", ".cm-editor .cm-openidea-lensAction"); +} + +/** Errors and warnings in the active editor: hints and notes come and go with the server's settings. */ +const problems = () => + (window as unknown as ProbeWindow).__openideaPerf + .diagnostics() + .filter((d) => d.severity === "error" || d.severity === "warning") + .map((d) => `L${d.line + 1}:${d.severity}:${d.message}`) + .sort(); + +/** Waits for what the previous run left in the active editor to go. */ +async function noErrors(s: Session) { + await s.page.waitForFunction(() => !document.querySelector(".cm-lintRange-error"), undefined, { timeout: TIMEOUT }); +} + +/** Opening a correct `file`: nothing is reported in it, once the server has had its say. */ +function diagnosticsOnOpen(project: string, id: string, title: string, file: string): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup() {}, + async step(s): Promise { + const timing = await track(s.page, { + openFile: join(s.root, file), + arg: null, + first: () => !!document.querySelector(".cm-content .cm-line"), + done: () => true, + signature: problems, + settleMs: 3000, + timeoutMs: TIMEOUT, + }); + return { timing, results: { problems: await s.page.evaluate(problems) } }; + }, + }; +} + +/** Typing `edit` into a correct file: the error shows under it. */ +function diagnosticsAfterEdit(project: string, id: string, title: string, at: Place, edit: Insertion): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup(s) { + await reach(s, at); + await noErrors(s); + await s.keepText(); + }, + async step(s): Promise { + const timing = await track(s.page, { + run: (edit) => (window as unknown as ProbeWindow).__openideaPerf.insertAfter(edit.after, edit.text), + arg: edit, + // The typed text answers the keystroke; the error is the result. + feedback: () => true, + first: () => !!document.querySelector(".cm-lintRange-error"), + done: () => true, + signature: problems, + settleMs: 2000, + timeoutMs: TIMEOUT, + }); + return { timing, results: { problems: await s.page.evaluate(problems) } }; + }, + }; +} + +/** Typing `fix` elsewhere repairs the mistake `edit` made: only the server can clear its error. */ +function diagnosticsAfterFix( + project: string, + id: string, + title: string, + at: Place, + edit: Insertion, + fix: Insertion, +): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup(s) { + await reach(s, at); + await noErrors(s); + await s.edit(edit.after, edit.text); + await s.page.waitForSelector(".cm-lintRange-error", { timeout: TIMEOUT }); + }, + async step(s): Promise { + const timing = await track(s.page, { + run: (fix) => (window as unknown as ProbeWindow).__openideaPerf.insertAfter(fix.after, fix.text), + arg: fix, + feedback: () => true, + first: () => !document.querySelector(".cm-lintRange-error"), + done: () => true, + signature: problems, + settleMs: 2000, + timeoutMs: TIMEOUT, + }); + return { timing, results: { problems: await s.page.evaluate(problems) } }; + }, + }; +} + +/** Code Completion right after typing `edit`: the lookup's labels. */ +function completion(project: string, id: string, title: string, file: string, edit: Insertion): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + setup: (s) => s.openFile(file), + async step(s): Promise { + const end = await s.edit(edit.after, edit.text); + await s.page.evaluate((o) => (window as unknown as ProbeWindow).__openideaPerf.setCursor(o), end); + const timing = await track(s.page, { + action: "code.completion", + arg: null, + first: () => (window as unknown as ProbeWindow).__openideaPerf.completions().length > 0, + // Quicker sources show first: every source has answered once it stops being pending. + done: () => !(window as unknown as ProbeWindow).__openideaPerf.completionPending(), + timeoutMs: TIMEOUT, + }); + const labels = await s.page.evaluate(() => (window as unknown as ProbeWindow).__openideaPerf.completions()); + return { timing, results: { completions: labels } }; + }, + }; +} + +/** Quick Documentation: the start of what the popup says. */ +function quickDoc(project: string, id: string, title: string, at: Place): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + setup: (s) => reach(s, at), + async step(s): Promise { + const shown = () => !!document.querySelector('[data-testid="doc-popup"], .cm-lsp-hint:not([data-pending])'); + const text = () => + document.querySelector('[data-testid="doc-popup"]')?.textContent?.replace(/\s+/g, " ").trim().slice(0, 300) ?? + ""; + const timing = await track(s.page, { + action: "code.quickDocumentation", + arg: null, + feedback: () => !!document.querySelector('[data-testid="doc-popup"]'), + first: shown, + done: shown, + signature: text, + settleMs: 500, + timeoutMs: TIMEOUT, + }); + const doc = await s.page.evaluate(text); + const hint = await finalHint(s); + return { + timing, + results: { documentation: doc ? [doc] : [] }, + degraded: hint && SYNTAX.test(hint) ? hint : null, + }; + }, + }; +} + +interface RenameAnswer { + edit: { files: Array<{ path?: string; edits?: Array<{ range: { start: { line: number } }; new_text: string }> }> }; +} + +/** Rename in place to `newName`, then ↵: every edit it makes (a preview is left unapplied). */ +function rename(project: string, id: string, title: string, at: Place, newName: string): EvalScenario { + return { + id, + title, + fixture: project, + server: "jdtls", + conditions: ["warm"], + async setup(s) { + await reach(s, at); + await s.keepText(); + await s.runAction("refactor.rename"); + await s.page.waitForSelector(".cm-lsp-inplace-primary", { timeout: TIMEOUT }); + await s.page.keyboard.type(newName); + }, + async step(s): Promise { + const answered = (newName: string) => { + if (document.querySelector('[data-testid="refactor-preview"]')) return true; + if (document.querySelector(".cm-lsp-hint:not([data-pending]):not(.cm-lsp-inplace-hint)")) return true; + return ( + !document.querySelector(".cm-lsp-inplace") && + !!(window as unknown as ProbeWindow).__openideaPerf.editorText()?.includes(newName) + ); + }; + const timing = await track(s.page, { + run: () => + document.querySelector(".cm-content")?.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Enter", + code: "Enter", + keyCode: 13, + bubbles: true, + cancelable: true, + }), + ), + arg: newName, + first: answered, + done: answered, + timeoutMs: TIMEOUT, + }); + // The preview only counts per file: where the name lands is in the answer it shows. A server + // may merge nearby occurrences into one edit, so every line of its new text counts. + const answer = s.lastAnswer.get("lsp_rename") as RenameAnswer | undefined; + const changes = (answer?.edit.files ?? []).flatMap((f) => + (f.edits ?? []).flatMap((e) => + e.new_text + .split("\n") + .flatMap((l, i) => + l.includes(newName) ? [`${normalizePath(s.root, f.path!)}:${e.range.start.line + i + 1}`] : [], + ), + ), + ); + return { timing, results: { changes } }; + }, + }; +} + +const APP = "java-app"; +const PETCLINIC = "spring-petclinic"; + +/** From `OrderBook` into the JDK's `ArrayList` / `Collection` sources. */ +const BOOK = `${SRC}/OrderBook.java`; +const TO_ARRAYLIST: [string, string] = ["private final List orders = new ArrayList<>();", "ArrayList"]; +/** A new method above the rest of `OrderBook`, calling `revenueCents`. */ +const TWICE = "\n public long twice() {\n return revenueCents();\n }\n"; +const TYPE_ERROR: Insertion = { after: "public int size() {", text: '\n int broken = "text";' }; +const TO_STREAM: [string, string] = ["return orders.stream().mapToLong(Order::totalCents).sum();", "stream"]; + +export const JAVA_SCENARIOS: EvalScenario[] = [ + declaration( + APP, + "java.app.declaration.project", + "Go to declaration: project method", + { file: `${SRC}/App.java`, line: "long revenue = book.revenueCents();", word: "revenueCents" }, + ["warm", "cold"], + ), + declaration(APP, "java.app.declaration.jdk", "Go to declaration: JDK class", { + file: BOOK, + line: TO_ARRAYLIST[0], + word: TO_ARRAYLIST[1], + }), + declaration(APP, "java.app.declaration.dependency", "Go to declaration: dependency method (JUnit)", { + file: `${TEST}/OrderBookTest.java`, + line: "assertEquals(2500, book.revenueCents());", + word: "assertEquals", + }), + showUsages(APP, "java.app.usages.record", "Show Usages: record used across files", { + file: `${SRC}/Order.java`, + line: "public record Order(", + word: "Order", + }), + showUsages(APP, "java.app.usages.method", "Show Usages: method used through calls and method references", { + file: `${SRC}/Order.java`, + line: "public long totalCents() {", + word: "totalCents", + }), + gutterOnOpen(APP, "java.app.gutter.interface", "Gutter markers: open an interface", `${SRC}/PricingRule.java`), + gutterOnOpen( + APP, + "java.app.gutter.subclass", + "Gutter markers: open an overriding subclass", + `${SRC}/BulkDiscount.java`, + ), + + // The JDK's sources: their lines depend on its build (`java` in .github/tool-versions.env). + declaration(APP, "java.app.declaration.jdkMethod", "Go to declaration: JDK static method", { + file: BOOK, + line: "return orders.stream().max(Comparator.comparingLong(Order::totalCents));", + word: "comparingLong", + }), + declaration(APP, "java.app.declaration.inJdkSource", "Go to declaration from a JDK class: its superclass", { + file: BOOK, + via: [TO_ARRAYLIST], + line: "extends AbstractList", + word: "AbstractList", + }), + showUsages( + APP, + "java.app.usages.jdkMethodLibraries", + "Show Usages in Project and Libraries: JDK method called once in the project", + { + file: BOOK, + line: "return orders.stream().max(Comparator.comparingLong(Order::totalCents));", + word: "comparingLong", + }, + true, + ), + showUsages(APP, "java.app.usages.inJdkSource", "Show Usages from a JDK class: a default method the project calls", { + file: BOOK, + via: [TO_STREAM], + line: "default Stream stream() {", + word: "stream", + }), + implementation(APP, "java.app.implementation.jdkInterface", "Go to Implementation: JDK interface", { + file: BOOK, + line: TO_ARRAYLIST[0], + word: "List", + }), + // From the import: `new ArrayList<>()` lands on a constructor, with no markers in view. + gutterOnOpen(APP, "java.app.gutter.jdkClass", "Gutter markers: open a JDK class", BOOK, [ + ["import java.util.ArrayList;", "ArrayList"], + ]), + lensOnOpen(APP, "java.app.lens.project", "Usage lenses: open a project class", `${SRC}/Order.java`), + lensOnOpen(APP, "java.app.lens.jdkClass", "Usage lenses: open a JDK class", BOOK, [ + ["public Optional largest() {", "Optional"], + ]), + + implementation(APP, "java.app.implementation.projectInterface", "Go to Implementation: project interface", { + file: `${SRC}/PricingRule.java`, + line: "public interface PricingRule {", + word: "PricingRule", + }), + implementation(APP, "java.app.implementation.interfaceMethod", "Go to Implementation: interface method", { + file: `${SRC}/PricingRule.java`, + line: "long price(Order order);", + word: "price", + }), + + // Right after an edit: results follow the new text, unsaved and with its lines moved. + declaration(APP, "java.app.edit.declaration", "Go to declaration right after typing above the target", { + file: BOOK, + typed: { after: "public class OrderBook {", text: TWICE }, + line: "return revenueCents();", + word: "revenueCents", + }), + showUsages(APP, "java.app.edit.usagesSameFile", "Show Usages right after typing a call in the same file", { + file: BOOK, + typed: { after: "public class OrderBook {", text: TWICE }, + line: "public long revenueCents() {", + word: "revenueCents", + }), + showUsages(APP, "java.app.edit.usagesOtherFile", "Show Usages after typing a call in another file", { + file: BOOK, + edit: { + after: "public class OrderBook {", + text: "\n public long firstTotal() {\n return orders.get(0).totalCents();\n }\n", + }, + via: [["return orders.get(0).totalCents();", "totalCents"]], + line: "public long totalCents() {", + word: "totalCents", + }), + + diagnosticsAfterEdit( + APP, + "java.app.diagnostics.afterError", + "Diagnostics: type a type error", + { file: BOOK, line: "public class OrderBook {", word: "OrderBook" }, + TYPE_ERROR, + ), + diagnosticsAfterFix( + APP, + "java.app.diagnostics.afterFix", + "Diagnostics: declare the missing variable", + { file: BOOK, line: "public class OrderBook {", word: "OrderBook" }, + { after: "public int size() {", text: "\n int used = missing;" }, + { after: "public class OrderBook {", text: "\n private final int missing = 0;" }, + ), + + completion(APP, "java.app.completion.jdkMember", "Code Completion: members of a JDK type", BOOK, { + after: "public int size() {", + text: "\n orders.", + }), + completion( + APP, + "java.app.completion.projectMember", + "Code Completion: members of a project class", + `${SRC}/App.java`, + { + after: "long revenue = book.revenueCents();", + text: "\n book.", + }, + ), + + quickDoc(APP, "java.app.doc.project", "Quick Documentation: project method", { + file: BOOK, + line: "public Order place(String customer, String product, int quantity, long unitPriceCents) {", + word: "place", + }), + quickDoc(APP, "java.app.doc.jdk", "Quick Documentation: JDK class", { + file: BOOK, + line: TO_ARRAYLIST[0], + word: TO_ARRAYLIST[1], + }), + + rename( + APP, + "java.app.rename.acrossFiles", + "Rename: method used across files", + { file: `${SRC}/Order.java`, line: "public long totalCents() {", word: "totalCents" }, + "orderTotalCents", + ), + rename( + APP, + "java.app.rename.local", + "Rename: local variable", + { file: BOOK, line: "Order order = new Order(customer, product, quantity, unitPriceCents);", word: "order" }, + "placedOrder", + ), + + declaration( + PETCLINIC, + "java.petclinic.declaration.project", + "Go to declaration: Spring Data query method in the project", + { + file: `${PET}/owner/OwnerController.java`, + line: "return owners.findByLastNameStartingWith(lastname, pageable);", + word: "findByLastNameStartingWith", + }, + ["warm", "cold"], + ), + declaration( + PETCLINIC, + "java.petclinic.declaration.inheritedLibrary", + "Go to declaration: method inherited from a Spring Data interface", + { file: `${PET}/owner/OwnerController.java`, line: "this.owners.save(owner);", word: "save" }, + ), + declaration(PETCLINIC, "java.petclinic.declaration.libraryClass", "Go to declaration: Spring Data class", { + file: `${PET}/owner/OwnerController.java`, + line: "Pageable pageable = PageRequest.of(validatedPage - 1, pageSize);", + word: "PageRequest", + }), + showUsages( + PETCLINIC, + "java.petclinic.usages.entity", + "Show Usages: entity class used across controllers, repositories and tests", + { file: `${PET}/owner/Owner.java`, line: "public class Owner extends Person {", word: "Owner" }, + ), + showUsages( + PETCLINIC, + "java.petclinic.usages.inheritedMethod", + "Show Usages: base-class getter called through subclasses", + { + file: `${PET}/model/BaseEntity.java`, + line: "public Integer getId() {", + word: "getId", + }, + ), + gutterOnOpen( + PETCLINIC, + "java.petclinic.gutter.baseClass", + "Gutter markers: open a base entity", + `${PET}/model/BaseEntity.java`, + ), + gutterOnOpen( + PETCLINIC, + "java.petclinic.gutter.midClass", + "Gutter markers: open a mid-hierarchy entity", + `${PET}/model/Person.java`, + ), + lensOnOpen(PETCLINIC, "java.petclinic.lens.entity", "Usage lenses: open an entity", `${PET}/owner/Owner.java`), + implementation(PETCLINIC, "java.petclinic.implementation.baseClass", "Go to Implementation: base entity", { + file: `${PET}/model/BaseEntity.java`, + line: "public class BaseEntity implements Serializable {", + word: "BaseEntity", + }), + diagnosticsOnOpen( + PETCLINIC, + "java.petclinic.diagnostics.cleanFile", + "Diagnostics: open a controller that compiles", + `${PET}/owner/OwnerController.java`, + ), + completion( + PETCLINIC, + "java.petclinic.completion.libraryClass", + "Code Completion: class from a dependency, not imported", + `${PET}/owner/PetValidator.java`, + { after: "public class PetValidator implements Validator {", text: "\n private Pageab" }, + ), + quickDoc(PETCLINIC, "java.petclinic.doc.library", "Quick Documentation: Spring Data class", { + file: `${PET}/owner/OwnerController.java`, + line: "Pageable pageable = PageRequest.of(validatedPage - 1, pageSize);", + word: "PageRequest", + }), + rename( + PETCLINIC, + "java.petclinic.rename.acrossFiles", + "Rename: entity method used across controllers and tests", + { file: `${PET}/owner/Owner.java`, line: "public void addVisit(Integer petId, Visit visit) {", word: "addVisit" }, + "recordVisit", + ), +]; diff --git a/evals/lib/golden.test.ts b/evals/lib/golden.test.ts new file mode 100644 index 000000000..6015e86ec --- /dev/null +++ b/evals/lib/golden.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "vitest"; +import { compare, toolDrift } from "./golden.ts"; + +describe("compare", () => { + test("a perfect answer", () => { + expect(compare({ usages: ["a:1", "b:2"] }, { usages: ["b:2", "a:1"] })).toEqual({ + recall: 1, + precision: 1, + missing: [], + extra: [], + }); + }); + + test("dropped and extra results lower recall and precision", () => { + const c = compare({ usages: ["a:1", "c:3"] }, { usages: ["a:1", "b:2"] }); + expect(c.recall).toBe(0.5); + expect(c.precision).toBe(0.5); + expect(c.missing).toEqual(["usages: b:2"]); + expect(c.extra).toEqual(["usages: c:3"]); + }); + + test("groups are part of the identity", () => { + expect(compare({ read: ["a:1"] }, { write: ["a:1"] }).recall).toBe(0); + }); + + test("nothing expected", () => { + expect(compare({}, {})).toMatchObject({ recall: 1, precision: 1 }); + expect(compare({ usages: ["a:1"] }, {})).toMatchObject({ recall: 1, precision: 0 }); + }); + + test("nothing found", () => { + expect(compare({}, { usages: ["a:1"] })).toMatchObject({ recall: 0, precision: 0 }); + }); +}); + +test("toolDrift lists tools that changed since the recording", () => { + expect(toolDrift({ tools: { jdtls: "1.63.0", java: "21" }, groups: {} }, { jdtls: "1.64.0", java: "21" })).toEqual([ + "jdtls: 1.63.0 → 1.64.0", + ]); +}); diff --git a/evals/lib/golden.ts b/evals/lib/golden.ts new file mode 100644 index 000000000..78b3f16f2 --- /dev/null +++ b/evals/lib/golden.ts @@ -0,0 +1,62 @@ +/** + * Golden results: what a scenario found when a person last confirmed it, keyed to the + * language server and JDK that found it. Compared as sets (recall / precision), not by order. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +/** Result group → items (e.g. `src/…/OrderBook.java:14`); repeated items count once. */ +export type ResultSet = Record; + +export interface Golden { + /** Tool versions the results were recorded with (`jdtls`, `java`). */ + tools: Record; + groups: ResultSet; +} + +export interface Comparison { + recall: number; + precision: number; + /** In the golden, not found (`group: item`). */ + missing: string[]; + /** Found, not in the golden. */ + extra: string[]; +} + +const keys = (set: ResultSet) => new Set(Object.entries(set).flatMap(([g, items]) => items.map((i) => `${g}: ${i}`))); + +export function compare(found: ResultSet, golden: ResultSet): Comparison { + const got = keys(found); + const want = keys(golden); + const missing = [...want].filter((k) => !got.has(k)); + const extra = [...got].filter((k) => !want.has(k)); + const hit = want.size - missing.length; + return { + // Nothing expected and nothing found is a perfect answer. + recall: want.size ? hit / want.size : 1, + precision: got.size ? hit / got.size : want.size ? 0 : 1, + missing: missing.sort(), + extra: extra.sort(), + }; +} + +export function readGolden(file: string): Golden | null { + return existsSync(file) ? (JSON.parse(readFileSync(file, "utf8")) as Golden) : null; +} + +export function writeGolden(file: string, golden: Golden) { + mkdirSync(dirname(file), { recursive: true }); + const sorted = Object.fromEntries( + Object.entries(golden.groups) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([g, items]) => [g, [...new Set(items)].sort()]), + ); + writeFileSync(file, JSON.stringify({ tools: golden.tools, groups: sorted }, null, 2) + "\n"); +} + +/** Tools that differ between the golden's recording and this run (`name: was → now`). */ +export function toolDrift(golden: Golden, tools: Record): string[] { + return Object.entries(golden.tools) + .filter(([k, v]) => tools[k] !== undefined && tools[k] !== v) + .map(([k, v]) => `${k}: ${v} → ${tools[k]}`); +} diff --git a/evals/lib/machine.ts b/evals/lib/machine.ts new file mode 100644 index 000000000..61660d09e --- /dev/null +++ b/evals/lib/machine.ts @@ -0,0 +1,86 @@ +/** The machine and tools a report was measured with: a baseline only holds for the same ones. */ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs"; +import { arch, cpus, platform, release, totalmem } from "node:os"; +import { dirname, join } from "node:path"; + +export interface Fingerprint { + cpu: string; + cores: number; + memoryGb: number; + os: string; +} + +export function fingerprint(): Fingerprint { + return { + cpu: cpus()[0]?.model.trim() ?? "unknown", + cores: cpus().length, + memoryGb: Math.round(totalmem() / 2 ** 30), + os: `${platform()} ${release()} ${arch()}`, + }; +} + +export function sameMachine(a: Fingerprint | undefined, b: Fingerprint): boolean { + return !!a && a.cpu === b.cpu && a.cores === b.cores && a.memoryGb === b.memoryGb && a.os === b.os; +} + +/** `KEY=value` lines (`#` comments), as GitHub Actions' `$GITHUB_ENV` takes them; keys lower-cased. */ +export function pinnedTools(file: string): Record { + if (!existsSync(file)) return {}; + const out: Record = {}; + for (const line of readFileSync(file, "utf8").split("\n")) { + const m = /^\s*([A-Z0-9_]+)\s*=\s*(\S+)/.exec(line); + if (m) out[m[1]!.toLowerCase()] = m[2]!; + } + return out; +} + +function which(cmd: string): string | null { + try { + const found = execFileSync(process.platform === "win32" ? "where" : "which", [cmd], { encoding: "utf8" }) + .split("\n")[0]! + .trim(); + return found ? realpathSync(found) : null; + } catch { + return null; + } +} + +/** jdtls's version from its core bundle (`org.eclipse.jdt.ls.core_1.63.0.202610191322.jar` → `1.63.0-202610191322`). */ +function jdtlsVersion(): string | null { + const bin = which("jdtls"); + if (!bin) return null; + for (let dir = dirname(bin), i = 0; i < 4; dir = dirname(dir), i++) { + for (const plugins of [join(dir, "plugins"), join(dir, "libexec", "plugins")]) { + if (!existsSync(plugins)) continue; + const core = readdirSync(plugins).find((f) => /^org\.eclipse\.jdt\.ls\.core_.*\.jar$/.test(f)); + const m = core && /_(\d+\.\d+\.\d+)\.(\d+)\.jar$/.exec(core); + if (m) return `${m[1]}-${m[2]}`; + } + } + return null; +} + +/** + * The JDK projects compile against, whose sources library navigation opens: `JAVA_HOME`'s (it + * wins for its Java version; jdtls itself runs on a newer one when it needs to), else the `java` + * on PATH. Runtime version and vendor (`-XshowSettings` prints to stderr). + */ +function javaVersion(): string | null { + const home = process.env.JAVA_HOME; + const java = home ? join(home, "bin", process.platform === "win32" ? "java.exe" : "java") : "java"; + const r = spawnSync(java, ["-XshowSettings:properties", "-version"], { encoding: "utf8" }); + const props = `${r.stdout ?? ""}${r.stderr ?? ""}`; + const version = /java\.runtime\.version = (\S+)/.exec(props)?.[1]; + const vendor = /java\.vendor = (.+)/.exec(props)?.[1]?.trim(); + return version ? `${version}${vendor ? ` (${vendor})` : ""}` : null; +} + +export function toolVersions(): Record { + const tools: Record = {}; + const jdtls = jdtlsVersion(); + if (jdtls) tools.jdtls = jdtls; + const java = javaVersion(); + if (java) tools.java = java; + return tools; +} diff --git a/evals/lib/measure.ts b/evals/lib/measure.ts new file mode 100644 index 000000000..844c8eb23 --- /dev/null +++ b/evals/lib/measure.ts @@ -0,0 +1,154 @@ +/** + * Timing one user action in the page: the action is run, then every animation frame checks + * what the user would see. A condition true at the start of a frame is painted by it. + */ +import type { Page } from "@playwright/test"; + +/** + * A check of the page, run in it every frame: self-contained (it is re-created from its + * source), given the arguments passed to `track` and the editor state before the action. + */ +export type Check = (arg: A, before: EditorSnapshot | null) => boolean; + +export interface EditorSnapshot { + path: string; + head: number; + line: number; + word: string; +} + +export interface Tracked { + /** Something visible changed in answer to the action (popup, hint, spinner, caret / editor moved). */ + feedbackMs: number; + /** The first result is shown. */ + firstResultMs: number; + /** The final result is shown and nothing is loading. */ + completeMs: number; +} + +export interface TrackSpec { + /** The action run (`actionArg` its argument), the file opened (`openFile`), or a check-like step in the page (`run`). */ + action?: string; + actionArg?: unknown; + openFile?: string; + run?: (arg: A) => unknown; + arg: A; + first: Check; + done: Check; + /** Extra feedback beyond the default (popup, hint, status spinner, editor or caret changed). */ + feedback?: Check; + /** + * For results with no loading indicator (gutter markers): done once `done` holds and + * `signature` has not changed for `settleMs`; complete is when it last changed. + */ + signature?: (arg: A) => unknown; + settleMs?: number; + timeoutMs: number; +} + +export class TimeoutError extends Error { + readonly partial: Partial; + constructor(message: string, partial: Partial) { + super(message); + this.partial = partial; + } +} + +export async function track(page: Page, spec: TrackSpec): Promise { + const r = await page.evaluate( + ({ action, actionArg, openFile, run, arg, first, done, feedback, signature, settleMs, timeoutMs }) => { + const w = window as unknown as { + __openideaPerf: { + runAction(id: string, arg?: unknown): Promise; + openFile(path: string): Promise; + editorState(): EditorSnapshot | null; + }; + }; + const make = (src: string | null) => + src + ? (new Function("arg", "before", `return (${src})(arg, before)`) as (a: unknown, b: unknown) => boolean) + : null; + const checks = { first: make(first)!, done: make(done)!, feedback: make(feedback) }; + const sign = signature + ? (new Function("arg", `return JSON.stringify((${signature})(arg))`) as (a: unknown) => string) + : null; + let lastSig: string | null = null; + let lastChange = 0; + const before = w.__openideaPerf.editorState(); + const visible = (sel: string) => + [...document.querySelectorAll(sel)].some((el) => (el as HTMLElement).getClientRects().length > 0); + const anyFeedback = () => { + if (visible('.cm-lsp-hint, [data-testid="usages-popup"], [role="dialog"], [role="listbox"]')) return true; + const now = w.__openideaPerf.editorState(); + if ((now?.path ?? null) !== (before?.path ?? null) || (now && before && now.head !== before.head)) return true; + return checks.feedback?.(arg, before) ?? false; + }; + const times: { feedbackMs?: number; firstResultMs?: number; completeMs?: number } = {}; + const t0 = performance.now(); + let failed: unknown = null; + const started = run + ? new Promise((resolve) => resolve(make(run)!(arg, before))) + : openFile + ? w.__openideaPerf.openFile(openFile) + : w.__openideaPerf.runAction(action!, actionArg); + void started.catch((e: unknown) => (failed = e)); + return new Promise<{ times: typeof times; timedOut: boolean; error: string | null }>((resolve) => { + const tick = () => { + const t = performance.now() - t0; + try { + if (times.feedbackMs === undefined && anyFeedback()) times.feedbackMs = t; + if (times.firstResultMs === undefined && checks.first(arg, before)) times.firstResultMs = t; + if (sign) { + const sig = sign(arg); + if (sig !== lastSig) { + lastSig = sig; + lastChange = t; + } + } + const settled = !sign || t - lastChange >= (settleMs ?? 0); + if (times.firstResultMs !== undefined && settled && checks.done(arg, before)) { + // Session.reset waits out the time in which invoking it again widens it. + if (action === "find.showUsages") + (window as unknown as { __openideaEvalShowUsagesAt?: number }).__openideaEvalShowUsagesAt = + performance.now(); + times.completeMs = sign ? Math.max(lastChange, times.firstResultMs) : t; + times.feedbackMs ??= t; + return resolve({ times, timedOut: false, error: null }); + } + } catch (e) { + return resolve({ times, timedOut: false, error: String(e) }); + } + if (failed) return resolve({ times, timedOut: false, error: String((failed as Error).message ?? failed) }); + if (t > timeoutMs) return resolve({ times, timedOut: true, error: null }); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + }); + }, + { + action: spec.action, + actionArg: spec.actionArg, + openFile: spec.openFile, + run: spec.run?.toString() ?? null, + arg: spec.arg, + first: spec.first.toString(), + done: spec.done.toString(), + feedback: spec.feedback?.toString() ?? null, + signature: spec.signature?.toString() ?? null, + settleMs: spec.settleMs, + timeoutMs: spec.timeoutMs, + }, + ); + const what = spec.action ?? (spec.openFile ? `open ${spec.openFile}` : "step"); + if (r.error) throw new Error(`${what}: ${r.error}`); + if (r.timedOut) { + const seen = + r.times.firstResultMs !== undefined + ? "a first result but no final one" + : r.times.feedbackMs !== undefined + ? "feedback but no result" + : "nothing"; + throw new TimeoutError(`${what} timed out after ${spec.timeoutMs} ms (${seen})`, r.times); + } + return r.times as Tracked; +} diff --git a/evals/lib/projects.ts b/evals/lib/projects.ts new file mode 100644 index 000000000..797f3b72f --- /dev/null +++ b/evals/lib/projects.ts @@ -0,0 +1,70 @@ +/** Where a scenario's project comes from: `fixtures/`, or an open-source project pinned in `evals/projects.json`. */ +import { execFileSync, spawnSync } from "node:child_process"; +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +interface Pinned { + repo: string; + commit: string; + remove?: string[]; + prepare?: string[]; +} + +function pinned(repoRoot: string, name: string): Pinned | null { + const all = JSON.parse(readFileSync(join(repoRoot, "evals", "projects.json"), "utf8")) as Record; + return all[name] ?? null; +} + +const quiet = { stdio: "ignore" as const }; + +/** The pinned checkout, cloned and prepared once; returns its directory. */ +function checkout(repoRoot: string, name: string, p: Pinned): string { + const dir = join(repoRoot, "target", "eval", "sources", name); + if (!existsSync(join(dir, ".git"))) { + console.log(`Cloning ${p.repo}…`); + mkdirSync(join(dir, ".."), { recursive: true }); + execFileSync("git", ["clone", "-q", "--filter=blob:none", p.repo, dir], quiet); + } + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: dir, encoding: "utf8" }).trim(); + if (head !== p.commit) { + const has = spawnSync("git", ["cat-file", "-e", `${p.commit}^{commit}`], { cwd: dir }).status === 0; + if (!has) execFileSync("git", ["fetch", "-q", "origin", p.commit], { cwd: dir, ...quiet }); + execFileSync("git", ["checkout", "-q", "--detach", p.commit], { cwd: dir, ...quiet }); + } + const marker = join(repoRoot, "target", "eval", "sources", `.${name}.prepared`); + if (p.prepare && (!existsSync(marker) || readFileSync(marker, "utf8") !== p.commit)) { + console.log(`Preparing ${name}: ${p.prepare.join(" ")}…`); + const r = spawnSync(p.prepare[0]!, p.prepare.slice(1), { cwd: dir, stdio: ["ignore", "ignore", "inherit"] }); + if (r.status !== 0) throw new Error(`preparing ${name} failed: ${p.prepare.join(" ")}`); + writeFileSync(marker, p.commit); + } + return dir; +} + +/** + * A fresh copy of `name` at `dest`, as its own git repository (edits and IDE state of an + * earlier session don't carry over; it isn't part of the checkout `target/` sits in). + */ +export function copyProject(repoRoot: string, name: string, dest: string) { + const fixture = join(repoRoot, "fixtures", name); + const p = existsSync(fixture) ? null : pinned(repoRoot, name); + if (!p && !existsSync(fixture)) throw new Error(`no fixtures/${name} and no ${name} in evals/projects.json`); + const src = p ? checkout(repoRoot, name, p) : fixture; + rmSync(dest, { recursive: true, force: true, maxRetries: 5, retryDelay: 500 }); + mkdirSync(join(dest, ".."), { recursive: true }); + cpSync(src, dest, { recursive: true, filter: (f) => !/[\\/](\.git|target|build|\.gradle|\.openidea)$/.test(f) }); + for (const r of p?.remove ?? []) rmSync(join(dest, r), { recursive: true, force: true }); + const env = { + ...process.env, + GIT_AUTHOR_NAME: "eval", + GIT_AUTHOR_EMAIL: "eval@localhost", + GIT_COMMITTER_NAME: "eval", + GIT_COMMITTER_EMAIL: "eval@localhost", + }; + for (const args of [ + ["init", "-q"], + ["add", "-A"], + ["commit", "-q", "--no-verify", "-m", "eval project"], + ]) + execFileSync("git", args, { cwd: dest, env, ...quiet }); +} diff --git a/evals/lib/scenario.ts b/evals/lib/scenario.ts new file mode 100644 index 000000000..5c0c89f74 --- /dev/null +++ b/evals/lib/scenario.ts @@ -0,0 +1,35 @@ +/** What an eval scenario is: a project, a step the user takes, and what they should see. */ +import type { ResultSet } from "./golden.ts"; +import type { Tracked } from "./measure.ts"; +import type { Session } from "./session.ts"; +import type { Condition } from "./verdict.ts"; + +export interface Measured { + timing: Tracked; + results?: ResultSet; + /** Why the answer is weaker than a semantic one, when the UI says so. */ + degraded?: string | null; +} + +export interface EvalScenario { + /** Stable id: `..`, used by `--only`, goldens and budgets. */ + id: string; + title: string; + /** `fixtures/`. */ + fixture: string; + /** The language server whose readiness the warm runs wait for. */ + server: string; + conditions: Condition[]; + /** Brings the idle workbench to where the step starts (open a file, place the caret). */ + setup(s: Session): Promise; + /** The measured step. */ + step(s: Session): Promise; +} + +/** Project-relative path, or a library class's URI without its machine-specific query. */ +export function normalizePath(root: string, path: string): string { + if (path.startsWith(root + "/")) return path.slice(root.length + 1); + const uri = /^[a-z]+:\/\//.exec(path); + if (uri) return path.replace(/[?#].*$/, ""); + return path; +} diff --git a/evals/lib/session.ts b/evals/lib/session.ts new file mode 100644 index 000000000..de1edd0c1 --- /dev/null +++ b/evals/lib/session.ts @@ -0,0 +1,425 @@ +/** + * One eval session: a release `openidea-server --stdio` with its own home, and the production + * workbench in Chromium connected to it as remote development connects a window (the Tauri + * runtime from `smoke/tauriShim.ts`, commands and notifications relayed by Playwright). + */ +import { mkdirSync, rmSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import type { Browser, BrowserContext, Page } from "@playwright/test"; +import { installInPage } from "../../perf/lib/inpage.ts"; +import { Server } from "../../smoke/server.ts"; +import { copyProject } from "./projects.ts"; +import { installTauriShim, type ShimWindow } from "../../smoke/tauriShim.ts"; + +/** How long after its answer Show Usages invoked again widens it (`HINT_MS` in code-insight/usages.ts), plus a margin. */ +const SHOW_USAGES_HINT_MS = 4200; + +export interface IpcCall { + cmd: string; + /** Milliseconds since the session started. */ + start: number; + ms: number; + ok: boolean; + /** JSON length of the result (or error). */ + bytes: number; + /** Array length of the result, when it is one. */ + items?: number; + /** Still unanswered when the window closed (`ms` is how long it had waited). */ + pending?: true; +} + +export interface TaskSpan { + id: number; + title: string; + start: number; + end: number | null; +} + +/** What the server did, for diagnostics: every command, background task and server status. */ +export class Timeline { + readonly t0 = Date.now(); + readonly calls: IpcCall[] = []; + readonly tasks = new Map(); + readonly servers: Array<{ at: number; name: string; status: string; message: string | null }> = []; + /** Last time anything happened (a call ended, a task or status changed). */ + lastActivity = 0; + inFlight = 0; + /** When the scenario's measured step started, for a failure's window. */ + stepStart: number | null = null; + /** Commands sent and not answered yet, by call number. */ + readonly unanswered = new Map(); + private nextCall = 0; + + now() { + return Date.now() - this.t0; + } + + onEvent(event: { type: string } & Record) { + if (event.type === "task") { + const id = event.id as number; + const span = this.tasks.get(id) ?? { id, title: String(event.title), start: this.now(), end: null }; + if (event.done) span.end = this.now(); + this.tasks.set(id, span); + this.lastActivity = this.now(); + } else if (event.type === "languageServer") { + this.servers.push({ + at: this.now(), + name: String(event.name), + status: String(event.status), + message: (event.message as string | null) ?? null, + }); + this.lastActivity = this.now(); + } + } + + /** Records `cmd` as sent; the returned function marks it answered. */ + sent(cmd: string, start: number): () => void { + const id = this.nextCall++; + this.unanswered.set(id, { cmd, start }); + return () => this.unanswered.delete(id); + } + + runningTasks(): TaskSpan[] { + return [...this.tasks.values()].filter((t) => t.end === null); + } + + serverStatus(name: string): string | null { + for (let i = this.servers.length - 1; i >= 0; i--) + if (this.servers[i]!.name === name) return this.servers[i]!.status; + return null; + } + + /** The calls and tasks overlapping `[from, to]` (session milliseconds). */ + window(from: number, to: number) { + return { + calls: [ + ...this.calls.filter((c) => c.start <= to && c.start + c.ms >= from), + // A command that never answers is what a timeout usually waits on. + ...[...this.unanswered.values()] + .filter((c) => c.start <= to) + .map((c): IpcCall => ({ ...c, ms: this.now() - c.start, ok: false, bytes: 0, pending: true })), + ], + tasks: [...this.tasks.values()].filter((t) => t.start <= to && (t.end ?? Infinity) >= from), + }; + } +} + +export interface SessionOptions { + /** `fixtures/` or a project pinned in `evals/projects.json`. */ + fixture: string; + /** Where the project copy and the server's home go. */ + workDir: string; + /** Keep the server's home (installed tools, language server workspaces) from an earlier session. */ + keepHome: boolean; + /** `OPENIDEA_LOG` for the server. */ + log?: string; +} + +export class Session { + readonly timeline = new Timeline(); + readonly root: string; + readonly home: string; + page!: Page; + private context!: BrowserContext; + private server!: Server; + private relayQueue: Promise = Promise.resolve(); + private pendingNotes: Array<[kind: "channel" | "event", a: unknown, b: unknown]> = []; + /** The last answer to each command that succeeded, for what the UI only summarises. */ + readonly lastAnswer = new Map(); + /** Editors changed by a scenario: path → the text to put back. */ + private readonly edited = new Map(); + readonly pageErrors: string[] = []; + + private constructor(opts: SessionOptions) { + this.root = join(opts.workDir, "projects", opts.fixture); + this.home = join(opts.workDir, "home"); + } + + static async start(browser: Browser, url: string, repoRoot: string, opts: SessionOptions): Promise { + const s = new Session(opts); + copyProject(repoRoot, opts.fixture, s.root); + if (!opts.keepHome) rmSync(s.home, { recursive: true, force: true, maxRetries: 5, retryDelay: 500 }); + mkdirSync(s.home, { recursive: true }); + + s.server = await Server.start(s.home, opts.log ? { OPENIDEA_LOG: opts.log } : {}); + s.server.onEvent = (target, event) => { + if (target !== null && target !== "main") return; + s.timeline.onEvent(event as { type: string }); + s.notify("event", target, event); + }; + s.server.onChannel = (id, data) => s.notify("channel", id, data); + await s.server.invoke("project_trust_set", { path: s.root, trusted: true, folder: false }); + + s.context = await browser.newContext({ + baseURL: url, + viewport: { width: 1400, height: 900 }, + deviceScaleFactor: 1, + }); + await s.context.exposeFunction("__openideaRelay", async (cmd: string, args: unknown) => { + const start = s.timeline.now(); + s.timeline.inFlight++; + const answered = s.timeline.sent(cmd, start); + try { + const value = await s.server.invoke(cmd, args); + s.record(cmd, start, true, value); + return { ok: value ?? null }; + } catch (err) { + s.record(cmd, start, false, err); + return { err }; + } finally { + answered(); + s.timeline.inFlight--; + s.timeline.lastActivity = s.timeline.now(); + } + }); + await s.context.addInitScript(installTauriShim, { + directories: { 12: tmpdir(), 21: homedir() }, + sep: sep as "/" | "\\", + }); + await s.context.addInitScript(() => { + const w = window as unknown as ShimWindow & { + __openideaRelay(cmd: string, args: unknown): Promise<{ ok: unknown } | { err: unknown }>; + }; + w.__openideaForward = async (cmd, args) => { + const r = await w.__openideaRelay(cmd, args); + if ("err" in r) throw r.err; + return r.ok; + }; + }); + await s.context.addInitScript(installInPage); + s.page = await s.context.newPage(); + s.page.on("pageerror", (e) => s.pageErrors.push(e.message)); + if (process.env.OPENIDEA_EVAL_DEBUG) { + s.page.on("console", (m) => console.log("[page]", m.type(), m.text().slice(0, 300))); + s.page.on("pageerror", (e) => console.log("[pageerror]", e.message)); + } + + await s.page.goto("/"); + await s.page.waitForFunction(() => "__openideaPerf" in window && "__openideaShim" in window); + await s.page.evaluate( + (root) => + (window as unknown as { __openideaPerf: { openProject(p: string): Promise } }).__openideaPerf.openProject( + root, + ), + s.root, + ); + await s.page.waitForSelector('[role="tree"][aria-label="Project"] [role="treeitem"]', { timeout: 60_000 }); + return s; + } + + private record(cmd: string, start: number, ok: boolean, value: unknown) { + let bytes = 0; + try { + bytes = JSON.stringify(value ?? null).length; + } catch { + // not serialisable: size unknown + } + const items = Array.isArray(value) + ? value.length + : value && typeof value === "object" && Array.isArray((value as { locations?: unknown }).locations) + ? (value as { locations: unknown[] }).locations.length + : undefined; + this.timeline.calls.push({ cmd, start, ms: this.timeline.now() - start, ok, bytes, items }); + if (ok) this.lastAnswer.set(cmd, value); + } + + /** Delivers server notifications in order, batched per turn of the event loop. */ + private notify(kind: "channel" | "event", a: unknown, b: unknown) { + this.pendingNotes.push([kind, a, b]); + if (this.pendingNotes.length > 1) return; + this.relayQueue = this.relayQueue + .then(() => new Promise((r) => setImmediate(r))) + .then(() => { + const batch = this.pendingNotes; + this.pendingNotes = []; + return this.page + ?.evaluate((notes) => { + const shim = (window as unknown as ShimWindow).__openideaShim; + for (const [kind, a, b] of notes) { + if (kind === "channel") shim.channel(a as number, b); + else shim.event(a as string | null, b); + } + }, batch) + .catch(() => {}); + }); + } + + runAction(id: string, arg?: unknown): Promise { + return this.page.evaluate( + async ({ id, arg }) => { + const w = window as unknown as { + __openideaPerf: { runAction(id: string, arg?: unknown): Promise }; + __openideaEvalShowUsagesAt?: number; + }; + const ran = await w.__openideaPerf.runAction(id, arg); + if (id === "find.showUsages") w.__openideaEvalShowUsagesAt = performance.now(); + return ran; + }, + { id, arg }, + ); + } + + /** Opens `rel` (project-relative, or a URI) in an editor and waits until its text is shown. */ + async openFile(rel: string) { + const path = /^[a-z]+:\/\//.test(rel) ? rel : join(this.root, rel); + const bare = rel.replace(/[?#].*$/, ""); + const name = bare.slice(bare.lastIndexOf("/") + 1); + await this.page.evaluate( + async ({ path, name }) => { + const w = window as unknown as { + __perf: { waitShown(s: string, t: string | null, ms: number): Promise }; + __openideaPerf: { openFile(p: string): Promise }; + }; + await w.__openideaPerf.openFile(path); + await w.__perf.waitShown('[role="tab"][aria-selected="true"]', name, 30_000); + await w.__perf.waitShown(".cm-content .cm-line", null, 30_000); + }, + { path, name }, + ); + } + + /** + * Goes to the declaration of `word` on `line` of the active editor and waits until another + * file shows (a library class: its `jdt://` URI, which carries what only the server knows). + */ + async declarationOf(line: string, word: string): Promise { + await this.caretAt(line, word); + type Probe = { __openideaPerf: { editorState(): { path: string } | null } }; + const from = await this.page.evaluate(() => (window as unknown as Probe).__openideaPerf.editorState()?.path); + await this.runAction("navigate.declaration"); + const handle = await this.page.waitForFunction( + (from) => { + const path = (window as unknown as Probe).__openideaPerf.editorState()?.path; + return path && path !== from && document.querySelector(".cm-content .cm-line") ? path : null; + }, + from, + { timeout: 30_000 }, + ); + return (await handle.jsonValue()) as string; + } + + /** Puts the caret inside `word` on the active editor's first line containing `line` (both literal). */ + async caretAt(line: string, word: string) { + const text = await this.editorText(); + const at = text.indexOf(line); + if (at < 0) throw new Error(`no line containing ${JSON.stringify(line)} in the active editor`); + const inLine = line.indexOf(word); + if (inLine < 0) throw new Error(`${JSON.stringify(word)} is not in ${JSON.stringify(line)}`); + // Inside the word, as a click on it would put it. + const target = at + inLine + Math.floor(word.length / 2); + await this.page.evaluate( + (o) => (window as unknown as { __openideaPerf: { setCursor(o: number): void } }).__openideaPerf.setCursor(o), + target, + ); + } + + /** The active editor's whole text. */ + editorText(): Promise { + return this.page.evaluate(() => { + const text = ( + window as unknown as { __openideaPerf: { editorText(): string | null } } + ).__openideaPerf.editorText(); + if (text === null) throw new Error("no active editor"); + return text; + }); + } + + /** Keeps the active editor's text, for `reset` to put back whatever the scenario changes. */ + async keepText() { + const { path, text } = await this.page.evaluate(() => { + const probe = ( + window as unknown as { + __openideaPerf: { editorState(): { path: string } | null; editorText(): string | null }; + } + ).__openideaPerf; + return { path: probe.editorState()?.path, text: probe.editorText() }; + }); + if (path === undefined || text === null) throw new Error("no active editor"); + if (!this.edited.has(path)) this.edited.set(path, text); + } + + /** Inserts `text` after the first `after` in the active editor; the offset after it. */ + async edit(after: string, text: string): Promise { + await this.keepText(); + return this.page.evaluate( + ({ after, text }) => + ( + window as unknown as { __openideaPerf: { insertAfter(a: string, t: string): number } } + ).__openideaPerf.insertAfter(after, text), + { after, text }, + ); + } + + private async restoreEdits() { + for (const [path, text] of this.edited) { + await this.page.evaluate( + async ({ path, text }) => { + const probe = ( + window as unknown as { + __openideaPerf: { + openFile(p: string): Promise; + editorState(): { path: string } | null; + editorText(): string | null; + setText(t: string): void; + }; + } + ).__openideaPerf; + await probe.openFile(path); + while (probe.editorState()?.path !== path) await new Promise(requestAnimationFrame); + if (probe.editorText() !== text) probe.setText(text); + }, + { path, text }, + ); + await this.runAction("file.save"); + } + this.edited.clear(); + } + + /** Waits until `server` is ready and nothing has happened for `quietMs`: no task, no command in flight. */ + async settle(server: string, quietMs = 2000, timeoutMs = 5 * 60_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const t = this.timeline; + const quiet = + t.serverStatus(server) === "ready" && + t.runningTasks().length === 0 && + t.inFlight === 0 && + t.now() - t.lastActivity >= quietMs; + if (quiet) return; + if (Date.now() > deadline) { + throw new Error( + `the IDE did not settle within ${timeoutMs / 1000}s: ${server} ${t.serverStatus(server) ?? "not started"}, ` + + `tasks ${JSON.stringify(t.runningTasks().map((x) => x.title))}, ${t.inFlight} command(s) in flight`, + ); + } + await new Promise((r) => setTimeout(r, 200)); + } + } + + /** Back to an idle workbench: popups dismissed, edits undone, every editor closed. */ + async reset() { + // Show Usages invoked again within 4 s of its hint widens the previous search: wait it out. + const since = await this.page.evaluate(() => { + const at = (window as unknown as { __openideaEvalShowUsagesAt?: number }).__openideaEvalShowUsagesAt; + return at === undefined ? Infinity : performance.now() - at; + }); + if (since < SHOW_USAGES_HINT_MS) await this.page.waitForTimeout(SHOW_USAGES_HINT_MS - since); + await this.page.keyboard.press("Escape"); + await this.page.keyboard.press("Escape"); + await this.restoreEdits(); + await this.runAction("editor.closeAllTabs"); + await this.page.waitForFunction(() => !document.querySelector('[role="tab"][aria-selected="true"]'), undefined, { + timeout: 10_000, + }); + } + + serverLog(): string { + return this.server.log(); + } + + async close() { + await this.context?.close().catch(() => {}); + await this.server?.close(); + } +} diff --git a/evals/lib/verdict.test.ts b/evals/lib/verdict.test.ts new file mode 100644 index 000000000..1b76349c0 --- /dev/null +++ b/evals/lib/verdict.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, test } from "vitest"; +import { judge, table, type Budgets, type Outcome } from "./verdict.ts"; + +const budgets: Budgets = { + regressionTolerance: 0.2, + regressionFloorMs: 200, + defaults: { feedbackMs: 100, recall: 1, precision: 1 }, + scenarios: { usages: { completeMs: 2000 }, "usages@cold": { completeMs: 10_000 } }, +}; + +const outcome = (o: Partial): Outcome => ({ + scenario: "usages", + title: "Show Usages", + condition: "warm", + metrics: { feedbackMs: 40, firstResultMs: 300, completeMs: 500, recall: 1, precision: 1 }, + samples: {}, + results: { usages: ["a:1"] }, + comparison: { recall: 1, precision: 1, missing: [], extra: [] }, + ...o, +}); + +const repro = () => "pnpm eval --only usages"; + +describe("judge", () => { + test("within budgets and matching the golden passes", () => { + const [v] = judge([outcome({})], budgets, null, repro); + expect(v).toMatchObject({ id: "usages@warm", status: "pass", failures: [], repro: "pnpm eval --only usages" }); + }); + + test("slow feedback, dropped results and silent degradation each fail", () => { + const [v] = judge( + [ + outcome({ + metrics: { feedbackMs: 250, firstResultMs: 300, completeMs: 2500, recall: 0.5, precision: 1 }, + comparison: { recall: 0.5, precision: 1, missing: ["usages: b:2"], extra: [] }, + degraded: "Based on syntax", + }), + ], + budgets, + null, + repro, + ); + expect(v!.status).toBe("fail"); + expect(v!.failures.map((f) => [f.kind, f.metric])).toEqual([ + ["degraded", undefined], + ["budget", "feedbackMs"], + ["correctness", "recall"], + ["budget", "completeMs"], + ]); + expect(v!.failures.find((f) => f.metric === "recall")!.detail).toContain("usages: b:2"); + }); + + test("condition budgets override scenario budgets", () => { + const [v] = judge( + [outcome({ condition: "cold", metrics: { feedbackMs: 40, completeMs: 5000, recall: 1, precision: 1 } })], + budgets, + null, + repro, + ); + expect(v!.status).toBe("pass"); + }); + + test("timeouts and missing goldens fail", () => { + const [timedOut, noGolden] = judge( + [outcome({ error: "find.showUsages timed out after 30000 ms" }), outcome({ comparison: undefined })], + budgets, + null, + repro, + ); + expect(timedOut!.failures[0]!.kind).toBe("timeout"); + expect(noGolden!.failures[0]!.kind).toBe("golden"); + }); + + test("regressions need both the relative tolerance and the absolute floor", () => { + const baseline = [outcome({ metrics: { feedbackMs: 40, firstResultMs: 300, completeMs: 500 } })]; + const [v] = judge( + [outcome({ metrics: { feedbackMs: 90, firstResultMs: 450, completeMs: 1200, recall: 1, precision: 1 } })], + budgets, + baseline, + repro, + ); + expect(v!.status).toBe("pass"); + expect(v!.regressions.map((r) => r.metric)).toEqual(["completeMs"]); + expect(table([v!], false)).toContain("regressed: completeMs"); + }); +}); diff --git a/evals/lib/verdict.ts b/evals/lib/verdict.ts new file mode 100644 index 000000000..7f177a626 --- /dev/null +++ b/evals/lib/verdict.ts @@ -0,0 +1,196 @@ +/** + * Verdicts: each scenario run under each condition passes or fails on its own numbers + * (budgets, correctness, degradation, errors), and is compared with the local baseline. + */ +import { readFileSync } from "node:fs"; +import type { Comparison, ResultSet } from "./golden.ts"; + +export type Condition = "warm" | "cold"; + +export interface Diagnostics { + /** Commands sent to the server during the slowest measured run (session ms). */ + calls: Array<{ cmd: string; start: number; ms: number; ok: boolean; bytes: number; items?: number; pending?: true }>; + tasks: Array<{ title: string; start: number; end: number | null }>; + servers: Array<{ at: number; name: string; status: string; message: string | null }>; + /** The slowest run's window in session milliseconds. */ + window: [number, number]; + serverLog: string; + trace?: string; +} + +export interface Outcome { + scenario: string; + title: string; + condition: Condition; + /** Medians of the measured runs (correctness: the worst run). */ + metrics: Record; + samples: Record; + results?: ResultSet; + comparison?: Comparison; + /** Notes of the tools whose versions differ from the golden's. */ + drift?: string[]; + /** Why the answer is weaker than a semantic one (syntax fallback, incomplete search), if it is. */ + degraded?: string | null; + error?: string; + diagnostics?: Diagnostics; +} + +export type FailureKind = "error" | "timeout" | "degraded" | "correctness" | "budget" | "golden"; + +export interface Failure { + kind: FailureKind; + metric?: string; + value?: number; + limit?: number; + detail: string; +} + +export interface Regression { + metric: string; + value: number; + baseline: number; +} + +export interface Verdict { + id: string; + scenario: string; + condition: Condition; + title: string; + status: "pass" | "fail"; + failures: Failure[]; + regressions: Regression[]; + improvements: Regression[]; + metrics: Record; + baseline?: Record; + repro: string; +} + +export interface Budgets { + regressionTolerance: number; + regressionFloorMs: number; + /** Applied to every scenario. `*Ms`: upper bounds; `recall` / `precision`: lower bounds. */ + defaults: Record; + /** By `` or `@` (the latter wins). */ + scenarios: Record>; +} + +export function loadBudgets(file: string): Budgets { + const raw = JSON.parse(readFileSync(file, "utf8")) as Partial; + return { + regressionTolerance: raw.regressionTolerance ?? 0.2, + regressionFloorMs: raw.regressionFloorMs ?? 200, + defaults: raw.defaults ?? {}, + scenarios: raw.scenarios ?? {}, + }; +} + +export const idOf = (o: { scenario: string; condition: Condition }) => `${o.scenario}@${o.condition}`; + +function budgetsFor(b: Budgets, o: Outcome): Record { + return { ...b.defaults, ...b.scenarios[o.scenario], ...b.scenarios[idOf(o)] }; +} + +const lowerBound = (metric: string) => metric === "recall" || metric === "precision"; + +export function judge( + outcomes: Outcome[], + budgets: Budgets, + baseline: Outcome[] | null, + repro: (o: Outcome) => string, +): Verdict[] { + const base = new Map((baseline ?? []).map((o) => [idOf(o), o.metrics])); + return outcomes.map((o) => { + const failures: Failure[] = []; + if (o.error) failures.push({ kind: /timed out/.test(o.error) ? "timeout" : "error", detail: o.error }); + if (o.degraded) failures.push({ kind: "degraded", detail: o.degraded }); + if (o.results && !o.comparison && !o.error) + failures.push({ kind: "golden", detail: "no golden recorded: run with --update-golden and review it" }); + for (const [metric, limit] of Object.entries(budgetsFor(budgets, o))) { + const value = o.metrics[metric]; + if (value === undefined || !Number.isFinite(value)) continue; + const over = lowerBound(metric) ? value < limit : value > limit; + if (!over) continue; + if (lowerBound(metric)) { + const c = o.comparison; + const list = metric === "recall" ? c?.missing : c?.extra; + const shown = (list ?? []).slice(0, 8).join("; "); + failures.push({ + kind: "correctness", + metric, + value, + limit, + detail: `${metric} ${value.toFixed(2)} < ${limit} (${metric === "recall" ? "missing" : "unexpected"}: ${shown}${(list?.length ?? 0) > 8 ? "; …" : ""})`, + }); + } else { + failures.push({ + kind: "budget", + metric, + value, + limit, + detail: `${metric} ${Math.round(value)} ms > ${limit} ms`, + }); + } + } + const before = base.get(idOf(o)); + const regressions: Regression[] = []; + const improvements: Regression[] = []; + if (before) { + for (const [metric, value] of Object.entries(o.metrics)) { + const was = before[metric]; + if (was === undefined || !metric.endsWith("Ms") || !Number.isFinite(value) || !Number.isFinite(was)) continue; + const slack = Math.max(was * budgets.regressionTolerance, budgets.regressionFloorMs); + if (value > was + slack) regressions.push({ metric, value, baseline: was }); + else if (value < was - slack) improvements.push({ metric, value, baseline: was }); + } + } + return { + id: idOf(o), + scenario: o.scenario, + condition: o.condition, + title: o.title, + status: failures.length ? "fail" : "pass", + failures, + regressions, + improvements, + metrics: o.metrics, + baseline: before, + repro: repro(o), + }; + }); +} + +const fmt = (metric: string, v: number | undefined) => + v === undefined || !Number.isFinite(v) ? "–" : metric.endsWith("Ms") ? String(Math.round(v)) : v.toFixed(2); + +const COLUMNS = ["feedbackMs", "firstResultMs", "completeMs", "recall", "precision"]; + +/** The verdicts as a table (Markdown when `md`), with the baseline in parentheses. */ +export function table(verdicts: Verdict[], md: boolean): string { + const head = ["scenario", ...COLUMNS, "verdict"]; + const rows = verdicts.map((v) => [ + v.id, + ...COLUMNS.map((m) => { + const now = fmt(m, v.metrics[m]); + const was = v.baseline?.[m]; + return was === undefined || !m.endsWith("Ms") ? now : `${now} (${fmt(m, was)})`; + }), + v.status === "fail" + ? `FAIL: ${v.failures.map((f) => f.kind + (f.metric ? ` ${f.metric}` : "")).join(", ")}` + : v.regressions.length + ? `regressed: ${v.regressions.map((r) => r.metric).join(", ")}` + : v.improvements.length + ? `improved: ${v.improvements.map((r) => r.metric).join(", ")}` + : "ok", + ]); + if (md) { + return [ + `| ${head.join(" | ")} |`, + `| ${head.map((_, i) => (i === 0 || i === head.length - 1 ? "---" : "---:")).join(" | ")} |`, + ...rows.map((r) => `| ${r.join(" | ")} |`), + ].join("\n"); + } + const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i]!.length))); + const line = (cells: string[]) => + cells.map((c, i) => (i === 0 || i === head.length - 1 ? c.padEnd(widths[i]!) : c.padStart(widths[i]!))).join(" "); + return [line(head), ...rows.map(line)].join("\n"); +} diff --git a/evals/projects.json b/evals/projects.json new file mode 100644 index 000000000..1752c00fd --- /dev/null +++ b/evals/projects.json @@ -0,0 +1,16 @@ +{ + "$comment": "Open-source projects the evals run on besides fixtures/: cloned at `commit` into target/eval/sources/, `remove`d paths dropped from the copy, `prepare` run once per checkout (dependencies downloaded before anything is timed; Maven needs to reach Central, a proxy goes in ~/.m2/settings.xml).", + "spring-petclinic": { + "repo": "https://github.com/spring-projects/spring-petclinic.git", + "commit": "500158f732419217507c7656904b8e6aa1bcc0d6", + "remove": ["build.gradle", "settings.gradle", "gradle", "gradlew", "gradlew.bat"], + "prepare": [ + "mvn", + "-q", + "-B", + "dependency:go-offline", + "dependency:sources", + "-DfailOnMissingClassifierArtifact=false" + ] + } +} diff --git a/evals/run.ts b/evals/run.ts new file mode 100644 index 000000000..fa8b07d82 --- /dev/null +++ b/evals/run.ts @@ -0,0 +1,365 @@ +/** + * Code-intelligence evals (`pnpm eval`): the production workbench in Chromium driving a + * release `openidea-server` with real language servers, each scenario timed as the user + * sees it (feedback, first result, complete) and checked against its golden results. + * + * pnpm eval every scenario (warm: 1 warm-up + median of 3; cold: once) + * pnpm eval --only java.usages --runs 5 scenarios whose id starts with java.usages + * pnpm eval --save-baseline store this run as target/eval/baseline.json + * pnpm eval --update-golden re-record evals//golden/ (review the diff) + * pnpm eval --only --trace Chrome trace of the slowest run → target/eval/traces/ + * + * Writes target/eval/report.json (verdicts, failures with diagnostics, machine fingerprint) + * and target/eval/report.md. Timings go over the remote-development route (one stdio hop + * more than Tauri IPC): compare them before / after on one machine, not as absolute numbers. + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { chromium, type Browser } from "@playwright/test"; +import { buildBundle, serveBundle } from "../perf/lib/bundle.ts"; +import { median } from "../perf/lib/stats.ts"; +import { JAVA_SCENARIOS } from "./java/scenarios.ts"; +import { compare, readGolden, toolDrift, writeGolden, type ResultSet } from "./lib/golden.ts"; +import { TimeoutError } from "./lib/measure.ts"; +import { fingerprint, pinnedTools, sameMachine, toolVersions, type Fingerprint } from "./lib/machine.ts"; +import type { EvalScenario } from "./lib/scenario.ts"; +import { Session } from "./lib/session.ts"; +import { idOf, judge, loadBudgets, table, type Condition, type Diagnostics, type Outcome } from "./lib/verdict.ts"; + +const ROOT = resolve(import.meta.dirname, ".."); +const OUT = join(ROOT, "target", "eval"); +const DIST = join(OUT, "dist"); +const SCENARIOS: EvalScenario[] = [...JAVA_SCENARIOS]; + +const { values: args } = parseArgs({ + options: { + only: { type: "string" }, + condition: { type: "string" }, + runs: { type: "string", default: "3" }, + warmup: { type: "string", default: "1" }, + channel: { type: "string", default: process.env.OPENIDEA_PERF_CHANNEL ?? "chromium" }, + headed: { type: "boolean", default: false }, + "no-build": { type: "boolean", default: false }, + "save-baseline": { type: "boolean", default: false }, + "update-golden": { type: "boolean", default: false }, + "fail-on-regression": { type: "boolean", default: false }, + trace: { type: "boolean", default: false }, + list: { type: "boolean", default: false }, + }, + allowPositionals: false, +}); + +if (args.list) { + for (const s of SCENARIOS) console.log(`${s.id.padEnd(44)} ${s.conditions.join(",").padEnd(10)} ${s.title}`); + process.exit(0); +} + +const runs = Math.max(1, Number(args.runs)); +const warmup = Math.max(0, Number(args.warmup)); +const only = args.only?.split(",").map((s) => s.trim()); +const conditionFilter = args.condition as Condition | undefined; +const selected = SCENARIOS.filter((s) => !only || only.some((o) => s.id === o || s.id.startsWith(`${o}.`))); +if (only && !selected.length) throw new Error(`no scenario matches ${args.only} (see --list)`); + +const goldenFile = (id: string) => join(ROOT, "evals", id.slice(0, id.indexOf(".")), "golden", `${id}.json`); + +function serverBinary(): string { + const exe = process.platform === "win32" ? ".exe" : ""; + const bin = process.env.OPENIDEA_SERVER ?? join(ROOT, "target", "release", `openidea-server${exe}`); + if (!args["no-build"]) { + console.log("Building openidea-server (cargo build --release)…"); + const r = spawnSync("cargo", ["build", "--release", "-p", "ide-host", "--bin", "openidea-server"], { + cwd: ROOT, + stdio: ["ignore", "ignore", "inherit"], + }); + if (r.status !== 0) throw new Error("cargo build failed"); + } + if (!existsSync(bin)) throw new Error(`no server at ${bin}; run without --no-build`); + return bin; +} + +function git(...a: string[]): string { + try { + return execFileSync("git", a, { cwd: ROOT, encoding: "utf8" }).trim(); + } catch { + return ""; + } +} + +/** Medians of the timings; correctness from the worst run. */ +function summarize(samples: Array>): Record { + const keys = [...new Set(samples.flatMap((s) => Object.keys(s)))]; + return Object.fromEntries( + keys.map((k) => { + const values = samples.map((s) => s[k]!).filter(Number.isFinite); + return [k, k === "recall" || k === "precision" ? Math.min(...values) : median(values)]; + }), + ); +} + +interface Run { + metrics: Record; + results?: ResultSet; + degraded?: string | null; + window: [number, number]; +} + +async function measureOnce(s: Session, scenario: EvalScenario, trace: string | null, browser: Browser): Promise { + await s.reset(); + await scenario.setup(s); + await s.settle(scenario.server, 1000); + const from = (s.timeline.stepStart = s.timeline.now()); + if (trace) await browser.startTracing(s.page, { screenshots: true }); + try { + const m = await scenario.step(s); + return { + metrics: { ...m.timing }, + results: m.results, + degraded: m.degraded, + window: [from, s.timeline.now()], + }; + } finally { + if (trace) writeFileSync(trace, await browser.stopTracing()); + } +} + +function diagnose(s: Session, window: [number, number], trace?: string): Diagnostics { + const w = s.timeline.window(window[0] - 500, window[1]); + return { + calls: w.calls.map((c) => ({ ...c })), + tasks: w.tasks.map((t) => ({ title: t.title, start: t.start, end: t.end })), + servers: s.timeline.servers.filter((x) => x.at <= window[1]), + window, + serverLog: s.serverLog(), + trace, + }; +} + +function outcomeOf( + scenario: EvalScenario, + condition: Condition, + done: Run[], + tools: Record, + extra: Record = {}, +): Outcome { + const golden = readGolden(goldenFile(scenario.id)); + const comparisons = golden ? done.map((r) => compare(r.results ?? {}, golden.groups)) : []; + const worst = comparisons.reduce<(typeof comparisons)[number] | undefined>( + (w, c) => (!w || c.recall + c.precision < w.recall + w.precision ? c : w), + undefined, + ); + const samples: Array> = done.map((r, i) => + comparisons[i] ? { ...r.metrics, recall: comparisons[i].recall, precision: comparisons[i].precision } : r.metrics, + ); + return { + scenario: scenario.id, + title: scenario.title, + condition, + metrics: { ...summarize(samples), ...extra }, + samples: Object.fromEntries(Object.keys(samples[0] ?? {}).map((k) => [k, samples.map((x) => x[k]!)])), + results: done.at(-1)?.results, + comparison: worst, + drift: golden ? toolDrift(golden, tools) : undefined, + degraded: done.find((r) => r.degraded)?.degraded ?? null, + }; +} + +async function runScenario( + s: Session, + scenario: EvalScenario, + condition: Condition, + browser: Browser, + tools: Record, + extra: Record = {}, +): Promise { + const total = condition === "cold" ? 1 : warmup + runs; + const done: Run[] = []; + process.stdout.write(`${idOf({ scenario: scenario.id, condition }).padEnd(52)} `); + try { + for (let i = 0; i < total; i++) { + const last = i === total - 1; + const trace = args.trace && last ? join(OUT, "traces", `${scenario.id}@${condition}.json`) : null; + const r = await measureOnce(s, scenario, trace, browser); + if (condition === "cold" || i >= warmup) done.push(r); + process.stdout.write(condition === "warm" && i < warmup ? "w" : "."); + } + console.log(); + const o = outcomeOf(scenario, condition, done, tools, extra); + const slowest = done.reduce((a, b) => ((b.metrics.completeMs ?? 0) > (a.metrics.completeMs ?? 0) ? b : a)); + o.diagnostics = diagnose( + s, + slowest.window, + args.trace ? join(OUT, "traces", `${scenario.id}@${condition}.json`) : undefined, + ); + return o; + } catch (e) { + console.log(" failed"); + const message = e instanceof Error ? e.message : String(e); + const partial = e instanceof TimeoutError ? (e.partial as Record) : {}; + const now = s.timeline.now(); + return { + ...outcomeOf(scenario, condition, done, tools, extra), + metrics: { ...partial, ...extra }, + error: message, + diagnostics: diagnose(s, [s.timeline.stepStart ?? Math.max(0, now - 35_000), now]), + }; + } +} + +async function main() { + mkdirSync(join(OUT, "traces"), { recursive: true }); + process.env.OPENIDEA_SERVER = serverBinary(); + if (!args["no-build"]) buildBundle(ROOT, DIST); + if (!existsSync(join(DIST, "index.html"))) throw new Error(`no build in ${DIST}; run without --no-build`); + + const tools = toolVersions(); + const pins = pinnedTools(join(ROOT, ".github", "tool-versions.env")); + const machine: Fingerprint = fingerprint(); + for (const [k, want] of Object.entries(pins)) { + if (tools[k] !== undefined && !tools[k]!.startsWith(want)) + console.warn(`warning: ${k} ${tools[k]} is not the pinned ${want} (.github/tool-versions.env)`); + } + + const { server, url } = await serveBundle(DIST); + const browser = await chromium.launch({ + channel: args.channel === "chromium" ? undefined : args.channel, + headless: !args.headed, + args: [ + "--disable-renderer-backgrounding", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + ], + }); + const outcomes: Outcome[] = []; + try { + // Warm: one session per fixture, its project imported once, scenarios one after another. + const warm = selected.filter( + (s) => s.conditions.includes("warm") && (!conditionFilter || conditionFilter === "warm"), + ); + for (const fixture of [...new Set(warm.map((s) => s.fixture))]) { + const scenarios = warm.filter((s) => s.fixture === fixture); + const s = await Session.start(browser, url, ROOT, { fixture, workDir: join(OUT, "warm"), keepHome: false }); + try { + await scenarios[0]!.setup(s); + await s.settle(scenarios[0]!.server); + for (const scenario of scenarios) outcomes.push(await runScenario(s, scenario, "warm", browser, tools)); + } finally { + await s.close(); + } + } + // Cold: a fresh server and project import per scenario, measured once after the import settles. + const cold = selected.filter( + (s) => s.conditions.includes("cold") && (!conditionFilter || conditionFilter === "cold"), + ); + for (const scenario of cold) { + const started = Date.now(); + const s = await Session.start(browser, url, ROOT, { + fixture: scenario.fixture, + workDir: join(OUT, "cold"), + keepHome: false, + }); + try { + await scenario.setup(s); + await s.settle(scenario.server); + // Settling waits out the quiet period at its end. + const readyMs = Date.now() - started - 2000; + outcomes.push(await runScenario(s, scenario, "cold", browser, tools, { readyMs })); + } finally { + await s.close(); + } + } + } finally { + await browser.close(); + server.close(); + } + + if (args["update-golden"]) { + for (const o of outcomes) { + if (o.error || !o.results) continue; + writeGolden(goldenFile(o.scenario), { tools, groups: o.results }); + const c = compare(o.results, o.results); + o.comparison = c; + o.metrics.recall = c.recall; + o.metrics.precision = c.precision; + o.drift = []; + } + console.log("Goldens re-recorded: review `git diff evals/` before committing."); + } + + const baselineFile = join(OUT, "baseline.json"); + let baseline: Outcome[] | null = null; + if (existsSync(baselineFile)) { + const saved = JSON.parse(readFileSync(baselineFile, "utf8")) as { machine: Fingerprint; outcomes: Outcome[] }; + if (sameMachine(saved.machine, machine)) baseline = saved.outcomes; + else console.warn("warning: target/eval/baseline.json is from another machine or OS: not compared"); + } + const repro = (o: Outcome) => `pnpm eval --only ${o.scenario} --condition ${o.condition}`; + const verdicts = judge(outcomes, loadBudgets(join(ROOT, "evals", "budgets.json")), baseline, repro); + + const meta = { + date: new Date().toISOString(), + commit: git("rev-parse", "--short", "HEAD"), + dirty: git("status", "--porcelain", "--untracked-files=no") !== "", + runs, + warmup, + machine, + tools, + pins, + browser: browser.version(), + }; + const report = { + meta, + pass: verdicts.every((v) => v.status === "pass"), + verdicts: verdicts.map((v) => { + const o = outcomes.find((x) => idOf(x) === v.id)!; + return { + ...v, + samples: o.samples, + results: o.results, + comparison: o.comparison, + drift: o.drift, + degraded: o.degraded, + error: o.error, + // Diagnostics only where something needs a look. + diagnostics: v.status === "fail" || v.regressions.length ? o.diagnostics : undefined, + }; + }), + }; + writeFileSync(join(OUT, "report.json"), JSON.stringify(report, null, 2)); + const failing = verdicts.filter((v) => v.status === "fail"); + const md = [ + "# OpenIDEA eval report", + "", + `${meta.date} · \`${meta.commit}\`${meta.dirty ? " (dirty)" : ""} · ${machine.cpu} · ${machine.os} · jdtls ${tools.jdtls ?? "?"} · java ${tools.java ?? "?"}`, + "", + table(verdicts, true), + "", + ...failing.flatMap((v) => [`- **${v.id}**: ${v.failures.map((f) => f.detail).join("; ")} — \`${v.repro}\``]), + ].join("\n"); + writeFileSync(join(OUT, "report.md"), md + "\n"); + + console.log(); + console.log(table(verdicts, false)); + for (const v of failing) { + console.log(`\n${v.id} FAILED (${v.repro})`); + for (const f of v.failures) console.log(` ${f.kind}: ${f.detail}`); + const o = outcomes.find((x) => idOf(x) === v.id)!; + if (o.drift?.length) console.log(` golden recorded with other tools: ${o.drift.join(", ")}`); + const slow = [...(o.diagnostics?.calls ?? [])].sort((a, b) => b.ms - a.ms).slice(0, 5); + if (slow.length) + console.log( + ` slowest commands: ${slow.map((c) => `${c.cmd} ${Math.round(c.ms)} ms${c.pending ? " (no answer)" : c.ok ? "" : " (failed)"}`).join(", ")}`, + ); + } + console.log(`\nReport: ${join(OUT, "report.json")}`); + if (args["save-baseline"]) { + writeFileSync(baselineFile, JSON.stringify({ machine, meta, outcomes }, null, 2)); + console.log(`Baseline saved to ${baselineFile}`); + } + const regressed = verdicts.some((v) => v.regressions.length); + process.exit(failing.length || (args["fail-on-regression"] && regressed) ? 1 : 0); +} + +await main(); diff --git a/evals/tsconfig.json b/evals/tsconfig.json new file mode 100644 index 000000000..7a302debd --- /dev/null +++ b/evals/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../perf/tsconfig.json", + "include": ["./**/*.ts", "../perf/lib/**/*.ts", "../smoke/server.ts", "../smoke/tauriShim.ts"] +} diff --git a/fixtures/java-app/src/main/java/com/example/orders/BulkDiscount.java b/fixtures/java-app/src/main/java/com/example/orders/BulkDiscount.java new file mode 100644 index 000000000..ee5b1329a --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/BulkDiscount.java @@ -0,0 +1,18 @@ +package com.example.orders; + +/** Takes a percentage off orders of at least {@code minQuantity} items. */ +public class BulkDiscount extends ListPrice { + private final int minQuantity; + private final int percent; + + public BulkDiscount(int minQuantity, int percent) { + this.minQuantity = minQuantity; + this.percent = percent; + } + + @Override + public long price(Order order) { + long list = super.price(order); + return order.quantity() >= minQuantity ? list * (100 - percent) / 100 : list; + } +} diff --git a/fixtures/java-app/src/main/java/com/example/orders/ListPrice.java b/fixtures/java-app/src/main/java/com/example/orders/ListPrice.java new file mode 100644 index 000000000..586f9f2bd --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/ListPrice.java @@ -0,0 +1,9 @@ +package com.example.orders; + +/** Charges the order's total. */ +public class ListPrice implements PricingRule { + @Override + public long price(Order order) { + return order.totalCents(); + } +} diff --git a/fixtures/java-app/src/main/java/com/example/orders/PricingRule.java b/fixtures/java-app/src/main/java/com/example/orders/PricingRule.java new file mode 100644 index 000000000..0fef0c51c --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/PricingRule.java @@ -0,0 +1,6 @@ +package com.example.orders; + +/** Turns an order into the price charged for it, in cents. */ +public interface PricingRule { + long price(Order order); +} diff --git a/knip.ts b/knip.ts index 616b53278..7714e4dab 100644 --- a/knip.ts +++ b/knip.ts @@ -12,6 +12,7 @@ const config: KnipConfig = { "!src/editor/core/cm/django/parser.terms.ts", "scripts/**/*.ts", "perf/**/*.ts", + "evals/**/*.ts", "e2e/**/*.ts", "smoke/**/*.ts", ], diff --git a/package.json b/package.json index 93cc83a4b..908dfcaa9 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "format": "oxfmt", "format:check": "oxfmt --check", "knip": "knip", - "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit && tsrs -p smoke --noEmit", + "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p evals --noEmit && tsrs -p scripts --noEmit && tsrs -p smoke --noEmit", "gen:bindings": "cargo test -p openidea --test export_bindings", "gen:windows-icons": "node scripts/gen-windows-icons.ts", "gen:angular-grammar": "lezer-generator src/editor/core/cm/angular/angular.grammar --typeScript -o src/editor/core/cm/angular/parser.ts", @@ -31,6 +31,7 @@ "e2e": "playwright test", "perf": "node perf/run.ts", "perf:rust": "node perf/rust.ts", + "eval": "node evals/run.ts", "release:prepare": "node scripts/release-prepare.ts", "nightly:prepare": "node scripts/nightly-prepare.ts" }, diff --git a/perf/lib/bundle.ts b/perf/lib/bundle.ts new file mode 100644 index 000000000..ff99c3e13 --- /dev/null +++ b/perf/lib/bundle.ts @@ -0,0 +1,47 @@ +/** The workbench's `--mode perf` production bundle: building it and serving it to Chromium. */ +import { spawnSync } from "node:child_process"; +import { createReadStream, existsSync, statSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import { extname, join } from "node:path"; + +export function buildBundle(root: string, outDir: string, flags: string[] = []) { + console.log("Building the workbench (vite build --mode perf)…"); + const r = spawnSync( + "pnpm", + ["exec", "vite", "build", "--mode", "perf", ...flags, "--outDir", outDir, "--emptyOutDir"], + { + cwd: root, + stdio: ["ignore", "ignore", "inherit"], + shell: process.platform === "win32", + }, + ); + if (r.status !== 0) throw new Error("vite build failed"); +} + +const MIME: Record = { + ".html": "text/html", + ".js": "text/javascript", + ".css": "text/css", + ".json": "application/json", + ".svg": "image/svg+xml", + ".png": "image/png", + ".woff2": "font/woff2", + ".wasm": "application/wasm", +}; + +/** Serves `dir` on a free local port (unknown paths: `index.html`). */ +export function serveBundle(dir: string): Promise<{ server: Server; url: string }> { + const server = createServer((req, res) => { + const path = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname); + let file = join(dir, path); + if (!file.startsWith(dir) || !existsSync(file) || statSync(file).isDirectory()) file = join(dir, "index.html"); + res.writeHead(200, { "content-type": MIME[extname(file)] ?? "application/octet-stream" }); + createReadStream(file).pipe(res); + }); + return new Promise((ok) => + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + ok({ server, url: `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}` }); + }), + ); +} diff --git a/perf/run.ts b/perf/run.ts index 98fa7f58b..2554b0f0c 100644 --- a/perf/run.ts +++ b/perf/run.ts @@ -11,13 +11,12 @@ * pnpm perf -- --only typing --trace Chrome trace of the last run → target/perf/traces/ * pnpm perf -- --only typing --profile --unminified CPU profile + top functions */ -import { spawnSync } from "node:child_process"; -import { createReadStream, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; -import { createServer, type Server } from "node:http"; -import { extname, join, resolve } from "node:path"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; import { parseArgs } from "node:util"; import { gzipSync } from "node:zlib"; import { chromium, type Browser } from "@playwright/test"; +import { buildBundle, serveBundle } from "./lib/bundle.ts"; import { Harness, type Metrics, type Scenario } from "./lib/harness.ts"; import { topFunctions, type CpuProfile } from "./lib/profile.ts"; import { installInPage } from "./lib/inpage.ts"; @@ -65,18 +64,7 @@ if (only && scenarios.length !== only.length) { } function build() { - console.log("Building the workbench (vite build --mode perf)…"); - const flags = args.unminified ? ["--minify", "false"] : []; - const r = spawnSync( - "pnpm", - ["exec", "vite", "build", "--mode", "perf", ...flags, "--outDir", DIST, "--emptyOutDir"], - { - cwd: ROOT, - stdio: ["ignore", "ignore", "inherit"], - shell: process.platform === "win32", - }, - ); - if (r.status !== 0) throw new Error("vite build failed"); + buildBundle(ROOT, DIST, args.unminified ? ["--minify", "false"] : []); } /** Initial-load JavaScript / CSS from the built index.html (gzip sizes). */ @@ -92,33 +80,6 @@ function bundleMetrics(): Metrics { }; } -const MIME: Record = { - ".html": "text/html", - ".js": "text/javascript", - ".css": "text/css", - ".json": "application/json", - ".svg": "image/svg+xml", - ".png": "image/png", - ".woff2": "font/woff2", - ".wasm": "application/wasm", -}; - -function serve(): Promise<{ server: Server; url: string }> { - const server = createServer((req, res) => { - const path = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname); - let file = join(DIST, path); - if (!file.startsWith(DIST) || !existsSync(file) || statSync(file).isDirectory()) file = join(DIST, "index.html"); - res.writeHead(200, { "content-type": MIME[extname(file)] ?? "application/octet-stream" }); - createReadStream(file).pipe(res); - }); - return new Promise((ok) => - server.listen(0, "127.0.0.1", () => { - const addr = server.address(); - ok({ server, url: `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}` }); - }), - ); -} - interface Capture { trace?: string; profile?: string; @@ -180,7 +141,7 @@ async function main() { const results: ScenarioResult[] = args.unminified ? [] : [{ id: "bundle", title: "Initial bundle (gzip)", metrics: bundleMetrics() }]; - const { server, url } = await serve(); + const { server, url } = await serveBundle(DIST); const browser = await chromium .launch({ channel: args.channel === "chromium" ? undefined : args.channel, diff --git a/smoke/desktop.ts b/smoke/desktop.ts index c8919e696..b2f1c1960 100644 --- a/smoke/desktop.ts +++ b/smoke/desktop.ts @@ -1,76 +1,24 @@ /** * The frontend's view of the desktop app, with a real backend: the window's Tauri runtime - * (callbacks, `Channel`s, the event plugin) in-process, and every app command sent to an - * `openidea-server` through the route remote development uses (`src/platform/tauriCore.ts`). + * in-process (`tauriShim.ts`), and every app command sent to an `openidea-server`, as + * remote development sends them. */ import { homedir, tmpdir } from "node:os"; -import { join } from "node:path"; -import { setInvokeRoute } from "@/platform/tauriCore"; -import { encodeArgs } from "@/platform/remote"; -import { Server } from "./server"; +import { sep } from "node:path"; +import { installTauriShim, type ShimWindow } from "./tauriShim"; +import type { Server } from "./server"; -type Callback = (message: unknown) => void; - -// One runtime per page, there from the start as in the app: the frontend registers its event -// listener once (`src/platform/ipc.ts`), possibly before a server is connected, and keeps it -// across the servers connected one after another. -const callbacks = new Map(); -const listeners: { event: string; handler: number }[] = []; -const channels = new Map void }>(); -let nextCallback = 1; -/** `BaseDirectory` values (`@tauri-apps/api/path`) the frontend resolves. */ -const DIRECTORIES: Record = { 12: tmpdir(), 21: homedir() }; - -{ - const internals = { - metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } }, - transformCallback(cb: Callback) { - const id = nextCallback++; - callbacks.set(id, cb); - return id; - }, - unregisterCallback(id: number) { - callbacks.delete(id); - }, - convertFileSrc: (path: string) => path, - // Only the window's own plugins reach this: app commands take the route below. - async invoke(cmd: string, args: Record = {}) { - if (cmd === "plugin:event|listen") { - listeners.push({ event: args.event as string, handler: args.handler as number }); - return listeners.length; - } - if (cmd === "plugin:window|get_all_windows") return []; - // This machine's directories (`@tauri-apps/api/path`), as on the desktop. - if (cmd === "plugin:path|resolve_directory") return DIRECTORIES[args.directory as number] ?? null; - if (cmd === "plugin:path|join") return join(...(args.paths as string[])); - if (cmd.startsWith("plugin:")) return null; - throw { kind: "unavailable", message: `${cmd} has no route`, detail: null }; - }, - }; - Object.assign(window, { - __TAURI_INTERNALS__: internals, - __TAURI_EVENT_PLUGIN_INTERNALS__: { unregisterListener: () => {} }, - }); -} +installTauriShim({ directories: { 12: tmpdir(), 21: homedir() }, sep: sep as "/" | "\\" }); /** Connects this page's frontend to `server`; returns a function that disconnects it. */ export function connectDesktop(server: Server): () => void { - channels.clear(); - server.onChannel = (id, data) => channels.get(id)?.onmessage(data); - server.onEvent = (target, event) => { - if (target !== null && target !== "main") return; - for (const l of listeners) { - if (l.event === "ide://event") callbacks.get(l.handler)?.({ event: l.event, id: 0, payload: event }); - } - }; - setInvokeRoute((cmd, args) => { - if (cmd.startsWith("plugin:")) return null; - const { json, channels: used } = encodeArgs(args); - for (const [id, channel] of used) channels.set(id, channel as unknown as { onmessage: (data: unknown) => void }); - return server.invoke(cmd, JSON.parse(json)); - }); + const w = window as unknown as ShimWindow; + w.__openideaShim.reset(); + server.onChannel = (id, data) => w.__openideaShim.channel(id, data); + server.onEvent = (target, event) => w.__openideaShim.event(target, event); + w.__openideaForward = (cmd, args) => server.invoke(cmd, args); return () => { - setInvokeRoute(null); + delete w.__openideaForward; server.onChannel = () => {}; server.onEvent = () => {}; }; diff --git a/smoke/server.ts b/smoke/server.ts index 9a7e66ed9..17594dd70 100644 --- a/smoke/server.ts +++ b/smoke/server.ts @@ -121,6 +121,7 @@ export class Server { onEvent: (window: string | null, event: unknown) => void = () => {}; private closing: Promise | undefined; + // A plain field, not a parameter property: the evals load this file with Node's type stripping. private readonly child: ChildProcessWithoutNullStreams; private constructor(child: ChildProcessWithoutNullStreams) { diff --git a/smoke/tauriShim.ts b/smoke/tauriShim.ts new file mode 100644 index 000000000..822e31a4f --- /dev/null +++ b/smoke/tauriShim.ts @@ -0,0 +1,111 @@ +/** + * The window's Tauri runtime as the frontend sees it (callbacks, `Channel`s, the event + * plugin, path resolution), with every app command handed to `window.__openideaForward`: + * a real `openidea-server` on the other end (`smoke/desktop.ts` in jsdom, `evals/` in + * Chromium). Self-contained: Playwright serialises this function into the page, so it + * may use nothing from outside its body. + */ + +export interface ShimConfig { + /** `BaseDirectory` values (`@tauri-apps/api/path`) the frontend resolves. */ + directories: Record; + sep: "/" | "\\"; +} + +/** What the host side calls to deliver the server's notifications. */ +export interface ShimBridge { + /** A value streamed to the `Channel` with this id. */ + channel(id: number, data: unknown): void; + /** An `IdeEvent` for a window (`null`: every window). */ + event(window: string | null, event: unknown): void; + /** Forgets the channels of an earlier server. */ + reset(): void; +} + +export interface ShimWindow { + __openideaShim: ShimBridge; + /** Runs an app command on the server; rejects with its `IdeError`. */ + __openideaForward?: (cmd: string, args: unknown) => Promise; +} + +export function installTauriShim(config: ShimConfig) { + type Callback = (message: unknown) => void; + const w = window as unknown as ShimWindow & Record; + if (w.__openideaShim) return; + // One runtime per page, there from the start as in the app: the frontend registers its + // event listener once (`src/platform/ipc.ts`), possibly before a server is connected, + // and keeps it across the servers connected one after another. + const callbacks = new Map(); + const listeners: { event: string; handler: number }[] = []; + const channels = new Map void }>(); + let nextCallback = 1; + + const join = (paths: string[]) => { + const absolute = paths[0]?.startsWith(config.sep) ?? false; + const parts: string[] = []; + for (const p of paths.join(config.sep).split(/[\\/]+/)) { + if (p === "" || p === ".") continue; + if (p === ".." && parts.length && parts[parts.length - 1] !== "..") parts.pop(); + else parts.push(p); + } + return (absolute ? config.sep : "") + parts.join(config.sep); + }; + + /** The arguments with each `Channel` replaced by `{ $channel: id }`, as remote development sends them. */ + const encode = (args: Record) => { + const plain: Record = {}; + for (const [k, v] of Object.entries(args)) { + const tag = + v && typeof v === "object" && typeof (v as { toJSON?: unknown }).toJSON === "function" + ? (v as { toJSON(): unknown }).toJSON() + : null; + if (typeof tag === "string" && tag.startsWith("__CHANNEL__:")) { + const id = Number(tag.slice("__CHANNEL__:".length)); + channels.set(id, v as { onmessage: (data: unknown) => void }); + plain[k] = { $channel: id }; + } else { + plain[k] = v; + } + } + return JSON.parse(JSON.stringify(plain)) as unknown; + }; + + const internals = { + metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } }, + transformCallback(cb: Callback) { + const id = nextCallback++; + callbacks.set(id, cb); + return id; + }, + unregisterCallback(id: number) { + callbacks.delete(id); + }, + convertFileSrc: (path: string) => path, + async invoke(cmd: string, args: Record = {}) { + if (cmd === "plugin:event|listen") { + listeners.push({ event: args.event as string, handler: args.handler as number }); + return listeners.length; + } + if (cmd === "plugin:window|get_all_windows") return []; + // This machine's directories (`@tauri-apps/api/path`), as on the desktop. + if (cmd === "plugin:path|resolve_directory") return config.directories[args.directory as number] ?? null; + if (cmd === "plugin:path|join") return join(args.paths as string[]); + if (cmd.startsWith("plugin:")) return null; + const forward = w.__openideaForward; + if (!forward) throw { kind: "unavailable", message: `${cmd} has no route`, detail: null }; + return forward(cmd, encode(args)); + }, + }; + w.__TAURI_INTERNALS__ = internals; + w.__TAURI_EVENT_PLUGIN_INTERNALS__ = { unregisterListener: () => {} }; + w.__openideaShim = { + channel: (id, data) => channels.get(id)?.onmessage(data), + event(target, event) { + if (target !== null && target !== "main") return; + for (const l of listeners) { + if (l.event === "ide://event") callbacks.get(l.handler)?.({ event: l.event, id: 0, payload: event }); + } + }, + reset: () => channels.clear(), + }; +} diff --git a/src/editor/lsp/Popups.tsx b/src/editor/lsp/Popups.tsx index 8f1228009..2df2d28a0 100644 --- a/src/editor/lsp/Popups.tsx +++ b/src/editor/lsp/Popups.tsx @@ -181,7 +181,12 @@ function ListPopup({ popup }: { popup: Extract }) { className={clsx(styles.listPopup, popup.preview && styles.withPreview)} style={popup.preview ? undefined : { width }} > -
+
{popup.title} {query && {query}} @@ -218,6 +223,9 @@ function ListPopup({ popup }: { popup: Extract }) {
{popup.numbered && {n > 0 && n <= 9 ? n : ""}} @@ -445,7 +453,7 @@ function DocPopup({ popup }: { popup: Extract }) { // As in IDEA the popup leaves the keyboard to the editor until F1 focuses it. autoFocus={false} > -
+
@@ -608,7 +616,7 @@ function PreviewDialog({ popup }: { popup: Extract ) : ( -
    +
      {summary.entries.map((e) => (
    • diff --git a/src/editor/lsp/actions.ts b/src/editor/lsp/actions.ts index 696e50935..a04ee117a 100644 --- a/src/editor/lsp/actions.ts +++ b/src/editor/lsp/actions.ts @@ -357,8 +357,8 @@ export function registerLspActions(): () => void { description: "Shows `{ text, tone? }` as a transient hint at the caret of the active editor.", run: ({ arg }) => { const view = activeEditor()?.view; - const a = (arg ?? {}) as { text?: string; tone?: "info" | "warning" | "error" }; - if (view && a.text) showHint(view, a.text, { tone: a.tone }); + const a = (arg ?? {}) as { text?: string; tone?: "info" | "warning" | "error"; data?: Record }; + if (view && a.text) showHint(view, a.text, { tone: a.tone, data: a.data }); }, }, ]); diff --git a/src/editor/lsp/hint.test.ts b/src/editor/lsp/hint.test.ts index f59ed2191..e98461dc6 100644 --- a/src/editor/lsp/hint.test.ts +++ b/src/editor/lsp/hint.test.ts @@ -29,10 +29,21 @@ describe("pending hints", () => { const done = showPendingHint(view, "Waiting…"); vi.advanceTimersByTime(PENDING_HINT_DELAY_MS); expect(hint()).toBe("Waiting…"); + expect(document.querySelector(".cm-lsp-hint")?.dataset.pending).toBe("true"); done(); expect(hint()).toBeNull(); }); + it("marks only progress as pending", () => { + showHint(view, "Done"); + expect(document.querySelector(".cm-lsp-hint")?.dataset.pending).toBeUndefined(); + }); + + it("puts what the hint answers in its data attributes", () => { + showHint(view, "Only one usage", { data: { usages: "1" } }); + expect(document.querySelector(".cm-lsp-hint")?.dataset.usages).toBe("1"); + }); + it("does not dismiss a newer hint when an old request finishes", () => { const done = showPendingHint(view, "Waiting…"); vi.advanceTimersByTime(PENDING_HINT_DELAY_MS); diff --git a/src/editor/lsp/hint.ts b/src/editor/lsp/hint.ts index ee7a7156f..a32e02248 100644 --- a/src/editor/lsp/hint.ts +++ b/src/editor/lsp/hint.ts @@ -13,6 +13,10 @@ interface Hint { /** Text, or DOM built when shown (with `destroy` when it holds e.g. a React root). */ content: string | (() => HTMLElement | { dom: HTMLElement; destroy: () => void }); tone: "info" | "error" | "warning"; + /** Progress of a request still running (`data-pending` on the hint). */ + pending?: boolean; + /** `data-*` attributes saying what the hint answers, for tests and tools. */ + data?: Record; } const setHint = StateEffect.define(); @@ -35,6 +39,8 @@ const hintField = StateField.define({ const dom = document.createElement("div"); dom.className = `cm-lsp-hint cm-lsp-hint-${h.tone}`; dom.setAttribute("role", "status"); + if (h.pending) dom.dataset.pending = "true"; + Object.assign(dom.dataset, h.data); if (typeof h.content === "string") { dom.textContent = txTemplate(h.content); return { dom }; @@ -57,10 +63,12 @@ const timers = new WeakMap>(); export function showHint( view: EditorView, content: Hint["content"], - opts: { pos?: number; tone?: Hint["tone"]; timeout?: number } = {}, + opts: { pos?: number; tone?: Hint["tone"]; timeout?: number; pending?: boolean; data?: Hint["data"] } = {}, ) { const pos = opts.pos ?? view.state.selection.main.head; - const effects: StateEffect[] = [setHint.of({ pos, content, tone: opts.tone ?? "info" })]; + const effects: StateEffect[] = [ + setHint.of({ pos, content, tone: opts.tone ?? "info", pending: opts.pending, data: opts.data }), + ]; if (!view.state.field(hintField, false)) effects.push(StateEffect.appendConfig.of(hintExtension())); view.dispatch({ effects }); clearTimeout(timers.get(view)); @@ -91,7 +99,7 @@ export function showPendingHint(view: EditorView, content: string): () => void { view.state.field(hintField, false) !== previous ) return; - showHint(view, content, { timeout: 30_000 }); + showHint(view, content, { timeout: 30_000, pending: true }); shown = view.state.field(hintField); }, PENDING_HINT_DELAY_MS); return () => { diff --git a/src/editor/lsp/navigation.ts b/src/editor/lsp/navigation.ts index 3c09256c3..95425e627 100644 --- a/src/editor/lsp/navigation.ts +++ b/src/editor/lsp/navigation.ts @@ -192,6 +192,7 @@ export async function locationItems(locs: readonly DeclarationTarget[]): Promise label: l.name, detail: [l.container, where].filter(Boolean).join(" · "), symbol: l.kind, + location: { path: l.path, line: l.range.start.line }, run: () => navigateTo(l), }; return { @@ -199,6 +200,7 @@ export async function locationItems(locs: readonly DeclarationTarget[]): Promise label: names[i] ?? basename(l.path), detail: [indexed.get(i) ?? enclosingType(files.get(l.path), l), where].filter(Boolean).join(" · "), icon: fileIconName(basename(l.path), l.path), + location: { path: l.path, line: l.range.start.line }, run: () => navigateTo(l), }; }); @@ -256,6 +258,7 @@ function streamedImplementations(view: EditorView, offset: number, onOpen: () => kind: "list", anchor: caretAnchor(view, offset), title: searchingTitle(name, first.length), + busy: true, items, pin: pinToFind(name ?? "", first), onClose: () => { @@ -282,6 +285,7 @@ function streamedImplementations(view: EditorView, offset: number, onOpen: () => if (closed || superseded()) return; updatePopup("list", { title: chooserTitle("implementation", name, all.length), + busy: false, items, pin: pinToFind(name ?? "", all), }); diff --git a/src/editor/lsp/popupStore.ts b/src/editor/lsp/popupStore.ts index 65d9be45c..2ac910c5c 100644 --- a/src/editor/lsp/popupStore.ts +++ b/src/editor/lsp/popupStore.ts @@ -26,6 +26,8 @@ export interface ListPopupItem { /** Disabled reason; disabled rows are shown but not activatable. */ disabled?: string | null; shortcut?: string; + /** The place the row goes to (`data-path` / `data-line`, 0-based, on the row). */ + location?: { path: string; line: number }; run: () => void | Promise; /** IDEA's right-arrow menu of the row (inspection options of a fix). */ submenu?: MenuItem[]; @@ -94,6 +96,8 @@ export type LspPopup = items: ListPopupItem[]; /** Rows 1-9 get a number; typing it runs the row (IDEA's numbered popups). */ numbered?: boolean; + /** More rows are still being looked for (`aria-busy`). */ + busy?: boolean; /** Header pin button (e.g. "Open in Find Tool Window"). */ pin?: { label: string; run: () => void }; /** Side panel for the selected row, fetched lazily; `signal` aborts when the selection moves. */ diff --git a/src/features/code-insight/UsagesPopup.tsx b/src/features/code-insight/UsagesPopup.tsx index 3657b0217..97e5e3b51 100644 --- a/src/features/code-insight/UsagesPopup.tsx +++ b/src/features/code-insight/UsagesPopup.tsx @@ -146,7 +146,16 @@ function UsagesPopupInner() { const keys = shortcutLabel("find.usages"); return ( -
      +
      {symbol ? tx`Usages of ${symbol}` : tx("Usages")}{" "} @@ -191,7 +200,11 @@ function UsagesPopupInner() {
      - {note &&
      {note}
      } + {note && ( +
      + {note} +
      + )}
      } renderItem={(u) => ( -
      +
      diff --git a/src/features/code-insight/ipc.ts b/src/features/code-insight/ipc.ts index faa629e37..8fdaccbce 100644 --- a/src/features/code-insight/ipc.ts +++ b/src/features/code-insight/ipc.ts @@ -21,9 +21,9 @@ export async function lspRequest(run: () => Promise>): Promise { export const isUnavailable = (e: unknown) => e instanceof IpcError && e.kind === "unavailable"; -/** Small hint at the caret (via editor/lsp), or a notification without an editor. */ -export function hint(text: string, tone: "info" | "warning" | "error" = "info") { - if (activeEditor()) void executeAction("editor.showHint", { text, tone }); +/** Small hint at the caret (via editor/lsp), or a notification without an editor; `data`: its `data-*` attributes. */ +export function hint(text: string, tone: "info" | "warning" | "error" = "info", data?: Record) { + if (activeEditor()) void executeAction("editor.showHint", { text, tone, data }); else notifications().notify({ group: "lsp", diff --git a/src/features/code-insight/usages.ts b/src/features/code-insight/usages.ts index 5da12694c..56d444afc 100644 --- a/src/features/code-insight/usages.ts +++ b/src/features/code-insight/usages.ts @@ -456,21 +456,33 @@ async function searchPopup(c: CaretContext, libraries: boolean, widenable: boole // As in IDEA, the hint offers to widen the search, which Show Usages does while it shows. const widen = scope === "project" && again !== ""; if (widen) popupTarget.hintUntil = Date.now() + HINT_MS; + // The count and scope, for tests: the hint is the whole answer. + const counted: Record = scope ? { usages: String(total), scope } : { usages: String(total) }; if (!total) { - if (widen) hint(`No usages found in Project Files. Press ${again} again to search in 'Project and Libraries'`); - else if (scope === "project") hint("No usages found in Project Files"); + if (widen) + hint( + `No usages found in Project Files. Press ${again} again to search in 'Project and Libraries'`, + "info", + counted, + ); + else if (scope === "project") hint("No usages found in Project Files", "info", counted); else { const none = c.word ? `No usages of '${c.word}' found` : "No usages found"; - hint(note ? `${none}. ${note}` : none); + hint(note ? `${none}. ${note}` : none, "info", counted); } return; } // IDEA jumps to the only usage and says so in the target editor. const u = groups[0]!.usages[0]!; await openLocation(u.path, u.line, u.col); - if (widen) hint(`Only one usage in Project Files. Press ${again} again to search in 'Project and Libraries'`); - else if (scope === "libraries") hint("Only one usage in Project and Libraries"); - else hint("Only one usage in Project Files"); + if (widen) + hint( + `Only one usage in Project Files. Press ${again} again to search in 'Project and Libraries'`, + "info", + counted, + ); + else if (scope === "libraries") hint("Only one usage in Project and Libraries", "info", counted); + else hint("Only one usage in Project Files", "info", counted); } catch (e) { if (popupRequest !== request || isCancelled(e)) return; if (early || reopened) useUsagesPopup.setState({ loading: false }); diff --git a/src/workbench/perfProbe.ts b/src/workbench/perfProbe.ts index a2e8d9138..0caf85171 100644 --- a/src/workbench/perfProbe.ts +++ b/src/workbench/perfProbe.ts @@ -3,12 +3,15 @@ * Loaded only in dev and in `vite build --mode perf`; release builds never * include it. Measurement itself lives in the harness so it stays out of the app. */ +import { completionStatus, currentCompletions } from "@codemirror/autocomplete"; import { syntaxParserRunning } from "@codemirror/language"; +import { forEachDiagnostic } from "@codemirror/lint"; import { executeAction } from "@/platform/actions"; import { bindingsFor } from "@/platform/keymap"; import { activeEditor } from "@/editor/api"; import { docBackend, type MemoryBackend } from "@/editor/core/backend"; import type { Commit, CompletionItem, CompletionKind, LogQuery } from "@/bindings"; +import { openProject } from "@/platform/project"; import { setInvokeRoute } from "@/platform/tauriCore"; import { editorService } from "./contributions"; @@ -16,6 +19,8 @@ export interface PerfProbe { /** Seeds file contents for the in-memory document backend (plain-browser mode). */ seed(path: string, text: string): void; openFile(path: string): Promise; + /** Opens project `path` in this window (as the welcome screen does), without the window chooser. */ + openProject(path: string): Promise; runAction(id: string, arg?: unknown): Promise; /** Effective keymap bindings of an action, e.g. `["Ctrl+Shift+A"]`. */ bindings(id: string): string[]; @@ -23,8 +28,22 @@ export interface PerfProbe { setCursor(offset: number): void; /** True while the active editor's background syntax parser still has work. */ parsing(): boolean; - /** Document length and caret of the active editor. */ - editorState(): { path: string; length: number; lines: number; head: number } | null; + /** The active editor's text. */ + editorText(): string | null; + /** Inserts `text` after the first `after` in the active editor, as an edit of the user's; the offset after it. */ + insertAfter(after: string, text: string): number; + /** Replaces the active editor's whole text, as an edit of the user's. */ + setText(text: string): void; + /** Labels of the completion lookup open in the active editor. */ + completions(): string[]; + /** True while a completion source of the active editor has not answered yet. */ + completionPending(): boolean; + /** The diagnostics in the active editor (0-based line). */ + diagnostics(): Array<{ line: number; severity: string; message: string }>; + /** 0-based line of the active editor shown at viewport `y` (e.g. a gutter marker's), or -1. */ + lineAtY(y: number): number; + /** Document length and caret (offset, 0-based line, the word around it) of the active editor. */ + editorState(): { path: string; length: number; lines: number; head: number; line: number; word: string } | null; /** * Answers code completion with `items` generated items (a language server's * list); other language server requests report no server. `0` removes it. @@ -126,6 +145,12 @@ function setRoute(name: string, next: Route | null) { ); } +function requireView() { + const view = activeEditor()?.view; + if (!view) throw new Error("no active editor"); + return view; +} + export function installPerfProbe(): () => void { const probe: PerfProbe = { seed(path, text) { @@ -134,11 +159,11 @@ export function installPerfProbe(): () => void { backend.files.set(path, text); }, openFile: (path) => editorService().openFile(path), + openProject: async (path) => void (await openProject(path)), runAction: (id, arg) => executeAction(id, arg), bindings: (id) => bindingsFor(id), setCursor(offset) { - const view = activeEditor()?.view; - if (!view) throw new Error("no active editor"); + const view = requireView(); view.dispatch({ selection: { anchor: Math.min(offset, view.state.doc.length) }, scrollIntoView: true }); view.focus(); }, @@ -177,11 +202,50 @@ export function installPerfProbe(): () => void { : null, ); }, + editorText: () => activeEditor()?.view.state.doc.toString() ?? null, + insertAfter(after, text) { + const view = requireView(); + const at = view.state.doc.toString().indexOf(after); + if (at < 0) throw new Error(`no ${JSON.stringify(after)} in the active editor`); + const from = at + after.length; + view.dispatch({ changes: { from, insert: text }, userEvent: "input" }); + return from + text.length; + }, + setText(text) { + const view = requireView(); + view.dispatch({ changes: { from: 0, to: view.state.doc.length, insert: text }, userEvent: "input" }); + }, + completions: () => currentCompletions(requireView().state).map((c) => c.label), + completionPending: () => completionStatus(requireView().state) === "pending", + diagnostics() { + const found: Array<{ line: number; severity: string; message: string }> = []; + const state = activeEditor()?.view.state; + if (!state) return found; + forEachDiagnostic(state, (d, from) => + found.push({ line: state.doc.lineAt(from).number - 1, severity: d.severity, message: d.message }), + ); + return found; + }, + lineAtY(y) { + const view = activeEditor()?.view; + if (!view) return -1; + const block = view.lineBlockAtHeight(y - view.documentTop); + return view.state.doc.lineAt(block.from).number - 1; + }, editorState() { const e = activeEditor(); if (!e) return null; const { doc, selection } = e.view.state; - return { path: e.ctx.path, length: doc.length, lines: doc.lines, head: selection.main.head }; + const head = selection.main.head; + const word = e.view.state.wordAt(head); + return { + path: e.ctx.path, + length: doc.length, + lines: doc.lines, + head, + line: doc.lineAt(head).number - 1, + word: word ? doc.sliceString(word.from, word.to) : "", + }; }, }; const w = window as unknown as { __openideaPerf?: PerfProbe }; diff --git a/vite.config.ts b/vite.config.ts index 366eed433..7cfbd7030 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -46,6 +46,6 @@ export default defineConfig(() => ({ environment: "jsdom", globals: false, setupFiles: ["./src/test/setup.ts"], - include: ["src/**/*.test.{ts,tsx}", "perf/**/*.test.ts", "scripts/**/*.test.ts"], + include: ["src/**/*.test.{ts,tsx}", "perf/**/*.test.ts", "evals/**/*.test.ts", "scripts/**/*.test.ts"], }, }));