diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 836b8710..6ad43075 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ -Closes # + ## What and why diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 180729a4..2a6b1f2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ concurrency: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 + # Line tables keep test backtraces readable; full debuginfo only adds build time and cache size. + CARGO_PROFILE_DEV_DEBUG: line-tables-only jobs: rust: @@ -25,10 +27,10 @@ jobs: os: [macos-latest, ubuntu-22.04, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@v5 + - uses: pnpm/action-setup@v5 with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: { node-version: 22, cache: pnpm } - uses: dtolnay/rust-toolchain@stable with: { components: "clippy, rustfmt" } @@ -62,7 +64,7 @@ jobs: name: Remote server (linux musl) runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: { targets: x86_64-unknown-linux-musl } - uses: Swatinem/rust-cache@v2 @@ -79,7 +81,7 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target-plugins steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: { targets: wasm32-wasip2, components: "clippy, rustfmt" } - uses: Swatinem/rust-cache@v2 @@ -100,9 +102,9 @@ jobs: cargo test --locked --manifest-path "$dir/Cargo.toml" echo "::endgroup::" done - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@v5 with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: { node-version: 22 } # Plugin web pages (`web-src/`, their own pnpm projects): typecheck and test. - name: Web pages typecheck and test @@ -126,7 +128,7 @@ jobs: name: Cargo deny runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-deny - run: cargo deny check @@ -135,10 +137,10 @@ jobs: name: Frontend runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@v5 + - uses: pnpm/action-setup@v5 with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: { node-version: 22, cache: pnpm } - run: pnpm install --frozen-lockfile - run: pnpm typecheck @@ -154,16 +156,16 @@ jobs: # Browsers and their system libraries are preinstalled; keep the tag in step with @playwright/test. container: mcr.microsoft.com/playwright:v1.64.1-noble steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@v5 + - uses: pnpm/action-setup@v5 with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: { node-version: 22, cache: pnpm } - run: pnpm install --frozen-lockfile - run: pnpm e2e env: HOME: /root - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 if: failure() with: name: playwright-report diff --git a/.github/workflows/language-smoke.yml b/.github/workflows/language-smoke.yml new file mode 100644 index 00000000..03f80a68 --- /dev/null +++ b/.github/workflows/language-smoke.yml @@ -0,0 +1,124 @@ +name: Language smoke +# Everyday code intelligence (completion, go to definition, Find Usages, hover, diagnostics +# with a quick fix, an intention, formatting, rename) per language, against the language +# server the app starts by default: crates/ide-host/tests/language_smoke.rs. +# Opening the fixture with its build tool and SDK (Maven / Gradle sync, the virtualenv, the +# JDK): the ignored tests of crates/ide-host/tests/project_smoke.rs. +# Then running and testing each fixture through the run store and openidea-server +# (Run, Run tests, the test tree, Rerun Failed Tests): smoke/run.test.ts; and debugging its +# entry point with the debugger the IDE installs (breakpoint, frames, variables, evaluate, +# Step Over, Resume): smoke/debug.test.ts. +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - "crates/**" + - "languages/**" + - "fixtures/**" + - "Cargo.toml" + - "Cargo.lock" + - "src/features/run/**" + - "src/features/debugger/**" + - "src/platform/**" + - "smoke/**" + - "package.json" + - "pnpm-lock.yaml" + - ".github/workflows/language-smoke.yml" + workflow_dispatch: + +concurrency: + group: language-smoke-${{ github.ref }} + # A new push supersedes a PR's run; on main every merge runs to the end (and saves the cache). + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + # Pinned so a server release can't fail an unrelated PR; bump them deliberately. + VTSLS: 0.3.0 + TYPESCRIPT: 6.0.3 + BASEDPYRIGHT: 1.40.2 + RUFF: 0.17.3 + GOPLS: v0.24.0 + JDTLS: 1.63.0-202610191322 + KOTLIN_LSP: 263.7084.0 + GRADLE: 8.14.3 + +jobs: + smoke: + name: Language smoke (${{ matrix.language }}) + strategy: + fail-fast: false + matrix: + language: [typescript, python, rust, go, java, kotlin] + runs-on: ubuntu-22.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: { components: "rust-analyzer, rustfmt" } + # One build cache for every language; main saves it. + - uses: Swatinem/rust-cache@v2 + with: + shared-key: language-smoke + save-if: ${{ github.ref == 'refs/heads/main' && matrix.language == 'rust' }} + + - uses: pnpm/action-setup@v4 + with: { version: 11 } + - uses: actions/setup-node@v4 + with: { node-version: 22, cache: pnpm } + - if: matrix.language == 'typescript' + run: npm install --global "@vtsls/language-server@$VTSLS" "typescript@$TYPESCRIPT" + + - if: matrix.language == 'python' + uses: actions/setup-python@v5 + with: { python-version: "3.12" } + - if: matrix.language == 'python' + run: pip install "basedpyright==$BASEDPYRIGHT" "ruff==$RUFF" + + - if: matrix.language == 'go' + uses: actions/setup-go@v5 + with: { go-version: "1.27", cache: false } + - if: matrix.language == 'go' + run: go install "golang.org/x/tools/gopls@$GOPLS" + + - if: matrix.language == 'java' || matrix.language == 'kotlin' + uses: actions/setup-java@v4 + with: { distribution: temurin, java-version: "21" } + - if: matrix.language == 'java' + run: | + mkdir -p "$RUNNER_TEMP/jdtls" + curl -fsSL "https://download.eclipse.org/jdtls/milestones/${JDTLS%-*}/jdt-language-server-$JDTLS.tar.gz" \ + | tar xz -C "$RUNNER_TEMP/jdtls" + echo "$RUNNER_TEMP/jdtls/bin" >> "$GITHUB_PATH" + + # The Kotlin fixture has no Gradle wrapper: the server imports it with the Gradle on PATH. + - if: matrix.language == 'kotlin' + uses: gradle/actions/setup-gradle@v4 + with: { gradle-version: "${{ env.GRADLE }}" } + - if: matrix.language == 'kotlin' + run: | + mkdir -p "$RUNNER_TEMP/kotlin-lsp" "$RUNNER_TEMP/bin" + curl -fsSL "https://download.jetbrains.com/language-server/kotlin-server/$KOTLIN_LSP/kotlin-server-$KOTLIN_LSP.tar.gz" \ + | tar xz -C "$RUNNER_TEMP/kotlin-lsp" --strip-components=1 + ln -s "$RUNNER_TEMP/kotlin-lsp/kotlin-lsp.sh" "$RUNNER_TEMP/bin/kotlin-lsp" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + + - run: cargo test --locked -p ide-host --test language_smoke --no-run + - run: cargo test --locked -p ide-host --test language_smoke -- --ignored --exact ${{ matrix.language }} + - if: matrix.language == 'java' + run: cargo test --locked -p ide-host --test project_smoke -- --ignored --exact maven_sync jdk + - if: matrix.language == 'kotlin' + run: cargo test --locked -p ide-host --test project_smoke -- --ignored --exact gradle_sync + - if: matrix.language == 'python' + run: cargo test --locked -p ide-host --test project_smoke -- --ignored --exact python_venv + + - run: pnpm install --frozen-lockfile + - run: cargo build --locked -p ide-host --bin openidea-server + - run: pnpm smoke smoke/run.test.ts -t "^${{ matrix.language }}$" + - run: pnpm smoke smoke/debug.test.ts -t "^${{ matrix.language }}$" + # The IDE installs the debugger from GitHub releases: authenticated, against rate limits. + env: { GITHUB_TOKEN: "${{ github.token }}" } diff --git a/.github/workflows/plugins-index.yml b/.github/workflows/plugins-index.yml new file mode 100644 index 00000000..bbf1d256 --- /dev/null +++ b/.github/workflows/plugins-index.yml @@ -0,0 +1,46 @@ +name: Plugin index + +# Refreshes the community plugins in the latest release's plugins-index.json +# (what released apps read) from the registry, l0ng-ai/openidea-plugins. The +# official plugins, their key and revocations stay as the release wrote them. +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + # Sent by the registry when its index changes. + repository_dispatch: + types: [community-plugins] + +permissions: + contents: write + +concurrency: + group: plugins-index + +env: + COMMUNITY_PLUGINS_INDEX: https://github.com/l0ng-ai/openidea-plugins/releases/download/index/index.json + +jobs: + merge: + runs-on: ubuntu-latest + steps: + - name: Merge the community index + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + tag="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName -q .tagName)" + tool=openidea-plugin-linux-x86_64 + if ! gh release download "$tag" --repo "$GITHUB_REPOSITORY" -p plugins-index.json -p "$tool.gz"; then + echo "::notice::$tag has no plugin index or openidea-plugin tool yet" + exit 0 + fi + gunzip "$tool.gz" && chmod +x "$tool" + cp plugins-index.json before.json + "./$tool" merge plugins-index.json "$COMMUNITY_PLUGINS_INDEX" --keep openidea + if cmp -s before.json plugins-index.json; then + echo "unchanged" + else + gh release upload "$tag" plugins-index.json --repo "$GITHUB_REPOSITORY" --clobber + fi diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index d1a74b7e..02949151 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -18,24 +18,15 @@ jobs: if: github.event.pull_request.user.type != 'Bot' && !github.event.pull_request.draft runs-on: ubuntu-latest steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: script: | const pr = context.payload.pull_request; const LABEL = 'needs-info'; const MARKER = ''; - // Docs, CI and pure refactors may skip the linked issue. - const NO_ISSUE_TYPES = /^(docs|ci|build|chore|refactor|style|test)(\(.+\))?!?:/; - const body = (pr.body || '').replace(//g, ''); const problems = []; - const linksIssue = - /\b(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+(#\d+|https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/issues\/\d+)/i.test(body); - if (!linksIssue && !NO_ISSUE_TYPES.test(pr.title)) { - problems.push('Link the issue this PR resolves, e.g. `Closes #123`.'); - } - const verified = body.match(/^##+\s*How it was verified\s*$([\s\S]*?)(?=^##+\s|(?![\s\S]))/im); if (!verified || !verified[1].trim()) { problems.push('Fill in **How it was verified**: the commands you ran and their results. UI changes need before/after screenshots.'); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3874f17e..1758e67a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,7 @@ env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 TAG: ${{ github.event.inputs.tag || github.ref_name }} + COMMUNITY_PLUGINS_INDEX: https://github.com/l0ng-ai/openidea-plugins/releases/download/index/index.json jobs: # Creates (or reuses) one draft release that every platform build uploads into. @@ -25,8 +26,9 @@ jobs: runs-on: ubuntu-latest outputs: release_id: ${{ steps.release.outputs.id }} + official_plugin_keys: ${{ steps.plugin-keys.outputs.keys }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: { ref: "${{ env.TAG }}" } - name: Tag matches app version run: | @@ -35,8 +37,22 @@ jobs: echo "::error::tag $TAG does not match tauri.conf.json version $app" exit 1 fi + # The official plugins' public key (from PLUGIN_SIGNING_KEY, the seed + # `openidea-plugin keygen` writes), built into the app: their packages + # must carry its signature whatever the marketplace index lists. Without + # the secret the app takes the index's word, as development builds do. + - id: plugin-keys + env: + SIGNING_KEY: ${{ secrets.PLUGIN_SIGNING_KEY }} + run: | + if [ -n "$SIGNING_KEY" ]; then + seed="$(printf %s "$SIGNING_KEY" | base64 -d | od -An -tx1 | tr -d ' \n')" + key="$(printf '302e020100300506032b657004220420%s' "$seed" | xxd -r -p | + openssl pkey -inform DER -pubout -outform DER | tail -c 32 | base64)" + echo "keys=$key" >> "$GITHUB_OUTPUT" + fi - id: release - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const tag = process.env.TAG; @@ -57,6 +73,8 @@ jobs: build: needs: create-release name: Build (${{ matrix.name }}) + env: + OPENIDEA_OFFICIAL_PLUGIN_KEYS: ${{ needs.create-release.outputs.official_plugin_keys }} strategy: fail-fast: false matrix: @@ -81,11 +99,11 @@ jobs: target: x86_64-pc-windows-msvc runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: { ref: "${{ env.TAG }}" } - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@v5 with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: { node-version: 22, cache: pnpm } - uses: dtolnay/rust-toolchain@stable with: { targets: "${{ matrix.target }}" } @@ -155,7 +173,7 @@ jobs: - { runner: macos-latest, target: x86_64-apple-darwin, os: macos, arch: x86_64 } runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: { ref: "${{ env.TAG }}" } - uses: dtolnay/rust-toolchain@stable with: { targets: "${{ matrix.target }}" } @@ -184,7 +202,11 @@ jobs: # (releases/latest/download/plugins-index.json). They are not in the # installers: users install them from Settings › Plugins. Without the # signing key (secret PLUGIN_SIGNING_KEY: the file `openidea-plugin keygen` - # writes) nothing is published and the release goes on. + # writes) nothing is published and the release goes on. The index also lists + # the community plugins of the registry (COMMUNITY_PLUGINS_INDEX; refreshed + # between releases by plugins-index.yml), and the release carries the + # openidea-plugin tool for Linux x64, which that workflow and the registry's + # checks run. plugins: needs: create-release name: Official plugins @@ -198,7 +220,7 @@ jobs: echo "::warning::PLUGIN_SIGNING_KEY is not set: the official plugins are not published" echo "skip=true" >> "$GITHUB_OUTPUT" fi - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 if: steps.key.outputs.skip != 'true' with: { ref: "${{ env.TAG }}" } - uses: dtolnay/rust-toolchain@stable @@ -207,10 +229,10 @@ jobs: - uses: Swatinem/rust-cache@v2 if: steps.key.outputs.skip != 'true' with: { key: release-plugins } - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@v5 if: steps.key.outputs.skip != 'true' with: { version: 11 } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 if: steps.key.outputs.skip != 'true' with: { node-version: 22 } # Builds the web pages; fails when a checked-in component isn't built from these sources. @@ -237,11 +259,17 @@ jobs: version="$(sed -n 's/^version *= *"\(.*\)".*/\1/p' "$dir/plugin.toml" | head -1)" "$tool" pack "$dir" --key "$key" --publisher openidea --out "$out/$id-$version.oiplugin" done - # The published index's revocations and earlier keys carry over. + # The published index's revocations, earlier keys and community plugins carry over. curl -fsSL "https://github.com/$GITHUB_REPOSITORY/releases/latest/download/plugins-index.json" \ -o "$out/plugins-index.json" || rm -f "$out/plugins-index.json" - "$tool" index "$out" --publisher openidea --name OpenIDEA --key "$key" --out "$out/plugins-index.json" - gh release upload "$TAG" "$out"/*.oiplugin "$out/plugins-index.json" --clobber + # Downloads point at this release: older versions kept for older IDEs stay where they were published. + "$tool" index "$out" --publisher openidea --name OpenIDEA --key "$key" --out "$out/plugins-index.json" \ + --download-base "https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG" + "$tool" merge "$out/plugins-index.json" "$COMMUNITY_PLUGINS_INDEX" --keep openidea || + echo "::warning::the community plugins were not refreshed" + gzip -9 -c "$tool" > "$out/openidea-plugin-linux-x86_64.gz" + gh release upload "$TAG" "$out"/*.oiplugin "$out/plugins-index.json" \ + "$out/openidea-plugin-linux-x86_64.gz" --clobber # The updater manifest (latest.json), built once from the signed bundles every build uploaded. # Fails when a platform is missing, so a release never ships a manifest without it. @@ -249,7 +277,7 @@ jobs: needs: [create-release, build] runs-on: ubuntu-latest steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 env: RELEASE_ID: ${{ needs.create-release.outputs.release_id }} with: @@ -309,7 +337,7 @@ jobs: needs: [create-release, build, server, plugins, updater-manifest] runs-on: ubuntu-latest steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 env: RELEASE_ID: ${{ needs.create-release.outputs.release_id }} with: diff --git a/AGENTS.md b/AGENTS.md index 02a191f1..54cc95ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,9 +18,12 @@ Visual style: dark, low-contrast greys, one cool accent - `src/ui/` — design-system primitives. `src/features//` — one feature each, exporting `activate()` registered in `src/features/index.ts`. - `languages/.toml` — language definitions (detection, comments, LSP, DAP). -- `crates/ide-host` — the `openidea-server` binary for remote development. -- `wit/` — plugin interface definitions; `plugins/` — sample and bundled plugins; - `templates/` — plugin starter templates. +- `crates/ide-host` — the `openidea-server` binary for remote development and the + `openidea-plugin` CLI (plugin starter templates live in its `templates.rs`). +- `wit/` — plugin interface definitions; `plugins/` — sample and bundled plugins. + Plugin docs: `plugins/README.md`, `docs/plugins/themes.md`. +- `java/` — JDT LS extension bundles (`java-usages`, `kotlin-bridge`) and the patched + Lombok agent, embedded into the app by `ide-sdk` / `ide-tools`. - `e2e/` — Playwright UI tests (`pnpm e2e`). `fixtures/` — sample projects per language for manual and automated testing. @@ -46,8 +49,8 @@ Visual style: dark, low-contrast greys, one cool accent ## Issues & PRs -Follow `CONTRIBUTING.md` and `.github/pull_request_template.md`: one issue per PR with -`Closes #`, reproduce first, keep the diff focused, short plain description, and a +Follow `CONTRIBUTING.md` and `.github/pull_request_template.md`: one problem per PR +(`Closes #` when it resolves an issue), reproduce first, keep the diff focused, short plain description, and a **How it was verified** section with commands run, results and (for UI) before/after screenshots. Unclear product decisions → `needs-decision` and stop. @@ -64,6 +67,13 @@ Shared agent skills live in `.agents/skills/` (`.claude/skills/` links to them): this worktree's running MCP bridges and actual ports. - `pnpm build` — typecheck + bundle frontend. - `cargo nextest run --workspace` (or `cargo test`), `cargo clippy --workspace --all-targets -- -D warnings`, `cargo fmt --all`. +- `cargo test -p ide-host --test language_smoke -- --ignored --exact ` — code intelligence + on `fixtures/` with that language's real server installed (CI: `Language smoke`, pinned versions). +- `pnpm smoke smoke/run.test.ts -t ` — run and test that language's fixture through the + run store and a real `openidea-server` (build it first: `cargo build -p ide-host --bin openidea-server`). +- `pnpm smoke smoke/debug.test.ts -t ` — debug that fixture's entry point the same way, with + the debugger the IDE installs (on macOS, CodeLLDB and Delve may need Developer Mode: + `sudo DevToolsSecurity -enable`). - `pnpm test`, `pnpm e2e`, `pnpm lint`, `pnpm format:check`, `pnpm knip`, `pnpm typecheck`, `pnpm gen:bindings`. - `pnpm perf` (UI interaction latency / frames / leaks), `pnpm perf:rust` (criterion @@ -111,8 +121,8 @@ Several worktrees can be developed and run side by side. default MCP session belongs to this worktree. - Keep each worktree's Cargo output directory separate; do not point `CARGO_TARGET_DIR` or `target` symlinks at a shared directory. -- Each worktree's first build is cold (around 5 minutes) and its Cargo output takes - 15–30 GB. Avoid running several heavy builds on one machine at the same time. +- Each worktree's first build is cold (several minutes) and `cargo build` output takes + around 9 GB; tests, clippy and incremental builds add more. Avoid running several heavy builds on one machine at the same time. - Stop only processes started for the current worktree/session. Never use broad process-name kills such as `pkill openidea`. - When done: stop that worktree's app and dev session, delete its build output, then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d10aacf..6e02bd6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,8 +16,7 @@ Project layout, coding rules, commands and checks are in [AGENTS.md](AGENTS.md). ## Pull requests -- One issue per PR, linked with `Closes #123`. Feature PRs need a `planned` issue. - Docs, CI and pure refactors may skip the issue. +- One problem per PR. If it resolves an issue, link it with `Closes #123`. - Reproduce the problem first, or write a test that fails before the fix. - Fix the cause, and check whether the same cause breaks anything nearby. - Keep the diff focused. Unrelated refactors go in their own PR. @@ -31,5 +30,9 @@ Project layout, coding rules, commands and checks are in [AGENTS.md](AGENTS.md). - If a change needs a product decision you can't make, ask in the issue and stop rather than guessing. -A check on each PR adds `needs-info` when the issue link or the verification -section is missing, and removes it once they're filled in. +Plugins: pull requests here add or change official plugins (`plugins/official/`) +and the plugin API. Community plugins live in their authors' repositories and +are listed through [l0ng-ai/openidea-plugins](https://github.com/l0ng-ai/openidea-plugins); +see [`plugins/README.md`](plugins/README.md) § 8.1. + +A check on each PR adds `needs-info` when the verification section is missing, and removes it once it's filled in. diff --git a/Cargo.lock b/Cargo.lock index 42891043..317a7da6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,17 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "apple-native-keyring-store" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b350bfd03649e07aa05c0a81b3e15934374e585c98204a57e20b9d49f49bb9a" +dependencies = [ + "keyring-core", + "log", + "security-framework", +] + [[package]] name = "arbitrary" version = "1.5.0" @@ -156,6 +167,27 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "ashpd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f3f79755c74fd155000314eb349864caa787c6592eace6c6882dad873d9c39" +dependencies = [ + "enumflags2", + "futures-channel", + "futures-util", + "rand 0.9.5", + "raw-window-handle", + "serde", + "serde_repr", + "tokio", + "url", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "zbus", +] + [[package]] name = "ast-grep-core" version = "0.40.1" @@ -1600,6 +1632,7 @@ dependencies = [ "lock_api", "once_cell", "parking_lot_core", + "serde", ] [[package]] @@ -1795,6 +1828,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + [[package]] name = "dlopen2" version = "0.8.2" @@ -2163,15 +2205,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "file-id" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc6a637b6dc58414714eddd9170ff187ecb0933d4c7024d1abbd23a3cc26e9" -dependencies = [ - "windows-sys 0.60.2", -] - [[package]] name = "filedescriptor" version = "0.8.3" @@ -4038,7 +4071,7 @@ checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] name = "ide-agent" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dunce", "encoding_rs", @@ -4061,7 +4094,7 @@ dependencies = [ [[package]] name = "ide-analysis" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ast-grep-core", "ide-core", @@ -4075,7 +4108,7 @@ dependencies = [ [[package]] name = "ide-build" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dirs 6.0.0", "dunce", @@ -4090,7 +4123,7 @@ dependencies = [ [[package]] name = "ide-clipboard" -version = "0.1.35" +version = "0.1.36" dependencies = [ "objc2 0.6.5", "objc2-app-kit 0.3.2", @@ -4099,7 +4132,7 @@ dependencies = [ [[package]] name = "ide-core" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dunce", "encoding_rs", @@ -4116,11 +4149,11 @@ dependencies = [ [[package]] name = "ide-coverage" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "ide-run", - "quick-xml", + "quick-xml 0.42.0", "serde", "serde_json", "specta", @@ -4130,7 +4163,7 @@ dependencies = [ [[package]] name = "ide-dap" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "ide-jsonrpc", @@ -4148,14 +4181,17 @@ dependencies = [ [[package]] name = "ide-db" -version = "0.1.35" +version = "0.1.36" dependencies = [ "bb8", "dirs 6.0.0", "futures", "ide-core", + "ide-http", "ide-ssh", "parking_lot", + "reqwest", + "rustls", "serde", "serde_json", "specta", @@ -4166,11 +4202,12 @@ dependencies = [ "tokio-util", "toml 0.9.12+spec-1.1.0", "tracing", + "uuid", ] [[package]] name = "ide-docker" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "serde", @@ -4183,7 +4220,7 @@ dependencies = [ [[package]] name = "ide-extension" -version = "0.1.35" +version = "0.1.36" dependencies = [ "base64 0.22.1", "blake3", @@ -4203,7 +4240,7 @@ dependencies = [ [[package]] name = "ide-fonts" -version = "0.1.35" +version = "0.1.36" dependencies = [ "objc2-core-foundation", "objc2-core-text", @@ -4215,7 +4252,7 @@ dependencies = [ [[package]] name = "ide-github" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "ide-vcs", @@ -4228,7 +4265,7 @@ dependencies = [ [[package]] name = "ide-host" -version = "0.1.35" +version = "0.1.36" dependencies = [ "base64 0.22.1", "criterion", @@ -4268,8 +4305,11 @@ dependencies = [ "ide-workspace", "ignore", "parking_lot", + "quick-xml 0.42.0", + "regex-syntax", "reqwest", "rustix", + "semver", "serde", "serde_json", "specta", @@ -4286,7 +4326,7 @@ dependencies = [ [[package]] name = "ide-http" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "parking_lot", @@ -4304,7 +4344,7 @@ dependencies = [ [[package]] name = "ide-index" -version = "0.1.35" +version = "0.1.36" dependencies = [ "blake3", "criterion", @@ -4326,7 +4366,7 @@ dependencies = [ [[package]] name = "ide-jsonrpc" -version = "0.1.35" +version = "0.1.36" dependencies = [ "encoding_rs", "futures", @@ -4334,6 +4374,7 @@ dependencies = [ "parking_lot", "serde", "serde_json", + "tempfile", "thiserror 2.0.21", "tokio", "tokio-util", @@ -4342,7 +4383,7 @@ dependencies = [ [[package]] name = "ide-language" -version = "0.1.35" +version = "0.1.36" dependencies = [ "blake3", "criterion", @@ -4398,7 +4439,7 @@ dependencies = [ [[package]] name = "ide-lsp" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dunce", "futures", @@ -4424,7 +4465,7 @@ dependencies = [ [[package]] name = "ide-notebook" -version = "0.1.35" +version = "0.1.36" dependencies = [ "futures", "ide-core", @@ -4447,7 +4488,7 @@ dependencies = [ [[package]] name = "ide-plugin" -version = "0.1.35" +version = "0.1.36" dependencies = [ "blake3", "ide-core", @@ -4467,7 +4508,7 @@ dependencies = [ [[package]] name = "ide-profiler" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "ide-run", @@ -4480,7 +4521,7 @@ dependencies = [ [[package]] name = "ide-remote" -version = "0.1.35" +version = "0.1.36" dependencies = [ "globset", "ide-core", @@ -4497,7 +4538,7 @@ dependencies = [ [[package]] name = "ide-run" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dunce", "encoding_rs", @@ -4506,7 +4547,7 @@ dependencies = [ "libc", "parking_lot", "portable-pty", - "quick-xml", + "quick-xml 0.42.0", "serde", "serde_json", "specta", @@ -4520,7 +4561,7 @@ dependencies = [ [[package]] name = "ide-sdk" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dunce", "ide-core", @@ -4539,7 +4580,7 @@ dependencies = [ [[package]] name = "ide-search" -version = "0.1.35" +version = "0.1.36" dependencies = [ "criterion", "globset", @@ -4559,7 +4600,7 @@ dependencies = [ [[package]] name = "ide-spell" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "parking_lot", @@ -4572,11 +4613,13 @@ dependencies = [ [[package]] name = "ide-ssh" -version = "0.1.35" +version = "0.1.36" dependencies = [ "dirs 6.0.0", "flate2", "ide-core", + "keyring", + "keyring-core", "parking_lot", "russh", "russh-config", @@ -4587,11 +4630,12 @@ dependencies = [ "tempfile", "tokio", "tracing", + "uuid", ] [[package]] name = "ide-syntax" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "ide-language", @@ -4606,7 +4650,7 @@ dependencies = [ [[package]] name = "ide-terminal" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "libc", @@ -4619,19 +4663,20 @@ dependencies = [ [[package]] name = "ide-text" -version = "0.1.35" +version = "0.1.36" dependencies = [ "criterion", "ide-core", "proptest", "ropey", "serde", + "similar", "specta", ] [[package]] name = "ide-tools" -version = "0.1.35" +version = "0.1.36" dependencies = [ "flate2", "hex", @@ -4652,14 +4697,16 @@ dependencies = [ [[package]] name = "ide-vcs" -version = "0.1.35" +version = "0.1.36" dependencies = [ + "blake3", "dunce", "encoding_rs", "gix", "ide-core", "ignore", "libc", + "redb", "regex-lite", "serde", "serde_json", @@ -4671,7 +4718,7 @@ dependencies = [ [[package]] name = "ide-vfs" -version = "0.1.35" +version = "0.1.36" dependencies = [ "chardetng", "dunce", @@ -4680,7 +4727,7 @@ dependencies = [ "globset", "ide-core", "ignore", - "notify-debouncer-full", + "notify", "parking_lot", "serde", "serde_json", @@ -4693,7 +4740,7 @@ dependencies = [ [[package]] name = "ide-web" -version = "0.1.35" +version = "0.1.36" dependencies = [ "ide-core", "open", @@ -4709,7 +4756,7 @@ dependencies = [ [[package]] name = "ide-workspace" -version = "0.1.35" +version = "0.1.36" dependencies = [ "blake3", "dunce", @@ -4727,6 +4774,7 @@ dependencies = [ "toml 0.9.12+spec-1.1.0", "toml_edit 0.25.17+spec-1.1.0", "tracing", + "uuid", "zip", ] @@ -5204,6 +5252,33 @@ dependencies = [ "serde", ] +[[package]] +name = "keyring" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2270074a3d26bcac93c1dc5d2845eb4c089e8d761ccf6e0ea266a16004640627" +dependencies = [ + "apple-native-keyring-store", + "keyring-core", + "windows-native-keyring-store", + "zbus-secret-service-keyring-store", +] + +[[package]] +name = "keyring-core" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fbf4b551298c67925f38cf341a1ada797a75f1837bdf8ae1fc8e2ebc6efb59" +dependencies = [ + "chrono", + "dashmap", + "log", + "regex", + "ron", + "serde", + "uuid", +] + [[package]] name = "kqueue" version = "1.2.1" @@ -5626,19 +5701,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "notify-debouncer-full" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375bd3a138be7bfeff3480e4a623df4cbfb55b79df617c055cd810ba466fa078" -dependencies = [ - "file-id", - "log", - "notify", - "notify-types", - "walkdir", -] - [[package]] name = "notify-types" version = "2.1.0" @@ -5667,6 +5729,30 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.5.1" @@ -5679,6 +5765,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -5694,6 +5789,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -6181,7 +6297,7 @@ dependencies = [ [[package]] name = "openidea" -version = "0.1.35" +version = "0.1.36" dependencies = [ "base64 0.22.1", "blake3", @@ -6582,7 +6698,7 @@ checksum = "65dca778f41c79ebbe8b2b2139fee86f01b766b67130d108c7c2b8ffbd949f77" dependencies = [ "base64 0.23.1", "indexmap 2.14.2", - "quick-xml", + "quick-xml 0.42.0", "serde", "time", ] @@ -6627,6 +6743,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "poly1305" version = "0.9.1" @@ -6887,6 +7009,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.42.0" @@ -7133,6 +7264,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -7163,18 +7295,18 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15ad77d9e70a92437d8f74c35d99b4e4691128df018833e99f90bcd36152672" dependencies = [ + "ashpd", "block2 0.6.2", "dispatch2", - "glib-sys", - "gobject-sys", - "gtk-sys", "js-sys", "log", "objc2 0.6.5", "objc2-app-kit 0.3.2", "objc2-core-foundation", "objc2-foundation 0.3.2", + "pollster", "raw-window-handle", + "urlencoding", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -7195,6 +7327,19 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3fe97c18ff24299ea56f802c5693eb34266fb0bc8317f4e101debc8a3ceaf" +dependencies = [ + "bitflags 2.13.2", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + [[package]] name = "ropey" version = "1.6.1" @@ -7261,7 +7406,7 @@ dependencies = [ "md5", "ml-kem", "module-lattice", - "num-bigint", + "num-bigint 0.5.1", "p256", "p384", "p521", @@ -7491,6 +7636,12 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "saa" version = "5.6.2" @@ -7586,6 +7737,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -7636,6 +7793,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secret-service" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5107b24b91445dd2aa449a258a1807b63240942157292354dc5bfdbeb8bc6db8" +dependencies = [ + "aes", + "cbc", + "futures-util", + "getrandom 0.4.3", + "hkdf", + "hybrid-array", + "num", + "once_cell", + "serde", + "sha2 0.11.1", + "zbus", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -7795,6 +7971,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "3.24.0" @@ -9112,7 +9300,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -9300,6 +9488,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.61.2", ] @@ -10636,6 +10825,66 @@ dependencies = [ "wasmtime", ] +[[package]] +name = "wayland-backend" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags 2.13.2", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml 0.41.0", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "log", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.106" @@ -10921,6 +11170,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-native-keyring-store" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063426e76fdec7438d56bb777f67e318a84a25c707b07e575cb8b78e10c028f8" +dependencies = [ + "byteorder", + "keyring-core", + "regex", + "windows-sys 0.61.2", + "zeroize", +] + [[package]] name = "windows-numerics" version = "0.3.1" @@ -11447,6 +11709,7 @@ dependencies = [ "rustix", "serde", "serde_repr", + "tokio", "tracing", "uds_windows", "uuid", @@ -11457,6 +11720,17 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zbus-secret-service-keyring-store" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74801d001b9e7729adb4f1825b67b398185fed424749aa3d8bacf70417137d9a" +dependencies = [ + "keyring-core", + "secret-service", + "zbus", +] + [[package]] name = "zbus_macros" version = "5.19.0" @@ -11662,6 +11936,7 @@ dependencies = [ "endi", "enumflags2", "serde", + "url", "winnow 1.0.4", "zcheapstr", "zvariant_derive", diff --git a/Cargo.toml b/Cargo.toml index a1c42bf2..ea4e1506 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = ["crates/*", "src-tauri"] exclude = ["sdk", "plugins"] [workspace.package] -version = "0.1.35" +version = "0.1.36" edition = "2024" rust-version = "1.96" license = "Apache-2.0" @@ -76,7 +76,6 @@ ignore = "0.4" base64 = "0.22" globset = "0.4" notify = "8" -notify-debouncer-full = "0.6" trash = "5" blake3 = "1" dunce = "1" @@ -86,6 +85,7 @@ zip = { version = "9", default-features = false, features = ["deflate"] } # search, index, language grep-searcher = "0.1" grep-regex = "0.1" +regex-syntax = "0.8" grep-matcher = "0.1" nucleo-matcher = "0.3" redb = "4" @@ -105,6 +105,9 @@ sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", tiberius = { version = "0.13", default-features = false, features = ["tds73", "rustls", "sql-browser-tokio"] } bb8 = "0.9" uuid = { version = "1", features = ["v4"] } +# OS credential stores +keyring = "4.2" +keyring-core = "1" # testing insta = { version = "1", features = ["yaml"] } @@ -114,8 +117,14 @@ tempfile = "3" [profile.dev] # Keep debug builds snappy while giving dependencies decent speed. opt-level = 0 +# Full debug info is most of target/. Line tables keep file:line in our backtraces, but a +# debugger can't show locals; re-enable with `--config profile.dev.debug=true`. +debug = "line-tables-only" [profile.dev.package."*"] opt-level = 2 +# Dependency frames keep function names only. Re-enable one with +# `--config 'profile.dev.package..debug=true'`. +debug = false [profile.release] codegen-units = 1 diff --git a/README.md b/README.md index 59b4f7c0..0e0ca9dc 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@  ·  Benchmark  ·  + Plugins +  ·  Discord

@@ -45,6 +47,16 @@ IDEA analyses Python more deeply, so "ready" doesn't mean the same work on both sides. Both are measured from the outside, the same way: see [`perf/compare`](perf/compare) to run it on your machine and projects. +## Plugins + +Plugins add actions, completion, quick fixes, tool windows, custom editors, +themes, file icons and languages. They run as sandboxed WebAssembly components; +themes, icon themes and languages need no code at all. + +Install them from Settings › Plugins, or start your own with +`openidea-plugin new` — see [Writing OpenIDEA plugins](plugins/README.md) and +the [official plugins](plugins/official). The plugin API is still a preview. + ## License [Apache 2.0](LICENSE). Bundled third-party assets are listed in [`THIRD_PARTY_LICENSES.md`](THIRD_PARTY_LICENSES.md). diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index ad504787..2ebfd19a 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -2,12 +2,12 @@ ## Project Lombok -`plugins/lombok/lombok-1.18.48-openidea.2.jar` contains Project Lombok v1.18.48 with the Equinox +`java/lombok/lombok-1.18.48-openidea.2.jar` contains Project Lombok v1.18.48 with the Equinox classloader fix from [projectlombok/lombok#4059](https://github.com/projectlombok/lombok/pull/4059) and the ECJ constructor-call fix from [projectlombok/lombok#4089](https://github.com/projectlombok/lombok/pull/4089). Copyright (C) 2009-2021 The Project Lombok Authors. MIT license and bundled component -notices are retained in [plugins/lombok/LICENSE](plugins/lombok/LICENSE) and inside the jar. -Source tag, patch commits and build instructions: [plugins/lombok/README.md](plugins/lombok/README.md). +notices are retained in [java/lombok/LICENSE](java/lombok/LICENSE) and inside the jar. +Source tag, patch commits and build instructions: [java/lombok/README.md](java/lombok/README.md). ## JetBrains IntelliJ Platform icons @@ -52,6 +52,15 @@ Their original copyright headers and Apache-2.0 licensing are retained. At render time `src/ui/icons.tsx` may recolour monochrome icons to `currentColor`. +## JetBrains IntelliJ Platform VCS log graph algorithms + +`crates/ide-vcs/src/bek.rs` and `src/features/vcs/graph.ts` adapt the Bek ordering, +graph layout and LinearBek merge linearization algorithms from +[intellij-community](https://github.com/JetBrains/intellij-community/tree/1d70d5c54302d655758edbd2f808fb8dea8b1764/platform/vcs-log/graph/src/com/intellij/vcs/log/graph) +(`impl/facade/sort/bek`, `impl/permanent/GraphLayoutBuilder.kt` and +`linearBek/LinearBekGraphBuilder.java`). Modified from the originals. +Copyright 2000-2026 JetBrains s.r.o. and contributors. Apache License 2.0 (below). + ## JetBrains IntelliJ Platform diff algorithm `src/features/vcs/comparison/**` is a TypeScript port of the text comparison of diff --git a/crates/ide-agent/src/lib.rs b/crates/ide-agent/src/lib.rs index a65400a6..7116047f 100644 --- a/crates/ide-agent/src/lib.rs +++ b/crates/ide-agent/src/lib.rs @@ -58,6 +58,8 @@ pub(crate) const LIMITS_UPDATE: &str = "openidea_limits"; const MAX_EMBED: u64 = 256 * 1024; /// How long a probe waits for the agent to answer `initialize`. const PROBE_TIMEOUT: Duration = Duration::from_secs(30); +/// How long deleting a task waits for its agent to exit before removing the worktree. +const EXIT_WAIT: Duration = Duration::from_secs(5); /// The agent tasks of one project. pub struct AgentManager { @@ -839,7 +841,7 @@ impl AgentManager { /// Forgets the task; with `remove_worktree` also deletes its worktree (the branch stays). pub async fn delete(&self, id: &str, remove_worktree: bool) -> IdeResult<()> { let task = self.task(id)?; - task.kill(); + let live = task.kill(); let (isolation, cwd, root) = { let s = task.state.lock(); ( @@ -851,6 +853,11 @@ impl AgentManager { // A copy of the task (a fork) may work in the same worktree. let shared = self.tasks.read().values().any(|t| t.id != id && t.state.lock().task.cwd.0 == cwd); if remove_worktree && !shared && isolation == Isolation::Worktree && cwd.exists() { + // On Windows the agent's processes lock the folder (their working directory, + // open files) until they have exited. + if let Some(peer) = live { + let _ = tokio::time::timeout(EXIT_WAIT, peer.exited()).await; + } let worktrees = self.shared.paths.worktrees_dir.clone(); blocking(move || worktree::remove(&root, &worktrees, &cwd)).await?; } @@ -1605,11 +1612,12 @@ impl Task { } } - fn kill(&self) { + /// Stops the agent; returns its connection, to wait for its process to exit. + fn kill(&self) -> Option { self.cancel_permissions(); - if let Some(live) = self.live.lock().take() { - live.peer.kill(); - } + let live = self.live.lock().take()?; + live.peer.kill(); + Some(live.peer) } } diff --git a/crates/ide-agent/src/tests.rs b/crates/ide-agent/src/tests.rs index e33338a7..e1da00d1 100644 --- a/crates/ide-agent/src/tests.rs +++ b/crates/ide-agent/src/tests.rs @@ -1109,3 +1109,114 @@ async fn idle_agents_stop_and_resume_their_session_on_the_next_message() { assert!(live(&mgr, &task.id)); mgr.shutdown(); } + +/// Deleting a task whose agent still runs in the worktree removes both and leaves no +/// process behind, whether it is waiting for permission or idle without session/load. +#[tokio::test(flavor = "multi_thread")] +async fn deleting_a_task_waits_for_its_agent_to_exit_before_removing_the_worktree() { + for no_load in [false, true] { + delete_running_agent(no_load).await; + } +} + +async fn delete_running_agent(no_load: bool) { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + let git = |args: &[&str]| { + let out = std::process::Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(&root) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + }; + git(&["init", "-q", "-b", "main"]); + std::fs::write(root.join("file.txt"), "original\n").unwrap(); + git(&["add", "."]); + git(&["commit", "-qm", "initial"]); + + let agent = Arc::new(FakeAgent { no_load, ..FakeAgent::default() }); + let pids: Arc>> = Arc::default(); + let (fake, started) = (agent.clone(), pids.clone()); + // The fake agent, served on behalf of a real process that exits only when killed. + let connector: Connector = Arc::new(move |_spec, cwd, client, _stderr| { + #[cfg(unix)] + let mut cmd = { + let mut cmd = tokio::process::Command::new("sleep"); + cmd.arg("30"); + cmd + }; + #[cfg(windows)] + let mut cmd = { + let mut cmd = tokio::process::Command::new("cmd"); + cmd.args(["/D", "/C", "echo ready>agent-ready & ping -n 30 127.0.0.1 >nul"]); + cmd + }; + ide_core::process::prepare(cmd.as_std_mut()); + ide_core::process::own_group(cmd.as_std_mut()); + cmd.current_dir(cwd).kill_on_drop(true); + let child = cmd.spawn()?; + started.lock().push(child.id().unwrap()); + let (a, b) = duplex(1 << 20); + let (ar, aw) = tokio::io::split(a); + let (br, bw) = tokio::io::split(b); + // `connect_with_child` frames with Content-Length. + fake.servers.lock().push(Peer::connect(br, bw, fake.clone())); + Ok(Peer::connect_with_child(ar, aw, client, child)) + }); + let mgr = AgentManager::with_connector( + AgentPaths { + project_root: root.clone(), + store_dir: root.join("store"), + worktrees_dir: root.join("worktrees"), + models_file: root.join("models.json"), + custom_agents_dir: None, + home_dir: None, + }, + tokio::runtime::Handle::current(), + connector, + ); + let task = mgr + .create(NewAgentTask { + project_root: None, + prompt: text(if no_load { "again" } else { "Fix the lexer" }), + provider: "claude".into(), + config: vec![], + isolation: Isolation::Worktree, + base: None, + }) + .await + .unwrap(); + wait_for(&mgr, &task.id, if no_load { AgentStatus::Ready } else { AgentStatus::NeedsInput }).await; + let pid = *pids.lock().last().expect("the agent was started"); + + #[cfg(windows)] + { + tokio::time::timeout(Duration::from_secs(5), async { + while !task.cwd.0.join("agent-ready").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!( + std::fs::rename(&task.cwd.0, task.cwd.0.with_extension("moved")).unwrap_err().raw_os_error(), + Some(32), + "the running agent really locks its worktree" + ); + } + + mgr.delete(&task.id, true).await.unwrap(); + assert!(!task.cwd.0.exists()); + // Killed and reaped: not even a zombie is left. + #[cfg(unix)] + let alive = std::process::Command::new("kill").args(["-0", &pid.to_string()]).status().unwrap().success(); + #[cfg(windows)] + let alive = { + let mut cmd = std::process::Command::new("tasklist"); + ide_core::process::prepare(&mut cmd).args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]); + String::from_utf8_lossy(&cmd.output().unwrap().stdout).contains(&format!("\"{pid}\"")) + }; + assert!(!alive, "the agent (pid {pid}) is still running"); +} diff --git a/crates/ide-agent/src/worktree.rs b/crates/ide-agent/src/worktree.rs index b0e2f62f..ddd45c41 100644 --- a/crates/ide-agent/src/worktree.rs +++ b/crates/ide-agent/src/worktree.rs @@ -5,7 +5,7 @@ use ide_core::{IdeError, IdeResult, IpcPath}; use ide_vcs::{ChangeKind, Repo}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use std::time::UNIX_EPOCH; +use std::time::{Duration, UNIX_EPOCH}; pub(crate) struct Created { pub path: PathBuf, @@ -65,7 +65,27 @@ pub(crate) fn remove(project_root: &Path, worktrees_dir: &Path, path: &Path) -> if !path.starts_with(worktrees_dir) || path == worktrees_dir { return Err(err); } - std::fs::remove_dir_all(path).map_err(|e| IdeError::internal(e.to_string())) + retrying(REMOVE_ATTEMPTS, REMOVE_PAUSE, || std::fs::remove_dir_all(path)) + .map_err(|e| IdeError::internal(format!("{}: {e}", path.display()))) +} + +/// On Windows a folder can't be deleted while a process still has a file in it open +/// (os error 32): an agent's helper that outlived it, an antivirus scan, the search indexer. +/// Those let go within moments, so removing is retried for a few seconds. +const REMOVE_ATTEMPTS: u32 = 10; +const REMOVE_PAUSE: Duration = Duration::from_millis(300); + +fn retrying(attempts: u32, pause: Duration, mut op: impl FnMut() -> std::io::Result<()>) -> std::io::Result<()> { + for _ in 1..attempts { + match op() { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => std::thread::sleep(pause), + _ => return Ok(()), + } + } + match op() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + r => r, + } } /// Files under `cwd` that differ from `base` (untracked files included), with line counts. @@ -149,6 +169,54 @@ fn unique_slug(base: &str, taken: impl Fn(&str) -> bool) -> String { mod tests { use super::*; + #[test] + fn removing_is_retried_while_a_file_is_still_in_use() { + use std::io::{Error, ErrorKind}; + // A sharing violation as Windows reports it (os error 32), a vanished folder, other errors. + let in_use = || Error::from_raw_os_error(32); + for (fails, outcome) in [(0, true), (3, true), (REMOVE_ATTEMPTS - 1, true), (REMOVE_ATTEMPTS, false)] { + let mut calls = 0; + let r = retrying(REMOVE_ATTEMPTS, Duration::ZERO, || { + calls += 1; + if calls <= fails { Err(in_use()) } else { Ok(()) } + }); + assert_eq!(r.is_ok(), outcome, "{fails} failures"); + assert_eq!(calls, (fails + 1).min(REMOVE_ATTEMPTS)); + } + let mut calls = 0; + let gone = retrying(REMOVE_ATTEMPTS, Duration::ZERO, || { + calls += 1; + if calls == 1 { Err(in_use()) } else { Err(Error::from(ErrorKind::NotFound)) } + }); + assert!(gone.is_ok(), "deleted meanwhile: done"); + assert_eq!(calls, 2); + } + + #[cfg(windows)] + #[test] + fn removing_waits_for_a_windows_file_lock_to_be_released() { + use std::os::windows::fs::OpenOptionsExt; + let tmp = tempfile::tempdir().unwrap(); + let repo = tmp.path().join("repo"); + std::fs::create_dir(&repo).unwrap(); + git(&repo, &["init", "-q", "-b", "main"]); + std::fs::write(repo.join("a.txt"), "one\n").unwrap(); + git(&repo, &["add", "."]); + git(&repo, &["commit", "-q", "-m", "init"]); + let worktrees = tmp.path().join("wt"); + let created = create(&repo, &worktrees, "Locked file", None).unwrap(); + let file = std::fs::OpenOptions::new().read(true).share_mode(0).open(created.path.join("a.txt")).unwrap(); + assert_eq!(std::fs::remove_file(created.path.join("a.txt")).unwrap_err().raw_os_error(), Some(32)); + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(800)); + drop(file); + }); + let result = remove(&repo, &worktrees, &created.path); + release.join().unwrap(); + result.unwrap(); + assert!(!created.path.exists()); + } + #[test] fn slugs_are_branch_safe() { assert_eq!(slugify("Fix lexer UTF-16 columns!"), "fix-lexer-utf-16-columns"); diff --git a/crates/ide-build/src/application_update.rs b/crates/ide-build/src/application_update.rs index cabd8a73..96a7ad0d 100644 --- a/crates/ide-build/src/application_update.rs +++ b/crates/ide-build/src/application_update.rs @@ -40,7 +40,7 @@ pub fn plan(root: &Path, model: &BuildModel, module: &Path, cache: &Path) -> Ide kind: "maven", }); } - if let Some(g) = &model.gradle + if let Some(g) = model.gradle_for_module(root, &module) && let Some(p) = g.projects.iter().find(|p| root.join(&p.dir) == module) { let script = cache.join("resources.init.gradle"); @@ -163,11 +163,11 @@ mod tests { assert!(maven.args.contains(&"-Dexpression=project.build.outputDirectory".into())); assert_eq!(maven.env["JAVA_HOME"], "/jdk"); model.maven = None; - model.gradle = Some(crate::GradleModel { + model.gradle = vec![crate::GradleModel { command: "/tools/gradlew".into(), projects: vec![crate::GradleProject { path: ":app".into(), dir: "app".into(), ..Default::default() }], ..Default::default() - }); + }]; let gradle = plan(&root, &model, Path::new("app"), &cache).unwrap(); assert_eq!(gradle.command, "/tools/gradlew"); assert_eq!(gradle.args.last().unwrap(), ":app:openideaUpdateResources"); diff --git a/crates/ide-build/src/compile/mod.rs b/crates/ide-build/src/compile/mod.rs index 603d5b3f..fcfa45f0 100644 --- a/crates/ide-build/src/compile/mod.rs +++ b/crates/ide-build/src/compile/mod.rs @@ -147,7 +147,9 @@ fn detect_dir(dir: &Path, model: &BuildModel, root: &Path) -> Option Option PathBuf { - rel.split('/').filter(|part| !part.is_empty()).fold(root.to_path_buf(), |path, part| path.join(part)) + root.join(rel).components().collect() } fn rel_str(root: &Path, dir: &Path) -> String { - dir.strip_prefix(root).map(|r| r.to_string_lossy().replace('\\', "/")).unwrap_or_default() + dir.strip_prefix(root).unwrap_or(dir).to_string_lossy().replace('\\', "/") } /// Directories from `file`'s folder up to `root` (inclusive). @@ -192,7 +191,14 @@ pub fn plan( let project_name = root.file_name().map_or_else(|| "project".to_owned(), |n| n.to_string_lossy().into_owned()); let module_dir = match file { Some(f) => { - let dir = ancestors_within(root, f) + let scope = model + .gradle + .iter() + .flat_map(|g| std::iter::once(root.join(&g.dir)).chain(g.projects.iter().map(|p| root.join(&p.dir)))) + .filter(|dir| f.starts_with(dir)) + .max_by_key(|dir| dir.components().count()) + .unwrap_or_else(|| root.to_path_buf()); + let dir = ancestors_within(&scope, f) .find(|d| detect_dir(d, model, root).is_some()) .ok_or_else(|| IdeError::not_found(format!("{} is not in a module", f.display())))?; Some(dir.to_path_buf()) @@ -202,7 +208,7 @@ pub fn plan( let dir = module_dir.clone().unwrap_or_else(|| root.to_path_buf()); // A folder of sibling projects has no build file itself; its Maven projects are built one by one. let system = detect_dir(&dir, model, root) - .or_else(|| if model.gradle.is_some() { Some(BuildSystem::Gradle) } else { None }) + .or((!model.gradle.is_empty()).then_some(BuildSystem::Gradle)) .or_else(|| model.maven.as_ref().filter(|m| !m.projects.is_empty()).map(|_| BuildSystem::Maven)) .ok_or_else(|| { IdeError::not_found("no supported build system (Maven, Gradle, Cargo, Go, TypeScript, CMake) found") @@ -270,14 +276,16 @@ fn maven_steps(root: &Path, model: &BuildModel, kind: BuildKind, module: Option< } fn gradle_steps(root: &Path, model: &BuildModel, kind: BuildKind, module: Option<&Path>) -> IdeResult> { - let fallback; - let g = match &model.gradle { - Some(g) => g, - None => { - fallback = crate::gradle::load(root).ok_or_else(|| IdeError::not_found("no Gradle build found"))?; - &fallback - } - }; + if module.is_none() && !model.gradle.is_empty() { + return Ok(model.gradle.iter().flat_map(|g| gradle_step(root, g, kind, None)).collect()); + } + let g = module + .and_then(|m| model.gradle_for_module(root, m)) + .ok_or_else(|| IdeError::not_found("module does not belong to a linked Gradle build"))?; + Ok(gradle_step(root, g, kind, module)) +} + +fn gradle_step(root: &Path, g: &crate::GradleModel, kind: BuildKind, module: Option<&Path>) -> Vec { let prefix = module .and_then(|m| { let rel = rel_str(root, m); @@ -296,7 +304,7 @@ fn gradle_steps(root: &Path, model: &BuildModel, kind: BuildKind, module: Option let mut s = BuildStep::new(&g.command, &["--console=plain"], &subdir(root, &g.dir), Format::Gradle); s.args.extend(tasks); s.env = g.env.clone(); - Ok(vec![s]) + vec![s] } fn cargo_steps(dir: &Path, kind: BuildKind) -> Vec { @@ -811,6 +819,77 @@ fn launch_class_dirs(system: BuildSystem, module: &Path, test: bool) -> Vec(root: &Path, dirs: impl IntoIterator) pub fn load(root: &Path) -> Option { let dir = build_dir(root)?; + load_build(root, dir.strip_prefix(root).unwrap_or(&dir)) +} + +/// Loads exactly this build, without searching its child folders. +pub fn load_build(root: &Path, dir: &Path) -> Option { + let dir = root.join(dir); + if !SETTINGS_FILES.iter().chain(BUILD_FILES).any(|name| dir.join(name).is_file()) { + return None; + } let settings = read_first(&dir, SETTINGS_FILES).map(|(_, t)| parse_settings(&t)).unwrap_or_default(); let root_name = settings.name.unwrap_or_else(|| dir.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()); @@ -455,6 +464,56 @@ pub fn load(root: &Path) -> Option { Some(GradleModel { dir: rel(root, &dir), projects, java_version, command: wrapper(&dir), ..GradleModel::default() }) } +/// Validates user-selected build folders and stores a stable, deduplicated identity. +pub fn normalize_linked_projects(root: &Path, dirs: &[String]) -> ide_core::IdeResult> { + let root = dunce::canonicalize(root)?; + let mut out = Vec::new(); + for dir in dirs { + let path = dunce::canonicalize(root.join(dir))?; + if !path.is_dir() || !SETTINGS_FILES.iter().chain(BUILD_FILES).any(|name| path.join(name).is_file()) { + return Err(ide_core::IdeError::invalid(format!("No Gradle build in {}", path.display()))); + } + let dir = rel(&root, &path); + if !out.contains(&dir) { + out.push(dir); + } + } + Ok(out) +} + +/// Linked build identities without reading their scripts (also used by index hot paths). +pub(crate) fn linked_dirs(root: &Path, settings: &ide_core::settings::GradleSettings) -> Vec { + match &settings.linked_projects { + None => build_dir(root).map(|dir| rel(root, &dir)).into_iter().collect(), + Some(dirs) => { + let mut out = Vec::new(); + for dir in dirs { + if let Ok(dirs) = normalize_linked_projects(root, std::slice::from_ref(dir)) { + for dir in dirs { + if !out.contains(&dir) { + out.push(dir); + } + } + } + } + out + } + } +} + +pub fn load_linked(root: &Path, settings: &ide_core::settings::GradleSettings) -> Vec { + linked_dirs(root, settings).iter().filter_map(|dir| load_build(root, Path::new(dir))).collect() +} + +/// A build's task paths are local to it, so sync caches must be local to its identity too. +pub fn cache_dir(cache: &Path, build_dir: &str) -> PathBuf { + if build_dir.is_empty() { + return cache.to_path_buf(); + } + let hash = build_dir.bytes().fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(b)).wrapping_mul(0x100000001b3)); + cache.join("gradle").join(format!("{hash:016x}")) +} + /// `JavaLanguageVersion.of(21)` / `sourceCompatibility = JavaVersion.VERSION_17` in a build script. pub fn script_java_version(text: &str) -> Option { let number = |s: &str| -> Option { @@ -616,6 +675,41 @@ pub(crate) fn has_war_plugin(script: &str) -> bool { #[cfg(test)] mod tests { + #[test] + fn explicit_linkage_preserves_default_discovery_and_can_unlink_every_build() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + for dir in ["a", "b"] { + std::fs::create_dir(root.join(dir)).unwrap(); + std::fs::write(root.join(dir).join("settings.gradle"), format!("rootProject.name = '{dir}'")).unwrap(); + } + assert_eq!(super::load_linked(root, &Default::default())[0].dir, "a"); + let mut settings = ide_core::settings::GradleSettings { + linked_projects: Some(vec!["b".into(), "./b".into(), "a".into()]), + ..Default::default() + }; + let models = super::load_linked(root, &settings); + assert_eq!(models.iter().map(|g| g.dir.as_str()).collect::>(), ["b", "a"]); + settings.linked_projects = Some(vec![]); + assert!(super::load_linked(root, &settings).is_empty()); + assert!(super::normalize_linked_projects(root, &["missing".into()]).is_err()); + assert!(super::normalize_linked_projects(root, &["".into()]).is_err()); + assert!(super::load_build(root, root).is_none(), "exact loading never picks a child"); + } + + #[cfg(unix)] + #[test] + fn linked_directories_deduplicate_symlinks_and_keep_external_builds() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("workspace"); + let external = temp.path().join("external"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&external).unwrap(); + std::fs::write(external.join("build.gradle"), "").unwrap(); + std::os::unix::fs::symlink(&external, root.join("alias")).unwrap(); + let dirs = super::normalize_linked_projects(&root, &["alias".into(), "../external".into()]).unwrap(); + assert_eq!(dirs, [external.canonicalize().unwrap().to_string_lossy().to_string()]); + } #[test] fn script_dependencies_read_string_and_map_notations() { let script = r#" diff --git a/crates/ide-build/src/gradle_deps.rs b/crates/ide-build/src/gradle_deps.rs new file mode 100644 index 00000000..109fea96 --- /dev/null +++ b/crates/ide-build/src/gradle_deps.rs @@ -0,0 +1,268 @@ +//! A Gradle project's resolved dependencies for the Dependency Analyzer: +//! `gradle :dependencies --configuration `, parsed from its tree report. + +use crate::gradle; +use crate::process; +use crate::sync::{SyncOptions, error_summary}; +use ide_core::{CancellationToken, IdeError, IdeResult}; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::path::Path; + +/// One line of the report: an occurrence of a dependency where it is requested. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct GradleDependency { + /// `group`, empty for a project dependency. + pub group: String, + /// Artifact name, or the project path (`:core`) of a project dependency. + pub name: String, + /// The version asked for here; `None` when a platform or constraint supplies it. + pub requested: Option, + /// The version Gradle selected (`-> 1.2`), when it differs from the request. + pub selected: Option, + /// A project of the build (`project :core`, or a module substituted by one). + pub project: bool, + /// `(*)`: listed with its dependencies earlier in the tree. + pub repeated: bool, + /// `(c)`: a dependency constraint rather than a dependency. + pub constraint: bool, + /// `(n)` / `FAILED`: not resolved. + pub unresolved: bool, + pub children: Vec, +} + +/// Parses the tree of a `dependencies --configuration` report (lines `+--- g:a:1.0 -> 1.1 (*)`). +pub fn parse_report(output: &str) -> Vec { + let mut roots: Vec = Vec::new(); + // Path of child indexes from the roots to the last node of each depth. + let mut path: Vec = Vec::new(); + for line in output.lines() { + let Some((depth, text)) = tree_line(line) else { continue }; + let node = parse_entry(text); + path.truncate(depth); + if depth > path.len() { + continue; // A line deeper than its parent allows: not part of a well-formed tree. + } + let siblings = if depth == 0 { + &mut roots + } else { + let mut list = &mut roots; + for &i in &path[..depth - 1] { + list = &mut list[i].children; + } + let Some(parent) = list.get_mut(path[depth - 1]) else { continue }; + &mut parent.children + }; + siblings.push(node); + path.push(siblings.len() - 1); + } + roots +} + +/// `| \--- g:a:1.0` → (1, "g:a:1.0"). +fn tree_line(line: &str) -> Option<(usize, &str)> { + let mut rest = line; + let mut depth = 0; + loop { + if let Some(r) = rest.strip_prefix("+--- ").or_else(|| rest.strip_prefix("\\--- ")) { + return Some((depth, r.trim())); + } + rest = rest.strip_prefix("| ").or_else(|| rest.strip_prefix(" "))?; + depth += 1; + } +} + +fn parse_entry(text: &str) -> GradleDependency { + let mut d = GradleDependency::default(); + let mut text = text.trim(); + loop { + if let Some(t) = text.strip_suffix(" (*)") { + d.repeated = true; + text = t; + } else if let Some(t) = text.strip_suffix(" (c)") { + d.constraint = true; + text = t; + } else if let Some(t) = text.strip_suffix(" (n)").or_else(|| text.strip_suffix(" FAILED")) { + d.unresolved = true; + text = t; + } else { + break; + } + text = text.trim_end(); + } + let (left, right) = match text.split_once(" -> ") { + Some((l, r)) => (l.trim(), Some(r.trim())), + None => (text, None), + }; + if let Some(project) = left.strip_prefix("project ") { + d.project = true; + d.name = project_path(project); + } else { + let mut parts = left.splitn(3, ':'); + d.group = parts.next().unwrap_or_default().to_owned(); + d.name = parts.next().unwrap_or_default().to_owned(); + d.requested = parts.next().filter(|v| !v.is_empty()).map(str::to_owned); + } + if let Some(right) = right { + if let Some(project) = right.strip_prefix("project ") { + // A module substituted by a project of the build. + d.project = true; + d.selected = Some(project_path(project)); + } else { + d.selected = Some(right.to_owned()); + } + } + d +} + +/// `:core`, or `':core'` as Gradle 9 quotes it. +fn project_path(text: &str) -> String { + let t = text.trim(); + t.strip_prefix('\'').and_then(|t| t.strip_suffix('\'')).unwrap_or(t).to_owned() +} + +/// Runs the `dependencies` report of `project` (a Gradle path, `:` for the root) for +/// `configuration` and returns its tree. +pub fn gradle_dependencies( + root: &Path, + model: &gradle::GradleModel, + options: &SyncOptions, + project: &str, + configuration: &str, + cancel: &CancellationToken, +) -> IdeResult> { + if !configuration.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') || configuration.is_empty() { + return Err(IdeError::invalid(format!("not a configuration name: {configuration}"))); + } + if !project.starts_with(':') || project.chars().any(char::is_whitespace) { + return Err(IdeError::invalid(format!("not a Gradle project path: {project}"))); + } + let task = if project == ":" { ":dependencies".to_owned() } else { format!("{project}:dependencies") }; + let mut args: Vec = + vec!["--console=plain".into(), "-q".into(), task, "--configuration".into(), configuration.into()]; + if options.offline { + args.push("--offline".into()); + } + let out = process::run( + options.gradle_command.as_deref().unwrap_or(&model.command), + &args, + &root.join(&model.dir), + &options.env, + cancel, + |_, _| {}, + )?; + if !out.success { + return Err(IdeError::unavailable(error_summary(&out.text))); + } + Ok(parse_report(&out.text)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dep(group: &str, name: &str, requested: Option<&str>) -> GradleDependency { + GradleDependency { + group: group.into(), + name: name.into(), + requested: requested.map(Into::into), + ..Default::default() + } + } + + #[test] + fn parses_the_tree_with_selected_versions_and_markers() { + let report = "\ +------------------------------------------------------------ +Project ':app' +------------------------------------------------------------ + +runtimeClasspath - Runtime classpath of source set 'main'. ++--- org.springframework.boot:spring-boot-starter-web -> 3.2.0 +| +--- org.springframework.boot:spring-boot-starter:3.2.0 +| | \\--- org.yaml:snakeyaml:2.2 +| \\--- com.fasterxml.jackson.core:jackson-databind:2.15.0 -> 2.15.3 (*) ++--- project ':core' +| \\--- com.google.guava:guava:{strictly 32.1.3-jre} -> 32.1.3-jre (c) ++--- org.example:legacy:1.0 -> project :legacy +\\--- org.missing:gone:9 FAILED + +(c) - A dependency constraint, not a dependency. The dependency affected by the constraint occurs elsewhere in the tree. +(*) - Indicates repeated occurrences of a transitive dependency subtree. Gradle expands transitive dependency subtrees only once per project; repeat occurrences only display the root of the subtree, followed by this annotation. +"; + let tree = parse_report(report); + assert_eq!(tree.len(), 4); + let web = &tree[0]; + assert_eq!((web.group.as_str(), web.name.as_str()), ("org.springframework.boot", "spring-boot-starter-web")); + assert_eq!((web.requested.as_deref(), web.selected.as_deref()), (None, Some("3.2.0"))); + assert_eq!(web.children.len(), 2); + assert_eq!(web.children[0].children, vec![dep("org.yaml", "snakeyaml", Some("2.2"))]); + let databind = &web.children[1]; + assert_eq!((databind.requested.as_deref(), databind.selected.as_deref()), (Some("2.15.0"), Some("2.15.3"))); + assert!(databind.repeated); + let core = &tree[1]; + assert!(core.project && core.group.is_empty()); + assert_eq!(core.name, ":core"); + let guava = &core.children[0]; + assert!(guava.constraint); + assert_eq!(guava.requested.as_deref(), Some("{strictly 32.1.3-jre}")); + let legacy = &tree[2]; + assert!(legacy.project); + assert_eq!((legacy.name.as_str(), legacy.selected.as_deref()), ("legacy", Some(":legacy"))); + assert!(tree[3].unresolved); + assert_eq!(tree[3].requested.as_deref(), Some("9")); + } + + #[test] + fn an_empty_configuration_has_no_dependencies() { + assert!( + parse_report("compileClasspath - Compile classpath for source set 'main'.\nNo dependencies\n").is_empty() + ); + } + + #[test] + fn rejects_names_that_are_not_configurations_or_projects() { + let tmp = tempfile::tempdir().unwrap(); + let options = SyncOptions::default(); + let cancel = CancellationToken::new(); + assert!( + gradle_dependencies(tmp.path(), &Default::default(), &options, ":", "x --init-script y", &cancel).is_err() + ); + assert!( + gradle_dependencies(tmp.path(), &Default::default(), &options, "app", "runtimeClasspath", &cancel).is_err() + ); + } + + #[cfg(unix)] + #[test] + fn runs_the_report_of_the_project_and_configuration() { + use std::os::unix::fs::PermissionsExt; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("settings.gradle"), "include 'app'").unwrap(); + std::fs::write(root.join("build.gradle"), "").unwrap(); + let launcher = root.join("fake-gradle"); + std::fs::write(&launcher, "#!/bin/sh\necho \"$@\" > args\necho '\\--- org.a:b:1.0'\n").unwrap(); + std::fs::set_permissions(&launcher, std::fs::Permissions::from_mode(0o755)).unwrap(); + let options = SyncOptions { + gradle_command: Some(launcher.to_string_lossy().into_owned()), + offline: true, + ..Default::default() + }; + let tree = gradle_dependencies( + root, + &gradle::load(root).unwrap(), + &options, + ":app", + "testRuntimeClasspath", + &CancellationToken::new(), + ) + .unwrap(); + assert_eq!(tree, vec![dep("org.a", "b", Some("1.0"))]); + assert_eq!( + std::fs::read_to_string(root.join("args")).unwrap().trim(), + "--console=plain -q :app:dependencies --configuration testRuntimeClasspath --offline" + ); + } +} diff --git a/crates/ide-build/src/lib.rs b/crates/ide-build/src/lib.rs index 7ca4789c..31541854 100644 --- a/crates/ide-build/src/lib.rs +++ b/crates/ide-build/src/lib.rs @@ -8,6 +8,7 @@ pub mod application_update; pub mod compile; pub mod gradle; +mod gradle_deps; pub mod kotlin_bridge; pub mod maven; pub mod pom_edit; @@ -17,8 +18,11 @@ pub mod web; mod xml; pub use gradle::{GradleModel, GradleProject, GradleTask}; +pub use gradle_deps::{GradleDependency, gradle_dependencies}; pub use maven::{MavenDependency, MavenModel, MavenPlugin, MavenProfile, MavenProject, Omission}; -pub use sync::{SyncOptions, SyncProgress, maven_profiles, save_maven_profiles, sync_gradle, sync_maven}; +pub use sync::{ + SyncOptions, SyncProgress, maven_profiles, save_maven_profiles, sync_gradle, sync_gradle_model, sync_maven, +}; use serde::{Deserialize, Serialize}; use specta::Type; @@ -36,13 +40,31 @@ pub enum BuildTool { #[serde(rename_all = "camelCase")] pub struct BuildModel { pub maven: Option, - pub gradle: Option, + #[serde(default)] + pub gradle: Vec, +} + +impl BuildModel { + /// Prefer the deepest linked build owning the module over a composite including it. + pub fn gradle_for_module(&self, root: &Path, module: &Path) -> Option<&GradleModel> { + let module = root.join(module); + self.gradle.iter().filter(|g| g.projects.iter().any(|p| root.join(&p.dir) == module)).min_by_key(|g| { + let dir = root.join(&g.dir); + std::cmp::Reverse(if module.starts_with(&dir) { dir.components().count() } else { 0 }) + }) + } } /// Static model of the builds under `root`, merged with the sync results cached in `cache_dir`. /// Parent POMs are looked up in `maven_repo` (the configured local repository), else the default one. pub fn load(root: &Path, cache_dir: &Path, maven_repo: Option) -> BuildModel { - load_linked(root, cache_dir, maven_repo, &ide_core::settings::MavenSettings::default()) + load_linked( + root, + cache_dir, + maven_repo, + &ide_core::settings::MavenSettings::default(), + &ide_core::settings::GradleSettings::default(), + ) } /// Loads only linked Maven reactors, retaining an empty model so projects can be re-added. @@ -51,6 +73,7 @@ pub fn load_linked( cache_dir: &Path, maven_repo: Option, settings: &ide_core::settings::MavenSettings, + gradle_settings: &ide_core::settings::GradleSettings, ) -> BuildModel { let maven = maven::is_maven_project(root).then(|| { let profiles = sync::maven_profiles(cache_dir); @@ -59,23 +82,50 @@ pub fn load_linked( sync::apply_maven_cache(root, cache_dir, &mut m); m }); - let gradle = gradle::load(root).map(|mut g| { - sync::apply_gradle_cache(root, cache_dir, &mut g); - g - }); + let gradle = gradle::load_linked(root, gradle_settings) + .into_iter() + .map(|mut g| { + sync::apply_gradle_cache(root, &gradle::cache_dir(cache_dir, &g.dir), &mut g); + g + }) + .collect(); BuildModel { maven, gradle } } /// Gradle project folders outside `root` as of the last sync (`includeFlat` siblings): /// extra content roots, as IDEA lists module roots beside the project folder. /// Reads only the sync's cached projects (no build script), so it is cheap enough for hot paths. -pub fn gradle_external_roots(root: &Path, cache_dir: &Path) -> Vec { - gradle::external_roots(root, sync::reported_gradle_projects(cache_dir).iter().map(|p| p.dir.as_str())) +pub fn gradle_external_roots( + root: &Path, + cache_dir: &Path, + settings: &ide_core::settings::GradleSettings, +) -> Vec { + let builds = gradle::linked_dirs(root, settings); + let dirs: Vec = builds + .iter() + .flat_map(|dir| { + std::iter::once(root.join(dir).to_string_lossy().into_owned()).chain( + sync::reported_gradle_projects(&gradle::cache_dir(cache_dir, dir)) + .iter() + .map(|p| p.dir.clone()) + .collect::>(), + ) + }) + .collect(); + gradle::external_roots(root, dirs.iter().map(String::as_str)) } /// Source, resource and build folders of the last Gradle sync, absolute. -pub fn gradle_source_roots(cache_dir: &Path) -> Vec<(PathBuf, gradle::ReportedRoot)> { - gradle::reported_source_roots(&sync::reported_gradle_projects(cache_dir)) +pub fn gradle_source_roots( + root: &Path, + cache_dir: &Path, + settings: &ide_core::settings::GradleSettings, +) -> Vec<(PathBuf, gradle::ReportedRoot)> { + let reported = gradle::linked_dirs(root, settings) + .iter() + .flat_map(|dir| sync::reported_gradle_projects(&gradle::cache_dir(cache_dir, dir)).as_ref().clone()) + .collect::>(); + gradle::reported_source_roots(&reported) } /// A library the project depends on, as `groupId:artifactId:version`. @@ -119,7 +169,7 @@ pub fn dependency_coordinates(root: &Path, model: &BuildModel) -> Vec bool { mod tests { use super::*; + #[test] + fn external_linked_build_is_indexed_before_sync_and_disappears_when_unlinked() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("workspace"); + let external = temp.path().join("external"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&external).unwrap(); + std::fs::write(external.join("settings.gradle"), "").unwrap(); + let mut settings = ide_core::settings::GradleSettings { + linked_projects: Some(vec![external.to_string_lossy().into_owned()]), + ..Default::default() + }; + assert_eq!( + gradle_external_roots(&root, &temp.path().join("cache"), &settings), + [external.canonicalize().unwrap()] + ); + settings.linked_projects = Some(vec![]); + assert!(gradle_external_roots(&root, &temp.path().join("cache"), &settings).is_empty()); + } + #[test] fn detects_build_systems_and_build_files() { let tmp = tempfile::tempdir().unwrap(); @@ -156,7 +226,7 @@ mod tests { std::fs::write(root.join("build.gradle"), "").unwrap(); let m = load(&root, &cache, None); assert_eq!(m.maven.unwrap().projects[0].pom, "svc/pom.xml"); - assert_eq!(m.gradle.unwrap().projects[0].path, ":"); + assert_eq!(m.gradle[0].projects[0].path, ":"); assert!(affects_build(Path::new("a/pom.xml")) && affects_build(Path::new("settings.gradle.kts"))); assert!(!affects_build(Path::new("src/Main.java"))); } @@ -203,7 +273,7 @@ mod tests { ], ..Default::default() }; - let gradle = gradle::load(root); + let gradle = gradle::load(root).into_iter().collect(); let model = BuildModel { maven: Some(maven), gradle }; let names: Vec = dependency_coordinates(root, &model) .into_iter() diff --git a/crates/ide-build/src/sync.rs b/crates/ide-build/src/sync.rs index 407df454..eca3daa4 100644 --- a/crates/ide-build/src/sync.rs +++ b/crates/ide-build/src/sync.rs @@ -315,7 +315,10 @@ pub fn sync_maven( for (n, top) in model.projects.iter().filter(|p| !p.aggregated).enumerate() { let effective = cache_dir.join(format!("effective-pom-{n}.xml")); let _ = std::fs::remove_file(&effective); - let mut args: Vec = vec!["-B".into(), "-fae".into(), "-Dstyle.color=never".into()]; + // `-T1` keeps mvnd serial: its default parallel build interleaves the + // `dependency:tree` output of concurrent modules, which the sync parses + // as one stream. mvn is serial anyway, so this only pins the daemon. + let mut args: Vec = vec!["-B".into(), "-fae".into(), "-Dstyle.color=never".into(), "-T1".into()]; args.extend(options.maven_options.iter().cloned()); if options.offline { args.push("-o".into()); @@ -370,8 +373,25 @@ pub fn sync_gradle( cancel: &CancellationToken, progress: &dyn Fn(SyncProgress), ) -> IdeResult<()> { - let mut model = gradle::load(root).ok_or_else(|| IdeError::not_found("no Gradle build in this project"))?; - std::fs::create_dir_all(cache_dir)?; + let model = gradle::load(root).ok_or_else(|| IdeError::not_found("no Gradle build in this project"))?; + sync_gradle_model(root, cache_dir, &model, options, cancel, progress) +} + +/// Syncs this exact linked build; another build's task paths and cache cannot interfere. +pub fn sync_gradle_model( + root: &Path, + cache_dir: &Path, + model: &GradleModel, + options: &SyncOptions, + cancel: &CancellationToken, + progress: &dyn Fn(SyncProgress), +) -> IdeResult<()> { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let mut model = model.clone(); + let cache_dir = gradle::cache_dir(cache_dir, &model.dir); + std::fs::create_dir_all(&cache_dir)?; let init_script = cache_dir.join("openidea-model.gradle"); std::fs::write(&init_script, MODEL_INIT_SCRIPT)?; let mut args: Vec = vec![ @@ -386,14 +406,24 @@ pub fn sync_gradle( args.push("--offline".into()); } progress(SyncProgress { message: "Loading Gradle tasks".into(), fraction: None }); - let out = run( + let out = match run( options.gradle_command.as_deref().unwrap_or(&model.command), &args, &root.join(&model.dir), &options.env, cancel, |_| {}, - )?; + ) { + Ok(out) => out, + Err(_) if cancel.is_cancelled() => return Err(IdeError::cancelled()), + Err(error) if error.kind == ide_core::error::ErrorKind::Unavailable => { + Output { success: false, text: error.message } + } + Err(error) => return Err(error), + }; + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } let projects = if out.success { parse_reported_projects(&out.text) } else { Vec::new() }; // Hash the build files of the reported projects, or the model is stale right after the sync. gradle::merge_reported_projects(root, &mut model, &projects); @@ -404,13 +434,128 @@ pub fn sync_gradle( tasks: if out.success { gradle::parse_tasks(&out.text) } else { Vec::new() }, projects, }; - write_cache(cache_dir, GRADLE_CACHE, &cache) + write_cache(&cache_dir, GRADLE_CACHE, &cache) } #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + #[test] + fn unavailable_gradle_launchers_are_cached_without_stopping_other_builds() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + for dir in ["a", "b"] { + std::fs::create_dir(root.join(dir)).unwrap(); + std::fs::write(root.join(dir).join("build.gradle"), "").unwrap(); + } + let wrapper = root.join("b/gradlew"); + std::fs::write(&wrapper, "#!/bin/sh\nprintf '%s\\n' 'Build tasks' '-----------' 'healthy - ok'\n").unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + let settings = ide_core::settings::GradleSettings { + linked_projects: Some(vec!["a".into(), "b".into()]), + ..Default::default() + }; + let mut models = gradle::load_linked(root, &settings); + models[0].command = root.join("a/missing-gradlew").to_string_lossy().into_owned(); + let cache = root.join("cache"); + let run = |model: &GradleModel| { + sync_gradle_model(root, &cache, model, &SyncOptions::default(), &CancellationToken::new(), &|_| {}) + }; + for model in &models { + run(model).unwrap(); + } + let loaded = crate::load_linked(root, &cache, None, &Default::default(), &settings); + assert!(loaded.gradle[0].sync_error.as_deref().unwrap().contains("cannot start")); + assert!(loaded.gradle[1].sync_error.is_none()); + assert_eq!(loaded.gradle[1].projects[0].tasks[0].name, "healthy"); + let bad_wrapper = root.join("a/gradlew"); + std::fs::write(&bad_wrapper, "#!/bin/sh\nexit 0\n").unwrap(); + std::fs::set_permissions(&bad_wrapper, std::fs::Permissions::from_mode(0o644)).unwrap(); + models[0].command = bad_wrapper.to_string_lossy().into_owned(); + run(&models[0]).unwrap(); + let loaded = crate::load_linked(root, &cache, None, &Default::default(), &settings); + assert!(loaded.gradle[0].sync_error.as_deref().unwrap().contains("cannot start")); + assert!(loaded.gradle[1].sync_error.is_none()); + let blocked_cache = root.join("blocked-cache"); + std::fs::write(&blocked_cache, "a file cannot contain sync caches").unwrap(); + assert!( + sync_gradle_model( + root, + &blocked_cache, + &models[0], + &SyncOptions::default(), + &CancellationToken::new(), + &|_| {} + ) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn independent_syncs_do_not_share_tasks_launchers_environments_or_failures() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + for dir in ["a", "b"] { + let build = root.join(dir); + std::fs::create_dir_all(build.join("app")).unwrap(); + std::fs::write(build.join("settings.gradle"), "include 'app'").unwrap(); + std::fs::write(build.join("app/build.gradle"), "").unwrap(); + let report = serde_json::json!({"path": ":app", "dir": build.join("app")}); + let launcher = build.join("gradlew"); + std::fs::write( + &launcher, + format!("#!/bin/sh\npwd > actual-cwd\nprintf '%s' \"$JAVA_HOME\" > actual-env\nprintf '%s\\n' '@openidea-model\t{report}' 'Build tasks' '-----------' 'app:custom{dir} - custom task'\n"), + ).unwrap(); + std::fs::set_permissions(launcher, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let settings = ide_core::settings::GradleSettings { + linked_projects: Some(vec!["a".into(), "b".into()]), + ..Default::default() + }; + let cache = root.join("cache"); + let models = gradle::load_linked(root, &settings); + for model in &models { + let options = SyncOptions { + env: BTreeMap::from([("JAVA_HOME".into(), format!("/jdk-{}", model.dir))]), + ..Default::default() + }; + sync_gradle_model(root, &cache, model, &options, &CancellationToken::new(), &|_| {}).unwrap(); + assert_eq!( + std::fs::read_to_string(root.join(&model.dir).join("actual-env")).unwrap(), + format!("/jdk-{}", model.dir) + ); + assert_eq!( + std::fs::read_to_string(root.join(&model.dir).join("actual-cwd")).unwrap().trim(), + root.join(&model.dir).canonicalize().unwrap().to_string_lossy() + ); + } + let load = || crate::load_linked(root, &cache, None, &Default::default(), &settings); + let model = load(); + assert_eq!(model.gradle[0].projects[0].tasks[0].name, "customa"); + assert_eq!(model.gradle[1].projects[0].tasks[0].name, "customb"); + assert_ne!(gradle::cache_dir(&cache, "a"), gradle::cache_dir(&cache, "b")); + let before = std::fs::read(gradle::cache_dir(&cache, "a").join(GRADLE_CACHE)).unwrap(); + std::fs::write(root.join("b/gradlew"), "#!/bin/sh\nprintf 'FAILURE: broken b\\n'\nexit 1\n").unwrap(); + sync_gradle_model(root, &cache, &models[1], &SyncOptions::default(), &CancellationToken::new(), &|_| {}) + .unwrap(); + let model = load(); + assert!(model.gradle[0].sync_error.is_none()); + assert!(model.gradle[1].sync_error.is_some()); + assert_eq!(std::fs::read(gradle::cache_dir(&cache, "a").join(GRADLE_CACHE)).unwrap(), before); + let cancel = CancellationToken::new(); + cancel.cancel(); + assert!(sync_gradle_model(root, &cache, &models[0], &SyncOptions::default(), &cancel, &|_| {}).is_err()); + assert_eq!(std::fs::read(gradle::cache_dir(&cache, "a").join(GRADLE_CACHE)).unwrap(), before); + let unlinked = ide_core::settings::GradleSettings { linked_projects: Some(vec![]), ..Default::default() }; + assert!(crate::load_linked(root, &cache, None, &Default::default(), &unlinked).gradle.is_empty()); + assert!(crate::gradle_source_roots(root, &cache, &unlinked).is_empty()); + } + #[cfg(unix)] #[test] fn gradle_sync_uses_resolved_command_and_environment() { @@ -598,7 +743,7 @@ mod tests { write(&projects); let first = reported_gradle_projects(&cache_dir); assert!(Arc::ptr_eq(&first, &reported_gradle_projects(&cache_dir)), "unchanged: not parsed again"); - assert_eq!(crate::gradle_external_roots(&root, &cache_dir), [billing]); + assert_eq!(crate::gradle_external_roots(&root, &cache_dir, &Default::default()), [billing]); write(&projects[..1]); assert_eq!(reported_gradle_projects(&cache_dir).len(), 1, "a new sync is read"); std::fs::remove_file(cache_dir.join(GRADLE_CACHE)).unwrap(); @@ -680,4 +825,25 @@ mod tests { assert_eq!(error_summary(text), "Failed to execute goal"); assert_eq!(error_summary("one\nlast line\n\n"), "last line"); } + + #[cfg(unix)] + #[test] + fn maven_sync_runs_the_daemon_command_serially() { + use std::os::unix::fs::PermissionsExt; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("p"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("pom.xml"), "ga") + .unwrap(); + let launcher = tmp.path().join("fake-mvnd"); + std::fs::write(&launcher, "#!/bin/sh\nprintf '%s\\n' \"$@\" > mvnd-args\n").unwrap(); + std::fs::set_permissions(&launcher, std::fs::Permissions::from_mode(0o755)).unwrap(); + let options = + SyncOptions { maven_command: Some(launcher.to_string_lossy().into_owned()), ..Default::default() }; + sync_maven(&root, &tmp.path().join("cache"), &options, &CancellationToken::new(), &|_| {}).unwrap(); + let args = std::fs::read_to_string(root.join("mvnd-args")).unwrap(); + // `-T1` pins the daemon serial so the dependency trees stay parseable. + assert!(args.contains("-T1"), "args: {args}"); + assert!(args.contains("help:effective-pom"), "args: {args}"); + } } diff --git a/crates/ide-build/src/web.rs b/crates/ide-build/src/web.rs index 9f25039a..fbff22b7 100644 --- a/crates/ide-build/src/web.rs +++ b/crates/ide-build/src/web.rs @@ -26,12 +26,12 @@ pub fn artifacts(root: &Path, model: &BuildModel) -> Vec { }); } } - if let Some(gradle) = &model.gradle { + for gradle in &model.gradle { for p in &gradle.projects { let script = p.build_file.as_ref().and_then(|f| fs::read_to_string(root.join(f)).ok()).unwrap_or_default(); if p.tasks.iter().any(|t| t.name == "war") || crate::gradle::has_war_plugin(&script) { out.push(WebArtifact { - id: format!("gradle:{}", p.path), + id: gradle_artifact_id(gradle, &p.path), name: p.name.clone(), module: p.dir.clone(), tool: "gradle".into(), @@ -42,6 +42,14 @@ pub fn artifacts(root: &Path, model: &BuildModel) -> Vec { out } +fn gradle_artifact_id(build: &crate::GradleModel, path: &str) -> String { + if build.dir.is_empty() { + format!("gradle:{path}") + } else { + format!("gradle:{}", serde_json::to_string(&(&build.dir, path)).unwrap()) + } +} + pub struct WebBuild { pub command: String, pub args: Vec, @@ -150,9 +158,14 @@ pub fn build(root: &Path, model: &BuildModel, id: &str, cache: &Path) -> IdeResu maven_project: Some((probe, output)), }); } - if let Some(path) = id.strip_prefix("gradle:") { - let g = model.gradle.as_ref().ok_or_else(|| IdeError::invalid("Choose a Gradle WAR module"))?; - if !artifacts(root, model).iter().any(|a| a.id == id) { + if let Some(legacy_path) = id.strip_prefix("gradle:") { + let identity = serde_json::from_str::<(String, String)>(legacy_path).ok(); + let (g, path) = match &identity { + Some((dir, path)) => (model.gradle.iter().find(|g| g.dir == *dir), path.as_str()), + None => (model.gradle.first(), legacy_path), + }; + let g = g.ok_or_else(|| IdeError::invalid("Choose a Gradle WAR module"))?; + if !artifacts(root, model).iter().any(|a| a.id == gradle_artifact_id(g, path)) { return Err(IdeError::invalid("Choose a Gradle WAR module")); } let script = cache.join("war.init.gradle"); @@ -186,6 +199,28 @@ pub fn build(root: &Path, model: &BuildModel, id: &str, cache: &Path) -> IdeResu #[cfg(test)] mod tests { + #[test] + fn independent_war_modules_have_distinct_ids_and_build_in_their_own_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + for dir in ["a", "b"] { + std::fs::create_dir(root.join(dir)).unwrap(); + std::fs::write(root.join(dir).join("build.gradle"), "plugins { id 'war' }").unwrap(); + } + let settings = ide_core::settings::GradleSettings { + linked_projects: Some(vec!["a".into(), "b".into()]), + ..Default::default() + }; + let model = crate::load_linked(root, &root.join("cache"), None, &Default::default(), &settings); + let artifacts = super::artifacts(root, &model); + assert_eq!(artifacts.len(), 2); + assert_ne!(artifacts[0].id, artifacts[1].id); + for (artifact, dir) in artifacts.iter().zip(["a", "b"]) { + let plan = super::build(root, &model, &artifact.id, &root.join(format!("cache/{dir}"))).unwrap(); + assert_eq!(plan.cwd, dir); + } + assert_eq!(super::build(root, &model, "gradle::", &root.join("legacy")).unwrap().cwd, "a"); + } use super::*; #[test] fn maven_war_modules_use_resolved_artifact_file() { diff --git a/crates/ide-core/Cargo.toml b/crates/ide-core/Cargo.toml index 4f9b43a6..3a4f5f6c 100644 --- a/crates/ide-core/Cargo.toml +++ b/crates/ide-core/Cargo.toml @@ -22,4 +22,4 @@ tempfile.workspace = true libc = "0.2" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Globalization", "Win32_Security", "Win32_System_Registry", "Win32_Foundation"] } +windows-sys = { version = "0.61", features = ["Win32_Globalization", "Win32_Security", "Win32_System_Registry", "Win32_Foundation", "Win32_System_Diagnostics_ToolHelp", "Win32_System_Threading"] } diff --git a/crates/ide-core/src/ignored.rs b/crates/ide-core/src/ignored.rs index fb6c20a3..efda11ac 100644 --- a/crates/ide-core/src/ignored.rs +++ b/crates/ide-core/src/ignored.rs @@ -2,6 +2,18 @@ //! treats as nonexistent — hidden from the project view and navigation, never //! indexed or searched, never offered as unversioned changes. +use std::path::Path; + +/// Whether `dir` lies in a Git (or jj) working tree: it or an ancestor holds `.git` / `.jj`. +/// +/// Walkers pass it to `ignore::WalkBuilder::require_git`: inside a repository, `.gitignore` +/// rules then stop at a nested repository's root as in git, so an outer repository's +/// `.gitignore` does not hide a project that is its own repository. A folder outside any +/// repository still honours its `.gitignore`. +pub fn inside_git_repo(dir: &Path) -> bool { + dir.ancestors().any(|d| d.join(".git").exists() || d.join(".jj").exists()) +} + /// IDEA's default list, plus `.jj` and the Windows shell files. pub const DEFAULT_IGNORED_NAMES: &[&str] = &[ "*.pyc", @@ -103,4 +115,15 @@ mod tests { assert!(m.is_ignored_path("pkg/__pycache__/mod.cpython-312.pyc")); assert!(!m.is_ignored_path("src/main.rs")); } + + #[test] + fn finds_the_repository_at_or_above_a_directory() { + let dir = tempfile::tempdir().unwrap(); + let sub = dir.path().join("a/b"); + std::fs::create_dir_all(&sub).unwrap(); + assert!(!inside_git_repo(&sub)); + std::fs::create_dir(dir.path().join("a/.git")).unwrap(); + assert!(inside_git_repo(&sub)); + assert!(inside_git_repo(&dir.path().join("a"))); + } } diff --git a/crates/ide-core/src/inspection_settings.rs b/crates/ide-core/src/inspection_settings.rs index f4522a2e..41dfb8f2 100644 --- a/crates/ide-core/src/inspection_settings.rs +++ b/crates/ide-core/src/inspection_settings.rs @@ -36,6 +36,23 @@ impl Default for InspectionSettings { } } +impl InspectionSettings { + /// Overrides of the profile in effect; `Default`'s when `profile` names none that exists. + pub fn active(&self) -> &BTreeMap { + let custom = |level: &str, key: &str| match level { + "ide" => self.ide_profiles.get(key), + "project" => self.project_profiles.get(key), + _ => None, + }; + match self.profile.as_str() { + "project" => &self.project, + id => { + id.split_once(':').and_then(|(level, key)| custom(level, key)).map_or(&self.default, |p| &p.overrides) + } + } + } +} + /// A named profile made with Copy. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(default, rename_all = "camelCase")] @@ -111,4 +128,20 @@ mod tests { let back: InspectionSettings = serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap(); assert_eq!(back, s); } + + #[test] + fn active_profile_falls_back_to_default() { + let off = |id: &str| { + BTreeMap::from([(id.to_owned(), InspectionOverride { enabled: Some(false), ..Default::default() })]) + }; + let mut s = InspectionSettings { default: off("a"), project: off("b"), ..Default::default() }; + s.ide_profiles.insert("strict".into(), CustomInspectionProfile { name: "Strict".into(), overrides: off("c") }); + assert!(s.active().contains_key("a")); + s.profile = "project".into(); + assert!(s.active().contains_key("b")); + s.profile = "ide:strict".into(); + assert!(s.active().contains_key("c")); + s.profile = "project:gone".into(); + assert!(s.active().contains_key("a")); + } } diff --git a/crates/ide-core/src/process.rs b/crates/ide-core/src/process.rs index 49565b3d..1778d0c0 100644 --- a/crates/ide-core/src/process.rs +++ b/crates/ide-core/src/process.rs @@ -43,19 +43,21 @@ pub fn own_group(cmd: &mut Command) -> &mut Command { /// Kills the child and everything it started, then reaps it. /// /// - Unix: `SIGKILL` to its process group, so spawn it with [`own_group`]. -/// - Windows: `taskkill /T /F`, which walks the tree from the child's process id. +/// - Windows: `taskkill /T /F`, which walks the tree from the child's process id, then a +/// wait for what the child started to be gone. pub fn kill_tree(child: &mut Child) { #[cfg(unix)] signal_group(child, libc::SIGKILL); #[cfg(windows)] - taskkill_tree(child); + taskkill_tree_of(child.id()); let _ = child.kill(); let _ = child.wait(); } /// [`kill_tree`] for a child known by its process id (a `tokio` child): `SIGKILL` to its -/// process group (spawn it with [`own_group`]) or `taskkill /T /F`. Call it before the child -/// is reaped, so the id still names it. The child itself is left to its owner to kill and reap. +/// process group (spawn it with [`own_group`]) or `taskkill /T /F` and a wait for what the +/// child started to be gone. Call it before the child is reaped, so the id still names it. +/// The child itself is left to its owner to kill and reap. pub fn kill_tree_of(pid: u32) { #[cfg(unix)] if let Ok(pid) = libc::pid_t::try_from(pid) { @@ -65,15 +67,7 @@ pub fn kill_tree_of(pid: u32) { } } #[cfg(windows)] - { - let mut taskkill = Command::new("taskkill"); - prepare(&mut taskkill) - .args(["/T", "/F", "/PID", &pid.to_string()]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = taskkill.status(); - } + taskkill_tree_of(pid); #[cfg(not(any(unix, windows)))] let _ = pid; } @@ -106,15 +100,107 @@ fn signal_group(child: &Child, signal: libc::c_int) { } } +/// How long killing a tree waits for the processes the child started to be gone. #[cfg(windows)] -fn taskkill_tree(child: &Child) { +const TREE_EXIT_WAIT: Duration = Duration::from_secs(5); + +/// `taskkill /T /F` from `pid`, then waits (up to [`TREE_EXIT_WAIT`]) until what the child +/// started is gone: termination is asynchronous, and until a process is gone the files it +/// holds open can't be opened or deleted. The child itself is left to its owner to reap. +#[cfg(windows)] +fn taskkill_tree_of(pid: u32) { + let started = windows_tree::descendants(pid); let mut taskkill = Command::new("taskkill"); prepare(&mut taskkill) - .args(["/T", "/F", "/PID", &child.id().to_string()]) + .args(["/T", "/F", "/PID", &pid.to_string()]) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); let _ = taskkill.status(); + windows_tree::wait_all(&started, TREE_EXIT_WAIT); +} + +#[cfg(windows)] +mod windows_tree { + use std::collections::HashSet; + use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; + use std::time::{Duration, Instant}; + use windows_sys::Win32::Foundation::{FILETIME, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, TH32CS_SNAPPROCESS, + }; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, WaitForSingleObject, + }; + + /// Every process `root` started, directly or not, opened so it can be waited for. A + /// parent's id is reused once it is gone, so a process counts only if it started after + /// its parent. + pub(super) fn descendants(root: u32) -> Vec { + let Some(root_started) = open(root).as_ref().and_then(started) else { return Vec::new() }; + let processes = processes(); + let mut seen = HashSet::from([root]); + let mut parents = vec![(root, root_started)]; + let mut found = Vec::new(); + while let Some((parent, parent_started)) = parents.pop() { + for &(pid, ppid) in &processes { + if ppid != parent || !seen.insert(pid) { + continue; + } + let Some(handle) = open(pid) else { continue }; + if let Some(at) = started(&handle).filter(|&at| at >= parent_started) { + parents.push((pid, at)); + found.push(handle); + } + } + } + found + } + + pub(super) fn wait_all(processes: &[OwnedHandle], within: Duration) { + let until = Instant::now() + within; + for process in processes { + let left = until.saturating_duration_since(Instant::now()); + // SAFETY: a live process handle opened with SYNCHRONIZE access. + unsafe { WaitForSingleObject(process.as_raw_handle(), left.as_millis().min(u32::MAX as u128 - 1) as u32) }; + } + } + + /// (process id, parent process id) of every running process. + fn processes() -> Vec<(u32, u32)> { + let mut out = Vec::new(); + // SAFETY: a snapshot handle owned here and an entry sized as the API requires. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if snapshot.is_null() || snapshot == INVALID_HANDLE_VALUE { + return out; + } + let snapshot = OwnedHandle::from_raw_handle(snapshot); + let mut entry = PROCESSENTRY32W { dwSize: size_of::() as u32, ..Default::default() }; + let mut more = Process32FirstW(snapshot.as_raw_handle(), &mut entry); + while more != 0 { + out.push((entry.th32ProcessID, entry.th32ParentProcessID)); + more = Process32NextW(snapshot.as_raw_handle(), &mut entry); + } + } + out + } + + fn open(pid: u32) -> Option { + // SAFETY: plain call; a non-null result is a process handle owned by the caller. + let handle = unsafe { OpenProcess(PROCESS_SYNCHRONIZE | PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + // SAFETY: as above. + (!handle.is_null()).then(|| unsafe { OwnedHandle::from_raw_handle(handle) }) + } + + /// When the process started, in 100 ns ticks since 1601. + fn started(process: &OwnedHandle) -> Option { + let (mut created, mut exited, mut kernel, mut user) = + (FILETIME::default(), FILETIME::default(), FILETIME::default(), FILETIME::default()); + // SAFETY: a live process handle opened with query access; four out-parameters. + let ok = unsafe { GetProcessTimes(process.as_raw_handle(), &mut created, &mut exited, &mut kernel, &mut user) }; + (ok != 0).then(|| (u64::from(created.dwHighDateTime) << 32) | u64::from(created.dwLowDateTime)) + } } /// Gives `cmd` the host's environment back when the IDE runs from an AppImage. `AppRun` and @@ -123,12 +209,14 @@ fn taskkill_tree(child: &Child) { /// inherits them loads the image's (older) libraries and plugins next to its own, e.g. /// `curl: symbol lookup error ... nghttp2_...`. The IDE process keeps its environment /// (WebKit and GTK still need it). Then applies the [`set_env_overlay`] variables and -/// those of a manual HTTP proxy ([`crate::proxy_settings::ProxySettings::child_env`]). -/// Variables the call site has already set are kept. A no-op outside an AppImage -/// without an overlay. +/// those of a manual HTTP proxy ([`crate::proxy_settings::ProxySettings::child_env`]), +/// and the login shell's variables this process lacks ([`shell_env`]: an app started from +/// Finder/Dock doesn't have what rc files export, such as `GOPATH`, `JAVA_HOME` or +/// `MAVEN_OPTS`). Variables the call site has already set are kept. pub fn host_env(cmd: &mut Command) -> &mut Command { let fixes = child_env_changes(); - if fixes.is_empty() { + let shell: Vec<_> = shell_env().collect(); + if fixes.is_empty() && shell.is_empty() { return cmd; } let explicit: Vec = cmd.get_envs().map(|(k, _)| k.to_owned()).collect(); @@ -138,6 +226,9 @@ pub fn host_env(cmd: &mut Command) -> &mut Command { None => cmd.env_remove(key), }; } + for (key, value) in shell.into_iter().filter(|(k, _)| !explicit.contains(k)) { + cmd.env(key, value); + } cmd } @@ -154,12 +245,13 @@ pub fn host_vars() -> impl Iterator { apply_changes(std::env::vars_os().collect(), &child_env_changes()).into_iter() } -/// [`std::env::var_os`] as a child should see it (see [`host_env`]). +/// [`std::env::var_os`] as a child should see it (see [`host_env`]), the login shell's value +/// included when this process lacks the variable. pub fn host_var_os(key: impl AsRef) -> Option { let key = key.as_ref(); match child_env_changes().into_iter().find(|(k, _)| k == key) { Some((_, fixed)) => fixed, - None => std::env::var_os(key), + None => std::env::var_os(key).or_else(|| shell_env().find(|(k, _)| k == key).map(|(_, v)| v)), } } @@ -572,6 +664,15 @@ mod tests { assert_eq!(get("HOME"), Some(None)); let shell: Vec<_> = shell_env().map(|(k, _)| k).collect(); assert_eq!(shell, [OsString::from("OPENIDEA_OVERLAY_B")]); + // Every child gets what the login shell exports and the IDE process lacks (a Finder + // launch: `GOPATH`, `JAVA_HOME`...), unless the call site sets it. + assert_eq!(get("OPENIDEA_OVERLAY_B"), Some(Some("b".into()))); + let mut own = Command::new("true"); + own.env("OPENIDEA_OVERLAY_B", "call site"); + host_env(&mut own); + let value = own.get_envs().find(|(n, _)| *n == "OPENIDEA_OVERLAY_B").and_then(|(_, v)| v); + assert_eq!(value, Some(OsStr::new("call site"))); + assert_eq!(host_var_os("OPENIDEA_OVERLAY_B"), Some("b".into())); assert_eq!(host_var_os("PATH"), Some("/reloaded".into())); assert!(host_vars().all(|(k, _)| k != "HOME")); assert!(host_vars().any(|(k, v)| k == "OPENIDEA_OVERLAY_A" && v == "project")); diff --git a/crates/ide-core/src/settings.rs b/crates/ide-core/src/settings.rs index 5991a79f..b0b86e6d 100644 --- a/crates/ide-core/src/settings.rs +++ b/crates/ide-core/src/settings.rs @@ -351,6 +351,7 @@ pub struct AppearanceSettings { pub preferred_dark_theme: String, /// A plugin's file icon theme (`/`); empty = the built-in icons. pub file_icon_theme: String, + pub file_icon_theme_options: BTreeMap, /// IDE zoom in percent (50–300). pub zoom: u32, /// Apply `ui_font_family` / `ui_font_size` (otherwise the theme's). @@ -386,6 +387,24 @@ pub struct AppearanceSettings { /// Main toolbar entries per side, in order: title widget ids (`project`, /// `vcs.branch`…) or `action:`. Widgets in neither list nor `hidden` /// (registered later) keep their default place. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(default, rename_all = "camelCase")] +pub struct FileIconThemeOptions { + pub groups: BTreeMap, + pub choices: BTreeMap, + pub hide_files: bool, + pub hide_folders: bool, + pub hollow_folders: bool, + pub hide_ui_icons: bool, + pub hide_psi_icons: bool, + pub monochrome: Option, + pub saturation: Option, + pub opacity: Option, + pub icon_size: Option, + pub row_height: Option, + pub accent: Option, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(default, rename_all = "camelCase")] pub struct MainToolbarSettings { @@ -406,6 +425,7 @@ impl Default for AppearanceSettings { preferred_light_theme: "light".into(), preferred_dark_theme: "dark".into(), file_icon_theme: String::new(), + file_icon_theme_options: BTreeMap::new(), zoom: 100, use_custom_font: false, ui_font_family: "Inter".into(), @@ -693,6 +713,15 @@ impl Default for PythonSettings { pub struct PhpSettings { /// `php` CLI executable; empty = the one on `PATH`. pub interpreter: String, + /// PHP version the language server checks the code against (`7.4`); empty = from + /// `composer.json` (`config.platform.php`, else the lowest version `require.php` allows). + pub language_level: String, + /// Folders outside the project whose code the language server indexes too (a platform a + /// module extends); absolute, `~/`, or relative to the project. + pub include_paths: Vec, + /// Intelephense stub sets loaded on top of its defaults and the `ext-*` entries of + /// `composer.json` (`wordpress`). + pub stubs: Vec, } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Type)] @@ -799,6 +828,8 @@ pub struct JavaSettings { #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Type)] #[serde(default, rename_all = "camelCase")] pub struct GradleSettings { + /// Linked build directories; absent preserves automatic discovery, empty unlinks all builds. + pub linked_projects: Option>, /// Empty = Project JDK; otherwise a JDK home. pub jvm: String, pub distribution: GradleDistribution, @@ -841,6 +872,12 @@ pub struct MavenSettings { pub user_settings: String, /// Local repository (`-Dmaven.repo.local`); empty = from `settings.xml`. pub local_repository: String, + /// Run Maven through the Maven Daemon (`mvnd`) for faster builds. mvnd embeds + /// its own Maven, so `home` and the project's `mvnw` are bypassed when it is on. + pub daemon: bool, + /// Maven Daemon home (or its `mvnd` executable); empty = `$MVND_HOME`, else + /// `mvnd` on `PATH` / common install dirs. Only used when `daemon` is on. + pub mvnd_home: String, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)] @@ -931,6 +968,36 @@ mod tests { assert_eq!(serde_json::from_value::(value).unwrap(), settings); } + #[test] + fn file_icon_theme_options_default_empty_and_round_trip() { + let old: Settings = serde_json::from_str(r#"{"appearance":{"fileIconTheme":"p/t"}}"#).unwrap(); + assert!(old.appearance.file_icon_theme_options.is_empty()); + let json = serde_json::json!({"appearance":{"fileIconThemeOptions":{"p/t":{ + "groups":{"tests":true},"choices":{"arrows":"plusminus"},"hideFiles":true,"hideFolders":false,"hollowFolders":true,"hideUiIcons":true,"hidePsiIcons":false, + "monochrome":"#546e7a","saturation":5,"opacity":80,"iconSize":20,"rowHeight":30,"accent":"#009688"}}}}); + let settings: Settings = serde_json::from_value(json.clone()).unwrap(); + let options = &settings.appearance.file_icon_theme_options["p/t"]; + assert!(options.hide_files && options.hollow_folders && !options.hide_folders); + assert!(options.hide_ui_icons && !options.hide_psi_icons); + assert!(options.groups["tests"]); + assert_eq!(options.choices["arrows"], "plusminus"); + assert_eq!( + (options.saturation, options.opacity, options.icon_size, options.row_height), + (Some(5), Some(80), Some(20), Some(30)) + ); + assert_eq!(options.monochrome.as_deref(), Some("#546e7a")); + assert_eq!(options.accent.as_deref(), Some("#009688")); + let value = serde_json::to_value(&settings).unwrap(); + let restored: Settings = serde_json::from_value(value).unwrap(); + assert_eq!(restored, settings); + let plain = serde_json::to_value(FileIconThemeOptions::default()).unwrap(); + assert_eq!(plain["hideFiles"], false); + assert_eq!(plain["hideUiIcons"], false); + assert_eq!(plain["hidePsiIcons"], false); + assert_eq!(plain["monochrome"], serde_json::Value::Null); + assert_eq!(plain["groups"], serde_json::json!({})); + } + #[test] fn ui_font_fallback_defaults_empty_and_round_trips() { let old: Settings = serde_json::from_str(r#"{"appearance":{"useCustomFont":true}}"#).unwrap(); @@ -1042,12 +1109,33 @@ mod maven_linkage_tests { assert!(encoded.contains("unlinkedProjects")); assert_eq!(serde_json::from_str::(&encoded).unwrap(), settings); } + + #[test] + fn daemon_settings_round_trip() { + // Old settings files predate the daemon fields; `serde(default)` keeps them loadable. + let old: MavenSettings = serde_json::from_str(r#"{"home":"/opt/maven"}"#).unwrap(); + assert!(!old.daemon && old.mvnd_home.is_empty()); + let settings = MavenSettings { daemon: true, mvnd_home: "/opt/mvnd".into(), ..old }; + let encoded = serde_json::to_string(&settings).unwrap(); + assert!(encoded.contains("\"daemon\":true") && encoded.contains("mvndHome")); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), settings); + } } #[cfg(test)] mod gradle_settings_tests { use super::*; + #[test] + fn absent_and_explicitly_empty_gradle_links_remain_distinct() { + let automatic: GradleSettings = serde_json::from_str("{}").unwrap(); + let unlinked: GradleSettings = serde_json::from_str(r#"{"linkedProjects":[]}"#).unwrap(); + assert_eq!(automatic.linked_projects, None); + assert_eq!(unlinked.linked_projects, Some(vec![])); + let restored: GradleSettings = serde_json::from_str(&serde_json::to_string(&unlinked).unwrap()).unwrap(); + assert_eq!(restored.linked_projects, Some(vec![])); + } + #[test] fn defaults_and_round_trip() { let old: JavaSettings = serde_json::from_str(r#"{"jdk":"/jdk"}"#).unwrap(); @@ -1060,6 +1148,7 @@ mod gradle_settings_tests { home: "/gradle".into(), user_home: "/cache/gradle".into(), run_using: GradleRunUsing::Ide, + linked_projects: Some(vec!["backend".into(), "../shared".into()]), }; let restored: Settings = serde_json::from_str(&serde_json::to_string(&settings).unwrap()).unwrap(); assert_eq!(restored.languages.java.gradle, settings.languages.java.gradle); diff --git a/crates/ide-dap/src/adapter.rs b/crates/ide-dap/src/adapter.rs index 2f4f88fc..25c44cdd 100644 --- a/crates/ide-dap/src/adapter.rs +++ b/crates/ide-dap/src/adapter.rs @@ -72,7 +72,13 @@ fn builtin(adapter: &str) -> Option { "stdio", "Install the Kotlin Debug Adapter from Settings > Language Servers", ), - "debugpy" => cfg("python3", &["-m", "debugpy.adapter"], "stdio", "pip install debugpy"), + // Windows installs `python.exe`; `python3.exe` there is at most a Store alias. + "debugpy" => cfg( + if cfg!(windows) { "python" } else { "python3" }, + &["-m", "debugpy.adapter"], + "stdio", + "pip install debugpy", + ), "js-debug" => cfg( "js-debug-adapter", // Without a host it listens on `localhost`, which may resolve to ::1 only. diff --git a/crates/ide-dap/src/lib.rs b/crates/ide-dap/src/lib.rs index 1be66239..f2790ad5 100644 --- a/crates/ide-dap/src/lib.rs +++ b/crates/ide-dap/src/lib.rs @@ -589,11 +589,12 @@ impl DebugManager { if let Some(command) = custom_command(&spec.command) { spec.command = command; } - // debugpy: run the adapter with the debuggee's interpreter (venvs carry their own debugpy). + // debugpy: run the adapter with the debuggee's interpreter (venvs carry their own debugpy), + // or with the launcher's when the debuggee runs through a manager (`uv run python`). if config.adapter == "debugpy" { - let python = match arguments.get("python") { - Some(Value::String(p)) => Some(p.clone()), - Some(Value::Array(a)) => a.first().and_then(Value::as_str).map(str::to_owned), + let python = match (arguments.get("debugLauncherPython"), arguments.get("python")) { + (Some(Value::String(p)), _) | (_, Some(Value::String(p))) => Some(p.clone()), + (_, Some(Value::Array(a))) => a.first().and_then(Value::as_str).map(str::to_owned), _ => None, }; if let Some(python) = python { @@ -651,8 +652,19 @@ impl DebugManager { { arguments["project"] = json!(real); } - if config.adapter == "codelldb" && arguments.get("sourceLanguages").is_none() { - arguments["sourceLanguages"] = json!(["rust"]); + if config.adapter == "codelldb" { + if arguments.get("sourceLanguages").is_none() { + arguments["sourceLanguages"] = json!(["rust"]); + } + // The Debug Console and Evaluate Expression take expressions, as with every other + // debugger; LLDB commands still run with a leading backtick. + match arguments.get_mut("_adapterSettings") { + None => arguments["_adapterSettings"] = json!({"consoleMode": "evaluate"}), + Some(Value::Object(settings)) => { + settings.entry("consoleMode").or_insert_with(|| json!("evaluate")); + } + Some(_) => {} + } } let port = debug_server.map(|s| s.port); self.start_session(id, owner, config.name, config.adapter, None, config.request, arguments, spec, port).await diff --git a/crates/ide-dap/src/session.rs b/crates/ide-dap/src/session.rs index 8ac55b92..f7bc69c3 100644 --- a/crates/ide-dap/src/session.rs +++ b/crates/ide-dap/src/session.rs @@ -111,8 +111,8 @@ impl Session { if let Some(path) = self.kotlin_mirror.lock().as_ref().and_then(|m| m.adapter_path(path)) { return path; } - if self.adapter == "java-debug" { - let native = java_source_path(&path.to_string_lossy()); + if self.adapter == "java-debug" || (cfg!(windows) && self.adapter == "netcoredbg") { + let native = windows_source_path(&path.to_string_lossy()); self.resolved_paths.lock().insert(native.clone(), path.to_path_buf()); return native; } @@ -133,7 +133,11 @@ impl Session { if let Some(real) = self.kotlin_mirror.lock().as_ref().and_then(|m| m.client_path(Path::new(path))) { return Some(real); } - let key = if self.adapter == "java-debug" { java_source_path(path) } else { PathBuf::from(path) }; + let key = if self.adapter == "java-debug" || (cfg!(windows) && self.adapter == "netcoredbg") { + windows_source_path(path) + } else { + PathBuf::from(path) + }; self.resolved_paths.lock().get(&key).cloned().or_else(|| { (self.adapter == "java-debug" && is_windows_source_path(path)) .then(|| PathBuf::from(path.replace('\\', "/"))) @@ -265,8 +269,8 @@ impl Session { } } -// java-debug treats a forward-slash drive path as a URI (scheme "C"). -// Keep DAP path mode, but use native Windows separators for Java source lookup. +// java-debug and Windows netcoredbg need native separators for source lookup. +// Keep DAP path mode, but normalize Windows drive and UNC paths. fn is_windows_source_path(path: &str) -> bool { let bytes = path.as_bytes(); (bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && matches!(bytes[2], b'/' | b'\\')) @@ -274,7 +278,7 @@ fn is_windows_source_path(path: &str) -> bool { || path.starts_with(r"\\") } -fn java_source_path(path: &str) -> PathBuf { +fn windows_source_path(path: &str) -> PathBuf { if is_windows_source_path(path) { PathBuf::from(path.replace('/', r"\")) } else { PathBuf::from(path) } } diff --git a/crates/ide-dap/src/tests.rs b/crates/ide-dap/src/tests.rs index 7f5930b3..1e5647b7 100644 --- a/crates/ide-dap/src/tests.rs +++ b/crates/ide-dap/src/tests.rs @@ -698,6 +698,37 @@ async fn js_debug_launch_gets_port_and_workspace_folder() { assert_eq!(launch["type"], "pwa-node"); } +/// CodeLLDB treats console input as LLDB commands unless told otherwise: Evaluate Expression +/// and the Debug Console send expressions, so its console evaluates them. A launch +/// configuration that picks a console mode keeps it. +#[tokio::test] +async fn codelldb_console_evaluates_expressions() { + let launch_with = |extra: Value| async move { + let h = harness(Script::default()); + let config = LaunchConfig { + name: "demo".into(), + adapter: "codelldb".into(), + request: "launch".into(), + program: Some("/p/target/debug/demo".into()), + args: vec![], + cwd: Some("/p".into()), + env: BTreeMap::new(), + extra_json: Some(extra.to_string()), + }; + h.start(config).await.unwrap(); + h.request("launch")["arguments"].clone() + }; + let launch = launch_with(json!({})).await; + assert_eq!(launch["_adapterSettings"]["consoleMode"], "evaluate"); + assert_eq!(launch["sourceLanguages"], json!(["rust"])); + + let launch = + launch_with(json!({"_adapterSettings": {"consoleMode": "commands", "showDisassembly": "never"}})).await; + assert_eq!(launch["_adapterSettings"], json!({"consoleMode": "commands", "showDisassembly": "never"})); + let launch = launch_with(json!({"_adapterSettings": {"showDisassembly": "never"}})).await; + assert_eq!(launch["_adapterSettings"], json!({"consoleMode": "evaluate", "showDisassembly": "never"})); +} + /// An adapter that is already listening (java-debug inside jdtls): the client /// connects over TCP to the `debugServer` port instead of spawning anything. #[tokio::test] @@ -923,6 +954,7 @@ fn godot_registry(port: u16, command: &str, args: &[&str]) -> Arc); + +struct State { + client: RwLock>, + endpoint: Url, + user: String, + password: String, + database: String, + settings: Vec<(String, String)>, + read_only: bool, + session_id: String, + session: Mutex, + closed: CancellationToken, +} + +/// The response format: a names line, a types line, then one JSON array per row, so rows past +/// the grid's limit are skipped without being parsed. +const FORMAT: &str = "JSONCompactEachRowWithNamesAndTypes"; + +pub(crate) fn transaction_error() -> IdeError { + IdeError::invalid("ClickHouse HTTP does not support SQL transactions; use auto-commit and run writes directly") +} + +impl ClickhousePool { + pub(crate) async fn connect(source: &DataSource, password: Option<&str>, setup: Setup) -> IdeResult { + if source.options.transaction == DbTxMode::Manual { + return Err(transaction_error()); + } + for (name, _) in &setup.params { + if reserved_setting(name) { + return Err(IdeError::invalid(format!("ClickHouse HTTP manages the {name} setting"))); + } + } + let secure = matches!(source.ssl, DbSslMode::Require | DbSslMode::VerifyCa | DbSslMode::VerifyFull); + let tls_files = [&source.ssl_ca_file, &source.ssl_cert_file, &source.ssl_key_file]; + if !secure && tls_files.iter().any(|p| !p.trim().is_empty()) { + return Err(IdeError::invalid( + "ClickHouse TLS certificate files require an SSL mode of Require, Verify CA, or Verify Full", + )); + } + let host = source.host.trim(); + if host.is_empty() || host.contains(['/', '@', '?', '#']) { + return Err(IdeError::invalid("ClickHouse host must contain only a hostname or IP address")); + } + let host = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { host.to_owned() }; + let scheme = if secure { "https" } else { "http" }; + let port = source.port.unwrap_or(if secure { 8443 } else { 8123 }); + let endpoint = Url::parse(&format!("{scheme}://{host}:{port}/")) + .map_err(|_| IdeError::invalid("invalid ClickHouse HTTP host or port"))?; + let timeout = setup.connect_timeout.unwrap_or(Duration::from_secs(30)); + let _ = rustls::crypto::ring::default_provider().install_default(); + let mut builder = Client::builder() + .connect_timeout(timeout) + .pool_max_idle_per_host(4) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(concat!("OpenIDEA/", env!("CARGO_PKG_VERSION"))); + let ca = source.ssl_ca_file.trim(); + if !ca.is_empty() { + let pem = read_pem(ca, "CA certificate")?; + let certificates = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|_| IdeError::invalid("invalid ClickHouse CA certificate PEM"))?; + if certificates.is_empty() { + return Err(IdeError::invalid("ClickHouse CA certificate PEM contains no certificates")); + } + builder = builder.tls_certs_only(certificates); + } + if source.ssl == DbSslMode::VerifyCa { + if ca.is_empty() { + return Err(IdeError::invalid("ClickHouse Verify CA requires a CA certificate file")); + } + builder = builder.tls_danger_accept_invalid_hostnames(true); + } + let cert = source.ssl_cert_file.trim(); + let key = source.ssl_key_file.trim(); + if cert.is_empty() != key.is_empty() { + return Err(IdeError::invalid("ClickHouse client TLS requires both certificate and key files")); + } + if !cert.is_empty() { + let mut pem = read_pem(cert, "client certificate")?; + pem.push(b'\n'); + pem.extend(read_pem(key, "client key")?); + let identity = reqwest::Identity::from_pem(&pem) + .map_err(|_| IdeError::invalid("invalid ClickHouse client certificate or key PEM"))?; + builder = builder.identity(identity); + } + let proxy = ide_core::proxy_settings::proxy(); + let client = ide_http::client::with_proxy(builder, &proxy).build().map_err(http_error)?; + let database = if source.options.default_schema.trim().is_empty() { + source.database.trim() + } else { + source.options.default_schema.trim() + }; + let pool = Self(Arc::new(State { + client: RwLock::new(Some(client)), + endpoint, + user: if source.user.is_empty() { "default".into() } else { source.user.clone() }, + password: password.unwrap_or_default().to_owned(), + database: if database.is_empty() { "default".into() } else { database.into() }, + settings: setup.params, + read_only: source.read_only, + session_id: uuid::Uuid::new_v4().to_string(), + session: Mutex::new(false), + closed: CancellationToken::new(), + })); + let cancel = CancellationToken::new(); + let initialize = async { + pool.execute("SELECT 1", 1, &cancel).await?; + for sql in setup.statements { + pool.execute(&sql, 0, &cancel).await?; + } + Ok(pool.clone()) + }; + tokio::pin!(initialize); + tokio::select! { + result = &mut initialize => { + if result.is_err() { + pool.close(); + } + result + }, + () = tokio::time::sleep(timeout) => { + cancel.cancel(); + let _ = initialize.await; + pool.close(); + Err(IdeError::new(ErrorKind::TimedOut, "timed out connecting to ClickHouse HTTP")) + } + } + } + + pub fn close(&self) { + self.0.closed.cancel(); + self.0.client.write().take(); + } + + pub fn is_closed(&self) -> bool { + self.0.closed.is_cancelled() + } + + fn client(&self) -> IdeResult { + self.0.client.read().clone().ok_or_else(|| IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")) + } + + fn request(&self, client: &Client, query_id: &str, max_rows: usize, initialized: bool) -> RequestBuilder { + let mut request = client + .post(self.0.endpoint.clone()) + .basic_auth(&self.0.user, Some(&self.0.password)) + .header(reqwest::header::CONTENT_TYPE, "text/plain; charset=utf-8") + .query(&self.0.settings) + .query(&[ + ("session_id", self.0.session_id.clone()), + ("session_timeout", "3600".into()), + ("query_id", query_id.to_owned()), + ("default_format", FORMAT.into()), + ("output_format_json_quote_denormals", "1".into()), + ("max_result_rows", max_rows.saturating_add(1).to_string()), + ("result_overflow_mode", "break".into()), + ]); + if initialized { + request = request.query(&[("session_check", "1")]); + } else { + request = request.query(&[("database", &self.0.database)]); + } + if self.0.read_only { + request = request.query(&[("readonly", "1")]); + } + request + } + + pub async fn execute(&self, sql: &str, max_rows: usize, cancel: &CancellationToken) -> IdeResult { + if transaction_statement(sql) { + return Err(transaction_error()); + } + let mut session = tokio::select! { + biased; + () = cancel.cancelled() => return Err(IdeError::cancelled()), + () = self.0.closed.cancelled() => return Err(IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")), + session = self.0.session.lock() => session, + }; + let client = self.client()?; + let query_id = uuid::Uuid::new_v4().to_string(); + let request = self.request(&client, &query_id, max_rows, *session).body(sql.to_owned()); + let started = Instant::now(); + let fetch = async { + let mut response = request.send().await.map_err(http_error)?; + let execution_ms = started.elapsed().as_secs_f64() * 1000.0; + let status = response.status(); + let header = |name: &str| response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_owned); + let exception_code = header("X-ClickHouse-Exception-Code"); + let format = header("X-ClickHouse-Format"); + let affected = header("X-ClickHouse-Summary") + .and_then(|v| serde_json::from_str::(&v).ok()) + .and_then(|v| { + v.get("written_rows") + .and_then(|n| n.as_str().and_then(|n| n.parse::().ok()).or_else(|| n.as_u64())) + }); + if !status.is_success() || exception_code.as_deref().is_some_and(|code| code != "0") { + let body = response.bytes().await.map_err(http_error)?; + return Err(self.server_error(status, exception_code.as_deref(), &body)); + } + let (columns, rows, has_more) = if format.as_deref().is_some_and(|f| f != FORMAT) { + let body = response.bytes().await.map_err(http_error)?; + if !body.iter().all(u8::is_ascii_whitespace) { + return Err(IdeError::invalid( + "ClickHouse result grids need the default output format; remove the query's FORMAT clause", + )); + } + (Vec::new(), Vec::new(), false) + } else { + let mut reader = RowReader::new(max_rows); + while let Some(chunk) = response.chunk().await.map_err(http_error)? { + reader.push(&chunk)?; + } + reader.finish().map_err(|error| self.redact_error(error))? + }; + let affected = if columns.is_empty() { affected.map(|n| n as f64) } else { None }; + Ok(QueryResult { + statement: sql.to_owned(), + columns, + rows, + affected, + has_more, + elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, + execution_ms, + }) + }; + let result = tokio::select! { + biased; + () = cancel.cancelled() => None, + () = self.0.closed.cancelled() => None, + result = fetch => Some(result), + }; + match result { + Some(result) => { + if result.is_ok() { + *session = true; + } + result + } + None => { + let killed = self.kill(&client, &query_id).await; + match killed { + Ok(()) => Err(IdeError::cancelled()), + Err(error) => Err(IdeError::new( + ErrorKind::Cancelled, + format!("cancelled locally; ClickHouse server cancellation failed: {}", error.message), + )), + } + } + } + } + + async fn kill(&self, client: &Client, query_id: &str) -> IdeResult<()> { + let response = client + .post(self.0.endpoint.clone()) + .basic_auth(&self.0.user, Some(&self.0.password)) + .timeout(Duration::from_secs(3)) + .body(format!("KILL QUERY WHERE query_id = '{query_id}' SYNC")) + .send() + .await + .map_err(http_error)?; + let status = response.status(); + let code = response + .headers() + .get("X-ClickHouse-Exception-Code") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body = response.bytes().await.map_err(http_error)?; + if status.is_success() && !code.as_deref().is_some_and(|code| code != "0") && !body.starts_with(b"Code:") { + Ok(()) + } else { + Err(self.server_error(status, code.as_deref(), &body)) + } + } + + fn redact_error(&self, mut error: IdeError) -> IdeError { + if !self.0.password.is_empty() { + error.message = error.message.replace(&self.0.password, "[redacted]"); + } + error + } + + fn server_error(&self, status: StatusCode, code: Option<&str>, body: &[u8]) -> IdeError { + let message = String::from_utf8_lossy(body); + let message: String = exception_text(&message).chars().take(16_384).collect(); + let auth = matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) + || matches!(code, Some("193" | "194" | "195" | "516" | "497")); + let detail = if message.is_empty() { status.canonical_reason().unwrap_or("request failed") } else { &message }; + let detail = if detail.contains("SESSION_NOT_FOUND") || detail.contains("Session not found") { + "ClickHouse HTTP session expired; reconnect to restore session settings" + } else { + detail + }; + self.redact_error(IdeError::new( + if auth { ErrorKind::PermissionDenied } else { ErrorKind::InvalidInput }, + format!("ClickHouse HTTP {}: {detail}", status.as_u16()), + )) + } +} + +type DecodedResult = (Vec, Vec>>, bool); + +/// Decodes a streamed [`FORMAT`] body. Keeps the first `max_rows` rows; later rows only set +/// `has_more` and are read past unparsed, so a grid limit bounds memory however large the +/// block `result_overflow_mode=break` lets the server send. A line that is not a JSON array +/// starts the exception ClickHouse writes when a query fails mid-stream. +struct RowReader { + max_rows: usize, + pending: Vec, + names: Option>, + types: Option>, + rows: Vec>>, + more: bool, + exception: Option, +} + +impl RowReader { + fn new(max_rows: usize) -> Self { + Self { max_rows, pending: Vec::new(), names: None, types: None, rows: Vec::new(), more: false, exception: None } + } + + fn push(&mut self, chunk: &[u8]) -> IdeResult<()> { + self.pending.extend_from_slice(chunk); + let buffer = std::mem::take(&mut self.pending); + let mut start = 0; + while let Some(end) = buffer[start..].iter().position(|&b| b == b'\n') { + self.line(&buffer[start..start + end])?; + start += end + 1; + } + self.pending = buffer[start..].to_vec(); + Ok(()) + } + + fn line(&mut self, line: &[u8]) -> IdeResult<()> { + if let Some(exception) = &mut self.exception { + if exception.len() < 16_384 { + exception.push_str(&String::from_utf8_lossy(line)); + exception.push('\n'); + } + return Ok(()); + } + let text = line.trim_ascii(); + if text.is_empty() { + return Ok(()); + } + if text[0] != b'[' { + self.exception = Some(format!("{}\n", String::from_utf8_lossy(line))); + return Ok(()); + } + if self.names.is_some() && self.types.is_some() && self.rows.len() >= self.max_rows { + self.more = true; + return Ok(()); + } + let Ok(values) = serde_json::from_slice::>>(text) else { + self.exception = Some(format!("{}\n", String::from_utf8_lossy(line))); + return Ok(()); + }; + let protocol = |message: String| IdeError::new(ErrorKind::Protocol, message); + let header = |values: Vec>| { + values + .iter() + .map(|v| serde_json::from_str::(v.get())) + .collect::, _>>() + .map_err(|e| protocol(format!("invalid ClickHouse result header: {e}"))) + }; + let Some(names) = &self.names else { + self.names = Some(header(values)?); + return Ok(()); + }; + if self.types.is_none() { + self.types = Some(header(values)?); + return Ok(()); + } + if values.len() != names.len() { + return Err(protocol("ClickHouse result row width does not match its metadata".into())); + } + let row = values + .into_iter() + .map(|value| { + let text = value.get(); + if text == "null" { + Ok(None) + } else if text.starts_with('"') { + serde_json::from_str::(text).map(Some).map_err(|e| protocol(e.to_string())) + } else { + Ok(Some(text.to_owned())) + } + }) + .collect::>()?; + self.rows.push(row); + Ok(()) + } + + fn finish(mut self) -> IdeResult { + let rest = std::mem::take(&mut self.pending); + self.line(&rest)?; + if let Some(exception) = self.exception { + return Err(IdeError::invalid(format!("ClickHouse: {}", exception_text(&exception)))); + } + let Some(names) = self.names else { return Ok((Vec::new(), Vec::new(), false)) }; + let types = self.types.unwrap_or_default(); + if types.len() != names.len() { + return Err(IdeError::new(ErrorKind::Protocol, "ClickHouse result types do not match its columns")); + } + let columns = names.into_iter().zip(types).map(|(name, type_name)| ResultColumn { name, type_name }).collect(); + Ok((columns, self.rows, self.more)) + } +} + +/// The server's message in a failed response: rows already streamed and the +/// `__exception__` block markers ClickHouse wraps a mid-stream error in are dropped. +fn exception_text(body: &str) -> &str { + let Some(at) = body.find("Code:") else { return body.trim() }; + let text = &body[at..]; + match text.find("\n__exception__") { + // The line before the closing marker is ` `. + Some(end) => text[..end].trim_end().rsplit_once('\n').map_or(text[..end].trim(), |(message, _)| message.trim()), + None => text.trim(), + } +} + +fn http_error(error: reqwest::Error) -> IdeError { + use std::error::Error; + let kind = if error.is_timeout() { ErrorKind::TimedOut } else { ErrorKind::Io }; + let error = error.without_url(); + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(error) = source { + let detail = error.to_string(); + if !message.contains(&detail) { + message.push_str(": "); + message.push_str(&detail); + } + source = error.source(); + } + IdeError::new(kind, format!("ClickHouse HTTP: {message}")) +} + +fn read_pem(path: &str, label: &str) -> IdeResult> { + std::fs::read(ide_ssh::expand(path)) + .map_err(|error| IdeError::new(ErrorKind::Io, format!("cannot read ClickHouse {label}: {error}"))) +} + +fn reserved_setting(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + name.starts_with("output_format_json_") + || matches!( + name.as_str(), + "user" + | "password" + | "query" + | "database" + | "session_id" + | "session_check" + | "session_timeout" + | "query_id" + | "default_format" + | "max_result_rows" + | "result_overflow_mode" + | "readonly" + ) +} + +fn transaction_statement(sql: &str) -> bool { + let mut sql = sql.trim_start(); + loop { + if let Some(rest) = sql.strip_prefix("--").or_else(|| sql.strip_prefix('#')) { + sql = rest.split_once('\n').map_or("", |(_, rest)| rest).trim_start(); + } else if let Some(rest) = sql.strip_prefix("/*") { + sql = rest.split_once("*/").map_or("", |(_, rest)| rest).trim_start(); + } else { + break; + } + } + let mut words = sql.split(|c: char| !c.is_ascii_alphanumeric() && c != '_').filter(|s| !s.is_empty()); + let first = words.next().unwrap_or_default(); + ["BEGIN", "COMMIT", "ROLLBACK", "SAVEPOINT", "RELEASE"].iter().any(|s| first.eq_ignore_ascii_case(s)) + || (first.eq_ignore_ascii_case("START") && words.next().is_some_and(|s| s.eq_ignore_ascii_case("TRANSACTION"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::conn::Pool; + use crate::source::{DbDriver, DbOptions, DbProperty}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + const ONE: &str = "[\"1\"]\n[\"UInt8\"]\n[1]\n"; + + async fn read_request(stream: &mut tokio::net::TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + loop { + let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer)).await.unwrap().unwrap(); + assert!(count > 0, "request ended early"); + bytes.extend_from_slice(&buffer[..count]); + if let Some(end) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + let header = String::from_utf8_lossy(&bytes[..end]); + let length: usize = header + .lines() + .find_map(|line| { + line.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()) + }) + .unwrap_or(0); + if bytes.len() >= end + 4 + length { + return String::from_utf8(bytes).unwrap(); + } + } + } + } + + async fn reply(stream: &mut tokio::net::TcpStream, status: &str, headers: &str, body: &str) { + let response = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{headers}\r\n{body}", body.len()); + stream.write_all(response.as_bytes()).await.unwrap(); + } + + async fn server( + responses: Vec<(&'static str, &'static str, &'static str)>, + ) -> (u16, tokio::task::JoinHandle>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for (status, headers, body) in responses { + let (mut stream, _) = + tokio::time::timeout(Duration::from_secs(5), listener.accept()).await.unwrap().unwrap(); + requests.push(read_request(&mut stream).await); + reply(&mut stream, status, headers, body).await; + } + requests + }); + (port, task) + } + + fn source(port: u16) -> DataSource { + DataSource { driver: DbDriver::Clickhouse, host: "127.0.0.1".into(), port: Some(port), ..Default::default() } + } + + fn decode(body: &[u8], max_rows: usize, chunk: usize) -> IdeResult { + let mut reader = RowReader::new(max_rows); + for part in body.chunks(chunk) { + reader.push(part)?; + } + reader.finish() + } + + #[test] + fn rows_preserve_precision_nulls_and_strings_across_chunks() { + let body = concat!( + r#"["n", "d", "v", "v", "v"]"#, + "\n", + r#"["UInt64", "Decimal(38, 9)", "Nullable(String)", "Nullable(String)", "Nullable(String)"]"#, + "\n", + r#"[18446744073709551615, 12345678901234567890.123456789, null, "null", "ᴺᵁᴸᴸ"]"#, + ); + for chunk in [1, 7, body.len()] { + let (columns, rows, more) = decode(body.as_bytes(), 1, chunk).unwrap(); + assert_eq!(columns.len(), 5); + assert_eq!(columns[1].type_name, "Decimal(38, 9)"); + assert_eq!( + rows[0], + vec![ + Some("18446744073709551615".into()), + Some("12345678901234567890.123456789".into()), + None, + Some("null".into()), + Some("ᴺᵁᴸᴸ".into()), + ] + ); + assert!(!more); + } + let (columns, rows, more) = decode(b"[\"x\"]\n[\"String\"]\n", 0, 64).unwrap(); + assert_eq!(columns[0].name, "x"); + assert!(rows.is_empty() && !more); + assert_eq!(decode(b"", 5, 64).unwrap().0.len(), 0); + assert!(decode(b"[\"x\"]\n[\"UInt8\"]\n[1, 2]\n", 1, 64).is_err()); + } + + #[test] + fn rows_past_the_limit_are_skipped_but_a_later_exception_still_fails() { + let mut body = String::from("[\"n\"]\n[\"UInt64\"]\n"); + for n in 0..1000 { + body.push_str(&format!("[{n}]\n")); + } + let (_, rows, more) = decode(body.as_bytes(), 2, 4096).unwrap(); + assert_eq!(rows, vec![vec![Some("0".into())], vec![Some("1".into())]]); + assert!(more); + body.push_str("\n__exception__\nabcdefgh\nCode: 395. DB::Exception: Value passed to 'throwIf' is non-zero\n"); + body.push_str("62 abcdefgh\n__exception__\n"); + let error = decode(body.as_bytes(), 2, 4096).unwrap_err(); + assert_eq!(error.message, "ClickHouse: Code: 395. DB::Exception: Value passed to 'throwIf' is non-zero"); + let multiline = "Code: 62. DB::Exception: Syntax error\nat line 2"; + assert_eq!(exception_text(multiline), multiline); + } + + #[tokio::test] + async fn http_requests_use_auth_settings_limits_and_a_shared_session() { + let (port, requests) = server(vec![ + ("200 OK", "X-ClickHouse-Format: JSONCompactEachRowWithNamesAndTypes\r\n", ONE), + ("200 OK", "", ""), + ( + "200 OK", + "X-ClickHouse-Format: JSONCompactEachRowWithNamesAndTypes\r\n", + "[\"n\"]\n[\"UInt64\"]\n[\"1\"]\n[\"2\"]\n[\"3\"]\n", + ), + ]) + .await; + let source = DataSource { + user: "reader".into(), + read_only: true, + options: DbOptions { default_schema: "analytics".into(), time_zone: "UTC".into(), ..Default::default() }, + properties: vec![DbProperty { name: "max_threads".into(), value: "2".into() }], + ..source(port) + }; + let pool = Pool::connect(&source, Some("mock-password"), None).await.unwrap(); + pool.execute("USE other", 0, &CancellationToken::new()).await.unwrap(); + let result = pool.execute("SELECT number FROM numbers(3)", 1, &CancellationToken::new()).await.unwrap(); + assert_eq!(result.rows, vec![vec![Some("1".into())]]); + assert!(result.has_more && result.affected.is_none()); + let requests = requests.await.unwrap(); + let target = |request: &String| { + Url::parse(&format!( + "http://localhost{}", + request.lines().next().unwrap().split_whitespace().nth(1).unwrap() + )) + .unwrap() + }; + let urls: Vec<_> = requests.iter().map(target).collect(); + let value = |url: &Url, key: &str| url.query_pairs().find(|(k, _)| k == key).map(|(_, v)| v.into_owned()); + assert_eq!(value(&urls[0], "database").as_deref(), Some("analytics")); + assert_eq!(value(&urls[1], "database"), None); + assert_eq!(value(&urls[1], "session_check").as_deref(), Some("1")); + assert_eq!(value(&urls[0], "session_id"), value(&urls[2], "session_id")); + assert_ne!(value(&urls[0], "query_id"), value(&urls[2], "query_id")); + assert_eq!(value(&urls[2], "max_result_rows").as_deref(), Some("2")); + for (request, url) in requests.iter().zip(&urls) { + assert!(request.starts_with("POST /?")); + assert!(request.contains("Basic cmVhZGVyOm1vY2stcGFzc3dvcmQ=")); + assert!(!url.as_str().contains("reader") && !url.as_str().contains("mock-password")); + assert_eq!(value(url, "readonly").as_deref(), Some("1")); + assert_eq!(value(url, "session_timezone").as_deref(), Some("UTC")); + assert_eq!(value(url, "max_threads").as_deref(), Some("2")); + } + assert!(requests[0].ends_with("SELECT 1")); + assert!(pool.transaction(&[]).await.is_err()); + let cancel = CancellationToken::new(); + cancel.cancel(); + assert_eq!(pool.execute("SELECT 1", 1, &cancel).await.unwrap_err().kind, ErrorKind::Cancelled); + pool.close().await; + assert!(pool.is_closed()); + assert!(pool.execute("SELECT 1", 1, &CancellationToken::new()).await.is_err()); + } + + #[tokio::test] + async fn sequential_queries_reuse_the_http_connection() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + for _ in 0..2 { + read_request(&mut stream).await; + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{ONE}", ONE.len()); + stream.write_all(response.as_bytes()).await.unwrap(); + } + }); + let pool = Pool::connect(&source(port), None, None).await.unwrap(); + assert_eq!( + pool.execute("SELECT 1", 1, &CancellationToken::new()).await.unwrap().rows, + vec![vec![Some("1".into())]] + ); + task.await.unwrap(); + } + + #[tokio::test] + async fn server_errors_keep_details_and_hide_passwords() { + let (port, requests) = server(vec![( + "403 Forbidden", + "X-ClickHouse-Exception-Code: 516\r\n", + "Code: 516. DB::Exception: bad mock-password", + )]) + .await; + let error = match Pool::connect(&source(port), Some("mock-password"), None).await { + Ok(_) => panic!("expected login failure"), + Err(error) => error, + }; + assert_eq!(error.kind, ErrorKind::PermissionDenied); + assert!(error.message.contains("516") && error.message.contains("[redacted]")); + assert!(!error.message.contains("mock-password")); + requests.await.unwrap(); + let (port, requests) = server(vec![("200 OK", "", "Code: 241. DB::Exception: memory limit")]).await; + let error = match Pool::connect(&source(port), None, None).await { + Ok(_) => panic!("expected late query failure"), + Err(error) => error, + }; + assert!(error.message.contains("memory limit")); + requests.await.unwrap(); + let (port, requests) = server(vec![( + "500 Internal Server Error", + "X-ClickHouse-Exception-Code: 395\r\n", + "[\"1\"]\n[\"UInt8\"]\n[1]\n\n__exception__\ntag\nCode: 395. DB::Exception: thrown\n32 tag\n__exception__\n", + )]) + .await; + let error = match Pool::connect(&source(port), None, None).await { + Ok(_) => panic!("expected mid-stream failure"), + Err(error) => error, + }; + assert_eq!(error.message, "ClickHouse HTTP 500: Code: 395. DB::Exception: thrown"); + requests.await.unwrap(); + } + + #[tokio::test] + async fn cancellation_sends_kill_outside_the_busy_session() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let (started, received) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut health, _) = listener.accept().await.unwrap(); + read_request(&mut health).await; + reply(&mut health, "200 OK", "", ONE).await; + drop(health); + let (mut query, _) = listener.accept().await.unwrap(); + let request = read_request(&mut query).await; + started.send(()).unwrap(); + let (mut kill, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()).await.unwrap().unwrap(); + let kill_request = read_request(&mut kill).await; + reply(&mut kill, "200 OK", "", "").await; + (request, kill_request) + }); + let pool = Pool::connect(&source(port), None, None).await.unwrap(); + let cancel = CancellationToken::new(); + let query = pool.execute("SELECT sleep(30)", 1, &cancel); + let cancelled = async { + received.await.unwrap(); + cancel.cancel(); + }; + let (result, ()) = tokio::join!(query, cancelled); + assert_eq!(result.unwrap_err().kind, ErrorKind::Cancelled); + let (query, kill) = server.await.unwrap(); + let target = query.lines().next().unwrap().split_whitespace().nth(1).unwrap(); + let url = Url::parse(&format!("http://localhost{target}")).unwrap(); + let query_id = url.query_pairs().find(|(k, _)| k == "query_id").unwrap().1.into_owned(); + assert!(kill.ends_with(&format!("KILL QUERY WHERE query_id = '{query_id}' SYNC"))); + assert!(!kill.contains("session_id=")); + } + + #[tokio::test] + async fn unsupported_settings_and_transactions_fail_before_connecting() { + for statement in ["BEGIN", "-- comment\n COMMIT", "/* comment */ ROLLBACK", "START TRANSACTION", "SAVEPOINT s"] + { + assert!(transaction_statement(statement)); + } + assert!(!transaction_statement("SELECT 'BEGIN'")); + let sources = [ + DataSource { options: DbOptions { transaction: DbTxMode::Manual, ..Default::default() }, ..source(1) }, + DataSource { + properties: vec![DbProperty { name: "statement_cache_capacity".into(), value: "4".into() }], + ..source(1) + }, + DataSource { properties: vec![DbProperty { name: "password".into(), value: "x".into() }], ..source(1) }, + DataSource { ssl_ca_file: "unused.pem".into(), ..source(1) }, + DataSource { ssl: DbSslMode::VerifyCa, ..source(1) }, + DataSource { ssl: DbSslMode::Require, ssl_cert_file: "unused.pem".into(), ..source(1) }, + ]; + for source in sources { + let error = match Pool::connect(&source, None, None).await { + Ok(_) => panic!("unsupported settings were accepted"), + Err(error) => error, + }; + assert_eq!(error.kind, ErrorKind::InvalidInput); + } + } + + #[tokio::test] + #[ignore = "requires CLICKHOUSE_TEST_HOST, CLICKHOUSE_TEST_USER, and CLICKHOUSE_TEST_PASSWORD"] + async fn clickhouse_cloud_transport() { + let source = DataSource { + driver: DbDriver::Clickhouse, + host: std::env::var("CLICKHOUSE_TEST_HOST").expect("CLICKHOUSE_TEST_HOST"), + port: Some(std::env::var("CLICKHOUSE_TEST_PORT").ok().and_then(|v| v.parse().ok()).unwrap_or(8443)), + user: std::env::var("CLICKHOUSE_TEST_USER").expect("CLICKHOUSE_TEST_USER"), + ssl: DbSslMode::Require, + options: DbOptions { + time_zone: "UTC".into(), + startup_script: "SET max_threads = 2".into(), + ..Default::default() + }, + properties: vec![DbProperty { name: "max_block_size".into(), value: "50".into() }], + ..Default::default() + }; + let password = std::env::var("CLICKHOUSE_TEST_PASSWORD").expect("CLICKHOUSE_TEST_PASSWORD"); + let pool = Pool::connect(&source, Some(&password), None).await.unwrap(); + let cancel = CancellationToken::new(); + let settings = pool + .execute("SELECT getSetting('max_threads'), getSetting('max_block_size'), timezone()", 1, &cancel) + .await + .unwrap(); + assert_eq!(settings.rows, vec![vec![Some("2".into()), Some("50".into()), Some("UTC".into())]]); + let result = pool.execute("SELECT toUInt64('18446744073709551615') AS n, CAST(NULL AS Nullable(String)) AS missing, CAST('ᴺᵁᴸᴸ' AS Nullable(String)) AS text, 'a\\nb' AS lines", 1, &cancel).await.unwrap(); + assert_eq!( + result.rows, + vec![vec![Some("18446744073709551615".into()), None, Some("ᴺᵁᴸᴸ".into()), Some("a\nb".into())]] + ); + assert_eq!(result.columns[1].type_name, "Nullable(String)"); + let limited = pool.execute("SELECT number FROM numbers(100)", 2, &cancel).await.unwrap(); + assert_eq!(limited.rows.len(), 2); + assert!(limited.has_more); + let marker = format!("transport_cancel_{}", uuid::Uuid::new_v4().simple()); + let sql = format!("SELECT sleepEachRow(0.05) FROM numbers(50) /* {marker} */"); + let stop = CancellationToken::new(); + let query = pool.execute(&sql, 1, &stop); + let cancel_query = async { + tokio::time::sleep(Duration::from_millis(500)).await; + stop.cancel(); + }; + let (cancelled, ()) = tokio::join!(query, cancel_query); + let cancelled = cancelled.unwrap_err(); + assert_eq!(cancelled.kind, ErrorKind::Cancelled); + assert_eq!(cancelled.message, "cancelled"); + let active = pool.execute(&format!("SELECT count() FROM system.processes WHERE position(query, '{marker}') > 0 AND query_id != currentQueryID()"), 1, &cancel).await.unwrap(); + assert_eq!(active.rows, vec![vec![Some("0".into())]]); + let table = format!("transport_test_{}", uuid::Uuid::new_v4().simple()); + pool.execute(&format!("CREATE TEMPORARY TABLE {table} (n UInt8)"), 0, &cancel).await.unwrap(); + let continuity = async { + pool.execute(&format!("INSERT INTO {table} VALUES (7)"), 0, &cancel).await?; + pool.execute(&format!("SELECT n FROM {table}"), 1, &cancel).await + } + .await; + let cleanup = pool.execute(&format!("DROP TEMPORARY TABLE {table}"), 0, &cancel).await; + assert_eq!(continuity.unwrap().rows, vec![vec![Some("7".into())]]); + cleanup.unwrap(); + pool.close().await; + assert!(pool.is_closed()); + } +} diff --git a/crates/ide-db/src/conn.rs b/crates/ide-db/src/conn.rs index aa9fd7f4..94bd0134 100644 --- a/crates/ide-db/src/conn.rs +++ b/crates/ide-db/src/conn.rs @@ -106,6 +106,7 @@ pub enum Pool { Postgres(PgPool), Mysql(MySqlPool), SqlServer(Bb8Pool), + Clickhouse(crate::clickhouse::ClickhousePool), } /// Runs `$body` with `$p` bound to the concrete pool of each driver. @@ -115,7 +116,7 @@ macro_rules! with_pool { Pool::Sqlite($p) => $body, Pool::Postgres($p) => $body, Pool::Mysql($p) => $body, - Pool::SqlServer(_) => unreachable!(), + Pool::SqlServer(_) | Pool::Clickhouse(_) => unreachable!(), } }; } @@ -503,6 +504,9 @@ impl Setup { value.parse::().map_err(|_| IdeError::invalid(format!("{what} must be a whole number: {value}"))) }; match (source.driver, name.to_ascii_lowercase().as_str()) { + (DbDriver::Clickhouse, "statement_cache_capacity") => { + return Err(IdeError::invalid("ClickHouse HTTP does not support statement caching")); + } (_, "statement_cache_capacity") => s.statement_cache = Some(number(&name)? as usize), (_, "connect_timeout") => s.connect_timeout = Some(Duration::from_secs(number(&name)?)), (DbDriver::Postgres, "application_name") => s.application_name = Some(value.to_owned()), @@ -512,6 +516,9 @@ impl Setup { _ => s.params.push((name, value.to_owned())), } } + if source.driver == DbDriver::Clickhouse && !o.time_zone.trim().is_empty() { + s.params.push(("session_timezone".into(), o.time_zone.trim().into())); + } let script = o.startup_script.trim(); if !script.is_empty() { s.statements.extend(crate::split::split(script, source.driver).into_iter().map(|st| st.text)); @@ -527,6 +534,7 @@ impl Pool { Pool::Postgres(_) => DbDriver::Postgres, Pool::Mysql(_) => DbDriver::Mysql, Pool::SqlServer(_) => DbDriver::SqlServer, + Pool::Clickhouse(_) => DbDriver::Clickhouse, } } @@ -534,6 +542,9 @@ impl Pool { pub async fn connect(source: &DataSource, password: Option<&str>, root: Option<&Path>) -> IdeResult { const MAX: u32 = 4; let setup = Setup::of(source)?; + if source.driver == DbDriver::Clickhouse { + return crate::clickhouse::ClickhousePool::connect(source, password, setup).await.map(Pool::Clickhouse); + } let timeout = setup.connect_timeout.unwrap_or(Duration::from_secs(30)); let init = Arc::new(setup.statements); // Runs the session statements (startup script…) on every new connection. @@ -695,12 +706,14 @@ impl Pool { conn.query("SELECT 1", &[]).await.map_err(mssql_err)?.into_results().await.map_err(mssql_err)?; Ok(Pool::SqlServer(pool)) } + DbDriver::Clickhouse => unreachable!(), } } pub async fn close(&self) { match self { Pool::SqlServer(_) => {} + Pool::Clickhouse(p) => p.close(), _ => with_pool!(self, p => p.close().await), } } @@ -708,12 +721,16 @@ impl Pool { pub fn is_closed(&self) -> bool { match self { Pool::SqlServer(_) => false, + Pool::Clickhouse(p) => p.is_closed(), _ => with_pool!(self, p => p.is_closed()), } } /// Runs `sql` (no row limit beyond `max`), for introspection: cells are text (see [`Cells::Text`]). pub async fn rows(&self, sql: &str, max: usize) -> IdeResult>>> { + if let Pool::Clickhouse(p) = self { + return Ok(p.execute(sql, max, &CancellationToken::new()).await?.rows); + } let fetched: IdeResult = match self { Pool::SqlServer(p) => { let mut conn = p.get_owned().await.map_err(mssql_pool_err)?; @@ -727,6 +744,9 @@ impl Pool { /// Runs one statement and keeps at most `max_rows` rows. Cancelling stops /// the statement on the server (PostgreSQL, MySQL) and drops the connection. pub async fn execute(&self, sql: &str, max_rows: usize, cancel: &CancellationToken) -> IdeResult { + if let Pool::Clickhouse(p) = self { + return p.execute(sql, max_rows, cancel).await; + } let mut conn = Held::acquire(self).await?; let r = conn.execute(self, sql, max_rows, cancel).await; if cancel.is_cancelled() { @@ -740,6 +760,7 @@ impl Pool { /// row when flagged (table editor submits). pub async fn transaction(&self, statements: &[(String, bool)]) -> IdeResult { match self { + Pool::Clickhouse(_) => Err(crate::clickhouse::transaction_error()), Pool::SqlServer(p) => { let mut conn = p.get_owned().await.map_err(mssql_pool_err)?; let result = async { @@ -790,6 +811,7 @@ pub enum Held { Postgres(PoolConnection), Mysql(PoolConnection), SqlServer(Box>), + Clickhouse(crate::clickhouse::ClickhousePool), } impl Held { @@ -799,6 +821,12 @@ impl Held { Pool::Postgres(p) => Held::Postgres(p.acquire().await.map_err(db_err)?), Pool::Mysql(p) => Held::Mysql(p.acquire().await.map_err(db_err)?), Pool::SqlServer(p) => Held::SqlServer(Box::new(p.get_owned().await.map_err(mssql_pool_err)?)), + Pool::Clickhouse(p) => { + if p.is_closed() { + return Err(IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")); + } + Held::Clickhouse(p.clone()) + } }) } @@ -806,13 +834,14 @@ impl Held { pub async fn command(&mut self, sql: &str) -> IdeResult<()> { match self { Held::SqlServer(c) => mssql_command(c, sql).await, + Held::Clickhouse(_) => Err(crate::clickhouse::transaction_error()), other => { let sql = sqlx::AssertSqlSafe(sql.to_owned()); match other { Held::Sqlite(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), Held::Postgres(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), Held::Mysql(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), - Held::SqlServer(_) => unreachable!(), + Held::SqlServer(_) | Held::Clickhouse(_) => unreachable!(), } .map_err(db_err) } @@ -833,6 +862,7 @@ impl Held { Held::SqlServer(mut c) => { c.0.take(); } + Held::Clickhouse(_) => {} } } @@ -845,6 +875,9 @@ impl Held { max_rows: usize, cancel: &CancellationToken, ) -> IdeResult { + if let Held::Clickhouse(c) = self { + return c.execute(sql, max_rows, cancel).await; + } let started = Instant::now(); // `$id`: the query naming the server process, `$kill`: how another connection stops it. macro_rules! run { @@ -887,6 +920,7 @@ impl Held { () = cancel.cancelled() => return Err(IdeError::cancelled()), } } + Held::Clickhouse(_) => unreachable!(), }; let (columns, rows, affected, has_more, first) = fetched; let elapsed_ms = started.elapsed().as_secs_f64() * 1000.0; @@ -907,7 +941,7 @@ pub fn begin_statement(driver: DbDriver) -> &'static str { match driver { DbDriver::Mysql => "START TRANSACTION", DbDriver::SqlServer => "BEGIN TRAN", - DbDriver::Sqlite | DbDriver::Postgres => "BEGIN", + DbDriver::Sqlite | DbDriver::Postgres | DbDriver::Clickhouse => "BEGIN", } } diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 5f7034c5..71545ecb 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -2,6 +2,7 @@ //! execution ([`conn`]), introspection, table data paging / editing ([`sql`]) //! and statement splitting ([`split`]), for SQLite, PostgreSQL and MySQL/MariaDB. +mod clickhouse; pub mod conn; pub mod history; pub mod plan; @@ -212,6 +213,8 @@ pub struct DbManager { /// App-wide SSH connections and host-key trust. ssh: Arc, saved_passwords: SecretStore, + /// Directory of the global data sources and their saved passwords ([`DbManager::with_global_sources`]). + global: Option<(PathBuf, SecretStore)>, history: Arc, /// Records executed statements in the history off the async threads, in order. recorder: std::sync::mpsc::Sender<(String, String, f64)>, @@ -338,6 +341,7 @@ impl DbManager { } Self { saved_passwords: SecretStore::new(passwords, &root), + global: None, history, recorder, root, @@ -350,8 +354,42 @@ impl DbManager { } } + /// Also lists the global data sources stored in `dir` (the app data directory). + pub fn with_global_sources(mut self, dir: PathBuf) -> Self { + let passwords = self.saved_passwords.scoped(&dir); + self.global = Some((dir, passwords)); + self + } + + /// The project's data sources, then the global ones (a project one wins on the same id). pub fn sources(&self) -> IdeResult> { - source::load(&self.root) + let mut all = source::load(&self.root)?; + let global: Vec<_> = self.global_sources().into_iter().filter(|g| all.iter().all(|s| s.id != g.id)).collect(); + all.extend(global); + Ok(all) + } + + fn global_sources(&self) -> Vec { + let Some((dir, _)) = &self.global else { return Vec::new() }; + source::load_global(dir).unwrap_or_else(|e| { + tracing::warn!("global data sources: {}", e.message); + Vec::new() + }) + } + + /// Where the saved passwords of data source `id` live: app-wide for a global one. + fn saved_for(&self, id: &str) -> &SecretStore { + match &self.global { + Some((_, store)) if self.global_sources().iter().any(|s| s.id == id) => store, + _ => &self.saved_passwords, + } + } + + fn store_of(&self, source: &DataSource) -> &SecretStore { + match &self.global { + Some((_, store)) if source.global => store, + _ => &self.saved_passwords, + } } /// The connection settings of target `target` ([`DataSource::resolved`]), with @@ -371,18 +409,43 @@ impl DbManager { } /// Saves the data source list; connections of removed or changed sources are closed. + /// Global ones (`global`) go to the app-wide list, the others to the project's. pub async fn save_sources(&self, sources: Vec) -> IdeResult<()> { let old = self.sources().unwrap_or_default(); - source::save(&self.root, &sources)?; + let (global, project): (Vec<_>, Vec<_>) = sources.iter().cloned().partition(|s| s.global); + match &self.global { + // Rewritten only when it changes: other projects' windows edit the same file. + Some((dir, _)) => { + if global != old.iter().filter(|s| s.global).cloned().collect::>() { + source::save_global(dir, &global)?; + } + } + None if !global.is_empty() => return Err(IdeError::invalid("global data sources are not available here")), + None => {} + } + source::save(&self.root, &project)?; for o in old { match sources.iter().find(|s| s.id == o.id) { Some(n) if *n == o => {} gone_or_changed => { - if gone_or_changed.is_none() { - for k in [o.id.clone(), secrets::ssh_key(&o.id)] { - self.session_passwords.lock().remove(&k); - self.saved_passwords.delete(&k); + let keys = [o.id.clone(), secrets::ssh_key(&o.id)]; + match gone_or_changed { + None => { + for k in &keys { + self.session_passwords.lock().remove(k); + self.store_of(&o).delete(k); + } } + // Made global or project-level: its saved passwords move along. + Some(n) if n.global != o.global => { + for k in &keys { + if let Some(p) = self.store_of(&o).get(k) { + self.store_of(n).set(k, &p)?; + self.store_of(&o).delete(k); + } + } + } + Some(_) => {} } self.disconnect_source(&o.id).await; } @@ -394,7 +457,7 @@ impl DbManager { /// Remembers `password` for the session, and in the saved passwords when /// the data source saves passwords (`None` forgets it). pub async fn set_password(&self, id: &str, password: Option, save: bool) -> IdeResult<()> { - self.remember(id.to_owned(), password, save).await + self.remember(id, id.to_owned(), password, save).await } /// [`DbManager::set_password`] for the SSH password / key passphrase. @@ -406,10 +469,10 @@ impl DbManager { self.ssh.set_secret(&c, secret, save); return Ok(()); } - self.remember(secrets::ssh_key(id), secret, save).await + self.remember(id, secrets::ssh_key(id), secret, save).await } - async fn remember(&self, key: String, secret: Option, save: bool) -> IdeResult<()> { + async fn remember(&self, id: &str, key: String, secret: Option, save: bool) -> IdeResult<()> { match &secret { Some(p) => { self.session_passwords.lock().insert(key.clone(), p.clone()); @@ -418,10 +481,11 @@ impl DbManager { self.session_passwords.lock().remove(&key); } } + let store = self.saved_for(id); let result = match (secret, save) { - (Some(p), true) => self.saved_passwords.set(&key, &p), + (Some(p), true) => store.set(&key, &p), _ => { - self.saved_passwords.delete(&key); + store.delete(&key); Ok(()) } }; @@ -432,18 +496,18 @@ impl DbManager { Ok(()) } - async fn secret(&self, key: String, saved: bool) -> Option { + async fn secret(&self, source: &DataSource, key: String, saved: bool) -> Option { if let Some(p) = self.session_passwords.lock().get(&key) { return Some(p.clone()); } if !saved { return None; } - self.saved_passwords.get(&key) + self.store_of(source).get(&key) } async fn password(&self, source: &DataSource) -> Option { - match self.secret(source.id.clone(), source.driver != DbDriver::Sqlite && source.save_password).await { + match self.secret(source, source.id.clone(), source.driver != DbDriver::Sqlite && source.save_password).await { Some(p) => Some(p), None => source.url_password(), } @@ -499,7 +563,7 @@ impl DbManager { let session = if config.is_empty() { let secret = match ssh_secret { Some(s) => Some(s), - None => self.secret(secrets::ssh_key(&source.id), ssh.save_secret).await, + None => self.secret(source, secrets::ssh_key(&source.id), ssh.save_secret).await, }; let target = SshTarget { host: ssh.host.clone(), @@ -547,12 +611,14 @@ impl DbManager { })?; let version = match source.driver { DbDriver::Sqlite => "SELECT 'SQLite ' || sqlite_version()", - DbDriver::Postgres => "SELECT version()", + DbDriver::Postgres | DbDriver::Clickhouse => "SELECT version()", DbDriver::Mysql => "SELECT CONCAT(@@version_comment, ' ', VERSION())", DbDriver::SqlServer => "SELECT @@VERSION", }; - let v = pool.rows(version, 1).await.ok().and_then(|r| r.into_iter().next()?.into_iter().next()?); + let result = pool.rows(version, 1).await; pool.close().await; + let rows = if source.driver == DbDriver::Clickhouse { result? } else { result.unwrap_or_default() }; + let v = rows.into_iter().next().and_then(|r| r.into_iter().next().flatten()); Ok(v.unwrap_or_else(|| "Connected".into())) } @@ -701,8 +767,11 @@ impl DbManager { _ => indexes.push(IndexInfo { name, unique: flag(&unique), columns: vec![col] }), } } - let fk_rows = - pool.rows(&sql::table_foreign_keys(driver, schema, table), MAX_META_ROWS).await.unwrap_or_default(); + let fk_rows = if driver == DbDriver::Clickhouse { + Vec::new() + } else { + pool.rows(&sql::table_foreign_keys(driver, schema, table), MAX_META_ROWS).await.unwrap_or_default() + }; let mut foreign_keys = group_foreign_keys(fk_rows, &[]); // SQLite keys that reference the primary key implicitly: look it up. for k in foreign_keys.iter_mut().filter(|k| k.ref_columns.is_empty()) { @@ -715,13 +784,15 @@ impl DbManager { .collect(); } foreign_keys.retain(|k| k.columns.len() == k.ref_columns.len()); - let triggers = pool - .rows(&sql::triggers(driver, schema, table), MAX_META_ROWS) - .await - .unwrap_or_else(|e| { + let trigger_rows = if driver == DbDriver::Clickhouse { + Vec::new() + } else { + pool.rows(&sql::triggers(driver, schema, table), MAX_META_ROWS).await.unwrap_or_else(|e| { tracing::warn!("triggers: {}", e.message); Vec::new() }) + }; + let triggers = trigger_rows .into_iter() .filter_map(|r| { let mut it = r.into_iter(); @@ -822,6 +893,11 @@ impl DbManager { let ddl = cells(pool.rows(&q, 1).await?, 1).into_iter().next(); ddl.map(|d| format!("{d};\n")).ok_or_else(missing) } + DbDriver::Clickhouse => { + let q = format!("SHOW CREATE TABLE {}", sql::qualified(driver, schema, name)); + let ddl = cells(pool.rows(&q, 1).await?, 0).into_iter().next(); + ddl.map(|d| format!("{};\n", d.trim_end().trim_end_matches(';'))).ok_or_else(missing) + } DbDriver::SqlServer => { let obj = format!("{}.{}", schema, name).replace('\'', "''"); let ddl = cells(pool.rows(&format!("SELECT OBJECT_DEFINITION(OBJECT_ID('{}'))", obj), 1).await?, 0) @@ -902,10 +978,14 @@ impl DbManager { } } // Foreign keys are a nicety (JOIN completion): a server that refuses the catalog query still completes. - let fk_rows = pool.rows(sql::foreign_keys(driver), MAX_META_ROWS).await.unwrap_or_else(|e| { - tracing::warn!("foreign keys: {}", e.message); + let fk_rows = if driver == DbDriver::Clickhouse { Vec::new() - }); + } else { + pool.rows(sql::foreign_keys(driver), MAX_META_ROWS).await.unwrap_or_else(|e| { + tracing::warn!("foreign keys: {}", e.message); + Vec::new() + }) + }; let foreign_keys = group_foreign_keys(fk_rows, &tables); let default_schema = match sql::current_schema(driver) { None => "main".to_owned(), @@ -980,6 +1060,9 @@ impl DbManager { let Some(session) = session else { return pool.execute(statement, max_rows, cancel).await; }; + if pool.driver() == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } let mut held = session.lock().await; if held.is_none() { let mut conn = Held::acquire(&pool).await?; @@ -998,8 +1081,11 @@ impl DbManager { /// Switches `id` between auto-commit and manual commit. Leaving manual /// mode rolls back what was not committed. - pub async fn set_manual_commit(&self, id: &str, manual: bool) { + pub async fn set_manual_commit(&self, id: &str, manual: bool) -> IdeResult<()> { if manual { + if self.source(id)?.driver == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } self.sessions.lock().entry(id.to_owned()).or_default(); } else { let session = self.sessions.lock().remove(id); @@ -1007,10 +1093,14 @@ impl DbManager { end_session(&s).await; } } + Ok(()) } /// Commits (or rolls back) the open transaction of a manual-commit data source. pub async fn finish_transaction(&self, id: &str, commit: bool) -> IdeResult<()> { + if self.source(id).is_ok_and(|s| s.driver == DbDriver::Clickhouse) { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } let session = self.sessions.lock().get(id).cloned(); let Some(session) = session else { return Ok(()) }; let mut held = session.lock().await; @@ -1052,6 +1142,9 @@ impl DbManager { /// Applies the table editor's pending changes in one transaction. pub async fn apply_edits(&self, id: &str, schema: &str, table: &str, edits: &[DbRowEdit]) -> IdeResult { + if self.source(id)?.driver == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse table editing is unsupported")); + } let pool = self.pool(id).await?; let driver = pool.driver(); let statements = edits @@ -1064,3 +1157,112 @@ impl DbManager { pool.transaction(&statements).await } } + +#[cfg(test)] +mod tests { + use super::*; + use ide_ssh::secrets::testing::files_containing; + use std::path::Path; + + fn manager(dir: &Path) -> DbManager { + ide_ssh::secrets::testing::use_memory_store(); + let data = dir.join("data"); + let ssh = Arc::new(SshManager::new(data.join("ssh-passwords.json"))); + DbManager::new(dir.join("project"), data.join("db-passwords.json"), data.join("db-history.json"), ssh) + } + + #[tokio::test] + async fn saved_passwords_stay_off_disk() { + let tmp = tempfile::tempdir().unwrap(); + manager(tmp.path()).set_password("s1", Some("hunter2-db".into()), true).await.unwrap(); + manager(tmp.path()).set_ssh_secret("s1", Some("hunter2-tunnel".into()), true).await.unwrap(); + assert_eq!(files_containing(tmp.path(), "hunter2-db"), Vec::::new()); + assert_eq!(files_containing(tmp.path(), "hunter2-tunnel"), Vec::::new()); + let reopened = manager(tmp.path()); + assert_eq!(reopened.secret(&DataSource::default(), "s1".into(), true).await.as_deref(), Some("hunter2-db")); + assert_eq!( + reopened.secret(&DataSource::default(), secrets::ssh_key("s1"), true).await.as_deref(), + Some("hunter2-tunnel") + ); + } + + #[tokio::test] + async fn moves_plaintext_passwords_of_older_versions_off_disk() { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("project").to_string_lossy().into_owned(); + let legacy = serde_json::json!({ project: { "s1": "old-db-pw" }, "/other": { "s2": "other-pw" } }); + std::fs::create_dir_all(tmp.path().join("data")).unwrap(); + std::fs::write(tmp.path().join("data/db-passwords.json"), legacy.to_string()).unwrap(); + let m = manager(tmp.path()); + assert_eq!(m.secret(&DataSource::default(), "s1".into(), true).await.as_deref(), Some("old-db-pw")); + assert_eq!(files_containing(tmp.path(), "old-db-pw"), Vec::::new()); + assert_eq!(files_containing(tmp.path(), "other-pw"), Vec::::new()); + let other = SecretStore::new(tmp.path().join("data/db-passwords.json"), Path::new("/other")); + assert_eq!(other.get("s2").as_deref(), Some("other-pw")); + } + + #[tokio::test] + async fn global_data_sources_are_shared_by_projects_with_their_passwords() { + ide_ssh::secrets::testing::use_memory_store(); + let tmp = tempfile::tempdir().unwrap(); + let (app, one, two) = (tmp.path().join("app"), tmp.path().join("one"), tmp.path().join("two")); + let manager = |root: &std::path::Path| { + let ssh = Arc::new(SshManager::new(app.join("ssh-passwords.json"))); + DbManager::new(root.to_path_buf(), app.join("passwords.json"), app.join("history.json"), ssh) + .with_global_sources(app.clone()) + }; + let (a, b) = (manager(&one), manager(&two)); + let source = |id: &str, global: bool| DataSource { + id: id.into(), + name: id.into(), + driver: DbDriver::Postgres, + save_password: true, + global, + ..Default::default() + }; + a.save_sources(vec![source("local", false), source("shared", false)]).await.unwrap(); + a.set_password("shared", Some("pw".into()), true).await.unwrap(); + // Made global: listed in the other project too, with its saved password. + a.save_sources(vec![source("local", false), source("shared", true)]).await.unwrap(); + let ids = |m: &DbManager| m.sources().unwrap().into_iter().map(|s| (s.id, s.global)).collect::>(); + assert_eq!(ids(&a), [("local".into(), false), ("shared".into(), true)]); + assert_eq!(ids(&b), [("shared".into(), true)]); + assert_eq!(b.password(&b.source("shared").unwrap()).await.as_deref(), Some("pw")); + let project_file = std::fs::read_to_string(one.join(".openidea/data-sources.toml")).unwrap(); + assert!(!project_file.contains("shared") && !project_file.contains("global"), "{project_file}"); + assert!(!std::fs::read_to_string(app.join("data-sources.toml")).unwrap().contains("global")); + // Made project-level again: gone from the other project, the password stays with it. + a.save_sources(vec![source("local", false), source("shared", false)]).await.unwrap(); + assert!(b.sources().unwrap().is_empty()); + assert_eq!(a.password(&a.source("shared").unwrap()).await.as_deref(), Some("pw")); + } + + #[tokio::test] + async fn clickhouse_rejects_manual_transactions_and_table_edits_without_connecting() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let ssh = Arc::new(SshManager::new(root.join("ssh-passwords.json"))); + let db = DbManager::new(root.clone(), root.join("passwords.json"), root.join("history.json"), ssh); + db.save_sources(vec![DataSource { + id: "clickhouse".into(), + name: "clickhouse".into(), + driver: DbDriver::Clickhouse, + host: "127.0.0.1".into(), + port: Some(1), + ..Default::default() + }]) + .await + .unwrap(); + let error = db.set_manual_commit("clickhouse", true).await.unwrap_err(); + assert!(error.message.contains("manual transactions are unsupported")); + assert!(db.sessions.lock().is_empty()); + for commit in [false, true] { + let error = db.finish_transaction("clickhouse", commit).await.unwrap_err(); + assert!(error.message.contains("manual transactions are unsupported")); + } + let error = db.apply_edits("clickhouse", "default", "events", &[]).await.unwrap_err(); + assert!(error.message.contains("table editing is unsupported")); + assert!(!db.is_connected("clickhouse")); + db.set_manual_commit("clickhouse", false).await.unwrap(); + } +} diff --git a/crates/ide-db/src/plan.rs b/crates/ide-db/src/plan.rs index af378913..32b217b6 100644 --- a/crates/ide-db/src/plan.rs +++ b/crates/ide-db/src/plan.rs @@ -35,6 +35,7 @@ pub fn explain_statement(driver: DbDriver, statement: &str) -> String { match driver { DbDriver::Sqlite => format!("EXPLAIN QUERY PLAN {s}"), DbDriver::Postgres => format!("EXPLAIN (FORMAT JSON) {s}"), + DbDriver::Clickhouse => format!("EXPLAIN json = 1 {s}"), DbDriver::Mysql => format!("EXPLAIN {s}"), DbDriver::SqlServer => format!("SET SHOWPLAN_TEXT ON; {s}"), } @@ -44,7 +45,7 @@ pub fn explain_statement(driver: DbDriver, statement: &str) -> String { pub fn parse(driver: DbDriver, result: &QueryResult) -> IdeResult> { match driver { DbDriver::Sqlite => Ok(sqlite(&result.rows)), - DbDriver::Postgres => { + DbDriver::Postgres | DbDriver::Clickhouse => { let text: String = result.rows.iter().filter_map(|r| r.first().cloned().flatten()).collect::>().join("\n"); let json: Value = serde_json::from_str(&text).map_err(|e| IdeError::invalid(format!("plan: {e}")))?; @@ -199,6 +200,7 @@ mod tests { assert_eq!(explain_statement(DbDriver::Sqlite, "select 1;"), "EXPLAIN QUERY PLAN select 1"); assert_eq!(explain_statement(DbDriver::Postgres, " select 1 "), "EXPLAIN (FORMAT JSON) select 1"); assert_eq!(explain_statement(DbDriver::Mysql, "select 1"), "EXPLAIN select 1"); + assert_eq!(explain_statement(DbDriver::Clickhouse, " select 1;"), "EXPLAIN json = 1 select 1"); assert_eq!(explain_statement(DbDriver::SqlServer, "select 1"), "SET SHOWPLAN_TEXT ON; select 1"); } @@ -254,6 +256,21 @@ mod tests { assert_eq!((plan[3].parent, plan[3].operation.as_str()), (Some(2), "Seq Scan")); } + #[test] + fn reads_clickhouse_json_plans_and_rejects_invalid_json() { + let json = r#"[{"Plan": {"Node Type": "Expression", "Node Id": "Expression_5", + "Description": "Project names", "Plans": [ + {"Node Type": "ReadFromMergeTree", "Node Id": "ReadFromMergeTree_0", + "Description": "default.events"} + ]}}]"#; + let plan = parse(DbDriver::Clickhouse, &result(&["explain"], vec![vec![Some(json)]])).unwrap(); + assert_eq!(plan.len(), 2); + assert_eq!((plan[0].parent, plan[0].operation.as_str()), (None, "Expression")); + assert_eq!((plan[1].parent, plan[1].operation.as_str()), (Some(0), "ReadFromMergeTree")); + assert!(plan[1].properties.iter().any(|p| p.name == "Description" && p.value == "default.events")); + assert!(parse(DbDriver::Clickhouse, &result(&["explain"], vec![vec![Some("invalid")]])).is_err()); + } + #[test] fn reads_mysql_rows() { let r = result( diff --git a/crates/ide-db/src/secrets.rs b/crates/ide-db/src/secrets.rs index 8de033f8..9889232a 100644 --- a/crates/ide-db/src/secrets.rs +++ b/crates/ide-db/src/secrets.rs @@ -1,20 +1,15 @@ -//! Saved data source passwords, in the app's data directory rather than the OS -//! credential store (no system prompts). One JSON file for all projects, keyed -//! by project root, then credential key; owner-only on Unix. +//! Saved data source passwords, in the OS credential store like the other saved +//! passwords ([`SecretFile`]). One index file for all projects; each credential is +//! keyed by project root, then credential key. -use ide_core::{IdeError, IdeResult}; -use parking_lot::Mutex; +use ide_core::IdeResult; +use ide_ssh::secrets::SecretFile; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -/// Project root → credential key → secret. -type Contents = BTreeMap>; - -/// Serialises read-modify-write across the projects sharing the file. -static LOCK: Mutex<()> = Mutex::new(()); - pub struct SecretStore { path: PathBuf, + file: SecretFile, project: String, } @@ -23,61 +18,41 @@ pub fn ssh_key(id: &str) -> String { format!("{id}#ssh") } +fn file_key(project: &str, key: &str) -> String { + format!("{project}|{key}") +} + +/// Older versions' file: project root → credential key → secret. +fn legacy(value: serde_json::Value) -> Option> { + let all: BTreeMap> = serde_json::from_value(value).ok()?; + Some( + all.into_iter() + .flat_map(|(project, keys)| keys.into_iter().map(move |(k, s)| (file_key(&project, &k), s))) + .collect(), + ) +} + impl SecretStore { pub fn new(path: PathBuf, root: &Path) -> Self { - Self { path, project: root.to_string_lossy().into_owned() } + let file = SecretFile::with_legacy(path.clone(), legacy); + Self { path, file, project: root.to_string_lossy().into_owned() } + } + + /// The same file, keyed by `scope` instead (global data sources: the app data directory). + pub fn scoped(&self, scope: &Path) -> Self { + Self::new(self.path.clone(), scope) } pub fn get(&self, key: &str) -> Option { - let _g = LOCK.lock(); - self.read().remove(&self.project)?.remove(key) + self.file.get(&file_key(&self.project, key)) } pub fn set(&self, key: &str, secret: &str) -> IdeResult<()> { - let _g = LOCK.lock(); - let mut all = self.read(); - all.entry(self.project.clone()).or_default().insert(key.to_owned(), secret.to_owned()); - self.write(&all) + self.file.set(&file_key(&self.project, key), secret) } pub fn delete(&self, key: &str) { - let _g = LOCK.lock(); - let mut all = self.read(); - let Some(project) = all.get_mut(&self.project) else { return }; - if project.remove(key).is_none() { - return; - } - if project.is_empty() { - all.remove(&self.project); - } - if let Err(e) = self.write(&all) { - tracing::warn!("saved passwords: {}", e.message); - } - } - - fn read(&self) -> Contents { - match std::fs::read(&self.path) { - Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| { - tracing::warn!("saved passwords: {}: {e}", self.path.display()); - Contents::new() - }), - Err(_) => Contents::new(), - } - } - - fn write(&self, all: &Contents) -> IdeResult<()> { - let err = |e: std::io::Error| IdeError::unavailable(format!("saved passwords: {e}")); - if let Some(dir) = self.path.parent() { - std::fs::create_dir_all(dir).map_err(err)?; - } - let tmp = self.path.with_extension("json.tmp"); - let mut options = std::fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); - let bytes = serde_json::to_vec_pretty(all).map_err(|e| IdeError::internal(e.to_string()))?; - std::io::Write::write_all(&mut options.open(&tmp).map_err(err)?, &bytes).map_err(err)?; - std::fs::rename(&tmp, &self.path).map_err(err) + self.file.delete(&file_key(&self.project, key)) } } @@ -87,6 +62,7 @@ mod tests { #[test] fn keeps_projects_apart_and_drops_empty_ones() { + ide_ssh::secrets::testing::use_memory_store(); let tmp = tempfile::tempdir().unwrap(); let file = tmp.path().join("db-passwords.json"); let a = SecretStore::new(file.clone(), Path::new("/p/a")); diff --git a/crates/ide-db/src/source.rs b/crates/ide-db/src/source.rs index a3590ae6..1acb4ea4 100644 --- a/crates/ide-db/src/source.rs +++ b/crates/ide-db/src/source.rs @@ -1,6 +1,7 @@ -//! Data sources: per-project connection settings in `.openidea/data-sources.toml`. -//! Passwords never go there: saved ones live in [`crate::secrets`], the others -//! only in memory for the session. +//! Data sources: per-project connection settings in `.openidea/data-sources.toml`, and +//! global ones (IDEA's Make Global: listed in every project) in `data-sources.toml` of the +//! app data directory. Passwords never go there: saved ones live in [`crate::secrets`], +//! the others only in memory for the session. use ide_core::{IdeError, IdeResult, IpcPath, project_dir}; use serde::{Deserialize, Serialize}; @@ -18,6 +19,7 @@ pub enum DbDriver { Mysql, /// Microsoft SQL Server. SqlServer, + Clickhouse, } impl DbDriver { @@ -27,6 +29,7 @@ impl DbDriver { DbDriver::Postgres => Some(5432), DbDriver::Mysql => Some(3306), DbDriver::SqlServer => Some(1433), + DbDriver::Clickhouse => Some(8123), } } } @@ -179,6 +182,9 @@ pub struct DataSource { pub options: DbOptions, pub introspection: DbIntrospection, pub properties: Vec, + /// Stored app-wide and listed in every project. Implied by the file it is read from, + /// so not written to it. + pub global: bool, } impl Default for DataSource { @@ -207,6 +213,7 @@ impl Default for DataSource { options: DbOptions::default(), introspection: DbIntrospection::default(), properties: Vec::new(), + global: false, } } } @@ -253,6 +260,7 @@ pub fn scheme_driver(scheme: &str) -> Option { "mysql" | "mariadb" | "tidb" | "singlestore" | "memsql" | "oceanbase" => Some(DbDriver::Mysql), "sqlserver" | "mssql" | "mssqlserver" => Some(DbDriver::SqlServer), "sqlite" | "sqlite3" => Some(DbDriver::Sqlite), + "clickhouse" | "http" | "https" => Some(DbDriver::Clickhouse), _ => None, } } @@ -260,7 +268,7 @@ pub fn scheme_driver(scheme: &str) -> Option { /// Parses `scheme://[user[:password]@]host[:port][/database][?k=v&…]`, `jdbc:` forms /// included, and `sqlite:path`. pub fn parse_url(raw: &str) -> IdeResult { - let bad = || IdeError::invalid(format!("not a database URL: {raw}")); + let bad = || IdeError::invalid("invalid database URL"); let url = raw.trim(); let url = url.strip_prefix("jdbc:").or_else(|| url.strip_prefix("JDBC:")).unwrap_or(url); let (scheme, rest) = url.split_once(':').ok_or_else(bad)?; @@ -367,6 +375,15 @@ impl DataSource { } properties.append(&mut s.properties); s.properties = properties; + if s.driver == DbDriver::Clickhouse { + let raw_url = self.url.trim().to_ascii_lowercase(); + let url = raw_url.strip_prefix("jdbc:").unwrap_or(&raw_url); + if url.starts_with("https://") { + s.ssl = DbSslMode::VerifyFull; + } else if url.starts_with("http://") { + s.ssl = DbSslMode::Disable; + } + } Ok(s) } @@ -385,9 +402,32 @@ struct SourcesFile { sources: Vec, } +/// The project's data sources. pub fn load(root: &Path) -> IdeResult> { - let path = project_dir::path(root, FILE); - let text = match std::fs::read_to_string(&path) { + read(&project_dir::path(root, FILE)) +} + +/// Saves the project's data sources. +pub fn save(root: &Path, sources: &[DataSource]) -> IdeResult<()> { + write(&project_dir::path(root, FILE), sources) +} + +/// The global data sources stored in `dir` (the app data directory). +pub fn load_global(dir: &Path) -> IdeResult> { + let mut sources = read(&dir.join(FILE))?; + for s in &mut sources { + s.global = true; + } + Ok(sources) +} + +/// Saves the global data sources in `dir`. +pub fn save_global(dir: &Path, sources: &[DataSource]) -> IdeResult<()> { + write(&dir.join(FILE), sources) +} + +fn read(path: &Path) -> IdeResult> { + let text = match std::fs::read_to_string(path) { Ok(t) => t, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), Err(e) => return Err(e.into()), @@ -396,19 +436,23 @@ pub fn load(root: &Path) -> IdeResult> { Ok(file.sources) } -pub fn save(root: &Path, sources: &[DataSource]) -> IdeResult<()> { +fn write(path: &Path, sources: &[DataSource]) -> IdeResult<()> { for s in sources { if s.id.is_empty() || s.name.trim().is_empty() { return Err(IdeError::invalid("every data source needs an id and a name")); } } - let path = project_dir::path(root, FILE); if let Some(dir) = path.parent() { std::fs::create_dir_all(dir)?; } - let text = toml::to_string_pretty(&SourcesFile { sources: sources.to_vec() }) - .map_err(|e| IdeError::internal(e.to_string()))?; - std::fs::write(path, text)?; + let internal = |e: &dyn std::fmt::Display| IdeError::internal(e.to_string()); + let mut value = toml::Value::try_from(SourcesFile { sources: sources.to_vec() }).map_err(|e| internal(&e))?; + if let Some(list) = value.get_mut("data-source").and_then(toml::Value::as_array_mut) { + for table in list.iter_mut().filter_map(toml::Value::as_table_mut) { + table.remove("global"); + } + } + std::fs::write(path, toml::to_string_pretty(&value).map_err(|e| internal(&e))?)?; Ok(()) } @@ -519,4 +563,31 @@ mod tests { assert!(DataSource { url: "mysql://h/db".into(), ..s.clone() }.resolved().is_err()); assert_eq!(DataSource { url_only: false, ..s }.resolved().unwrap().host, "localhost"); } + + #[test] + fn clickhouse_urls_resolve_http_https_and_credentials() { + assert_eq!(DbDriver::Clickhouse.default_port(), Some(8123)); + let source = DataSource { + driver: DbDriver::Clickhouse, + url_only: true, + url: "https://reader:s%40cret@db.example:8443/analytics?max_execution_time=30".into(), + ..Default::default() + }; + let resolved = source.resolved().unwrap(); + assert_eq!(resolved.host, "db.example"); + assert_eq!(resolved.port, Some(8443)); + assert_eq!(resolved.database, "analytics"); + assert_eq!(resolved.user, "reader"); + assert_eq!(resolved.ssl, DbSslMode::VerifyFull); + assert_eq!(source.url_password().as_deref(), Some("s@cret")); + assert_eq!(resolved.properties[0].name, "max_execution_time"); + let http = DataSource { url: "http://localhost/default".into(), ..source.clone() }.resolved().unwrap(); + assert_eq!(http.ssl, DbSslMode::Disable); + let jdbc = DataSource { url: "jdbc:clickhouse://localhost/analytics".into(), ..source.clone() }; + assert_eq!(jdbc.resolved().unwrap().database, "analytics"); + let secure = DataSource { url: "https://db.example/default?sslmode=disable".into(), ..source }; + assert_eq!(secure.resolved().unwrap().ssl, DbSslMode::VerifyFull); + let error = parse_url("clickhouse://reader:secret@db.example:invalid/default").unwrap_err(); + assert!(!error.message.contains("secret")); + } } diff --git a/crates/ide-db/src/split.rs b/crates/ide-db/src/split.rs index e9cdbc84..b2ee3a10 100644 --- a/crates/ide-db/src/split.rs +++ b/crates/ide-db/src/split.rs @@ -28,6 +28,7 @@ enum State { /// Every statement of `text` that contains more than whitespace and comments. pub fn split(text: &str, driver: DbDriver) -> Vec { let mysql = driver == DbDriver::Mysql; + let clickhouse = driver == DbDriver::Clickhouse; let chars: Vec<(usize, char)> = text.char_indices().collect(); // UTF-16 offset of each char index (plus the end). let mut u16_at = Vec::with_capacity(chars.len() + 1); @@ -81,7 +82,7 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { state = State::Double; has_code = true; } - '`' if mysql => { + '`' if mysql || clickhouse => { state = State::Backtick; has_code = true; } @@ -117,7 +118,7 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { _ => {} }, State::Single => { - if mysql && c == '\\' { + if (mysql || clickhouse) && c == '\\' { i += 1; } else if c == '\'' { if next == Some('\'') { @@ -128,7 +129,9 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { } } State::Double => { - if c == '"' { + if clickhouse && c == '\\' { + i += 1; + } else if c == '"' { if next == Some('"') { i += 1; } else { @@ -137,7 +140,9 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { } } State::Backtick => { - if c == '`' { + if clickhouse && (c == '\\' || (c == '`' && next == Some('`'))) { + i += 1; + } else if c == '`' { state = State::Code; } } @@ -224,6 +229,12 @@ mod tests { assert_eq!(texts(sql, DbDriver::Mysql), vec!["select `a;b`, 'it\\'s;' from t # x;y", "select 2"]); } + #[test] + fn clickhouse_backticks_and_escaped_quotes_keep_statements_intact() { + let query = r#"SELECT `a\`;b`, "c\";d", 'it\'s;safe'; SELECT 2"#; + assert_eq!(texts(query, DbDriver::Clickhouse), vec![r#"SELECT `a\`;b`, "c\";d", 'it\'s;safe'"#, "SELECT 2"]); + } + #[test] fn postgres_dollar_quotes() { let sql = diff --git a/crates/ide-db/src/sql.rs b/crates/ide-db/src/sql.rs index 79d6c1ec..bd281a85 100644 --- a/crates/ide-db/src/sql.rs +++ b/crates/ide-db/src/sql.rs @@ -7,6 +7,7 @@ use specta::Type; pub fn quote_ident(driver: DbDriver, name: &str) -> String { match driver { + DbDriver::Clickhouse => format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")), DbDriver::Mysql => format!("`{}`", name.replace('`', "``")), DbDriver::SqlServer => format!("[{}]", name.replace(']', "]]")), _ => format!("\"{}\"", name.replace('"', "\"\"")), @@ -15,7 +16,7 @@ pub fn quote_ident(driver: DbDriver, name: &str) -> String { pub fn quote_literal(driver: DbDriver, value: &str) -> String { let mut s = value.replace('\'', "''"); - if driver == DbDriver::Mysql { + if matches!(driver, DbDriver::Mysql | DbDriver::Clickhouse) { s = s.replace('\\', "\\\\"); } format!("'{s}'") @@ -48,6 +49,10 @@ pub const INTERNAL_DATABASES: &[&str] = &["rdsadmin", "azure_maintenance", "azur pub fn schemas(driver: DbDriver, system: bool) -> &'static str { match driver { DbDriver::Sqlite => "SELECT name FROM pragma_database_list WHERE name <> 'temp' ORDER BY seq", + DbDriver::Clickhouse if system => "SELECT name FROM system.databases ORDER BY name", + DbDriver::Clickhouse => { + "SELECT name FROM system.databases WHERE lower(name) NOT IN ('system', 'information_schema') ORDER BY name" + } DbDriver::Postgres if system => { "SELECT schema_name FROM information_schema.schemata \ WHERE schema_name NOT LIKE 'pg\\_toast%' AND schema_name NOT LIKE 'pg\\_temp%' ORDER BY 1" @@ -73,6 +78,12 @@ pub fn schemas(driver: DbDriver, system: bool) -> &'static str { /// Rows: `name, kind` where kind is `table`, `view` or `materialized view`. pub fn tables(driver: DbDriver, schema: &str) -> String { match driver { + DbDriver::Clickhouse => format!( + "SELECT name, CASE engine WHEN 'View' THEN 'view' WHEN 'MaterializedView' THEN 'materialized view' \ + WHEN 'WindowView' THEN 'view' ELSE 'table' END FROM system.tables \ + WHERE database = {} AND is_temporary = 0 ORDER BY name", + quote_literal(driver, schema) + ), DbDriver::Sqlite => format!( "SELECT name, type FROM {}.sqlite_master WHERE type IN ('table', 'view') \ AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name", @@ -101,6 +112,12 @@ pub fn tables(driver: DbDriver, schema: &str) -> String { pub fn columns(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => format!( + "SELECT name, type, CASE WHEN startsWith(type, 'Nullable(') \ + OR startsWith(type, 'LowCardinality(Nullable(') THEN 'YES' ELSE 'NO' END, \ + if(default_kind = '', NULL, default_expression), is_in_primary_key \ + FROM system.columns WHERE database = {s} AND table = {t} ORDER BY position" + ), DbDriver::Sqlite => format!( "SELECT name, type, CASE WHEN \"notnull\" = 0 THEN 'YES' ELSE 'NO' END, dflt_value, \ CASE WHEN pk > 0 THEN 1 ELSE 0 END FROM pragma_table_info({t}, {s}) ORDER BY cid" @@ -138,6 +155,12 @@ pub fn columns(driver: DbDriver, schema: &str, table: &str) -> String { pub fn indexes(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => format!( + "SELECT * FROM (SELECT 'PRIMARY KEY' AS index_name, 0 AS is_unique, primary_key AS expression \ + FROM system.tables WHERE database = {s} AND name = {t} AND primary_key != '' \ + UNION ALL SELECT name, 0, expr FROM system.data_skipping_indices \ + WHERE database = {s} AND table = {t}) ORDER BY index_name" + ), DbDriver::Sqlite => format!( "SELECT il.name, il.\"unique\", ii.name FROM pragma_index_list({t}, {s}) il \ JOIN pragma_index_info(il.name, {s}) ii ORDER BY il.name, ii.seqno" @@ -169,6 +192,15 @@ pub fn indexes(driver: DbDriver, schema: &str, table: &str) -> String { /// for SQL completion, grouped by table in column order. pub fn completion_columns(driver: DbDriver) -> &'static str { match driver { + DbDriver::Clickhouse => { + "SELECT c.database, c.table, CASE t.engine WHEN 'View' THEN 'view' \ + WHEN 'MaterializedView' THEN 'materialized view' WHEN 'WindowView' THEN 'view' ELSE 'table' END, \ + c.name, c.type, CASE WHEN startsWith(c.type, 'Nullable(') \ + OR startsWith(c.type, 'LowCardinality(Nullable(') THEN 'YES' ELSE 'NO' END, c.is_in_primary_key, 1 \ + FROM system.columns c JOIN system.tables t ON t.database = c.database AND t.name = c.table \ + WHERE lower(c.database) NOT IN ('system', 'information_schema') AND t.is_temporary = 0 \ + ORDER BY c.database, c.table, c.position LIMIT 50000" + } DbDriver::Sqlite => { "SELECT 'main', m.name, m.type, p.name, p.type, CASE WHEN p.\"notnull\" = 0 THEN 'YES' ELSE 'NO' END, \ CASE WHEN p.pk > 0 THEN 1 ELSE 0 END, \ @@ -221,6 +253,7 @@ pub fn completion_columns(driver: DbDriver) -> &'static str { /// when the key references the primary key implicitly. pub fn foreign_keys(driver: DbDriver) -> &'static str { match driver { + DbDriver::Clickhouse => "SELECT '', '', '', '', '', '', '' WHERE 0", DbDriver::Sqlite => { "SELECT 'main', m.name, f.id, f.\"from\", 'main', f.\"table\", f.\"to\" \ FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f \ @@ -258,6 +291,7 @@ pub fn foreign_keys(driver: DbDriver) -> &'static str { pub fn table_foreign_keys(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => foreign_keys(driver).to_owned(), DbDriver::Sqlite => format!( "SELECT {s}, {t}, f.id, f.\"from\", {s}, f.\"table\", f.\"to\" FROM pragma_foreign_key_list({t}, {s}) f ORDER BY f.id, f.seq" ), @@ -276,6 +310,7 @@ pub fn table_foreign_keys(driver: DbDriver, schema: &str, table: &str) -> String /// The schema unqualified names resolve to (`None`: always `main`). pub fn current_schema(driver: DbDriver) -> Option<&'static str> { match driver { + DbDriver::Clickhouse => Some("SELECT currentDatabase()"), DbDriver::Sqlite => None, DbDriver::Postgres => Some("SELECT current_schema()"), DbDriver::Mysql => Some("SELECT DATABASE()"), @@ -362,6 +397,9 @@ fn where_key(driver: DbDriver, key: &[CellValue]) -> String { /// The DML statement for `edit`. Values are written as quoted literals, which /// every supported database coerces to the column type. pub fn edit_statement(driver: DbDriver, schema: &str, table: &str, edit: &DbRowEdit) -> Result { + if driver == DbDriver::Clickhouse { + return Err("ClickHouse table editing is unsupported: primary keys do not uniquely identify rows and edits require transactions".into()); + } let target = qualified(driver, schema, table); match edit { DbRowEdit::Update { key, values } => { @@ -450,6 +488,7 @@ pub fn postgres_ddl_indexes(oid: u32) -> String { pub fn triggers(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => "SELECT '', '', '' WHERE 0".to_owned(), DbDriver::Sqlite => format!( "SELECT name, sql FROM {}.sqlite_master WHERE type = 'trigger' AND tbl_name = {t} ORDER BY name", quote_ident(driver, schema) @@ -492,7 +531,7 @@ pub fn sqlite_trigger_event(sql: &str) -> (String, String) { pub fn routines(driver: DbDriver, schema: &str) -> Option { let s = quote_literal(driver, schema); match driver { - DbDriver::Sqlite => None, + DbDriver::Sqlite | DbDriver::Clickhouse => None, // Functions of extensions are left out, as in IDEA. DbDriver::Postgres => Some(format!( "SELECT p.proname, CASE WHEN p.prokind = 'p' THEN 'procedure' ELSE 'function' END, \ @@ -625,6 +664,71 @@ mod tests { assert_eq!(quote_literal(DbDriver::Mysql, "it's \\"), "'it''s \\\\'"); } + #[test] + fn clickhouse_escapes_backslashes_and_delimiters() { + let driver = DbDriver::Clickhouse; + assert_eq!(quote_literal(driver, r"path\O'Brien"), r"'path\\O''Brien'"); + assert_eq!(quote_literal(driver, r"\' OR 1 = 1 --"), r"'\\'' OR 1 = 1 --'"); + assert_eq!(quote_ident(driver, r"a\`b"), r"`a\\\`b`"); + assert_eq!(qualified(driver, "data base", "event`log"), r"`data base`.`event\`log`"); + } + + #[test] + fn clickhouse_metadata_preserves_types_defaults_and_nonunique_keys() { + let driver = DbDriver::Clickhouse; + assert_eq!(schemas(driver, true), "SELECT name FROM system.databases ORDER BY name"); + assert!(schemas(driver, false).contains("lower(name) NOT IN ('system', 'information_schema')")); + assert_eq!(current_schema(driver), Some("SELECT currentDatabase()")); + let relations = tables(driver, "event's\\db"); + assert!(relations.contains("WHEN 'MaterializedView' THEN 'materialized view'")); + assert!(relations.contains(r"WHERE database = 'event''s\\db' AND is_temporary = 0")); + let fields = columns(driver, "default", "events"); + assert!(fields.starts_with("SELECT name, type, CASE WHEN startsWith(type, 'Nullable(')")); + assert!(fields.contains("startsWith(type, 'LowCardinality(Nullable(')")); + assert!(fields.contains("if(default_kind = '', NULL, default_expression), is_in_primary_key")); + assert!(fields.ends_with("WHERE database = 'default' AND table = 'events' ORDER BY position")); + let keys = indexes(driver, "default", "events"); + assert!(keys.contains("'PRIMARY KEY' AS index_name, 0 AS is_unique, primary_key AS expression")); + assert!(keys.contains("SELECT name, 0, expr FROM system.data_skipping_indices")); + let completion = completion_columns(driver); + assert!(completion.contains("c.is_in_primary_key, 1")); + assert!(completion.contains("ORDER BY c.database, c.table, c.position LIMIT 50000")); + } + + #[test] + fn clickhouse_has_no_foreign_key_trigger_or_schema_routine_catalogs() { + let driver = DbDriver::Clickhouse; + assert_eq!(foreign_keys(driver), "SELECT '', '', '', '', '', '', '' WHERE 0"); + assert_eq!(table_foreign_keys(driver, "default", "events"), foreign_keys(driver)); + assert_eq!(triggers(driver, "default", "events"), "SELECT '', '', '' WHERE 0"); + assert!(routines(driver, "default").is_none()); + assert!(sequences(driver, "default").is_none()); + assert!(object_ddl(driver, "default", crate::DbObjectKind::Trigger, "t").is_none()); + } + + #[test] + fn clickhouse_pages_quote_keys_and_fetch_one_extra_row() { + let sort = [SortKey { column: "event`id".into(), descending: true }]; + assert_eq!( + table_page(DbDriver::Clickhouse, "default", "events", Some("WHERE id > 3"), &sort, 50, 25), + r"SELECT * FROM `default`.`events` WHERE id > 3 ORDER BY `event\`id` DESC LIMIT 26 OFFSET 50" + ); + } + + #[test] + fn clickhouse_keys_never_enable_transactional_table_edits() { + let key = vec![cv("id", Some("1"))]; + for edit in [ + DbRowEdit::Update { key: key.clone(), values: vec![cv("name", Some("new"))] }, + DbRowEdit::Delete { key }, + DbRowEdit::Insert { values: vec![cv("name", Some("new"))] }, + DbRowEdit::Insert { values: vec![] }, + ] { + let error = edit_statement(DbDriver::Clickhouse, "default", "events", &edit).unwrap_err(); + assert!(error.contains("primary keys do not uniquely identify rows")); + } + } + #[test] fn pages() { let sort = [SortKey { column: "id".into(), descending: true }]; diff --git a/crates/ide-db/tests/clickhouse.rs b/crates/ide-db/tests/clickhouse.rs new file mode 100644 index 00000000..d19a9ff1 --- /dev/null +++ b/crates/ide-db/tests/clickhouse.rs @@ -0,0 +1,136 @@ +use ide_core::{CancellationToken, IdeError, IdeResult}; +use ide_db::{CellValue, DataSource, DbDriver, DbManager, DbRowEdit, DbSslMode, SortKey, sql}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn check(condition: bool, message: &str) -> IdeResult<()> { + if condition { Ok(()) } else { Err(IdeError::invalid(message)) } +} + +#[tokio::test] +#[ignore = "requires CLICKHOUSE_TEST_HOST, CLICKHOUSE_TEST_USER, and CLICKHOUSE_TEST_PASSWORD"] +async fn clickhouse_cloud_manager_catalogs_and_session_table() -> IdeResult<()> { + let tmp = tempfile::tempdir()?; + let root = tmp.path().to_path_buf(); + let ssh = Arc::new(ide_ssh::SshManager::new(root.join("ssh-passwords.json"))); + let db = DbManager::new(root.clone(), root.join("passwords.json"), root.join("history.json"), ssh); + let mut source = DataSource { + id: "clickhouse".into(), + name: "clickhouse".into(), + driver: DbDriver::Clickhouse, + host: std::env::var("CLICKHOUSE_TEST_HOST").expect("set CLICKHOUSE_TEST_HOST"), + port: Some( + std::env::var("CLICKHOUSE_TEST_PORT") + .map(|value| value.parse().expect("numeric CLICKHOUSE_TEST_PORT")) + .unwrap_or(8443), + ), + user: std::env::var("CLICKHOUSE_TEST_USER").expect("set CLICKHOUSE_TEST_USER"), + database: "default".into(), + ssl: DbSslMode::VerifyFull, + ..Default::default() + }; + source.introspection.show_system = true; + let password = std::env::var("CLICKHOUSE_TEST_PASSWORD").expect("set CLICKHOUSE_TEST_PASSWORD"); + check(!db.test(&source, Some(password.clone()), None).await?.is_empty(), "missing server version")?; + db.save_sources(vec![source]).await?; + db.set_password("clickhouse", Some(password), false).await?; + let cancel = CancellationToken::new(); + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let name = format!("manager_test_{}_{suffix}", std::process::id()); + let table = sql::quote_ident(DbDriver::Clickhouse, &name); + db.execute( + "clickhouse", + &format!( + "CREATE TEMPORARY TABLE {table} (id UInt64, label Nullable(String) DEFAULT NULL, \ + category LowCardinality(Nullable(String)) DEFAULT NULL, items Array(Nullable(UInt8)), \ + label_size UInt64 ALIAS length(ifNull(label, ''))) ENGINE = Memory" + ), + 0, + &cancel, + ) + .await?; + let checked: IdeResult<()> = async { + db.execute( + "clickhouse", + &format!("INSERT INTO {table} (id, label, items) VALUES (1, 'first', [1, NULL]), (1, NULL, [])"), + 0, + &cancel, + ) + .await?; + let schemas = db.schemas("clickhouse").await?; + check(schemas.iter().any(|schema| schema == "default"), "default database missing")?; + check(schemas.iter().any(|schema| schema == "system"), "system database missing")?; + let tables = db.tables("clickhouse", "system").await?; + check(tables.iter().any(|table| table.name == "one"), "system table listing missing one")?; + let details = db.table_details("clickhouse", "", &name).await?; + check(details.columns.len() == 5, "temporary table columns missing")?; + for (name, data_type, nullable) in [ + ("id", "UInt64", false), + ("label", "Nullable(String)", true), + ("category", "LowCardinality(Nullable(String))", true), + ("items", "Array(Nullable(UInt8))", false), + ] { + check( + details + .columns + .iter() + .any(|column| column.name == name && column.data_type == data_type && column.nullable == nullable), + "column type or nullability mismatch", + )?; + } + check( + details.columns.iter().any(|column| column.name == "label" && column.default.as_deref() == Some("NULL")), + "explicit default expression missing", + )?; + check(details.indexes.iter().all(|index| !index.unique), "ClickHouse index reported unique")?; + check(details.foreign_keys.is_empty() && details.triggers.is_empty(), "unsupported table objects returned")?; + let sort = [SortKey { column: "id".into(), descending: false }]; + let first = db.table_page("clickhouse", "", &name, None, &sort, 0, 1, &cancel).await?; + check(first.result.rows.len() == 1 && first.result.has_more, "first page limit failed")?; + let second = db.table_page("clickhouse", "", &name, None, &sort, 1, 1, &cancel).await?; + check(second.result.rows.len() == 1 && !second.result.has_more, "page offset failed")?; + let ddl = db.ddl("clickhouse", "", &name).await?; + check(ddl.contains(&name) && ddl.contains("Memory"), "temporary table DDL missing")?; + let completion = db.completion("clickhouse").await?; + check(completion.default_schema == "default", "completion default database mismatch")?; + check(completion.foreign_keys.is_empty(), "completion returned unsupported foreign keys")?; + if let Some(table) = completion.tables.first() { + let details = db.table_details("clickhouse", &table.schema, &table.name).await?; + check( + table.columns.iter().all(|column| { + details.columns.iter().any(|field| field.name == column.name && field.data_type == column.data_type) + }), + "completion columns differ from catalog", + )?; + } + let plan = db.explain("clickhouse", &format!("SELECT id FROM {table}"), &cancel).await?; + check(!plan.is_empty(), "query plan missing")?; + let value = r"path\O'Brien"; + let result = db + .execute("clickhouse", &format!("SELECT {}", sql::quote_literal(DbDriver::Clickhouse, value)), 1, &cancel) + .await?; + check(result.rows == vec![vec![Some(value.to_owned())]], "quoted literal changed value")?; + let edits = [DbRowEdit::Update { + key: vec![CellValue { column: "id".into(), value: Some("1".into()) }], + values: vec![CellValue { column: "label".into(), value: Some("changed".into()) }], + }]; + check( + db.apply_edits("clickhouse", "", &name, &edits) + .await + .err() + .is_some_and(|error| error.message.contains("table editing is unsupported")), + "supplied row key authorized table editing", + )?; + check(db.set_manual_commit("clickhouse", true).await.is_err(), "manual transactions were accepted")?; + for commit in [false, true] { + check(db.finish_transaction("clickhouse", commit).await.is_err(), "transaction control was accepted")?; + } + Ok(()) + } + .await; + let cleanup = db.execute("clickhouse", &format!("DROP TEMPORARY TABLE {table}"), 0, &cancel).await; + db.disconnect("clickhouse").await; + checked?; + cleanup?; + Ok(()) +} diff --git a/crates/ide-db/tests/sqlite.rs b/crates/ide-db/tests/sqlite.rs index f340ee53..b800cd15 100644 --- a/crates/ide-db/tests/sqlite.rs +++ b/crates/ide-db/tests/sqlite.rs @@ -197,7 +197,7 @@ async fn count(db: &DbManager) -> String { async fn manual_commit_keeps_a_transaction_until_commit_or_rollback() -> IdeResult<()> { let (_tmp, db) = setup().await; let c = CancellationToken::new(); - db.set_manual_commit("s1", true).await; + db.set_manual_commit("s1", true).await.unwrap(); db.execute("s1", "INSERT INTO users (name) VALUES ('dee')", 1, &c).await?; // The session sees its own change. assert_eq!(count(&db).await, "4"); @@ -208,10 +208,10 @@ async fn manual_commit_keeps_a_transaction_until_commit_or_rollback() -> IdeResu db.finish_transaction("s1", true).await?; // Leaving manual mode rolls back what is still open. db.execute("s1", "DELETE FROM users", 1, &c).await?; - db.set_manual_commit("s1", false).await; + db.set_manual_commit("s1", false).await.unwrap(); assert_eq!(count(&db).await, "4"); // Disconnecting with an open transaction does not hang. - db.set_manual_commit("s1", true).await; + db.set_manual_commit("s1", true).await.unwrap(); db.execute("s1", "DELETE FROM users", 1, &c).await?; db.disconnect("s1").await; assert_eq!(count(&db).await, "4"); diff --git a/crates/ide-extension/src/ledger.rs b/crates/ide-extension/src/ledger.rs index 9bed117b..19128165 100644 --- a/crates/ide-extension/src/ledger.rs +++ b/crates/ide-extension/src/ledger.rs @@ -364,7 +364,7 @@ pub fn package_hash(dir: &Path, files: &[PathBuf]) -> std::io::Result { sorted.dedup(); let mut hasher = blake3::Hasher::new(); for rel in sorted { - hasher.update(rel.to_string_lossy().as_bytes()); + hasher.update(hashed_path(rel).as_bytes()); hasher.update(&[0]); hasher.update_reader(std::fs::File::open(dir.join(rel))?)?; hasher.update(&[0]); @@ -372,6 +372,12 @@ pub fn package_hash(dir: &Path, files: &[PathBuf]) -> std::io::Result { Ok(hasher.finalize().to_hex().to_string()) } +/// A package-relative path as hashed: `/`-separated on every OS, so a package has one digest +/// (a walked `web\index.html` on Windows is `web/index.html` everywhere else). +fn hashed_path(rel: &Path) -> String { + rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::>().join("/") +} + #[cfg(test)] mod tests { use super::*; @@ -480,4 +486,20 @@ mod tests { let groups = by_risk(&caps(&["storage", "process:git", "net:x.io"])); assert_eq!(groups.iter().map(|g| g.0).collect::>(), [Risk::High, Risk::Medium, Risk::Low]); } + + #[test] + fn hashes_paths_with_forward_slashes_on_every_os() { + assert_eq!(hashed_path(Path::new("web/index.html")), "web/index.html"); + assert_eq!(hashed_path(Path::new("plugin.wasm")), "plugin.wasm"); + #[cfg(windows)] + assert_eq!(hashed_path(Path::new(r"web\index.html")), "web/index.html"); + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir(tmp.path().join("web")).unwrap(); + std::fs::write(tmp.path().join("web").join("index.html"), "

").unwrap(); + // A walked path (native separators) hashes like the manifest's `/` spelling. + assert_eq!( + package_hash(tmp.path(), &[Path::new("web").join("index.html")]).unwrap(), + package_hash(tmp.path(), &["web/index.html".into()]).unwrap() + ); + } } diff --git a/crates/ide-extension/src/lib.rs b/crates/ide-extension/src/lib.rs index 9089067a..fdf63d95 100644 --- a/crates/ide-extension/src/lib.rs +++ b/crates/ide-extension/src/lib.rs @@ -27,8 +27,8 @@ mod when; pub use capability::{Capability, CapabilityInfo, Risk, host_matches}; pub use index::ContributionIndex; pub use manifest::{ - ActivationEvent, Capabilities, ENGINE_VERSION, EVENT_KINDS, LEGACY_API, MANIFEST_FILE, Manifest, check_relative, - engine_compatible, validate_item_id, validate_plugin_id, validate_point_id, + ActivationEvent, CATEGORIES, Capabilities, ENGINE_VERSION, EVENT_KINDS, LEGACY_API, MANIFEST_FILE, MAX_CATEGORIES, + Manifest, check_relative, engine_compatible, validate_item_id, validate_plugin_id, validate_point_id, }; pub use point::{Commit, Disposable, ExtensionPoint, MAX_PACKAGE_FILE, Package, Placement, Prepared, decode}; pub use registry::{ContributionInfo, ExtensionRegistry, PreparedSet}; diff --git a/crates/ide-extension/src/manifest.rs b/crates/ide-extension/src/manifest.rs index 99820d5f..d079cff2 100644 --- a/crates/ide-extension/src/manifest.rs +++ b/crates/ide-extension/src/manifest.rs @@ -35,6 +35,11 @@ pub const MANIFEST_FILE: &str = "plugin.toml"; pub const ENGINE_VERSION: &str = "1.0.0"; /// The 0.1 API, still accepted for contribution-only packages and 0.1 components. pub const LEGACY_API: &str = "0.1"; +/// Marketplace categories, in the order they are shown. +pub const CATEGORIES: &[&str] = + &["languages", "themes", "icon-themes", "linters", "vcs", "ai", "frameworks", "tools", "other"]; +/// Categories a plugin is listed under; later ones are ignored. +pub const MAX_CATEGORIES: usize = 3; #[derive(Clone, Debug, PartialEq)] pub struct Manifest { @@ -47,6 +52,8 @@ pub struct Manifest { pub license: Option, /// Release notes of this version (Settings › Plugins › What's New). pub change_notes: Option, + /// Marketplace categories as declared (see [`Manifest::categories`]). + pub declared_categories: Vec, /// `engines.openidea` (a semver requirement), or the legacy `openidea-api`. pub engine: String, /// Written in the 0.1 format. @@ -166,6 +173,8 @@ struct Raw { #[serde(default)] change_notes: Option, #[serde(default)] + categories: Vec, + #[serde(default)] engines: Option, /// 0.1 format. #[serde(default)] @@ -243,6 +252,7 @@ impl Manifest { homepage: raw.homepage, license: raw.license, change_notes: raw.change_notes, + declared_categories: raw.categories, engine, legacy, wasm: raw.wasm, @@ -290,6 +300,9 @@ impl Manifest { return Err(format!("duplicate activation event {event}")); } } + for c in &self.declared_categories { + validate_item_id(c, "category")?; + } for (point, decls) in &self.contributes { validate_point_id(point)?; if !matches!(decls, toml::Value::Array(_) | toml::Value::Table(_)) { @@ -325,6 +338,39 @@ impl Manifest { } } +impl Manifest { + /// Its marketplace categories, in [`CATEGORIES`] order: the first + /// [`MAX_CATEGORIES`] declared that this IDE knows; without any, those its + /// contributions imply (themes, icon themes, languages), else `other`. + pub fn categories(&self) -> Vec { + let pick = |f: &dyn Fn(&str) -> bool| CATEGORIES.iter().filter(|c| f(c)).map(|c| c.to_string()).collect(); + let mut known: Vec<&str> = Vec::new(); + for d in &self.declared_categories { + if CATEGORIES.contains(&d.as_str()) && !known.contains(&d.as_str()) && known.len() < MAX_CATEGORIES { + known.push(d); + } + } + let mut out: Vec = pick(&|c| known.contains(&c)); + if out.is_empty() { + out = pick(&|c| match c { + "themes" => self.contributes.contains_key("themes"), + "icon-themes" => self.contributes.contains_key("iconThemes"), + "languages" => self.contributes.contains_key("languages"), + _ => false, + }); + } + if out.is_empty() { + out.push("other".into()); + } + out + } + + /// Declared categories this IDE doesn't know (ignored). + pub fn unknown_categories(&self) -> Vec<&str> { + self.declared_categories.iter().map(String::as_str).filter(|c| !CATEGORIES.contains(c)).collect() + } +} + /// Whether this host can load a plugin written for `engine`. pub fn engine_compatible(engine: &str, legacy: bool) -> bool { if legacy { @@ -417,6 +463,26 @@ mod tests { assert!(ActivationEvent::parse("on-event:scheduled").is_err(), "declared with on-schedule"); } + #[test] + fn categories_are_declared_or_implied() { + let head = "id = \"acme.x\"\nname = \"X\"\nversion = \"1.0.0\"\nengines.openidea = \"^1.0\"\n"; + let m = |rest: &str| Manifest::parse(&format!("{head}{rest}")); + // Declared, in the catalogue's order; unknown ones (a newer IDE's) are ignored. + let declared = m("categories = [\"tools\", \"vcs\", \"robots\"]\n").unwrap(); + assert_eq!(declared.categories(), ["vcs", "tools"]); + assert_eq!(declared.unknown_categories(), ["robots"]); + // None declared: implied by the contributions, else `other`. + let theme = m("[[contributes.themes]]\nid = \"t\"\n[[contributes.languages]]\nid = \"l\"\n").unwrap(); + assert_eq!(theme.categories(), ["languages", "themes"]); + let declared_wins = m("categories = [\"tools\"]\n[[contributes.languages]]\nid = \"l\"\n").unwrap(); + assert_eq!(declared_wins.categories(), ["tools"]); + assert_eq!(m("").unwrap().categories(), ["other"]); + // At most three: the first ones declared. + let many = m("categories = [\"vcs\", \"ai\", \"vcs\", \"tools\", \"themes\"]\n").unwrap(); + assert_eq!(many.categories(), ["vcs", "ai", "tools"]); + assert!(m("categories = [\"Bad Name\"]\n").is_err()); + } + #[test] fn parses_the_02_format() { let m = Manifest::parse(FULL).unwrap(); diff --git a/crates/ide-extension/src/package.rs b/crates/ide-extension/src/package.rs index b8d3988d..f1bc9c04 100644 --- a/crates/ide-extension/src/package.rs +++ b/crates/ide-extension/src/package.rs @@ -9,9 +9,19 @@ //! proves who published the bytes; whether to trust that publisher is the //! index's call. //! - **Index** (`index.json`): the publishers and their keys (the trust root), -//! the published plugins, and revocations. A publisher owns the id prefix -//! `.`; a revoked key invalidates everything it signed, a -//! revoked plugin version is disabled where it is installed. +//! the published plugin versions, and revocations. A publisher owns the id +//! prefix `.`; a revoked key invalidates everything it signed, a +//! revoked plugin version is disabled where it is installed. A plugin may be +//! listed in several versions, each with the plugin API it needs: the IDE +//! offers the newest one it can load. +//! - **Key continuity**: an installed plugin updates silently only to a +//! package signed with the same key, or with a key the installed one +//! endorsed ([`Endorsement`], a publisher rotating keys); any other signer +//! is the user's call. An IDE can pin a publisher to keys it was built with +//! ([`Index::pin_publisher`]): the index then can't replace them. +//! - **Format**: fields are only ever added (IDEs ignore those they don't +//! know); `format` grows only for changes older IDEs can't read, and they +//! still apply such an index's revocations ([`Index::parse`]). use crate::MANIFEST_FILE; use base64::Engine as _; @@ -40,6 +50,24 @@ fn signed_message(id: &str, version: &str, digest: &str) -> Vec { format!("openidea-plugin/1\n{id}\n{version}\n{digest}").into_bytes() } +fn endorsement_message(publisher: &str, key: &str) -> Vec { + format!("openidea-key/1\n{publisher}\n{key}").into_bytes() +} + +/// Whether `signature` (base64) is `key`'s (base64) Ed25519 signature of +/// `message`; malformed keys and signatures are errors. +fn verify_ed25519(key: &str, message: &[u8], signature: &str) -> Result { + let key: [u8; 32] = B64 + .decode(key) + .ok() + .and_then(|b| b.try_into().ok()) + .ok_or_else(|| "signature: malformed public key".to_owned())?; + let key = VerifyingKey::from_bytes(&key).map_err(|_| "signature: invalid public key".to_owned())?; + let sig: [u8; 64] = + B64.decode(signature).ok().and_then(|b| b.try_into().ok()).ok_or_else(|| "signature: malformed".to_owned())?; + Ok(key.verify_strict(message, &Signature::from_bytes(&sig)).is_ok()) +} + /// `SIGNATURE.json` of a signed package. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -74,6 +102,12 @@ impl PublisherKey { B64.encode(self.0.verifying_key().to_bytes()) } + /// Vouches for `publisher`'s new key `key` (base64), the one that follows this one. + pub fn endorse(&self, publisher: &str, key: &str) -> Endorsement { + let sig = self.0.sign(&endorsement_message(publisher, key)); + Endorsement { key: self.public_base64(), signature: B64.encode(sig.to_bytes()) } + } + pub fn sign(&self, publisher: &str, id: &str, version: &str, digest: &str) -> PackageSignature { let sig = self.0.sign(&signed_message(id, version, digest)); PackageSignature { @@ -88,19 +122,11 @@ impl PackageSignature { /// Whether the signature is valid for `id`, `version` and `digest` (the /// key's validity is checked against an [`Index`]). pub fn check(&self, id: &str, version: &str, digest: &str) -> Result<(), String> { - let key: [u8; 32] = B64 - .decode(&self.key) - .ok() - .and_then(|b| b.try_into().ok()) - .ok_or_else(|| "signature: malformed public key".to_owned())?; - let key = VerifyingKey::from_bytes(&key).map_err(|_| "signature: invalid public key".to_owned())?; - let sig: [u8; 64] = B64 - .decode(&self.signature) - .ok() - .and_then(|b| b.try_into().ok()) - .ok_or_else(|| "signature: malformed".to_owned())?; - key.verify_strict(&signed_message(id, version, digest), &Signature::from_bytes(&sig)) - .map_err(|_| "the signature does not match the package (it was changed after signing)".to_owned()) + if verify_ed25519(&self.key, &signed_message(id, version, digest), &self.signature)? { + Ok(()) + } else { + Err("the signature does not match the package (it was changed after signing)".to_owned()) + } } } @@ -242,6 +268,10 @@ pub struct Index { pub plugins: Vec, #[serde(default)] pub revoked: Vec, + /// The marketplace moved: its index is at this URL (relative to this + /// one, or absolute), and the IDE reads that one instead. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub moved: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)] @@ -253,7 +283,7 @@ pub struct Publisher { pub keys: Vec, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, specta::Type)] #[serde(rename_all = "camelCase")] pub struct KeyEntry { /// Ed25519 public key, base64. @@ -261,6 +291,27 @@ pub struct KeyEntry { /// Packages signed with it are no longer trusted. #[serde(default)] pub revoked: bool, + /// The publisher's earlier key vouching for this one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub endorsement: Option, +} + +impl KeyEntry { + pub fn new(key: String) -> Self { + Self { key, ..Self::default() } + } +} + +/// A publisher's earlier key vouching for a newer one: plugins installed from +/// packages signed with the earlier key update to ones signed with the newer +/// key without asking the user. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)] +#[serde(rename_all = "camelCase")] +pub struct Endorsement { + /// The endorsing key, base64. + pub key: String, + /// Its Ed25519 signature of `openidea-key/1\n\n`, base64. + pub signature: String, } /// A published plugin version. @@ -281,6 +332,34 @@ pub struct IndexEntry { pub digest: String, #[serde(default)] pub capabilities: Vec, + /// The plugin API versions it loads in (a semver requirement): its + /// manifest's `engines.openidea` and the API its component was built for. + /// Required: an entry without one is offered to no IDE. + #[serde(default)] + pub engine: String, + /// Marketplace categories (`CATEGORIES`); an IDE ignores those it doesn't know. + #[serde(default)] + pub categories: Vec, +} + +impl IndexEntry { + /// Whether this IDE can load the version. + pub fn compatible(&self) -> bool { + crate::engine_compatible(&self.engine, false) + } + + fn semver(&self) -> Option { + semver::Version::parse(&self.version).ok() + } +} + +/// `engines.openidea` of a manifest narrowed to the API its component was built +/// for (`major.minor`, if it has code): the requirement an index lists. +pub fn engine_requirement(manifest_engine: &str, built_for: Option<&semver::Version>) -> String { + match built_for { + Some(v) => format!("{manifest_engine}, ^{}.{}", v.major, v.minor), + None => manifest_engine.to_owned(), + } } /// A plugin (all versions when `versions` is empty) that must not run. @@ -294,12 +373,71 @@ pub struct Revocation { } impl Index { + /// Parses an index. One in a newer format keeps what must reach every IDE — + /// revocations, publisher keys (revoked ones) and `moved` — and lists no + /// plugins ([`Index::is_newer`]): the IDE needs updating to install from it. pub fn parse(text: &str) -> Result { - let index: Index = serde_json::from_str(text).map_err(|e| format!("index: {e}"))?; - if index.format != INDEX_FORMAT { - return Err(format!("index format {} is not supported (this IDE reads {INDEX_FORMAT})", index.format)); + let value: serde_json::Value = serde_json::from_str(text).map_err(|e| format!("index: {e}"))?; + let format = value.get("format").and_then(serde_json::Value::as_u64).ok_or("index: `format` is missing")?; + if format == u64::from(INDEX_FORMAT) { + return serde_json::from_value(value).map_err(|e| format!("index: {e}")); } - Ok(index) + if format < u64::from(INDEX_FORMAT) { + return Err(format!("index format {format} is not supported (this IDE reads {INDEX_FORMAT})")); + } + fn each(value: &serde_json::Value, field: &str) -> Vec { + let items = value.get(field).and_then(serde_json::Value::as_array); + items.into_iter().flatten().filter_map(|v| serde_json::from_value(v.clone()).ok()).collect() + } + Ok(Index { + format: u32::try_from(format).unwrap_or(u32::MAX), + publishers: each(&value, "publishers"), + plugins: Vec::new(), + revoked: each(&value, "revoked"), + moved: value.get("moved").and_then(serde_json::Value::as_str).map(str::to_owned), + }) + } + + /// Written in a format newer than this IDE reads: it lists no plugins. + pub fn is_newer(&self) -> bool { + self.format > INDEX_FORMAT + } + + /// The version of each plugin to offer, and whether this IDE can load it: + /// the newest it can load, else the newest. Revoked versions are skipped; + /// plugins come in the order they are first listed. + pub fn offered(&self) -> Vec<(&IndexEntry, bool)> { + let mut ids: Vec<&str> = Vec::new(); + for e in &self.plugins { + if !ids.contains(&e.id.as_str()) { + ids.push(&e.id); + } + } + ids.into_iter() + .filter_map(|id| { + let versions: Vec<&IndexEntry> = + self.plugins.iter().filter(|e| e.id == id && self.revocation(id, &e.version).is_none()).collect(); + let newest = |loadable: bool| { + versions.iter().copied().filter(|e| !loadable || e.compatible()).max_by_key(|e| e.semver()) + }; + newest(true).map(|e| (e, true)).or_else(|| newest(false).map(|e| (e, false))) + }) + .collect() + } + + /// Drops the versions no IDE would pick: revoked ones, and of each plugin + /// all but the newest version per API requirement. + pub fn prune_versions(&mut self) { + let plugins = std::mem::take(&mut self.plugins); + let mut kept: Vec = Vec::new(); + for e in plugins.into_iter().filter(|e| self.revocation(&e.id, &e.version).is_none()) { + match kept.iter_mut().find(|k| k.id == e.id && k.engine == e.engine) { + Some(k) if e.semver() > k.semver() => *k = e, + Some(_) => {} + None => kept.push(e), + } + } + self.plugins = kept; } /// The revocation that applies to `id` at `version`. @@ -335,6 +473,127 @@ impl Index { sig.check(id, version, digest)?; Ok(publisher.id.clone()) } + + /// Whether `to` is `from`, or a key of `publisher` that `from` endorsed, + /// directly or through keys it endorsed. Endorsements by revoked keys don't + /// count: whoever stole a key could vouch for their own. + pub fn succeeds(&self, publisher: &str, from: &str, to: &str) -> bool { + let keys = self.publishers.iter().find(|p| p.id == publisher).map_or(&[][..], |p| &p.keys[..]); + let mut key = to; + // Each step moves to an earlier key; more steps than keys is a loop. + for _ in 0..=keys.len() { + if key == from { + return true; + } + let Some(e) = keys.iter().find(|k| k.key == key).and_then(|k| k.endorsement.as_ref()) else { + return false; + }; + let endorser_valid = keys.iter().any(|k| k.key == e.key && !k.revoked); + if !endorser_valid || verify_ed25519(&e.key, &endorsement_message(publisher, key), &e.signature) != Ok(true) + { + return false; + } + key = &e.key; + } + false + } + + /// Makes `roots` (base64) the keys `publisher` signs with, whatever this + /// index says: another key it lists stays only when a root endorsed it (a + /// rotation the roots vouched for). Revocations of the keys are kept. + pub fn pin_publisher(&mut self, publisher: &str, name: &str, roots: &[String]) { + if roots.is_empty() { + return; + } + let i = match self.publishers.iter().position(|p| p.id == publisher) { + Some(i) => i, + None => { + self.publishers.push(Publisher { id: publisher.into(), name: name.into(), keys: Vec::new() }); + self.publishers.len() - 1 + } + }; + for root in roots { + if !self.publishers[i].keys.iter().any(|k| &k.key == root) { + self.publishers[i].keys.push(KeyEntry::new(root.clone())); + } + } + let keys = self.publishers[i].keys.clone(); + let trusted = keys + .into_iter() + .filter(|k| roots.iter().any(|r| self.succeeds(publisher, r, &k.key))) + .map(|k| if roots.contains(&k.key) { KeyEntry { endorsement: None, ..k } } else { k }) + .collect(); + self.publishers[i].keys = trusted; + } + + /// Replaces what this index lists for every publisher but `keep` with what + /// `other` lists: its publishers, their plugins and their revocations + /// (`other`'s downloads should already be absolute). `other` may only + /// speak for its own publishers, and none of them may be in `keep`. + pub fn include(&mut self, other: Index, keep: &[&str]) -> Result<(), String> { + let mut theirs: Vec<&str> = Vec::new(); + for p in &other.publishers { + if !publisher_id_ok(&p.id) { + return Err(format!("invalid publisher id {:?}: use a-z, 0-9, '-' and '_'", p.id)); + } + if keep.contains(&p.id.as_str()) { + return Err(format!("may not list publisher {:?}", p.id)); + } + if theirs.contains(&p.id.as_str()) { + return Err(format!("publisher {:?} is listed twice", p.id)); + } + theirs.push(&p.id); + } + let owned_by = |id: &str| theirs.iter().any(|p| owns(p, id)); + for e in &other.plugins { + if !theirs.contains(&e.publisher.as_str()) || !owns(&e.publisher, &e.id) { + return Err(format!("{}: not a plugin of publisher {:?}", e.id, e.publisher)); + } + if semver::VersionReq::parse(&e.engine).is_err() { + return Err(format!("{} {}: `engine` must be the plugin API it needs", e.id, e.version)); + } + } + if let Some(r) = other.revoked.iter().find(|r| !owned_by(&r.id)) { + return Err(format!("{}: revoked by an index that doesn't publish it", r.id)); + } + let kept = |id: &str| keep.iter().any(|p| owns(p, id)); + self.publishers.retain(|p| keep.contains(&p.id.as_str())); + self.plugins.retain(|e| keep.contains(&e.publisher.as_str())); + self.revoked.retain(|r| kept(&r.id)); + self.publishers.extend(other.publishers); + self.plugins.extend(other.plugins); + self.revoked.extend(other.revoked); + Ok(()) + } +} + +/// Whether a package signed `new` (`None`: unsigned) may replace an installed +/// one signed `installed` without asking the user: signed with the same key, +/// or with one the installed key endorsed (per `index`). An unsigned installed +/// copy pins nothing. +pub fn signer_continues( + index: Option<&Index>, + installed: Option<&PackageSignature>, + new: Option<&PackageSignature>, +) -> bool { + match (installed, new) { + (None, _) => true, + (Some(_), None) => false, + (Some(old), Some(new)) => { + old.publisher == new.publisher + && (old.key == new.key || index.is_some_and(|i| i.succeeds(&new.publisher, &old.key, &new.key))) + } + } +} + +/// Whether the plugin `id` is in `publisher`'s namespace. +fn owns(publisher: &str, id: &str) -> bool { + id.strip_prefix(publisher).is_some_and(|rest| rest.starts_with('.')) +} + +/// A publisher id is a single segment of a plugin id. +pub fn publisher_id_ok(id: &str) -> bool { + !id.contains('.') && crate::validate_plugin_id(id).is_ok() } #[cfg(test)] @@ -354,7 +613,7 @@ mod tests { publishers: vec![Publisher { id: "acme".into(), name: "ACME".into(), - keys: vec![KeyEntry { key: key.public_base64(), revoked: false }], + keys: vec![KeyEntry::new(key.public_base64())], }], plugins: Vec::new(), revoked: vec![Revocation { @@ -362,6 +621,7 @@ mod tests { versions: vec!["1.0.0".into()], reason: "malware".into(), }], + moved: None, } } @@ -399,6 +659,138 @@ mod tests { assert_eq!(PublisherKey::from_base64(&key.seed_base64()).unwrap().public_base64(), key.public_base64()); } + fn entry(id: &str, publisher: &str) -> IndexEntry { + IndexEntry { + id: id.into(), + name: id.into(), + version: "1.0.0".into(), + description: String::new(), + authors: Vec::new(), + publisher: publisher.into(), + download: format!("https://example.com/{id}.oiplugin"), + digest: "d".into(), + capabilities: Vec::new(), + engine: "^1.0".into(), + categories: Vec::new(), + } + } + + fn version(id: &str, version: &str, engine: &str) -> IndexEntry { + IndexEntry { version: version.into(), engine: engine.into(), ..entry(id, "acme") } + } + + #[test] + fn the_ide_is_offered_the_newest_version_it_can_load() { + let host = crate::ENGINE_VERSION; + let newer = semver::Version::parse(host).unwrap().major + 1; + let mut index = Index { + format: INDEX_FORMAT, + plugins: vec![ + version("acme.a", "1.0.0", &format!("^{host}")), + version("acme.a", "1.2.0", &format!("^{host}")), + version("acme.a", "2.0.0", &format!("^{newer}.0")), + version("acme.b", "0.1.0", &format!("^{newer}.0")), + version("acme.b", "0.2.0", &format!("^{newer}.1")), + version("acme.c", "1.0.0", ""), + version("acme.c", "1.1.0", ""), + ], + revoked: vec![Revocation { id: "acme.c".into(), versions: vec!["1.1.0".into()], reason: "r".into() }], + ..Index::default() + }; + let offered = |index: &Index| { + index.offered().into_iter().map(|(e, ok)| (e.id.clone(), e.version.clone(), ok)).collect::>() + }; + let expected = [ + ("acme.a".to_owned(), "1.2.0".to_owned(), true), + ("acme.b".to_owned(), "0.2.0".to_owned(), false), + // No `engine`: listed, but no IDE installs it. + ("acme.c".to_owned(), "1.0.0".to_owned(), false), + ]; + assert_eq!(offered(&index), expected); + + // Pruning keeps the newest version per requirement, which changes nothing offered. + index.prune_versions(); + let left: Vec<_> = index.plugins.iter().map(|e| format!("{} {}", e.id, e.version)).collect(); + assert_eq!(left, ["acme.a 1.2.0", "acme.a 2.0.0", "acme.b 0.1.0", "acme.b 0.2.0", "acme.c 1.0.0"]); + assert_eq!(offered(&index), expected); + + let built_for = semver::Version::new(1, 3, 0); + assert_eq!(engine_requirement("^1.0", Some(&built_for)), "^1.0, ^1.3"); + assert_eq!(engine_requirement("^1.0", None), "^1.0"); + assert!(semver::VersionReq::parse(&engine_requirement("^1.0", Some(&built_for))).is_ok()); + } + + fn publisher(id: &str) -> Publisher { + Publisher { id: id.into(), name: id.into(), keys: vec![KeyEntry::new(format!("{id}-key"))] } + } + + fn revocation(id: &str) -> Revocation { + Revocation { id: id.into(), versions: Vec::new(), reason: "r".into() } + } + + #[test] + fn including_an_index_replaces_everything_but_the_kept_publishers() { + let mut official = Index { + format: INDEX_FORMAT, + publishers: vec![publisher("openidea"), publisher("gone")], + plugins: vec![entry("openidea.a", "openidea"), entry("gone.x", "gone")], + revoked: vec![revocation("openidea.old"), revocation("gone.x")], + moved: None, + }; + let community = Index { + format: INDEX_FORMAT, + publishers: vec![publisher("alice")], + plugins: vec![entry("alice.arthas", "alice")], + revoked: vec![revocation("alice.bad")], + moved: None, + }; + official.include(community.clone(), &["openidea"]).unwrap(); + let ids = |v: &[IndexEntry]| v.iter().map(|e| e.id.clone()).collect::>(); + assert_eq!(official.publishers.iter().map(|p| p.id.as_str()).collect::>(), ["openidea", "alice"]); + assert_eq!(ids(&official.plugins), ["openidea.a", "alice.arthas"]); + assert_eq!(official.revoked.iter().map(|r| r.id.as_str()).collect::>(), ["openidea.old", "alice.bad"]); + + // Including again is idempotent. + let once = official.clone(); + official.include(community.clone(), &["openidea"]).unwrap(); + assert_eq!(official, once); + } + + #[test] + fn an_included_index_speaks_only_for_its_own_publishers() { + let base = Index { format: INDEX_FORMAT, publishers: vec![publisher("openidea")], ..Index::default() }; + let with = |publishers: Vec, plugins: Vec, revoked: Vec| Index { + format: INDEX_FORMAT, + publishers, + plugins, + revoked, + moved: None, + }; + let cases = [ + (with(vec![publisher("openidea")], vec![], vec![]), "may not list"), + (with(vec![publisher("alice"), publisher("alice")], vec![], vec![]), "listed twice"), + (with(vec![publisher("openidea.x")], vec![], vec![]), "invalid publisher"), + (with(vec![publisher("alice")], vec![entry("openidea.a", "alice")], vec![]), "not a plugin of"), + (with(vec![publisher("alice")], vec![entry("alice.a", "bob")], vec![]), "not a plugin of"), + (with(vec![publisher("alice")], vec![entry("alicex.a", "alice")], vec![]), "not a plugin of"), + (with(vec![publisher("alice")], vec![], vec![revocation("openidea.a")]), "doesn't publish"), + ( + with( + vec![publisher("alice")], + vec![IndexEntry { engine: String::new(), ..entry("alice.a", "alice") }], + vec![], + ), + "`engine`", + ), + ]; + for (other, error) in cases { + let mut index = base.clone(); + let err = index.include(other, &["openidea"]).unwrap_err(); + assert!(err.contains(error), "{err}"); + assert_eq!(index, base); + } + } + #[test] fn unpacking_refuses_paths_leaving_the_package() { let tmp = tempfile::tempdir().unwrap(); @@ -413,7 +805,7 @@ mod tests { let err = unpack(&out, &tmp.path().join("dest")).unwrap_err(); assert!(err.contains("invalid path"), "{err}"); assert!(!tmp.path().join("outside.txt").exists()); - assert!(Index::parse(r#"{"format": 2}"#).is_err()); + assert!(Index::parse(r#"{"format": 2}"#).unwrap().is_newer()); } fn zip_of(path: &Path, entries: &[(&str, &str)]) { @@ -462,4 +854,102 @@ mod tests { assert!(err.contains("is not an OpenIDEA plugin"), "{name}: {err}"); } } + + #[test] + fn a_newer_index_still_revokes_and_moves() { + let newer = format!( + r#"{{"format": {}, "plugins": [{{"anything": "new"}}], "shape": "changed", + "publishers": [{{"id": "acme", "name": "ACME", "keys": [{{"key": "k1", "revoked": true}}]}}, {{"id": 5}}], + "revoked": [{{"id": "acme.bad", "reason": "malware"}}, {{"oops": 1}}], + "moved": "https://example.com/next.json"}}"#, + INDEX_FORMAT + 1 + ); + let index = Index::parse(&newer).unwrap(); + assert!(index.is_newer()); + assert!(index.plugins.is_empty() && index.offered().is_empty()); + assert_eq!(index.revocation("acme.bad", "2.0.0").unwrap().reason, "malware"); + assert!(index.key_revoked("k1")); + assert_eq!(index.moved.as_deref(), Some("https://example.com/next.json")); + // Fields this IDE doesn't know are ignored in its own format. + let current = format!(r#"{{"format": {INDEX_FORMAT}, "later": true, "plugins": []}}"#); + assert!(!Index::parse(¤t).unwrap().is_newer()); + assert!(Index::parse(r#"{"format": 0}"#).unwrap_err().contains("not supported")); + assert!(Index::parse(r#"{"plugins": []}"#).is_err()); + } + + fn keys<'a>(index: &'a mut Index, publisher: &str) -> &'a mut Vec { + &mut index.publishers.iter_mut().find(|p| p.id == publisher).unwrap().keys + } + + #[test] + fn rotated_keys_continue_through_endorsements() { + let (k1, k2, k3) = + (PublisherKey::from_seed([1; 32]), PublisherKey::from_seed([2; 32]), PublisherKey::from_seed([3; 32])); + let mut index = index(&k1); + let endorsed = |by: &PublisherKey, key: &PublisherKey| KeyEntry { + endorsement: Some(by.endorse("acme", &key.public_base64())), + ..KeyEntry::new(key.public_base64()) + }; + keys(&mut index, "acme").push(endorsed(&k1, &k2)); + keys(&mut index, "acme").push(endorsed(&k2, &k3)); + let (p1, p2, p3) = (k1.public_base64(), k2.public_base64(), k3.public_base64()); + assert!(index.succeeds("acme", &p1, &p1)); + assert!(index.succeeds("acme", &p1, &p3), "through k2"); + assert!(!index.succeeds("acme", &p3, &p1), "not backwards"); + assert!(!index.succeeds("other", &p1, &p2)); + // An endorsement for another publisher, or by a revoked key, doesn't count. + let mut foreign = index.clone(); + keys(&mut foreign, "acme")[1].endorsement = Some(k1.endorse("evil", &p2)); + assert!(!foreign.succeeds("acme", &p1, &p2)); + let mut stolen = index.clone(); + keys(&mut stolen, "acme")[0].revoked = true; + assert!(!stolen.succeeds("acme", &p1, &p2)); + + let sig = |key: &PublisherKey| key.sign("acme", "acme.tools", "1.0.0", "d"); + assert!(signer_continues(Some(&index), Some(&sig(&k1)), Some(&sig(&k3)))); + assert!(signer_continues(None, Some(&sig(&k1)), Some(&sig(&k1)))); + assert!(!signer_continues(None, Some(&sig(&k1)), Some(&sig(&k2))), "no index to vouch"); + let stranger = PublisherKey::from_seed([4; 32]); + assert!(!signer_continues(Some(&index), Some(&sig(&k1)), Some(&sig(&stranger)))); + assert!(!signer_continues(Some(&index), Some(&sig(&k1)), None), "signed, then unsigned"); + assert!(signer_continues(Some(&index), None, Some(&sig(&stranger))), "unsigned pins nothing"); + } + + #[test] + fn pinned_publishers_keep_their_built_in_keys() { + let (root, next, intruder) = + (PublisherKey::from_seed([1; 32]), PublisherKey::from_seed([2; 32]), PublisherKey::from_seed([5; 32])); + let mut index = index(&intruder); + index.publishers[0].id = "openidea".into(); + keys(&mut index, "openidea").push(KeyEntry { + endorsement: Some(root.endorse("openidea", &next.public_base64())), + ..KeyEntry::new(next.public_base64()) + }); + let roots = [root.public_base64()]; + let listed = |index: &Index| -> Vec<(String, bool)> { + let mut index = index.clone(); + index.pin_publisher("openidea", "OpenIDEA", &roots); + keys(&mut index, "openidea").iter().map(|k| (k.key.clone(), k.revoked)).collect() + }; + // The intruder's key goes, the successor the root endorsed stays, the root is added. + assert_eq!(listed(&index), [(next.public_base64(), false), (root.public_base64(), false)]); + let mut pinned = index.clone(); + pinned.pin_publisher("openidea", "OpenIDEA", &roots); + let digest = "d"; + let signed = |k: &PublisherKey| k.sign("openidea", "openidea.tools", "1.0.0", digest); + assert!(pinned.verify(&signed(&intruder), "openidea.tools", "1.0.0", digest).is_err()); + assert!(pinned.verify(&signed(&next), "openidea.tools", "1.0.0", digest).is_ok()); + // A revoked root stays revoked, and what it endorsed is no longer trusted. + keys(&mut index, "openidea").push(KeyEntry { revoked: true, ..KeyEntry::new(root.public_base64()) }); + assert_eq!(listed(&index), [(root.public_base64(), true)]); + + // A publisher missing from the index is added with its roots. + let mut empty = Index { format: INDEX_FORMAT, ..Index::default() }; + empty.pin_publisher("openidea", "OpenIDEA", &[root.public_base64()]); + assert_eq!(empty.verify(&signed(&root), "openidea.tools", "1.0.0", digest).unwrap(), "openidea"); + // No roots: the index decides. + let mut open = index.clone(); + open.pin_publisher("openidea", "OpenIDEA", &[]); + assert_eq!(open, index); + } } diff --git a/crates/ide-fonts/src/lib.rs b/crates/ide-fonts/src/lib.rs index 341febde..e52a9396 100644 --- a/crates/ide-fonts/src/lib.rs +++ b/crates/ide-fonts/src/lib.rs @@ -21,6 +21,18 @@ pub struct FontFamily { pub monospaced: bool, } +/// PANOSE bProportion value 9 means "Monospaced"; DirectWrite's +/// `IsMonospacedFont` alone misses fonts that only carry this classification. +const PANOSE_MONOSPACED: u8 = 9; + +/// One face counts as monospaced when the font system flags it fixed pitch, its +/// PANOSE proportion is "Monospaced", or its glyph advances are uniform +/// (fonts with wrong/missing metadata, e.g. bitmap-style TTFs). +#[cfg_attr(not(windows), allow(dead_code))] +fn face_is_mono(is_monospaced_font: bool, panose_proportion: u8, advances_equal: bool) -> bool { + is_monospaced_font || panose_proportion == PANOSE_MONOSPACED || advances_equal +} + type Cache = Mutex>>; /// The installed families, sorted case-insensitively, read on first use and @@ -195,7 +207,9 @@ mod imp { // SAFETY: CoreText descriptors carry documented CFString, CFURL and trait dictionary values. unsafe { let collection = CTFontCollection::from_available_fonts(None); - let Some(descriptors) = collection.matching_font_descriptors() else { return Vec::new() }; + let Some(descriptors) = collection.matching_font_descriptors() else { + return Vec::new(); + }; let mut seen = std::collections::BTreeSet::new(); descriptors .cast_unchecked::() @@ -271,14 +285,16 @@ mod imp { #[cfg(windows)] mod imp { + use std::os::windows::ffi::OsStrExt; + use windows::Win32::Graphics::DirectWrite::IDWriteFontFace1; use windows::Win32::Graphics::DirectWrite::{ - DWRITE_FACTORY_TYPE_SHARED, DWRITE_FONT_STRETCH_NORMAL, DWRITE_FONT_STYLE_NORMAL, DWRITE_FONT_WEIGHT_NORMAL, - DWriteCreateFactory, IDWriteFactory, IDWriteFont1, IDWriteLocalizedStrings, + DWRITE_FACTORY_TYPE_SHARED, DWriteCreateFactory, IDWriteFactory, IDWriteFactory3, IDWriteFactory5, IDWriteFont, + IDWriteFont1, IDWriteFontCollection, IDWriteFontFamily, IDWriteLocalizedStrings, }; - use windows::core::{BOOL, Interface, w}; + use windows::core::{BOOL, Interface, PCWSTR, w}; - /// The system font collection's families, named in English when the - /// family has an English name (what CSS matches), else its first name. + /// The installed families, named in English when the family has an + /// English name (what CSS matches), else its first name. pub(crate) fn list(refresh: bool) -> Vec<(String, bool)> { // SAFETY: plain DirectWrite COM calls on interfaces we own. unsafe { read(refresh) }.unwrap_or_default() @@ -286,11 +302,85 @@ mod imp { unsafe fn read(refresh: bool) -> windows::core::Result> { let factory: IDWriteFactory = unsafe { DWriteCreateFactory(DWRITE_FACTORY_TYPE_SHARED) }?; + let mut faces = Vec::new(); + // Windows 10+: the newer collection covers registered per-user fonts. + // Exclude downloadable fonts that are not installed on this machine. + // Bitmap .FON fonts (e.g. the stock raster Fixedsys) are invisible to + // DirectWrite either way: it only reads TrueType/OpenType files. let mut collection = None; - unsafe { factory.GetSystemFontCollection(&mut collection, refresh) }?; - let Some(collection) = collection else { - return Ok(Vec::new()); + if let Ok(factory3) = factory.cast::() { + let mut local = None; + if unsafe { factory3.GetSystemFontCollection(false, &mut local, refresh) }.is_ok() { + collection = local.map(Into::into); + } + } + if collection.is_none() { + unsafe { factory.GetSystemFontCollection(&mut collection, refresh) }?; + } + if let Some(collection) = collection { + faces.extend(collect(&collection)); + } + // DirectWrite enumerates via the font registry, so font files copied + // into the per-user directory without registering (no HKCU entry) + // are invisible to the calls above; scan the directory directly. + faces.extend(unsafe { user_dir_faces(&factory) }); + Ok(faces) + } + + /// Families from `%LOCALAPPDATA%\Microsoft\Windows\Fonts` files, registered + /// or not, via a font set built from the files on disk. + unsafe fn user_dir_faces(factory: &IDWriteFactory) -> Vec<(String, bool)> { + let Some(local_app_data) = std::env::var_os("LOCALAPPDATA") else { + return Vec::new(); }; + let dir = std::path::Path::new(&local_app_data).join(r"Microsoft\Windows\Fonts"); + let Ok(entries) = std::fs::read_dir(&dir) else { + return Vec::new(); + }; + let files: Vec<_> = entries + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + matches!( + p.extension().and_then(|e| e.to_str()).map(|e| e.to_ascii_lowercase()).as_deref(), + Some("ttf" | "otf" | "ttc" | "otc") + ) + }) + .collect(); + unsafe { file_faces(factory, &files) } + } + + unsafe fn file_faces(factory: &IDWriteFactory, files: &[std::path::PathBuf]) -> Vec<(String, bool)> { + if files.is_empty() { + return Vec::new(); + } + let Ok(factory3) = factory.cast::() else { + return Vec::new(); + }; + let Ok(factory5) = factory.cast::() else { + return Vec::new(); + }; + let Ok(builder) = (unsafe { factory5.CreateFontSetBuilder() }) else { + return Vec::new(); + }; + for path in files { + let wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + // SAFETY: wide is a valid NUL-terminated string for the calls. + let Ok(file) = (unsafe { factory.CreateFontFileReference(PCWSTR(wide.as_ptr()), None) }) else { + continue; + }; + let _ = unsafe { builder.AddFontFile(&file) }; + } + let Ok(set) = (unsafe { builder.CreateFontSet() }) else { + return Vec::new(); + }; + let Ok(collection) = (unsafe { factory3.CreateFontCollectionFromFontSet(&set) }) else { + return Vec::new(); + }; + collect(&collection) + } + + fn collect(collection: &IDWriteFontCollection) -> Vec<(String, bool)> { let mut out = Vec::new(); for i in 0..unsafe { collection.GetFontFamilyCount() } { let Ok(family) = (unsafe { collection.GetFontFamily(i) }) else { @@ -299,19 +389,120 @@ mod imp { let Some(name) = unsafe { family.GetFamilyNames() }.ok().and_then(|n| unsafe { english(&n) }) else { continue; }; - let mono = unsafe { - family.GetFirstMatchingFont( - DWRITE_FONT_WEIGHT_NORMAL, - DWRITE_FONT_STRETCH_NORMAL, - DWRITE_FONT_STYLE_NORMAL, - ) - } - .ok() - .and_then(|f| f.cast::().ok()) - .is_some_and(|f| unsafe { f.IsMonospacedFont() }.as_bool()); - out.push((name, mono)); + out.push((name, unsafe { family_is_mono(&family) })); + } + out + } + + /// A family is monospaced when any face reports fixed pitch, PANOSE + /// proportion 9, or uniform glyph advances; probing every face matches + /// `normalize`'s any-face merge (macOS and Linux already flag every face, + /// not just the regular one). + unsafe fn family_is_mono(family: &IDWriteFontFamily) -> bool { + (0..unsafe { family.GetFontCount() }).any(|i| { + let Ok(font) = (unsafe { family.GetFont(i) }) else { + return false; + }; + if let Ok(font1) = font.cast::() { + let is_monospaced = unsafe { font1.IsMonospacedFont() }.as_bool(); + // The proportion byte is defined for text/display fonts only. + let panose = unsafe { font1.GetPanose().values }; + let panose_proportion = if panose[0] == 2 { panose[3] } else { 0 }; + if super::face_is_mono(is_monospaced, panose_proportion, false) { + return true; + } + } + unsafe { advances_equal(&font) } + }) + } + + /// Some monospaced fonts carry no fixed-pitch metadata at all (bitmap-style + /// TTFs like Fixedsys Excelsior): compare the advance widths of 'i' and + /// 'W'. Dual-width CJK monospace fonts pass too (both half-width). + unsafe fn advances_equal(font: &IDWriteFont) -> bool { + let Ok(face) = (unsafe { font.CreateFontFace() }) else { + return false; + }; + let Ok(face1) = face.cast::() else { + return false; + }; + let codepoints = ['i' as u32, 'W' as u32]; + let mut indices = [0u16; 2]; + if unsafe { face.GetGlyphIndices(codepoints.as_ptr(), 2, indices.as_mut_ptr()) }.is_err() + || indices.contains(&0) + { + return false; + } + let mut advances = [0i32; 2]; + unsafe { face1.GetDesignGlyphAdvances(2, indices.as_ptr(), advances.as_mut_ptr(), false) }.is_ok() + && advances[0] > 0 + && advances[0] == advances[1] + } + + #[test] + fn glyph_probe_distinguishes_monospace_proportional_and_symbol_fonts() { + unsafe { + let factory: IDWriteFactory = DWriteCreateFactory(DWRITE_FACTORY_TYPE_SHARED).unwrap(); + let mut collection = None; + factory.GetSystemFontCollection(&mut collection, false).unwrap(); + let collection = collection.unwrap(); + for (name, expected) in [ + ("Consolas", true), + ("Arial", false), + ("Times New Roman", false), + ("Wingdings", false), + ("Webdings", false), + ("Segoe MDL2 Assets", false), + ] { + let wide: Vec<_> = name.encode_utf16().chain(Some(0)).collect(); + let mut index = 0; + let mut exists = BOOL(0); + collection.FindFamilyName(PCWSTR(wide.as_ptr()), &mut index, &mut exists).unwrap(); + assert!(exists.as_bool(), "{name} ships with Windows"); + let family = collection.GetFontFamily(index).unwrap(); + for i in 0..family.GetFontCount() { + let font = family.GetFont(i).unwrap(); + assert_eq!(advances_equal(&font), expected, "{name} face {i}"); + } + } + } + } + + #[test] + fn font_files_include_every_family_in_a_collection() { + let path = std::path::PathBuf::from(std::env::var_os("WINDIR").unwrap()).join(r"Fonts\cambria.ttc"); + unsafe { + let factory: IDWriteFactory = DWriteCreateFactory(DWRITE_FACTORY_TYPE_SHARED).unwrap(); + let faces = file_faces(&factory, &[path]); + for name in ["Cambria", "Cambria Math"] { + assert!(faces.iter().any(|(family, _)| family == name), "missing {name}: {faces:?}"); + } + } + } + + #[test] + fn excludes_families_available_only_as_downloads() { + use windows::Win32::Graphics::DirectWrite::{DWRITE_E_REMOTEFONT, DWRITE_LOCALITY_LOCAL, IDWriteFont3}; + + unsafe { + let factory: IDWriteFactory = DWriteCreateFactory(DWRITE_FACTORY_TYPE_SHARED).unwrap(); + let factory3: IDWriteFactory3 = factory.cast().unwrap(); + let set = factory3.GetSystemFontSet().unwrap(); + let collection = factory3.CreateFontCollectionFromFontSet(&set).unwrap(); + let listed = super::normalize(read(false).unwrap()); + let user = super::normalize(user_dir_faces(&factory)); + for i in 0..collection.GetFontFamilyCount() { + let family = collection.GetFontFamily(i).unwrap(); + let name = english(&family.GetFamilyNames().unwrap()).unwrap(); + let remote_only = (0..family.GetFontCount()).all(|j| match family.GetFont(j) { + Ok(font) => font.cast::().unwrap().GetLocality() != DWRITE_LOCALITY_LOCAL, + Err(error) => error.code() == DWRITE_E_REMOTEFONT, + }); + if remote_only && !user.iter().any(|f| f.name == name) { + assert!(!listed.iter().any(|f| f.name == name), "{name} is not installed locally"); + } + } } - Ok(out) } unsafe fn english(names: &IDWriteLocalizedStrings) -> Option { @@ -476,6 +667,15 @@ mod tests { ); } + #[test] + fn panose_monospaced_proportion_counts_as_mono() { + assert!(face_is_mono(true, 0, false)); + assert!(face_is_mono(false, PANOSE_MONOSPACED, false)); + assert!(face_is_mono(false, 0, true)); + assert!(!face_is_mono(false, 0, false)); + assert!(!face_is_mono(false, 3, false)); + } + #[test] fn fc_list_spacing_marks_monospace() { let out = "DejaVu Sans Mono\t100\nDejaVu Sans\t\nNoto Sans Mono CJK SC\t90\nTerminus\t110\nCantarell\nFira\\-Code\tmono\n\t100\n"; diff --git a/crates/ide-host/Cargo.toml b/crates/ide-host/Cargo.toml index 5f7267d7..a8473b6b 100644 --- a/crates/ide-host/Cargo.toml +++ b/crates/ide-host/Cargo.toml @@ -14,6 +14,7 @@ encoding_rs.workspace = true ide-core.workspace = true ide-text.workspace = true ide-vfs.workspace = true +regex-syntax.workspace = true ide-workspace.workspace = true ide-language.workspace = true ide-index.workspace = true @@ -45,6 +46,7 @@ ide-extension.workspace = true ide-plugin.workspace = true ide-tools = { path = "../ide-tools" } ide-docker = { path = "../ide-docker" } +semver = "1" serde.workspace = true serde_json.workspace = true toml.workspace = true @@ -58,12 +60,13 @@ dunce.workspace = true dirs.workspace = true uuid.workspace = true tempfile.workspace = true +zip.workspace = true +quick-xml = "0.42" [dev-dependencies] ide-ssh = { workspace = true, features = ["testing"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "io-util", "test-util"] } criterion = { version = "0.5", default-features = false } -zip.workspace = true [target.'cfg(unix)'.dependencies] rustix = { version = "1.1", features = ["fs", "process"] } diff --git a/crates/ide-host/src/bin/openidea-plugin/main.rs b/crates/ide-host/src/bin/openidea-plugin/main.rs index 99b73630..ef9a7fa7 100644 --- a/crates/ide-host/src/bin/openidea-plugin/main.rs +++ b/crates/ide-host/src/bin/openidea-plugin/main.rs @@ -3,13 +3,15 @@ //! ```text //! openidea-plugin new

[--template command|theme|icon-theme|language] [--id ID] [--name NAME] [--sdk PATH] //! openidea-plugin build [dir] [--debug] # cargo → dist/plugin.wasm, then `check` -//! openidea-plugin check [dir] # validate the package as the IDE would install it +//! openidea-plugin check [dir|package] # validate the package as the IDE would install it //! openidea-plugin link [dir] # load the folder in the IDE without installing it //! openidea-plugin unlink [dir] //! openidea-plugin links # list linked folders //! openidea-plugin keygen # a publisher's Ed25519 key +//! openidea-plugin endorse --key OLD --publisher ID # vouch for a new key //! openidea-plugin pack [dir] [--key F --publisher ID] [--out FILE] # -.oiplugin -//! openidea-plugin index --publisher ID --name NAME --key F # marketplace index.json +//! openidea-plugin index [--publisher ID --name NAME --key F [--previous-key F]] # marketplace index.json +//! openidea-plugin merge --keep ID # list another index's publishers //! ``` //! //! `link` writes `links.json` in the IDE's plugins directory @@ -20,7 +22,7 @@ mod templates; use ide_extension::{Manifest, validate_plugin_id}; -use ide_host::plugins::PluginHost; +use ide_host::plugins::{PluginHost, read_market_url, resolve_market_href}; use ide_plugin::{Limits, PluginEngine}; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; @@ -31,17 +33,31 @@ usage: openidea-plugin [options] commands: new [--template T] [--id ID] [--name NAME] [--sdk PATH] - create a plugin; T = command (default), theme, icon-theme, language + create a plugin; T = command (default), theme, icon-theme, language. A Rust + plugin depends on the SDK of this tool's release (git); --sdk PATH uses a + local sdk/rust/openidea-plugin instead build [dir] [--debug] compile a Rust plugin to dist/plugin.wasm and check it - check [dir] validate the package as the IDE would install it + check [dir|package] validate the package (folder or .oiplugin) as the IDE would install it link [dir] load the folder in the IDE without installing it unlink [dir] stop loading a linked folder links list linked folders keygen create a publisher signing key (keep the file secret) + endorse --key F --publisher ID + print an earlier key's endorsement of a new one (JSON), for an + index the publisher doesn't write with `index --previous-key` pack [dir] [--key F --publisher ID] [--out FILE] write -.oiplugin, signed with the key - index --publisher ID --name NAME --key F [--out FILE] - write the marketplace index.json for the packages in dir + index [--publisher ID --name NAME --key F [--previous-key F]] [--out FILE] + [--download-base URL] + write the marketplace index.json for the packages in dir: with a + key, register it and list that publisher's plugins (earlier + versions stay for older IDEs); without, list only these packages, + each checked against the index's publishers. A new key replacing + an earlier one is endorsed with --previous-key, so installed + plugins update to it without asking + merge --keep ID [--out FILE] + replace every publisher but ID in index with those of other-index + (a path or URL), their plugins and revocations environment: OPENIDEA_PLUGINS_DIR the IDE's plugins directory (default: the app data directory)"; @@ -51,13 +67,15 @@ fn main() -> ExitCode { let result = match args.first().map(String::as_str) { Some("new") => cmd_new(&args[1..]), Some("build") => cmd_build(&args[1..]), - Some("check") => dir_arg(&args[1..]).and_then(|d| check(&d)), + Some("check") => dir_arg(&args[1..]).and_then(|d| check_path(&d)), Some("link") => dir_arg(&args[1..]).and_then(|d| link(&d, true)), Some("unlink") => dir_arg(&args[1..]).and_then(|d| link(&d, false)), Some("links") => list_links(), Some("keygen") => cmd_keygen(&args[1..]), + Some("endorse") => cmd_endorse(&args[1..]), Some("pack") => cmd_pack(&args[1..]), Some("index") => cmd_index(&args[1..]), + Some("merge") => cmd_merge(&args[1..]), Some("--version" | "-V") => { println!("openidea-plugin {} (plugin API {})", env!("CARGO_PKG_VERSION"), ide_extension::ENGINE_VERSION); Ok(()) @@ -161,7 +179,7 @@ fn cmd_new(args: &[String]) -> Result<(), String> { std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; let dir = canonical(&dir)?; let sdk = match template { - templates::Template::Command => Some(sdk_path(a.value("sdk"), &dir)?), + templates::Template::Command => Some(sdk_dependency(a.value("sdk"), &dir)?), _ => None, }; let vars = templates::Vars { id: &id, name: &name, sdk: sdk.as_deref() }; @@ -216,12 +234,16 @@ fn title_case(s: &str) -> String { .join(" ") } -/// The SDK dependency path written into a new plugin's Cargo.toml: `--sdk`, -/// or the SDK of the source tree this tool was built from. -fn sdk_path(explicit: Option<&str>, plugin_dir: &Path) -> Result { +/// The SDK dependency written into a new plugin's Cargo.toml: by path with `--sdk`, or +/// when the plugin is inside the source tree this tool was built from; otherwise the +/// repository at this tool's release tag. +fn sdk_dependency(explicit: Option<&str>, plugin_dir: &Path) -> Result { let sdk = match explicit { Some(p) => PathBuf::from(p), - None => PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/../../sdk/rust/openidea-plugin")), + None => match dunce::canonicalize(concat!(env!("CARGO_MANIFEST_DIR"), "/../..")) { + Ok(tree) if plugin_dir.starts_with(&tree) => tree.join("sdk/rust/openidea-plugin"), + _ => return Ok(format!("{{ git = \"{REPOSITORY}\", tag = \"v{}\" }}", env!("CARGO_PKG_VERSION"))), + }, }; let sdk = dunce::canonicalize(&sdk).map_err(|_| { format!("the plugin SDK was not found at {}; pass --sdk ", sdk.display()) @@ -230,9 +252,11 @@ fn sdk_path(explicit: Option<&str>, plugin_dir: &Path) -> Result return Err(format!("{} is not the openidea-plugin SDK (no Cargo.toml)", sdk.display())); } let path = relative(plugin_dir, &sdk).unwrap_or(sdk); - Ok(path.to_string_lossy().replace('\\', "/")) + Ok(format!("{{ path = \"{}\" }}", path.to_string_lossy().replace('\\', "/"))) } +const REPOSITORY: &str = "https://github.com/l0ng-ai/openidea"; + /// `to` relative to `from` when they share a root (keeps packages in a repo movable). fn relative(from: &Path, to: &Path) -> Option { let (f, t): (Vec<_>, Vec<_>) = (from.components().collect(), to.components().collect()); @@ -323,10 +347,21 @@ fn ensure_wasm_target() { /// Validates `dir` exactly as Install from Disk would, and checks the /// component imports only the plugin API. +/// [`check`] of a folder, or of a package file unpacked as installing would. +fn check_path(path: &Path) -> Result<(), String> { + if !path.is_file() { + return check(path); + } + let tmp = scratch_dir("unpack")?; + let dir = tmp.path().join("package"); + package::unpack(path, &dir).and_then(|_| check(&dir)) +} + fn check(dir: &Path) -> Result<(), String> { let dir = canonical(dir)?; - let scratch = std::env::temp_dir().join(format!("openidea-plugin-check-{}", std::process::id())); - let result = (|| { + let tmp = scratch_dir("check")?; + let scratch = tmp.path().join("host"); + (|| { let host = PluginHost::new(scratch.clone(), Arc::new(|_: Option<&str>, _| {}), Limits::default()) .map_err(|e| e.to_string())?; let preview = host.preview_install(&dir).map_err(|e| e.to_string())?; @@ -345,6 +380,17 @@ fn check(dir: &Path) -> Result<(), String> { } else { println!(" code: none (contributions only)"); } + println!(" categories: {}", manifest.categories().join(", ")); + for c in manifest.unknown_categories() { + println!(" warning: unknown category {c:?} (ignored); use {}", ide_extension::CATEGORIES.join(", ")); + } + if manifest.declared_categories.len() > ide_extension::MAX_CATEGORIES { + println!( + " warning: only the first {} known categories are used; {} are declared", + ide_extension::MAX_CATEGORIES, + manifest.declared_categories.len() + ); + } if !manifest.activation.is_empty() { let events: Vec = manifest.activation.iter().map(ToString::to_string).collect(); println!(" activation: {}", events.join(", ")); @@ -361,9 +407,7 @@ fn check(dir: &Path) -> Result<(), String> { } println!("ok"); Ok(()) - })(); - let _ = std::fs::remove_dir_all(&scratch); - result + })() } fn human_size(bytes: u64) -> String { @@ -479,14 +523,31 @@ fn cmd_keygen(args: &[String]) -> Result<(), String> { Ok(()) } +fn cmd_endorse(args: &[String]) -> Result<(), String> { + let a = Args::parse(args, &[], &["key", "publisher"])?; + let usage = "usage: openidea-plugin endorse --key --publisher ID"; + let ([new], Some(key), Some(publisher)) = (a.positional.as_slice(), a.value("key"), a.value("publisher")) else { + return Err(usage.into()); + }; + if !package::publisher_id_ok(publisher) { + return Err(format!("invalid publisher id {publisher:?}")); + } + let endorsement = read_key(key)?.endorse(publisher, new.trim()); + println!("{}", serde_json::to_string_pretty(&endorsement).map_err(|e| e.to_string())?); + Ok(()) +} + +/// A fresh scratch directory, removed when dropped. +fn scratch_dir(what: &str) -> Result { + tempfile::Builder::new().prefix(&format!("openidea-plugin-{what}-")).tempdir().map_err(|e| e.to_string()) +} + /// A scratch host to list a package's files as the IDE would. fn with_scratch_host(f: impl FnOnce(&PluginHost) -> Result) -> Result { - let scratch = std::env::temp_dir().join(format!("openidea-plugin-pack-{}", std::process::id())); - let result = PluginHost::new(scratch.clone(), Arc::new(|_: Option<&str>, _| {}), Limits::default()) + let tmp = scratch_dir("host")?; + PluginHost::new(tmp.path().join("host"), Arc::new(|_: Option<&str>, _| {}), Limits::default()) .map_err(|e| e.to_string()) - .and_then(|h| f(&h)); - let _ = std::fs::remove_dir_all(&scratch); - result + .and_then(|h| f(&h)) } fn cmd_pack(args: &[String]) -> Result<(), String> { @@ -522,29 +583,28 @@ fn cmd_pack(args: &[String]) -> Result<(), String> { } fn cmd_index(args: &[String]) -> Result<(), String> { - let a = Args::parse(args, &[], &["publisher", "name", "key", "out"])?; + let a = Args::parse(args, &[], &["publisher", "name", "key", "previous-key", "out", "download-base"])?; let dir = canonical(&a.dir()?)?; - let publisher = a.value("publisher").ok_or("--publisher is required")?; - let name = a.value("name").unwrap_or(publisher); - let key = read_key(a.value("key").ok_or("--key is required")?)?; let out = a.value("out").map(PathBuf::from).unwrap_or_else(|| dir.join("index.json")); + let base = a.value("download-base").map(|b| b.trim_end_matches('/')); // Revocations and other publishers of an existing index are kept. let mut index = match std::fs::read_to_string(&out) { - Ok(text) => Index::parse(&text)?, + Ok(text) => current_format(Index::parse(&text)?, &out.to_string_lossy())?, Err(_) => Index { format: package::INDEX_FORMAT, ..Index::default() }, }; - // Earlier keys of the publisher stay listed (packages they signed remain valid until revoked). - let current = key.public_base64(); - let earlier: Vec = index - .publishers - .iter() - .filter(|p| p.id == publisher) - .flat_map(|p| p.keys.iter().filter(|k| k.key != current).cloned()) - .collect(); - index.publishers.retain(|p| p.id != publisher); - let keys: Vec = std::iter::once(KeyEntry { key: current, revoked: false }).chain(earlier).collect(); - index.publishers.push(package::Publisher { id: publisher.into(), name: name.into(), keys }); - index.plugins.clear(); + let publisher = match (a.value("publisher"), a.value("key")) { + (Some(publisher), Some(key)) => { + let key = read_key(key)?; + let name = a.value("name").unwrap_or(publisher); + let previous = a.value("previous-key").map(read_key).transpose()?; + index.publishers = + register_key(std::mem::take(&mut index.publishers), publisher, name, &key, previous.as_ref())?; + Some(publisher) + } + (None, None) if a.has("previous-key") => return Err("--previous-key goes with --publisher and --key".into()), + (None, None) => None, + _ => return Err("--publisher and --key go together".into()), + }; let mut packages: Vec = std::fs::read_dir(&dir) .map_err(|e| e.to_string())? .flatten() @@ -552,35 +612,144 @@ fn cmd_index(args: &[String]) -> Result<(), String> { .filter(|p| p.extension().is_some_and(|e| e == package::PACKAGE_EXTENSION)) .collect(); packages.sort(); + let engine = PluginEngine::new()?; + let mut added: Vec = Vec::new(); for path in packages { - let scratch = std::env::temp_dir().join(format!("openidea-plugin-index-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&scratch); - let unpacked = package::unpack(&path, &scratch)?; + let tmp = scratch_dir("index")?; + let scratch = tmp.path().join("package"); let entry = (|| { + let unpacked = package::unpack(&path, &scratch)?; let m = Manifest::load(&scratch)?; let files = with_scratch_host(|h| h.package_files_of(&scratch).map_err(|e| e.to_string()))?; let digest = package::package_digest(&scratch, &files).map_err(|e| e.to_string())?; - let sig = unpacked.signature.as_ref().ok_or_else(|| format!("{}: not signed", path.display()))?; - sig.check(&m.id, &m.version, &digest).map_err(|e| format!("{}: {e}", path.display()))?; + let sig = unpacked.signature.as_ref().ok_or("not signed")?; + let signer = index.verify(sig, &m.id, &m.version, &digest)?; + if publisher.is_some_and(|p| p != signer) { + return Err(format!("signed by {signer}, not {}", publisher.unwrap_or_default())); + } + if added.iter().any(|e| (&e.id, &e.version) == (&m.id, &m.version)) { + return Err(format!("another package of {} {} is in the folder", m.id, m.version)); + } + if m.legacy { + return Err("plugin.toml uses the 0.1 format; see plugins/README.md".into()); + } + let built_for = match &m.wasm { + Some(wasm) => engine.check_component(&engine.compile(&scratch.join(wasm))?)?, + None => None, + }; + let file = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); Ok::<_, String>(IndexEntry { id: m.id.clone(), name: m.name.clone(), version: m.version.clone(), description: m.description.clone(), authors: m.authors.clone(), - publisher: sig.publisher.clone(), - download: path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), + publisher: signer, + download: base.map_or_else(|| file.clone(), |b| format!("{b}/{file}")), digest, capabilities: m.capabilities.all().map(|c| c.to_string()).collect(), + engine: package::engine_requirement(&m.engine, built_for.as_ref()), + categories: m.categories(), }) })(); - let _ = std::fs::remove_dir_all(&scratch); - let entry = entry?; - println!(" {} {}", entry.id, entry.version); - index.plugins.push(entry); + let entry = entry.map_err(|e| format!("{}: {e}", path.display()))?; + println!(" {} {} (plugin API {})", entry.id, entry.version, entry.engine); + added.push(entry); } - let json = serde_json::to_string_pretty(&index).map_err(|e| e.to_string())?; - std::fs::write(&out, json).map_err(|e| format!("{}: {e}", out.display()))?; + // A publisher's earlier versions of these plugins stay for IDEs that can't load the new + // ones (pruned below); plugins no longer in the folder go. + index.plugins.retain(|e| match publisher { + Some(p) => { + e.publisher != p + || (added.iter().any(|n| n.id == e.id) + && !added.iter().any(|n| (&n.id, &n.version) == (&e.id, &e.version))) + } + None => false, + }); + index.plugins.extend(added); + index.prune_versions(); + write_index(&index, &out) +} + +/// `publishers` with `key` as `publisher`'s current key. Earlier keys stay listed +/// (packages they signed remain valid until revoked); a key new to the +/// publisher is endorsed by `previous`, one of them, so installed plugins +/// signed with it update without asking. +fn register_key( + mut publishers: Vec, + publisher: &str, + name: &str, + key: &PublisherKey, + previous: Option<&PublisherKey>, +) -> Result, String> { + let current = key.public_base64(); + let earlier: Vec = publishers.iter().filter(|p| p.id == publisher).flat_map(|p| p.keys.clone()).collect(); + let entry = match earlier.iter().find(|k| k.key == current) { + Some(listed) => listed.clone(), + None => { + let endorsement = match previous { + Some(prev) => { + let prev_key = prev.public_base64(); + if !earlier.iter().any(|k| k.key == prev_key && !k.revoked) { + return Err(format!("--previous-key is not one of {publisher}'s valid keys in the index")); + } + Some(prev.endorse(publisher, ¤t)) + } + None => { + if earlier.iter().any(|k| !k.revoked) { + println!( + " warning: {publisher}'s new key is not endorsed (--previous-key): users who installed \ + its plugins will be asked to confirm the change when they update" + ); + } + None + } + }; + KeyEntry { endorsement, ..KeyEntry::new(current.clone()) } + } + }; + let keys = std::iter::once(entry).chain(earlier.into_iter().filter(|k| k.key != current)).collect(); + publishers.retain(|p| p.id != publisher); + publishers.push(package::Publisher { id: publisher.into(), name: name.into(), keys }); + Ok(publishers) +} + +/// `index`, unless a newer `openidea-plugin` wrote it (rewriting it would drop what this one can't read). +fn current_format(index: Index, file: &str) -> Result { + if index.is_newer() { + return Err(format!( + "{file}: index format {} is newer than this tool writes; update openidea-plugin", + index.format + )); + } + Ok(index) +} + +fn cmd_merge(args: &[String]) -> Result<(), String> { + let a = Args::parse(args, &[], &["keep", "out"])?; + let [base, other] = a.positional.as_slice() else { + return Err("usage: openidea-plugin merge --keep ID [--out FILE]".into()); + }; + let keep = a.value("keep").ok_or("--keep is required")?; + let out = a.value("out").unwrap_or(base); + let mut index = + current_format(Index::parse(&std::fs::read_to_string(base).map_err(|e| format!("{base}: {e}"))?)?, base)?; + let bytes = read_market_url(other, 16 << 20)?; + let theirs = Index::parse(&String::from_utf8(bytes).map_err(|_| format!("{other}: not UTF-8"))?)?; + let mut theirs = current_format(theirs, other)?; + // Their downloads are relative to their index, not to ours. + for e in &mut theirs.plugins { + e.download = resolve_market_href(other, &e.download); + } + let count = theirs.plugins.len(); + index.include(theirs, &[keep]).map_err(|e| format!("{other}: {e}"))?; + println!(" {count} plugins from {other}"); + write_index(&index, Path::new(out)) +} + +fn write_index(index: &Index, out: &Path) -> Result<(), String> { + let json = serde_json::to_string_pretty(index).map_err(|e| e.to_string())?; + std::fs::write(out, json).map_err(|e| format!("{}: {e}", out.display()))?; println!("-> {} ({} plugins)", out.display(), index.plugins.len()); Ok(()) } @@ -589,6 +758,25 @@ fn cmd_index(args: &[String]) -> Result<(), String> { mod tests { use super::*; + #[test] + fn a_new_key_is_endorsed_by_the_previous_one() { + let (k1, k2, k3) = + (PublisherKey::from_seed([1; 32]), PublisherKey::from_seed([2; 32]), PublisherKey::from_seed([3; 32])); + let first = register_key(Vec::new(), "acme", "ACME", &k1, None).unwrap(); + let rotated = register_key(first.clone(), "acme", "ACME", &k2, Some(&k1)).unwrap(); + let index = Index { format: package::INDEX_FORMAT, publishers: rotated.clone(), ..Index::default() }; + assert!(index.succeeds("acme", &k1.public_base64(), &k2.public_base64())); + assert_eq!(rotated[0].keys.len(), 2, "the earlier key stays listed"); + // Registering the current key again keeps its endorsement. + let again = register_key(rotated.clone(), "acme", "ACME", &k2, None).unwrap(); + assert_eq!(again, rotated); + // Only a key the publisher has can endorse. + assert!(register_key(first.clone(), "acme", "ACME", &k2, Some(&k3)).unwrap_err().contains("--previous-key")); + // Without one the new key is listed unendorsed. + let unendorsed = register_key(first, "acme", "ACME", &k3, None).unwrap(); + assert!(unendorsed[0].keys[0].endorsement.is_none()); + } + #[test] fn names() { assert_eq!(slugify("My Cool_Plugin"), "my-cool-plugin"); @@ -601,13 +789,189 @@ mod tests { assert_eq!(relative(Path::new("/x/y"), Path::new("/z")), None); } + #[test] + fn new_plugins_depend_on_the_sdk_by_path_only_in_its_source_tree() { + let tree = dunce::canonicalize(concat!(env!("CARGO_MANIFEST_DIR"), "/../..")).unwrap(); + assert_eq!( + sdk_dependency(None, &tree.join("plugins/official/new")).unwrap(), + "{ path = \"../../../sdk/rust/openidea-plugin\" }" + ); + let outside = dunce::canonicalize(std::env::temp_dir()).unwrap().join("my-plugin"); + assert_eq!( + sdk_dependency(None, &outside).unwrap(), + format!("{{ git = \"https://github.com/l0ng-ai/openidea\", tag = \"v{}\" }}", env!("CARGO_PKG_VERSION")) + ); + let sdk = tree.join("sdk/rust/openidea-plugin"); + assert!(sdk_dependency(Some(sdk.to_str().unwrap()), &outside).unwrap().starts_with("{ path = ")); + assert!( + sdk_dependency(Some(tree.join("wit").to_str().unwrap()), &outside) + .unwrap_err() + .contains("not the openidea-plugin SDK") + ); + } + + /// A theme package `id` signed by `publisher` with `key`, packed into `out`. + fn signed_theme(tmp: &Path, id: &str, publisher: &str, key: &str, out: &Path) { + signed_theme_version(tmp, id, "0.1.0", "^1.0", publisher, key, out); + } + + fn signed_theme_version(tmp: &Path, id: &str, version: &str, engine: &str, publisher: &str, key: &str, out: &Path) { + let src = tmp.join("src").join(format!("{id}-{version}")); + let vars = templates::Vars { id, name: id, sdk: None }; + for (rel, content) in templates::render(templates::Template::Theme, &vars) { + let path = src.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let content = content + .replace("version = \"0.1.0\"", &format!("version = \"{version}\"")) + .replace("engines.openidea = \"^1.0\"", &format!("engines.openidea = \"{engine}\"")); + std::fs::write(path, content).unwrap(); + } + std::fs::create_dir_all(out).unwrap(); + let args = [src.to_str().unwrap(), "--key", key, "--publisher", publisher, "--out"] + .into_iter() + .map(String::from) + .chain([out.join(format!("{id}-{version}.oiplugin")).to_string_lossy().into_owned()]) + .collect::>(); + cmd_pack(&args).unwrap(); + } + + #[test] + fn a_release_keeps_the_versions_older_ides_need() { + let tmp = tempfile::tempdir().unwrap(); + let t = tmp.path(); + let key = t.join("acme.key"); + std::fs::write(&key, PublisherKey::from_seed([5; 32]).seed_base64()).unwrap(); + let key = key.to_string_lossy().into_owned(); + let index_file = t.join("index.json"); + let release = |n: usize, packages: &[(&str, &str, &str)]| { + let dir = t.join(format!("release-{n}")); + for (id, version, engine) in packages { + signed_theme_version(t, id, version, engine, "acme", &key, &dir); + } + let base = format!("https://example.com/download/v{n}"); + let args = [dir.to_str().unwrap(), "--publisher", "acme", "--key", &key, "--download-base", &base]; + cmd_index(&strings(&[&args[..], &["--out", index_file.to_str().unwrap()]].concat())).unwrap(); + let index = Index::parse(&std::fs::read_to_string(&index_file).unwrap()).unwrap(); + index.plugins.iter().map(|e| format!("{} {} {}", e.id, e.version, e.download)).collect::>() + }; + release(1, &[("acme.a", "1.0.0", "^1.0"), ("acme.b", "1.0.0", "^1.0")]); + // Same requirement: the newer version replaces it; acme.b is gone from the release. + assert_eq!( + release(2, &[("acme.a", "1.1.0", "^1.0")]), + ["acme.a 1.1.0 https://example.com/download/v2/acme.a-1.1.0.oiplugin"] + ); + // Another API requirement: the last version for the earlier one stays, where it was published. + assert_eq!( + release(3, &[("acme.a", "2.0.0", ">=1.0, <2")]), + [ + "acme.a 1.1.0 https://example.com/download/v2/acme.a-1.1.0.oiplugin", + "acme.a 2.0.0 https://example.com/download/v3/acme.a-2.0.0.oiplugin", + ] + ); + let index = Index::parse(&std::fs::read_to_string(&index_file).unwrap()).unwrap(); + assert_eq!(index.offered()[0].0.version, "2.0.0", "the newest this IDE can load"); + } + + fn strings(args: &[&str]) -> Vec { + args.iter().map(|a| a.to_string()).collect() + } + + #[test] + fn official_and_community_indexes_merge() { + let tmp = tempfile::tempdir().unwrap(); + let t = tmp.path(); + let mut keys = Vec::new(); + for (name, seed) in [("openidea", 1), ("alice", 2), ("carol", 3)] { + let file = t.join(format!("{name}.key")); + let key = PublisherKey::from_seed([seed; 32]); + std::fs::write(&file, key.seed_base64()).unwrap(); + keys.push((file.to_string_lossy().into_owned(), key.public_base64())); + } + let (official, community) = (t.join("official"), t.join("community")); + signed_theme(t, "openidea.night", "openidea", &keys[0].0, &official); + signed_theme(t, "alice.dawn", "alice", &keys[1].0, &community); + let official_index = official.join("index.json"); + let community_index = community.join("index.json"); + let release = strings(&[official.to_str().unwrap(), "--publisher", "openidea", "--key", &keys[0].0]); + cmd_index(&release).unwrap(); + + // The community index lists its publishers; `index` without a key checks packages against them. + let skeleton = Index { + format: package::INDEX_FORMAT, + publishers: vec![package::Publisher { + id: "alice".into(), + name: "Alice".into(), + keys: vec![KeyEntry::new(keys[1].1.clone())], + }], + ..Index::default() + }; + write_index(&skeleton, &community_index).unwrap(); + cmd_index(&strings(&[community.to_str().unwrap()])).unwrap(); + + cmd_merge(&strings(&[ + official_index.to_str().unwrap(), + community_index.to_str().unwrap(), + "--keep", + "openidea", + ])) + .unwrap(); + // A release rebuilds the official plugins and keeps the community's. + cmd_index(&release).unwrap(); + let index = Index::parse(&std::fs::read_to_string(&official_index).unwrap()).unwrap(); + let listed: Vec<_> = index.plugins.iter().map(|e| (e.id.as_str(), e.publisher.as_str())).collect(); + assert_eq!(listed, [("alice.dawn", "alice"), ("openidea.night", "openidea")]); + assert_eq!(index.plugins[1].categories, ["themes"], "implied by the theme"); + assert_eq!(index.plugins[1].download, "openidea.night-0.1.0.oiplugin"); + assert!(Path::new(&index.plugins[0].download).is_absolute(), "{}", index.plugins[0].download); + + // A package by a publisher the index doesn't know is refused. + signed_theme(t, "carol.noon", "carol", &keys[2].0, &community); + let err = cmd_index(&strings(&[community.to_str().unwrap()])).unwrap_err(); + assert!(err.contains("unknown publisher"), "{err}"); + } + + /// `cargo test` of a new Rust plugin (its own tests against the mock host) with this SDK. + #[test] + fn the_command_template_builds_against_the_sdk() { + let dir = std::env::temp_dir().join(format!("openidea-plugin-template-build-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let dir = dunce::canonicalize(&dir).unwrap(); + let plugin = dir.join("plugin"); + let sdk = concat!(env!("CARGO_MANIFEST_DIR"), "/../../sdk/rust/openidea-plugin"); + let sdk = sdk_dependency(Some(sdk), &plugin).unwrap(); + let vars = templates::Vars { id: "acme.sample", name: "Sample", sdk: Some(&sdk) }; + for (rel, content) in templates::render(templates::Template::Command, &vars) { + let path = plugin.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); + } + // The SDK's lockfile keeps the versions its own CI builds with (the resolver also + // looks up the wasm-only dependencies, which may not be downloaded yet). + let sdk_lock = concat!(env!("CARGO_MANIFEST_DIR"), "/../../sdk/rust/openidea-plugin/Cargo.lock"); + std::fs::copy(sdk_lock, plugin.join("Cargo.lock")).unwrap(); + let out = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into())) + .args(["test", "--manifest-path"]) + .arg(plugin.join("Cargo.toml")) + .env("CARGO_TARGET_DIR", dir.join("target")) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn every_template_is_a_valid_package() { for t in templates::Template::ALL { let dir = std::env::temp_dir().join(format!("openidea-plugin-template-{}-{}", t.label(), std::process::id())); let _ = std::fs::remove_dir_all(&dir); - let vars = templates::Vars { id: "acme.sample", name: "Sample", sdk: Some("../sdk") }; + let vars = templates::Vars { id: "acme.sample", name: "Sample", sdk: Some("{ path = \"../sdk\" }") }; for (rel, content) in templates::render(t, &vars) { let path = dir.join(rel); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); diff --git a/crates/ide-host/src/bin/openidea-plugin/templates.rs b/crates/ide-host/src/bin/openidea-plugin/templates.rs index fc23cdd0..44613eea 100644 --- a/crates/ide-host/src/bin/openidea-plugin/templates.rs +++ b/crates/ide-host/src/bin/openidea-plugin/templates.rs @@ -32,6 +32,7 @@ impl Template { pub struct Vars<'a> { pub id: &'a str, pub name: &'a str, + /// The SDK dependency, as a TOML value (`{ path = "…" }`). pub sdk: Option<&'a str>, } @@ -82,6 +83,9 @@ name = "__NAME__" version = "0.1.0" description = "Adds two sample commands." authors = [] +# Marketplace categories (up to 3): languages, themes, icon-themes, linters, +# vcs, ai, frameworks, tools, other. +categories = ["tools"] engines.openidea = "^1.0" # Built by `openidea-plugin build`. The code is loaded the first time one of # its commands runs; the commands below show in Find Action before that. @@ -115,7 +119,7 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] -openidea-plugin = { path = "__SDK__" } +openidea-plugin = __SDK__ [profile.release] opt-level = "s" @@ -127,7 +131,7 @@ panic = "abort" const COMMAND_LIB: &str = r#"//! __NAME__: an OpenIDEA plugin. -use openidea_plugin::{Invocation, NotificationLevel, Plugin, host, register_plugin}; +use openidea_plugin::{Error, Invocation, NotificationLevel, Plugin, host, register_plugin}; #[derive(Default)] pub struct Main { @@ -135,7 +139,7 @@ pub struct Main { } impl Plugin for Main { - fn run_command(&mut self, id: &str, ctx: &Invocation, _arg: Option<&str>) -> Result, String> { + fn run_command(&mut self, id: &str, ctx: &Invocation, _arg: Option<&str>) -> Result, Error> { match id { "say-hello" => { // Instance state lasts until the plugin is idle for a while; @@ -150,7 +154,7 @@ impl Plugin for Main { let text = editor.document()?.text; Ok(Some(format!("{} lines in {}", text.lines().count(), editor.path()))) } - _ => Err(format!("unknown command {id}")), + _ => Err(format!("unknown command {id}").into()), } } } diff --git a/crates/ide-host/src/lib.rs b/crates/ide-host/src/lib.rs index edaafb43..2685d7bf 100644 --- a/crates/ide-host/src/lib.rs +++ b/crates/ide-host/src/lib.rs @@ -25,7 +25,7 @@ use ide_language::{LanguageRegistry, Syntax}; use ide_run::ProcessManager; use ide_terminal::TerminalManager; use ide_workspace::SettingsStore; -use parking_lot::RwLock; +use parking_lot::{Mutex, RwLock}; use std::collections::HashMap; use std::path::PathBuf; use std::sync::{Arc, OnceLock}; @@ -74,15 +74,23 @@ pub struct Host { pub environment: environment::Environment, /// The HTTP proxy's saved password (see `proxy.rs`). proxy_password: ide_ssh::secrets::SecretFile, + /// The proxy password when it could not be saved (no credential store). + proxy_session_password: Mutex>, pub(crate) sink_for: SinkFactory, owner: fn(&str) -> &str, pub runtime: tokio::runtime::Handle, /// One project per window, keyed by window label. projects: RwLock>>, + pub(crate) external_logs: RwLock>>>, requests: requests::Requests, project_requests: RwLock>>, + /// Shutdowns of projects closed with [`Host::close_project_detached`]. + closing: parking_lot::Mutex>>, } +/// How long exit waits for project shutdown before giving up. +const EXIT_SHUTDOWN_WAIT: std::time::Duration = std::time::Duration::from_secs(10); + impl Host { pub fn new(config: HostConfig) -> Self { let HostConfig { @@ -102,6 +110,7 @@ impl Host { let host = Self { trust: trust::TrustStore::load(&config_dir, &data_dir), proxy_password: ide_ssh::secrets::SecretFile::new(data_dir.join("proxy-password.json")), + proxy_session_password: Mutex::default(), remote: Arc::new(ide_remote::RemoteService::new(ssh.clone(), data_dir.join("remote"))), ssh, environment: environment::Environment::new(project_environment, local_data_dir.clone()), @@ -122,8 +131,10 @@ impl Host { owner, runtime, projects: RwLock::default(), + external_logs: RwLock::default(), requests: Default::default(), project_requests: RwLock::default(), + closing: Default::default(), }; host.apply_proxy(); host @@ -177,45 +188,69 @@ impl Host { } } + /// Closes `window`'s project and waits until its language servers, kernels and + /// debug sessions have shut down. Never call this on the UI event-loop thread: + /// the shutdown runs on the async runtime, whose workers may themselves be + /// waiting for the event loop (IPC responses, window queries). Use + /// [`Self::close_project_detached`] there. pub fn close_project(&self, window: &str) { - let project = self.projects.write().remove(window); - if let Some(project) = project { - if !project.light { - self.environment.project_closed(); - } - if let Some(bridge) = &project.kotlin_bridge { - bridge.stop(); - } - project.documents.clear_workspace_edit_history(); - project.agents.shutdown(); - project.notebooks.stop_admission(); - // What the closing window started goes with it — the builds and searches it - // asked for, its runs, its shells and its debug sessions — and the language - // servers finish shutting down before the window is gone, as IDEA closes a - // project's processes. - for request in self.project_requests.write().remove(self.owner_window(window)).unwrap_or_default() { - self.cancel_request(request); - } - self.processes.kill_project(&project.root); - self.terminals.kill_key(self.owner_window(window)); - self.shutdown_project_workers_blocking( - vec![project.notebooks.clone()], - project.kotlin_bridge.iter().cloned().collect(), - Some(project.lsp.clone()), - Some((self.debugger.clone(), window.to_owned())), - ); + if let Some(shutdown) = self.begin_close(window) { + self.wait_blocking(shutdown); } } - /// Synchronous project/window destruction must finish owned kernel cleanup, - /// not enqueue work which can be lost as the desktop runtime exits. - fn shutdown_project_workers_blocking( + /// Like [`Self::close_project`], but returns once the project is detached from + /// the window; its workers finish shutting down in the background, and + /// [`Self::shutdown_notebooks_on_exit`] waits for them before the process exits. + pub fn close_project_detached(&self, window: &str) { + if let Some(shutdown) = self.begin_close(window) { + let mut closing = self.closing.lock(); + closing.retain(|handle| !handle.is_finished()); + closing.push(shutdown); + } + } + + /// Detaches the project from `window` and stops what it started, synchronously, + /// so the window can open another project at once. Returns the thread that + /// finishes shutting down its workers. + fn begin_close(&self, window: &str) -> Option> { + self.external_logs.write().remove(self.owner_window(window)); + for request in self.project_requests.write().remove(self.owner_window(window)).unwrap_or_default() { + self.cancel_request(request); + } + let project = self.projects.write().remove(window)?; + if !project.light { + self.environment.project_closed(); + } + if let Some(bridge) = &project.kotlin_bridge { + bridge.stop(); + } + project.documents.clear_workspace_edit_history(); + project.agents.shutdown(); + project.notebooks.stop_admission(); + // What the closing window started goes with it — the builds and searches it + // asked for, its runs, its shells and its debug sessions — and its language + // servers shut down. + self.processes.kill_project(&project.root); + self.terminals.kill_key(self.owner_window(window)); + Some(self.spawn_project_shutdown( + vec![project.notebooks.clone()], + project.kotlin_bridge.iter().cloned().collect(), + Some(project.lsp.clone()), + Some((self.debugger.clone(), window.to_owned())), + )) + } + + /// Project/window destruction must finish owned kernel cleanup, not enqueue + /// work which can be lost as the desktop runtime exits: the returned thread is + /// joined by the closer or, for a detached close, before exit. + fn spawn_project_shutdown( &self, notebooks: Vec>, bridges: Vec>, lsp: Option>, debug: Option<(Arc, String)>, - ) { + ) -> std::thread::JoinHandle<()> { for bridge in &bridges { bridge.stop(); } @@ -223,25 +258,37 @@ impl Host { service.stop_admission(); } let runtime = self.runtime.clone(); - let wait = move || { - let worker = std::thread::spawn(move || { - runtime.block_on(async move { - for bridge in bridges { - bridge.shutdown().await; - } - for service in notebooks { - service.shutdown().await; - } - if let Some(lsp) = lsp { - lsp.shutdown_all().await; - } - if let Some((debug, key)) = debug { - debug.stop_key(&key).await; - } - }); + std::thread::spawn(move || { + runtime.block_on(async move { + for bridge in bridges { + bridge.shutdown().await; + } + for service in notebooks { + service.shutdown().await; + } + if let Some(lsp) = lsp { + lsp.shutdown_all().await; + } + if let Some((debug, key)) = debug { + debug.stop_key(&key).await; + } }); - if let Err(error) = worker.join() { - tracing::error!("notebook shutdown thread failed: {error:?}"); + }) + } + + /// Waits for detached closes, so a project opened next never races the + /// language servers of one still shutting down (e.g. over a shared workspace). + pub(crate) fn wait_detached_closes(&self) { + let pending: Vec<_> = self.closing.lock().drain(..).collect(); + for shutdown in pending { + self.wait_blocking(shutdown); + } + } + + fn wait_blocking(&self, shutdown: std::thread::JoinHandle<()>) { + let wait = move || { + if let Err(error) = shutdown.join() { + tracing::error!("project shutdown thread failed: {error:?}"); } }; if tokio::runtime::Handle::try_current() @@ -253,15 +300,32 @@ impl Host { } } - /// Called before the desktop runtime exits, including update/restart exits. + /// Called before the desktop runtime exits, including update/restart exits, on + /// the event-loop thread: waits for detached closes and the open projects' + /// kernels, but no longer than [`EXIT_SHUTDOWN_WAIT`], so a runtime worker + /// stuck on the event loop cannot keep the process from exiting. pub fn shutdown_notebooks_on_exit(&self) { let projects = self.projects(); - self.shutdown_project_workers_blocking( + let mut pending: Vec<_> = self.closing.lock().drain(..).collect(); + pending.push(self.spawn_project_shutdown( projects.iter().map(|project| project.notebooks.clone()).collect(), projects.iter().filter_map(|project| project.kotlin_bridge.clone()).collect(), None, None, - ); + )); + let deadline = std::time::Instant::now() + EXIT_SHUTDOWN_WAIT; + while pending.iter().any(|handle| !handle.is_finished()) { + if std::time::Instant::now() >= deadline { + tracing::warn!("project shutdown did not finish within {EXIT_SHUTDOWN_WAIT:?}; exiting anyway"); + return; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for handle in pending { + if let Err(error) = handle.join() { + tracing::error!("project shutdown thread failed: {error:?}"); + } + } } /// Called before desktop exit, which may bypass document destructors. @@ -311,14 +375,20 @@ impl Host { } pub fn begin_project_request(&self, window: &str, id: RequestId) -> CancellationToken { - let token = self.begin_request(id); - self.project_requests.write().entry(self.owner_window(window).to_owned()).or_default().push(id); + let token = self.begin_project_read_request(window, id); if self.project(window).is_ok_and(|p| !p.is_trusted()) { token.cancel(); } token } + /// Read-only work remains cancellable/disposed with its window in restricted projects. + pub fn begin_project_read_request(&self, window: &str, id: RequestId) -> CancellationToken { + let token = self.begin_request(id); + self.project_requests.write().entry(self.owner_window(window).to_owned()).or_default().push(id); + token + } + pub fn end_request(&self, id: RequestId) { for ids in self.project_requests.write().values_mut() { ids.retain(|entry| *entry != id); diff --git a/crates/ide-host/src/ops/build.rs b/crates/ide-host/src/ops/build.rs index 472411a3..5d0fb052 100644 --- a/crates/ide-host/src/ops/build.rs +++ b/crates/ide-host/src/ops/build.rs @@ -32,6 +32,7 @@ pub fn load(root: &Path, cache: &Path, settings: &Settings) -> BuildModel { cache, ide_sdk::apply::maven_local_repository(root, settings), &settings.languages.java.maven, + &settings.languages.java.gradle, ); let jdk = ide_sdk::apply::project_jdk(root, settings); if let Some(m) = &mut model.maven { @@ -45,7 +46,7 @@ pub fn load(root: &Path, cache: &Path, settings: &Settings) -> BuildModel { p.options = options.clone(); } } - if let Some(g) = &mut model.gradle { + for g in &mut model.gradle { g.env = ide_sdk::apply::gradle_env(root, settings, java_env(jdk.as_ref(), g.java_version)); g.command = ide_sdk::apply::gradle_command(root, &root.join(&g.dir), settings).to_string_lossy().into_owned(); } @@ -65,11 +66,12 @@ pub fn load(root: &Path, cache: &Path, settings: &Settings) -> BuildModel { let hint = m.projects.iter().find_map(|p| missing(p.java_version)); explain(&mut m.sync_error, hint); } - if let Some(g) = &mut model.gradle - && g.sync_error.is_some() - { - let hint = if settings.languages.java.gradle.jvm.trim().is_empty() { missing(g.java_version) } else { None }; - explain(&mut g.sync_error, hint); + for g in &mut model.gradle { + if g.sync_error.is_some() { + let hint = + if settings.languages.java.gradle.jvm.trim().is_empty() { missing(g.java_version) } else { None }; + explain(&mut g.sync_error, hint); + } } model } @@ -94,9 +96,88 @@ pub async fn build_model(host: &Host, window: &str) -> IdeResult { tokio::task::spawn_blocking(move || load(&root, &cache, &settings)).await.map_err(join_err) } +fn linked_gradle_paths( + root: &Path, + settings: &ide_core::settings::GradleSettings, + paths: &[String], + link: bool, +) -> IdeResult> { + let existing = settings + .linked_projects + .clone() + .unwrap_or_else(|| ide_build::gradle::load_linked(root, settings).into_iter().map(|g| g.dir).collect()); + let mut linked = Vec::new(); + for path in existing { + let normalized = ide_build::gradle::normalize_linked_projects(root, std::slice::from_ref(&path)) + .unwrap_or_else(|_| vec![path]); + for path in normalized { + if !linked.contains(&path) { + linked.push(path); + } + } + } + if link { + let directories: Vec = paths + .iter() + .map(|path| { + let path = root.join(path); + let is_script = path.file_name().and_then(|n| n.to_str()).is_some_and(|name| { + ide_build::gradle::SETTINGS_FILES.contains(&name) || ide_build::gradle::BUILD_FILES.contains(&name) + }); + if is_script && path.is_file() { + path.parent().unwrap_or(root).to_string_lossy().into_owned() + } else { + path.to_string_lossy().into_owned() + } + }) + .collect(); + for path in ide_build::gradle::normalize_linked_projects(root, &directories)? { + if !linked.contains(&path) { + linked.push(path); + } + } + } else { + linked.retain(|path| !paths.contains(path)); + } + Ok(linked) +} + +async fn set_gradle_projects(host: &Host, window: &str, paths: Vec, link: bool) -> IdeResult { + host.require_trust(window)?; + let project = host.project(window)?; + tokio::task::spawn_blocking(move || { + if ide_workspace::workspace::Workspace::load(&project.root)?.is_some() { + return Err(IdeError::invalid("Open a workspace member as a project to link Gradle builds")); + } + let linked = + linked_gradle_paths(&project.root, &project.settings.effective().languages.java.gradle, &paths, link)?; + let key = "languages.java.gradle.linkedProjects"; + let settings = + project.settings.set(ide_workspace::settings::SettingsScope::Project, key, serde_json::json!(linked))?; + project.refresh_index_scope(); + project.lsp.reconfigure_soon(); + project.sink.emit(IdeEvent::SettingsChanged { key: key.into() }); + project.sink.emit(IdeEvent::ProjectRootsChanged { root: project.root.clone().into() }); + Ok::(load(&project.root, &project.build_cache_dir(), &settings)) + }) + .await + .map_err(join_err)? +} + +/// Links independent builds without changing their Gradle scripts. +pub async fn gradle_link_projects(host: &Host, window: &str, paths: Vec) -> IdeResult { + set_gradle_projects(host, window, paths, true).await +} + +/// Stops importing the selected independent builds; files and saved run configurations stay intact. +pub async fn gradle_unlink_projects(host: &Host, window: &str, paths: Vec) -> IdeResult { + set_gradle_projects(host, window, paths, false).await +} + /// Runs Maven (`help:effective-pom dependency:tree`) or Gradle (`tasks --all`) in the /// background, reporting progress on `on_progress`; cancellable with `cancel_request(request)`. /// Returns the refreshed model; a failed build is reported in its `syncError`. +#[allow(clippy::too_many_arguments)] pub async fn build_sync( host: &Host, window: &str, @@ -104,6 +185,7 @@ pub async fn build_sync( request: RequestId, offline: bool, profiles: Vec, + build_dir: Option, on_progress: Channel, ) -> IdeResult { host.require_trust(window)?; @@ -122,6 +204,8 @@ pub async fn build_sync( project.sink.emit(ide_core::IdeEvent::ProjectRootsChanged { root: project.root.clone().into() }); } let token = host.begin_project_request(window, request); + let completion_token = token.clone(); + let original_links = settings.languages.java.gradle.linked_projects.clone(); let result = tokio::task::spawn_blocking(move || { let current = load(&root, &cache, &settings); let env = match tool { @@ -131,7 +215,7 @@ pub async fn build_sync( .and_then(|m| m.projects.iter().find(|p| !p.env.is_empty())) .map(|p| p.env.clone()) .unwrap_or_default(), - BuildTool::Gradle => current.gradle.as_ref().map(|g| g.env.clone()).unwrap_or_default(), + BuildTool::Gradle => BTreeMap::new(), }; let options = SyncOptions { offline, @@ -140,14 +224,31 @@ pub async fn build_sync( maven_command: ide_sdk::apply::maven_command(&root, &settings).map(|p| p.to_string_lossy().into_owned()), maven_options: ide_sdk::apply::maven_options(&root, &settings), maven_settings: settings.languages.java.maven.clone(), - gradle_command: current.gradle.as_ref().map(|g| g.command.clone()), + gradle_command: None, }; let progress = |p: SyncProgress| { let _ = on_progress.send(p); }; match tool { BuildTool::Maven => ide_build::sync_maven(&root, &cache, &options, &token, &progress)?, - BuildTool::Gradle => ide_build::sync_gradle(&root, &cache, &options, &token, &progress)?, + BuildTool::Gradle => { + if let Some(dir) = &build_dir + && !current.gradle.iter().any(|g| &g.dir == dir) + { + return Err(IdeError::not_found("Gradle project is no longer linked")); + } + for g in current.gradle.iter().filter(|g| build_dir.as_ref().is_none_or(|dir| dir == &g.dir)) { + if token.is_cancelled() { + return Err(IdeError::cancelled()); + } + let options = + SyncOptions { env: g.env.clone(), gradle_command: Some(g.command.clone()), ..options.clone() }; + ide_build::sync_gradle_model(&root, &cache, g, &options, &token, &progress)?; + } + if token.is_cancelled() { + return Err(IdeError::cancelled()); + } + } } Ok::(load(&root, &cache, &settings)) }) @@ -155,6 +256,18 @@ pub async fn build_sync( .map_err(join_err); host.end_request(request); let model = result??; + if completion_token.is_cancelled() { + return Err(IdeError::cancelled()); + } + if tool == BuildTool::Gradle && original_links != project.settings.effective().languages.java.gradle.linked_projects + { + // A late sync must not re-import a build that was unlinked while it ran. + return tokio::task::spawn_blocking(move || { + load(&project.root, &project.build_cache_dir(), &project.settings.effective()) + }) + .await + .map_err(join_err); + } if tool == BuildTool::Gradle { // The sync may report project folders outside the project (`includeFlat`): index and list them. project.refresh_index_scope(); @@ -238,6 +351,42 @@ pub async fn gradle_published_catalog( .map_err(join_err) } +/// The resolved dependency tree of the Gradle project `gradle_path` (`:`, `:app`) for +/// `configuration`, from Gradle's `dependencies` report; cancellable with `cancel_request(request)`. +pub async fn gradle_dependencies( + host: &Host, + window: &str, + gradle_path: String, + configuration: String, + offline: bool, + request: RequestId, + build_dir: Option, +) -> IdeResult> { + host.require_trust(window)?; + let project = host.project(window)?; + let (root, cache, settings) = (project.root.clone(), project.build_cache_dir(), project.settings.effective()); + let token = host.begin_project_request(window, request); + let result = tokio::task::spawn_blocking(move || { + let model = load(&root, &cache, &settings); + let gradle = model + .gradle + .iter() + .find(|g| build_dir.as_ref().is_none_or(|dir| &g.dir == dir)) + .ok_or_else(|| IdeError::not_found("Gradle project is no longer linked"))?; + let options = SyncOptions { + offline, + env: gradle.env.clone(), + gradle_command: Some(gradle.command.clone()), + ..Default::default() + }; + ide_build::gradle_dependencies(&root, gradle, &options, &gradle_path, &configuration, &token) + }) + .await + .map_err(join_err); + host.end_request(request); + result? +} + /// Where the direct dependency `dependency` of the Maven module `pom` is declared. pub async fn maven_dependency_declaration( host: &Host, @@ -352,6 +501,11 @@ pub async fn build_run( /// Commands of this module the remote server answers. pub(crate) fn register(d: &mut Dispatcher) { d.add("build_model", |c| async move { build_model(&c.host, &c.window).await }); + d.add("gradle_link_projects", |c| async move { gradle_link_projects(&c.host, &c.window, c.arg("paths")?).await }); + d.add( + "gradle_unlink_projects", + |c| async move { gradle_unlink_projects(&c.host, &c.window, c.arg("paths")?).await }, + ); d.add("build_sync", |c| async move { build_sync( &c.host, @@ -360,6 +514,7 @@ pub(crate) fn register(d: &mut Dispatcher) { c.arg("request")?, c.arg("offline")?, c.arg("profiles")?, + c.arg("buildDir")?, c.sender("onProgress")?, ) .await @@ -367,6 +522,18 @@ pub(crate) fn register(d: &mut Dispatcher) { d.add("gradle_published_catalog", |c| async move { gradle_published_catalog(&c.host, &c.window, c.arg("coordinates")?).await }); + d.add("gradle_dependencies", |c| async move { + gradle_dependencies( + &c.host, + &c.window, + c.arg("gradlePath")?, + c.arg("configuration")?, + c.arg("offline")?, + c.arg("request")?, + c.arg("buildDir")?, + ) + .await + }); d.add("maven_dependency_declaration", |c| async move { maven_dependency_declaration(&c.host, &c.window, c.arg("pom")?, c.arg("dependency")?).await }); @@ -391,6 +558,48 @@ pub(crate) fn register(d: &mut Dispatcher) { mod gradle_settings_tests { use super::*; + #[test] + fn linking_retains_the_implicit_build_and_unlinking_all_stays_empty() { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + for name in ["a", "b"] { + std::fs::create_dir(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("settings.gradle"), format!("rootProject.name = '{name}'")).unwrap(); + } + let mut settings = ide_core::settings::GradleSettings::default(); + let linked = linked_gradle_paths(&root, &settings, &["b/settings.gradle".into()], true).unwrap(); + assert_eq!(linked, ["a", "b"]); + settings.linked_projects = Some(linked); + assert_eq!(linked_gradle_paths(&root, &settings, &["b".into()], true).unwrap(), ["a", "b"]); + assert!(linked_gradle_paths(&root, &settings, &["missing".into()], true).is_err()); + settings.linked_projects = Some(vec!["./a".into(), root.join("b").to_string_lossy().into_owned()]); + settings.linked_projects = Some(linked_gradle_paths(&root, &settings, &["a".into()], false).unwrap()); + assert_eq!(settings.linked_projects.as_ref().unwrap(), &["b"]); + settings.linked_projects = Some(linked_gradle_paths(&root, &settings, &["b".into()], false).unwrap()); + assert!(ide_build::gradle::load_linked(&root, &settings).is_empty()); + assert!(root.join("a/settings.gradle").is_file()); + assert!(root.join("b/settings.gradle").is_file()); + } + + #[test] + fn settings_resolve_each_linked_builds_own_wrapper() { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + let mut settings = Settings::default(); + settings.languages.java.gradle.linked_projects = Some(vec!["a".into(), "b".into()]); + let executable = if cfg!(windows) { "gradlew.bat" } else { "gradlew" }; + for dir in ["a", "b"] { + std::fs::create_dir(root.join(dir)).unwrap(); + std::fs::write(root.join(dir).join("build.gradle"), "plugins { id 'java' }").unwrap(); + std::fs::write(root.join(dir).join(executable), "").unwrap(); + } + let model = load(&root, &root.join("cache"), &settings); + assert_eq!(model.gradle.len(), 2); + for g in model.gradle { + assert_eq!(g.command, root.join(&g.dir).join(executable).to_string_lossy()); + } + } + #[test] fn model_for_sync_tasks_and_compile_uses_gradle_settings() { let tmp = tempfile::tempdir().unwrap(); @@ -399,13 +608,13 @@ mod gradle_settings_tests { let wrapper = root.join(if cfg!(windows) { "gradlew.bat" } else { "gradlew" }); std::fs::write(&wrapper, "").unwrap(); let mut settings = Settings::default(); - let default = load(root, &root.join("cache"), &settings).gradle.unwrap(); + let default = load(root, &root.join("cache"), &settings).gradle.remove(0); assert_eq!(default.command, wrapper.to_string_lossy()); settings.languages.java.gradle.jvm = "jdk21".into(); settings.languages.java.gradle.user_home = "user-home".into(); settings.languages.java.gradle.distribution = ide_core::settings::GradleDistribution::Local; settings.languages.java.gradle.home = "gradle-install".into(); - let model = load(root, &root.join("cache"), &settings).gradle.unwrap(); + let model = load(root, &root.join("cache"), &settings).gradle.remove(0); assert_eq!( model.command, root.join("gradle-install") diff --git a/crates/ide-host/src/ops/db.rs b/crates/ide-host/src/ops/db.rs index 63c8419a..102bf274 100644 --- a/crates/ide-host/src/ops/db.rs +++ b/crates/ide-host/src/ops/db.rs @@ -216,8 +216,7 @@ pub async fn connect(host: &Host, window: &str, id: &str) -> IdeResult<()> { /// manual mode rolls back what was not committed). pub async fn set_manual_commit(host: &Host, window: &str, id: &str, manual: bool) -> IdeResult<()> { let db = host.project(window)?.db.clone(); - db.set_manual_commit(id, manual).await; - Ok(()) + db.set_manual_commit(id, manual).await } /// Commits or rolls back the open transaction of a manual-commit data source. diff --git a/crates/ide-host/src/ops/doc/commands.rs b/crates/ide-host/src/ops/doc/commands.rs index e1fab8f9..ddb9c8a6 100644 --- a/crates/ide-host/src/ops/doc/commands.rs +++ b/crates/ide-host/src/ops/doc/commands.rs @@ -18,7 +18,7 @@ pub async fn blocking(f: impl FnOnce() -> IdeResult + Send /// editor buffer, so syntax features and language-server requests work in it. pub async fn doc_open_virtual(host: &Host, window: &str, uri: String) -> IdeResult { let project = host.project(window)?; - let vd = project.lsp.virtual_document(&uri).await?; + let vd = crate::ops::lsp::lsp_virtual_document(host, window, uri).await?; blocking(move || project.documents.open_virtual(&vd.uri, vd.language_id, &vd.text)).await } diff --git a/crates/ide-host/src/ops/fs.rs b/crates/ide-host/src/ops/fs.rs index 0277dfc4..e9807500 100644 --- a/crates/ide-host/src/ops/fs.rs +++ b/crates/ide-host/src/ops/fs.rs @@ -137,16 +137,23 @@ pub fn set_writable(host: &Host, window: &str, path: PathBuf, writable: bool) -> pub async fn delete(host: &Host, window: &str, path: PathBuf) -> IdeResult<()> { let history = host.project(window).ok().map(|p| p.history.clone()); host.blocking(move || { + let paths = ide_workspace::file_ops::deletion_paths(&path)?; if let Some(h) = history { - h.record_deletion(&path, DELETE_SNAPSHOT_LIMIT); + for p in paths { + h.record_deletion(&p, DELETE_SNAPSHOT_LIMIT); + } } - ide_vfs::delete(&path) + ide_workspace::file_ops::delete(&path) }) .await } pub async fn copy(host: &Host, from: PathBuf, to: PathBuf) -> IdeResult<()> { - host.blocking(move || ide_vfs::copy(&from, &to)).await + host.blocking(move || ide_workspace::file_ops::copy(&from, &to)).await +} + +pub async fn rename(host: &Host, from: PathBuf, to: PathBuf) -> IdeResult<()> { + host.blocking(move || ide_workspace::file_ops::rename(&from, &to)).await } /// Reads a text file without opening it as a document; prefers the unsaved buffer. diff --git a/crates/ide-host/src/ops/lsp.rs b/crates/ide-host/src/ops/lsp.rs index f8b27063..2e2402f3 100644 --- a/crates/ide-host/src/ops/lsp.rs +++ b/crates/ide-host/src/ops/lsp.rs @@ -76,9 +76,17 @@ async fn plugin_call(f: impl FnOnce() -> T + Send + 'static) tokio::task::spawn_blocking(f).await.ok() } -/// Items from plugins carry no server (resolve and commands don't apply). +/// Items from plugins carry no server: commands don't apply, and a code +/// action is resolved by the plugin named in its `data` ([`PluginActionRef`]). const PLUGIN_SERVER: ServerId = ServerId(0); +/// `data` of a plugin's code action whose edits it computes when picked. +#[derive(serde::Serialize, serde::Deserialize)] +struct PluginActionRef { + plugin: String, + data: String, +} + /// A plugin's `completion-item.kind` (wit/providers.wit lists them). fn plugin_completion_kind(kind: &str) -> Option { Some(match kind { @@ -120,14 +128,14 @@ fn plugin_completion(item: ide_plugin::CompletionItem) -> CompletionItem { kind, documentation: item.documentation.map(|value| Markup { kind: "markdown".into(), value }), sort_text: item.sort_text, - filter_text: None, + filter_text: item.filter_text, preselect: false, deprecated: false, insert_text: item.insert_text, - is_snippet: false, + is_snippet: item.snippet, replace_range: item.replace, replace_range_tab: None, - additional_edits: Vec::new(), + additional_edits: item.additional_edits, commit_characters: Vec::new(), command: None, data: None, @@ -252,6 +260,9 @@ pub async fn lsp_implementations( /// Contents of a read-only language-server document, including JDT library /// classes and disk class files. Cached per project. pub async fn lsp_virtual_document(host: &Host, window: &str, uri: String) -> IdeResult { + if uri.starts_with("git-revision:") { + return crate::ops::vcs::revision_virtual_document(host, window, &uri).await; + } host.project(window)?.lsp.virtual_document(&uri).await } @@ -314,7 +325,7 @@ pub async fn lsp_code_actions( }; let plugin_actions: Vec = plugin_actions .into_iter() - .map(|(_, a)| plugin_code_action(a)) + .map(|(plugin, a)| plugin_code_action(&plugin, a)) .filter(|a| { only.is_empty() || a.kind.as_ref().is_some_and(|k| only.iter().any(|o| k == o || k.starts_with(&format!("{o}.")))) @@ -330,24 +341,53 @@ pub async fn lsp_code_actions( prepare_actions(project, doc, version, actions).await } -fn plugin_code_action(a: ide_plugin::CodeAction) -> CodeAction { - let files = a +fn plugin_code_action(plugin: &str, a: ide_plugin::CodeAction) -> CodeAction { + // With `data` and no edits, the plugin computes them when the action is picked. + let lazy = a.edits.is_empty() && a.data.is_some(); + let files: Vec<_> = a .edits .into_iter() .map(|(path, edits)| ide_core::FileEdit::Edit { path: path.into(), edits, version: None }) .collect(); + let data = a + .data + .filter(|_| lazy) + .and_then(|data| serde_json::to_value(PluginActionRef { plugin: plugin.to_owned(), data }).ok()); CodeAction { title: a.title, kind: a.kind, is_preferred: a.preferred, disabled_reason: None, - edit: Some(WorkspaceEdit { files, preconditions: Vec::new() }), + edit: (!lazy).then(|| WorkspaceEdit { files, preconditions: Vec::new() }), command: None, - data: None, + data, server: PLUGIN_SERVER, } } +/// A code action resolved by whoever offered it: its language server, or +/// the plugin its `data` names. +async fn resolve_action(host: &Host, window: &str, doc: DocId, action: CodeAction) -> IdeResult { + if action.server != PLUGIN_SERVER { + return host.project(window)?.lsp.resolve_code_action(doc, action).await; + } + let Some(r) = action.data.clone().and_then(|d| serde_json::from_value::(d).ok()) else { + return Ok(action); + }; + let (plugins, d) = plugin_providers(host, window, doc, ProviderKind::CodeActions) + .ok_or_else(|| ide_core::IdeError::unavailable("the plugin that offered this action is gone"))?; + let pending = ide_plugin::CodeAction { + title: action.title.clone(), + kind: action.kind.clone(), + preferred: action.is_preferred, + edits: Vec::new(), + data: Some(r.data), + }; + let plugin = r.plugin.clone(); + let resolved = blocking(move || plugins.resolve_code_action(&plugin, &d, pending)).await?; + Ok(plugin_code_action(&r.plugin, ide_plugin::CodeAction { data: None, ..resolved })) +} + /// Cancellable code actions with the kinds the servers advertise (Refactor This, refactoring availability). pub async fn lsp_code_action_query( host: &Host, @@ -381,7 +421,7 @@ pub async fn lsp_resolve_code_action( let project = host.project(window)?; let documents = project.documents.clone(); let version = blocking(move || documents.version(doc)).await?; - let action = project.lsp.resolve_code_action(doc, action).await?; + let action = resolve_action(host, window, doc, action).await?; Ok(prepare_actions(project, doc, version, vec![action]).await?.remove(0)) } @@ -391,7 +431,7 @@ pub async fn lsp_apply_code_action(host: &Host, window: &str, doc: DocId, action let action = if action.edit.is_none() && action.command.is_none() { let documents = project.documents.clone(); let version = blocking(move || documents.version(doc)).await?; - let action = project.lsp.resolve_code_action(doc, action).await?; + let action = resolve_action(host, window, doc, action).await?; prepare_actions(project.clone(), doc, version, vec![action]).await?.remove(0) } else { action @@ -401,17 +441,85 @@ pub async fn lsp_apply_code_action(host: &Host, window: &str, doc: DocId, action let edit = edit.clone(); blocking(move || apply_edit(&documents, &edit)).await?; } - if let Some(command) = action.command { + if let Some(command) = action.command + && !(action.edit.is_some() && is_client_follow_up(&project, doc, None, &command).await) + { project.lsp.execute_command(doc, command).await?; } Ok(()) } +/// A command after a code action's edit that no server provides: what other editors' +/// extensions do once the edit is applied (vtsls' `_typescript.didOrganizeImports`). The edit +/// is the action; there is nothing to run. +async fn is_client_follow_up( + project: &crate::project::Project, + doc: DocId, + server: Option, + command: &Command, +) -> bool { + let provided = project.lsp.provides_command(doc, server, &command.command).await; + if !provided { + tracing::debug!("skipping `{}` after the code action's edit: no server provides it", command.command); + } + !provided +} + pub async fn lsp_prepare_rename(host: &Host, window: &str, doc: DocId, pos: Pos) -> IdeResult> { host.project(window)?.lsp.prepare_rename(doc, pos).await } /// Computes the rename edit without applying it, so the UI can preview conflicts. +/// The class at `pos` and the members a class-hierarchy refactoring can move. +pub async fn lsp_hierarchy_members( + host: &Host, + window: &str, + request: RequestId, + doc: DocId, + pos: Pos, + kind: HierarchyRefactoring, +) -> IdeResult { + let project = host.project(window)?; + let cancel = host.begin_request(request); + let result = project.lsp.hierarchy_members(doc, pos, kind, cancel).await; + host.end_request(request); + result +} + +/// Checks a class-hierarchy refactoring and computes its edit without applying it. +#[allow(clippy::too_many_arguments)] +pub async fn lsp_hierarchy_refactor( + host: &Host, + window: &str, + request: RequestId, + doc: DocId, + pos: Pos, + kind: HierarchyRefactoring, + target: Option, + choices: Vec, +) -> IdeResult { + let project = host.project(window)?; + let documents = project.documents.clone(); + let version = blocking(move || documents.version(doc)).await?; + let cancel = host.begin_request(request); + let result = project.lsp.hierarchy_refactor(doc, pos, kind, target, choices, cancel).await; + host.end_request(request); + let mut refactored = result?; + blocking(move || { + if project.documents.version(doc)? != version { + return Err(ide_core::IdeError::new( + ide_core::error::ErrorKind::OutOfSync, + "document changed while computing the refactoring; request it again", + )); + } + if let Some(edit) = &mut refactored.edit { + project.documents.prepare_workspace_edit(edit)?; + } + Ok(refactored) + }) + .await +} + pub async fn lsp_rename( host: &Host, window: &str, @@ -589,6 +697,7 @@ pub async fn lsp_execute_code_action( if cancel.is_cancelled() { return Err(ide_core::IdeError::cancelled()); } + let edited = action.edit.is_some(); if let Some(edit) = action.edit { let documents = project.documents.clone(); tokio::task::spawn_blocking(move || apply_edit(&documents, &edit)) @@ -596,6 +705,9 @@ pub async fn lsp_execute_code_action( .map_err(|e| ide_core::IdeError::internal(e.to_string()))??; } match action.command { + Some(command) if edited && is_client_follow_up(&project, doc, Some(action.server), &command).await => { + Ok(Vec::new()) + } Some(command) => project .lsp .execute_action_command(doc, action.server, command, answers, cancel) @@ -676,6 +788,22 @@ pub(crate) fn register(d: &mut Dispatcher) { d.add("lsp_rename", |c| async move { lsp_rename(&c.host, &c.window, c.arg("request")?, c.arg("doc")?, c.arg("pos")?, c.arg("newName")?).await }); + d.add("lsp_hierarchy_members", |c| async move { + lsp_hierarchy_members(&c.host, &c.window, c.arg("request")?, c.arg("doc")?, c.arg("pos")?, c.arg("kind")?).await + }); + d.add("lsp_hierarchy_refactor", |c| async move { + lsp_hierarchy_refactor( + &c.host, + &c.window, + c.arg("request")?, + c.arg("doc")?, + c.arg("pos")?, + c.arg("kind")?, + c.arg("target")?, + c.arg("choices")?, + ) + .await + }); d.add("lsp_format", |c| async move { lsp_format(&c.host, &c.window, c.arg("doc")?, c.arg("range")?, c.arg("options")?).await }); @@ -735,6 +863,25 @@ pub(crate) fn register(d: &mut Dispatcher) { mod tests { use super::*; + #[test] + fn a_plugin_action_with_data_and_no_edits_is_resolved_by_its_plugin() { + let action = |edits: Vec<(std::path::PathBuf, Vec)>, data: Option<&str>| ide_plugin::CodeAction { + title: "Fix".into(), + kind: None, + preferred: false, + edits, + data: data.map(str::to_owned), + }; + let lazy = plugin_code_action("acme.x", action(Vec::new(), Some("ref"))); + assert!(lazy.edit.is_none()); + let r: PluginActionRef = serde_json::from_value(lazy.data.unwrap()).unwrap(); + assert_eq!((r.plugin.as_str(), r.data.as_str()), ("acme.x", "ref")); + // With edits, `data` is the plugin's business: applied as is. + let edit = TextEdit { range: Range::default(), new_text: "x".into() }; + let eager = plugin_code_action("acme.x", action(vec![("/a".into(), vec![edit])], Some("ref"))); + assert!(eager.edit.is_some() && eager.data.is_none()); + } + #[test] fn every_completion_kind_the_plugin_api_documents_is_understood() { let wit = include_str!("../../../../wit/providers.wit"); diff --git a/crates/ide-host/src/ops/nav.rs b/crates/ide-host/src/ops/nav.rs index 5a4c3b32..646e3ab8 100644 --- a/crates/ide-host/src/ops/nav.rs +++ b/crates/ide-host/src/ops/nav.rs @@ -3,7 +3,7 @@ use crate::Host; use crate::Project; use crate::rpc::Dispatcher; use ide_core::{DocId, IdeResult, IpcPath, Location, Pos}; -use ide_index::{FileMatch, IndexStatus, SymbolFilter, SymbolMatch}; +use ide_index::{ExtensionCount, FileMatch, IndexStatus, SymbolFilter, SymbolMatch}; use ide_language::{FileReference, WebReference}; use ide_workspace::file_refs::{self, PathCompletion}; use serde::{Deserialize, Serialize}; @@ -32,10 +32,19 @@ pub fn index_status(host: &Host, window: &str) -> IdeResult { Ok(host.project(window)?.index.status()) } +/// The project's file extensions, most common first. +pub fn index_file_extensions(host: &Host, window: &str) -> IdeResult> { + Ok(host.project(window)?.index.file_extensions()) +} + pub fn find_files(host: &Host, window: &str, query: String, limit: u32) -> IdeResult> { Ok(host.project(window)?.index.find_files(&query, limit as usize)) } +pub fn find_dirs(host: &Host, window: &str, query: String, limit: u32) -> IdeResult> { + Ok(host.project(window)?.index.find_dirs(&query, limit as usize)) +} + pub fn find_symbols( host: &Host, window: &str, @@ -156,7 +165,9 @@ pub async fn web_unresolved_references(host: &Host, window: &str, doc: DocId) -> /// Commands of this module the remote server answers. pub(crate) fn register(d: &mut Dispatcher) { d.add("index_status", |c| async move { index_status(&c.host, &c.window) }); + d.add("index_file_extensions", |c| async move { index_file_extensions(&c.host, &c.window) }); d.add("find_files", |c| async move { find_files(&c.host, &c.window, c.arg("query")?, c.arg("limit")?) }); + d.add("find_dirs", |c| async move { find_dirs(&c.host, &c.window, c.arg("query")?, c.arg("limit")?) }); d.add("find_symbols", |c| async move { find_symbols(&c.host, &c.window, c.arg("query")?, c.arg("filter")?, c.arg("limit")?) }); diff --git a/crates/ide-host/src/ops/project.rs b/crates/ide-host/src/ops/project.rs index e6632c1d..f739d95c 100644 --- a/crates/ide-host/src/ops/project.rs +++ b/crates/ide-host/src/ops/project.rs @@ -17,6 +17,8 @@ pub struct ProjectInfo { pub settings: Settings, #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unity: Option, } pub fn info(project: &Project) -> ProjectInfo { @@ -33,6 +35,7 @@ pub fn info(project: &Project) -> ProjectInfo { settings, root: project.root.clone().into(), workspace: ide_workspace::workspace::Workspace::load(&project.root).ok().flatten(), + unity: ide_workspace::unity::detect(&project.root), } } @@ -172,7 +175,12 @@ pub fn set_settings( host.apply_proxy(); } // A new interpreter / toolchain / server executable restarts the language servers it affects. - let code_style = key == "editor" || key == "editor.codeStyle" || key.starts_with("editor.codeStyle."); + // So do inspections that are compiler options (`ide-sdk` `configure_java_compiler`). + let code_style = key == "editor" + || key == "editor.codeStyle" + || key.starts_with("editor.codeStyle.") + || key == "inspections" + || key.starts_with("inspections."); let toolchain = ["languages", "tools", "languageServers", "projectStructure"] .iter() .any(|k| key == *k || key.starts_with(&format!("{k}."))); @@ -188,7 +196,8 @@ pub fn set_settings( } let roots_changed = key == "projectStructure" || key.starts_with("projectStructure.") - || "languages.java.maven.unlinkedProjects".starts_with(key); + || "languages.java.maven.unlinkedProjects".starts_with(key) + || "languages.java.gradle.linkedProjects".starts_with(key); // Content roots and excludes (files.exclude, ignored names, excluded folders) reach the // index and the watchers without reopening the project (the rescan runs in the background). if roots_changed || key == "files" || key.starts_with("files.") { diff --git a/crates/ide-host/src/ops/run.rs b/crates/ide-host/src/ops/run.rs index ae24f9f8..a71a8da5 100644 --- a/crates/ide-host/src/ops/run.rs +++ b/crates/ide-host/src/ops/run.rs @@ -155,6 +155,9 @@ pub async fn start(host: &Host, window: &str, config: RunConfig, on_output: Emit let (root, settings) = settings_for(host, &project, &config)?; let root = &root; // Configurations made in the UI name bare tools (`godot`) that may live outside PATH. + // Maven re-applies too: a config listed before a settings change (daemon toggle, + // Maven home) keeps the launcher that was current then; re-resolving at launch is + // what makes the setting take effect without reopening the project. let mut config = config; if config.kind == "maven" { ide_sdk::apply::prepare_maven_run(&mut config, root, &settings); diff --git a/crates/ide-host/src/ops/run/commands.rs b/crates/ide-host/src/ops/run/commands.rs index 6d2ba830..b1a6c867 100644 --- a/crates/ide-host/src/ops/run/commands.rs +++ b/crates/ide-host/src/ops/run/commands.rs @@ -142,6 +142,21 @@ pub async fn run_gradle_jvm_args_script(_host: &Host, _window: &str, jvm_args: V .map_err(|e| IdeError::internal(e.to_string()))? } +/// Writes one of the IDE's Gradle init scripts (`text`; the file is named `-`) on +/// the machine that runs the build, for `--init-script`, and returns its path. +pub async fn run_gradle_init_script(_host: &Host, _window: &str, name: String, text: String) -> IdeResult { + if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') { + return Err(IdeError::invalid(format!("invalid init script name {name:?}"))); + } + tokio::task::spawn_blocking(move || { + ide_run::gradle_jvm::write_script(&ide_run::gradle_jvm::default_dir(), &name, &text) + .map(|p| p.to_string_lossy().into_owned()) + .map_err(|e| IdeError::internal(format!("cannot write the Gradle init script: {e}"))) + }) + .await + .map_err(|e| IdeError::internal(e.to_string()))? +} + /// Results from the JUnit XML reports (Surefire, Failsafe, Gradle) of the module at /// `module` (relative to the project root) written since `since_ms` (Unix epoch, ms). pub async fn run_test_reports( @@ -308,6 +323,9 @@ pub(crate) fn register(d: &mut Dispatcher) { d.add("run_gradle_jvm_args_script", |c| async move { run_gradle_jvm_args_script(&c.host, &c.window, c.arg("jvmArgs")?).await }); + d.add("run_gradle_init_script", |c| async move { + run_gradle_init_script(&c.host, &c.window, c.arg("name")?, c.arg("text")?).await + }); d.add("run_test_reports", |c| async move { run_test_reports(&c.host, &c.window, c.arg("module")?, c.arg("sinceMs")?).await }); diff --git a/crates/ide-host/src/ops/sdk.rs b/crates/ide-host/src/ops/sdk.rs index 2db3148d..535426d8 100644 --- a/crates/ide-host/src/ops/sdk.rs +++ b/crates/ide-host/src/ops/sdk.rs @@ -34,6 +34,42 @@ pub fn sdk_python_effective(host: &Host, window: &str) -> IdeResult, + /// The one Auto picks, from `composer.json`. + pub auto: Option, + /// The workspace member `path` belongs to: its own settings choose the level, not the + /// workspace's. `None`: the opened project. + pub member: Option, +} + +pub async fn sdk_php_language_level(host: &Host, window: &str, path: PathBuf) -> IdeResult { + let project = host.project(window)?; + let (project_root, project_settings) = (project.root.clone(), project.settings.effective()); + let config_dir = host.config_dir.clone(); + tokio::task::spawn_blocking(move || { + let member = ide_workspace::workspace::Workspace::load(&project_root).ok().flatten().and_then(|w| { + w.projects.into_iter().filter(|p| path.starts_with(p)).max_by_key(|p| p.components().count()) + }); + let (root, settings) = match &member { + Some(member) => (member, ide_workspace::SettingsStore::load(&config_dir, Some(member)).effective()), + None => (&project_root, project_settings), + }; + PhpLanguageLevels { + effective: ide_sdk::php::language_level(root, &settings), + auto: ide_sdk::php::composer_level(root), + member: member.map(ide_core::IpcPath), + } + }) + .await + .map_err(join_err) +} + /// SDK / library source folders shown under "External Libraries" in the Project view, /// with the Maven / Gradle dependency jars of the build model. pub async fn sdk_project_libraries(host: &Host, window: &str) -> IdeResult> { @@ -144,6 +180,10 @@ pub async fn sdk_python_pip( pub(crate) fn register(d: &mut Dispatcher) { d.add("sdk_detect", |c| async move { sdk_detect(&c.host, &c.window, c.arg("kind")?).await }); d.add("sdk_python_effective", |c| async move { sdk_python_effective(&c.host, &c.window) }); + d.add( + "sdk_php_language_level", + |c| async move { sdk_php_language_level(&c.host, &c.window, c.arg("path")?).await }, + ); d.add("sdk_project_libraries", |c| async move { sdk_project_libraries(&c.host, &c.window).await }); d.add("sdk_locate_tools", |c| async move { sdk_locate_tools(&c.host, &c.window, c.arg("names")?, c.arg("near")?).await diff --git a/crates/ide-host/src/ops/vcs/commands.rs b/crates/ide-host/src/ops/vcs/commands.rs index 1ed60e50..4075dd9d 100644 --- a/crates/ide-host/src/ops/vcs/commands.rs +++ b/crates/ide-host/src/ops/vcs/commands.rs @@ -12,6 +12,76 @@ use std::path::{Path, PathBuf}; use crate::apply_git_executable; +/// Read-only repository access also works in a standalone log window. +struct LogAccess { + project: Option>, + external: Option>, + cache: PathBuf, +} + +/// Immutable Git objects exposed through the editor's ordinary read-only VFS. +pub async fn revision_virtual_document( + host: &Host, + window: &str, + uri: &str, +) -> IdeResult { + #[derive(Deserialize)] + struct Target { + repo: PathBuf, + revision: String, + path: PathBuf, + } + let encoded = uri + .strip_prefix("git-revision:") + .and_then(|s| s.split('/').next()) + .ok_or_else(|| ide_core::IdeError::invalid("invalid Git revision URI"))?; + if encoded.len() > 32768 || encoded.len() % 2 != 0 || !encoded.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(ide_core::IdeError::invalid("invalid Git revision URI")); + } + let bytes = (0..encoded.len()) + .step_by(2) + .map(|i| { + u8::from_str_radix(&encoded[i..i + 2], 16) + .map_err(|_| ide_core::IdeError::invalid("invalid Git revision URI")) + }) + .collect::>>()?; + let target: Target = serde_json::from_slice(&bytes).map_err(|e| ide_core::IdeError::invalid(e.to_string()))?; + let path = target.repo.join(&target.path); + let text = vcs_file_at(host, window, path, Some(target.revision.clone())) + .await? + .ok_or_else(|| ide_core::IdeError::not_found("file is absent at this revision"))?; + Ok(ide_lsp::types::VirtualDocument { + uri: uri.into(), + language_id: host.languages.detect(&target.path, text.lines().next()), + text, + title: target.path.file_name().unwrap_or_default().to_string_lossy().into_owned(), + description: Some(format!("{} · {}", target.repo.display(), target.revision)), + decompiled: false, + }) +} +impl LogAccess { + fn new(host: &Host, window: &str, at: Option<&Path>) -> Self { + let project = host.project(window).ok(); + let external = host + .external_logs + .read() + .get(host.owner_window(window)) + .and_then(|rs| rs.iter().find(|r| Some(r.root()) == at).cloned()) + .filter(|repo| project.as_ref().is_none_or(|p| p.repos().at_root(repo.root()).is_none())); + Self { project, external, cache: host.local_data_dir.join("cache/git-log") } + } + fn with_log_repo_at(&self, at: Option<&Path>, f: impl FnOnce(&Repo) -> IdeResult) -> IdeResult { + if let Some(repo) = &self.external { + repo.enable_log_index(&self.cache); + return f(repo); + } + self.project + .as_ref() + .ok_or_else(|| ide_core::IdeError::unavailable("external Git log is not open"))? + .with_log_repo_at(at, f) + } +} + /// A repository of the project (IDEA's VCS root). #[derive(Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] @@ -182,8 +252,8 @@ pub fn vcs_roots(host: &Host, window: &str) -> IdeResult> { } pub async fn vcs_branch_info(host: &Host, window: &str, repo: Option) -> IdeResult { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.branch_info())).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.branch_info())).await } /// Gutter change markers for a file, computed against its current (unsaved) text. @@ -197,9 +267,21 @@ pub async fn vcs_line_hunks(host: &Host, window: &str, path: PathBuf) -> IdeResu } pub async fn vcs_file_at(host: &Host, window: &str, path: PathBuf, rev: Option) -> IdeResult> { - let project = host.project(window)?; + let project = host.project(window).ok(); + let external = host.external_logs.read().get(host.owner_window(window)).and_then(|rs| { + rs.iter().filter(|r| path.starts_with(r.root())).max_by_key(|r| r.root().components().count()).cloned() + }); blocking(move || { - project.with_repo_of(&path, |r| r.file_at(&path, rev.as_deref(), project.vcs_encoding(&path, rev.as_deref())?)) + if let Some(project) = project { + return project.with_log_repo_of(&path, |r| { + r.file_at(&path, rev.as_deref(), project.vcs_encoding(&path, rev.as_deref())?) + }); + } + external.ok_or_else(|| ide_core::IdeError::unavailable("external Git log is not open"))?.file_at( + &path, + rev.as_deref(), + encoding_rs::UTF_8, + ) }) .await } @@ -222,9 +304,148 @@ pub async fn vcs_blame( .await } -pub async fn vcs_log(host: &Host, window: &str, repo: Option, query: LogQuery) -> IdeResult> { +/// History for Selection: the commits that changed `lines` (0-based, inclusive) of the +/// editor's text of `path`. Cancellable. +pub async fn vcs_block_history( + host: &Host, + window: &str, + request: RequestId, + path: PathBuf, + text: Option, + lines: (u32, u32), +) -> IdeResult { let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.log(&query))).await + let cancel = host.begin_request(request); + let result = blocking(move || { + let text = text.or_else(|| project.documents.text_of(&path)); + project.with_repo_of(&path, |r| { + r.block_history(&path, text.as_deref(), lines, project.vcs_encoding(&path, None)?, &cancel) + }) + }) + .await; + host.end_request(request); + result +} + +pub async fn vcs_log(host: &Host, window: &str, repo: Option, query: LogQuery) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + let request = query.request; + let cancel = + request.map_or_else(ide_core::CancellationToken::new, |id| host.begin_project_read_request(window, id)); + let result = blocking(move || { + project.with_log_repo_at(repo.as_deref(), |r| { + r.enable_log_index(&project.cache); + r.log_cancellable(&query, &cancel) + }) + }) + .await; + if let Some(id) = request { + host.end_request(id); + } + result +} + +pub async fn vcs_log_refs( + host: &Host, + window: &str, + repo: Option, +) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.log_refs())).await +} + +pub async fn vcs_log_users(host: &Host, window: &str, repo: Option) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.log_users())).await +} + +pub async fn vcs_open_external_log(host: &Host, window: &str, path: PathBuf) -> IdeResult { + let path = dunce::canonicalize(path)?; + let repo = Repo::discover(&path)?.ok_or_else(|| ide_core::IdeError::unavailable("folder is not a Git worktree"))?; + let root = repo.root().to_path_buf(); + repo.set_execution_allowed(false); + repo.enable_log_index(&host.local_data_dir.join("cache/git-log")); + if let Ok(project) = host.project(window) { + project.open_external_log(&root)?; + } + let mut logs = host.external_logs.write(); + let roots = logs.entry(host.owner_window(window).to_string()).or_default(); + if !roots.iter().any(|r| r.root() == root) { + roots.push(std::sync::Arc::new(repo)); + } + Ok(root.into()) +} + +pub async fn vcs_log_details( + host: &Host, + window: &str, + repo: Option, + revision: String, +) -> IdeResult { + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.log_details(&revision))).await +} + +pub async fn vcs_log_details_batch( + host: &Host, + window: &str, + repo: Option, + revisions: Vec, + request: RequestId, +) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + let cancel = host.begin_project_read_request(window, request); + let result = blocking(move || { + project.with_log_repo_at(repo.as_deref(), |r| { + revisions.iter().map(|id| r.log_details_cancellable(id, &cancel)).collect() + }) + }) + .await; + host.end_request(request); + result +} + +pub async fn vcs_log_index( + host: &Host, + window: &str, + repo: Option, + request: Option, +) -> IdeResult { + let project = LogAccess::new(host, window, repo.as_deref()); + let cancel = + request.map_or_else(ide_core::CancellationToken::new, |id| host.begin_project_read_request(window, id)); + let result = blocking(move || { + project.with_log_repo_at(repo.as_deref(), |r| { + r.enable_log_index(&project.cache); + if request.is_some() { r.index_log(&cancel) } else { Ok(r.log_index_progress()) } + }) + }) + .await; + if let Some(request) = request { + host.end_request(request); + } + result +} + +pub async fn vcs_revision_children( + host: &Host, + window: &str, + repo: Option, + revision: String, + directory: String, +) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.revision_children(&revision, &directory))).await +} + +pub async fn vcs_revision_files( + host: &Host, + window: &str, + repo: Option, + revision: String, +) -> IdeResult> { + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.revision_files(&revision))).await } pub async fn vcs_commit_files( @@ -233,8 +454,8 @@ pub async fn vcs_commit_files( repo: Option, commit: String, ) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.commit_files(&commit))).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.commit_files(&commit))).await } /// Combined changes of several commits, `commits` oldest first. @@ -244,13 +465,13 @@ pub async fn vcs_commits_changes( repo: Option, commits: Vec, ) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.commits_changes(&commits))).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.commits_changes(&commits))).await } pub async fn vcs_branches(host: &Host, window: &str, repo: Option) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.branches())).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.branches())).await } pub async fn vcs_stashes(host: &Host, window: &str, repo: Option) -> IdeResult> { @@ -605,8 +826,8 @@ pub async fn vcs_config_get( repo: Option, key: String, ) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.config_get(&key))).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.config_get(&key))).await } /// Local history operation that may take a while (hooks, large checkouts); git's @@ -667,8 +888,8 @@ pub async fn vcs_create_tag( } pub async fn vcs_tags(host: &Host, window: &str, repo: Option) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.tags())).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.tags())).await } /// Merge dialog: merges `branches` into the current branch. @@ -704,8 +925,8 @@ pub async fn vcs_rebase_with( } pub async fn vcs_remotes(host: &Host, window: &str, repo: Option) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.remotes())).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.remotes())).await } pub async fn vcs_edit_remote(host: &Host, window: &str, repo: Option, edit: RemoteEdit) -> IdeResult<()> { @@ -792,8 +1013,8 @@ pub async fn vcs_diff_with_worktree( repo: Option, rev: String, ) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.diff_with_worktree(&rev))).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.diff_with_worktree(&rev))).await } /// Files that differ between two revisions (the "N files updated" of Update Project). @@ -804,8 +1025,8 @@ pub async fn vcs_diff_revisions( from: String, to: String, ) -> IdeResult> { - let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.diff_revisions(&from, &to))).await + let project = LogAccess::new(host, window, repo.as_deref()); + blocking(move || project.with_log_repo_at(repo.as_deref(), |r| r.diff_revisions(&from, &to))).await } /// The best common ancestor of `a` and `b` (`git merge-base`), if any: the New Pull @@ -987,7 +1208,30 @@ pub(crate) fn register(d: &mut Dispatcher) { d.add("vcs_blame", |c| async move { vcs_blame(&c.host, &c.window, c.arg("path")?, c.arg("text")?, c.arg("revision")?).await }); + d.add("vcs_block_history", |c| async move { + let (request, path, text, lines) = (c.arg("request")?, c.arg("path")?, c.arg("text")?, c.arg("lines")?); + vcs_block_history(&c.host, &c.window, request, path, text, lines).await + }); d.add("vcs_log", |c| async move { vcs_log(&c.host, &c.window, c.arg("repo")?, c.arg("query")?).await }); + d.add("vcs_log_refs", |c| async move { vcs_log_refs(&c.host, &c.window, c.arg("repo")?).await }); + d.add("vcs_log_users", |c| async move { vcs_log_users(&c.host, &c.window, c.arg("repo")?).await }); + d.add("vcs_open_external_log", |c| async move { vcs_open_external_log(&c.host, &c.window, c.arg("path")?).await }); + d.add("vcs_log_details", |c| async move { + vcs_log_details(&c.host, &c.window, c.arg("repo")?, c.arg("revision")?).await + }); + d.add("vcs_log_details_batch", |c| async move { + vcs_log_details_batch(&c.host, &c.window, c.arg("repo")?, c.arg("revisions")?, c.arg("request")?).await + }); + d.add( + "vcs_log_index", + |c| async move { vcs_log_index(&c.host, &c.window, c.arg("repo")?, c.arg("request")?).await }, + ); + d.add("vcs_revision_children", |c| async move { + vcs_revision_children(&c.host, &c.window, c.arg("repo")?, c.arg("revision")?, c.arg("directory")?).await + }); + d.add("vcs_revision_files", |c| async move { + vcs_revision_files(&c.host, &c.window, c.arg("repo")?, c.arg("revision")?).await + }); d.add("vcs_commit_files", |c| async move { vcs_commit_files(&c.host, &c.window, c.arg("repo")?, c.arg("commit")?).await }); diff --git a/crates/ide-host/src/plugins/api.rs b/crates/ide-host/src/plugins/api.rs index d1f8f1a1..1b1c997e 100644 --- a/crates/ide-host/src/plugins/api.rs +++ b/crates/ide-host/src/plugins/api.rs @@ -667,6 +667,17 @@ impl HostApi for PluginApi { Ok(()) } + fn open_location(&self, client: Option<&str>, path: &Path, line: u32, column: u32) -> Result<(), HostError> { + let event = PluginEvent::OpenLocation { + plugin_id: self.id().to_owned(), + path: path.to_path_buf().into(), + line, + column, + }; + self.shared.sink.emit(client, event); + Ok(()) + } + fn setting(&self, root: Option<&Path>, key: &str) -> Option { let schema: Vec = self .shared diff --git a/crates/ide-host/src/plugins/icon_extensions.rs b/crates/ide-host/src/plugins/icon_extensions.rs new file mode 100644 index 00000000..fc970b6b --- /dev/null +++ b/crates/ide-host/src/plugins/icon_extensions.rs @@ -0,0 +1,207 @@ +use super::icon_patterns::validate_pattern; +use ide_extension::validate_item_id; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +const MAX_GROUPS: usize = 64; +const MAX_CHOICES: usize = 16; +const MAX_CHOICE_OPTIONS: usize = 16; +const MAX_NAME_LEN: usize = 255; +pub(super) const DARK_SUFFIX: &str = "_dark"; + +#[derive(Deserialize)] +#[serde(untagged)] +pub(super) enum IconDefinition { + Path(PathBuf), + Variants(IconVariants), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct IconVariants { + light: PathBuf, + #[serde(default)] + dark: Option, +} + +impl IconDefinition { + pub(super) fn light(&self) -> &PathBuf { + match self { + IconDefinition::Path(p) => p, + IconDefinition::Variants(v) => &v.light, + } + } + + pub(super) fn dark(&self) -> Option<&PathBuf> { + match self { + IconDefinition::Path(_) => None, + IconDefinition::Variants(v) => v.dark.as_ref(), + } + } +} + +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum Target { + #[default] + File, + Folder, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Pattern { + pattern: String, + icon: String, + #[serde(default)] + target: Target, + #[serde(default)] + path: bool, + #[serde(default)] + ignore_case: bool, + #[serde(default)] + group: Option, + #[serde(default)] + open_icon: Option, +} + +fn enabled_by_default() -> bool { + true +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Group { + pub(super) id: String, + name: String, + #[serde(default = "enabled_by_default")] + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ChoiceOption { + pub(super) id: String, + pub(super) name: String, + #[serde(default)] + overrides: BTreeMap, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Choice { + pub(super) id: String, + pub(super) name: String, + pub(super) default: String, + pub(super) options: Vec, +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct Extensions { + #[serde(default)] + pub(super) patterns: Vec, + #[serde(default)] + pub(super) overrides: BTreeMap, + #[serde(default)] + pub(super) groups: Vec, + #[serde(default)] + pub(super) choices: Vec, + #[serde(default)] + pub(super) accent: bool, +} + +pub(super) fn check_icon_id(id: &str) -> Result<(), String> { + validate_item_id(id, "icon id")?; + if id.ends_with(DARK_SUFFIX) { + return Err(format!("icon id {id:?}: the {DARK_SUFFIX:?} suffix is reserved for dark variants")); + } + Ok(()) +} + +fn check_named(id: &str, name: &str, what: &str) -> Result<(), String> { + validate_item_id(id, what)?; + if name.trim().is_empty() || name.len() > MAX_NAME_LEN { + return Err(format!("{what} {id:?}: name must have 1 to {MAX_NAME_LEN} characters")); + } + Ok(()) +} + +fn check_unique<'a>(ids: impl Iterator, what: &str) -> Result<(), String> { + let mut seen = BTreeSet::new(); + match ids.into_iter().find(|id| !seen.insert(*id)) { + Some(id) => Err(format!("duplicate {what} {id:?}")), + None => Ok(()), + } +} + +impl Extensions { + pub(super) fn mappings(&self) -> usize { + let chosen: usize = self.choices.iter().flat_map(|c| &c.options).map(|o| o.overrides.len()).sum(); + self.patterns.len() + self.overrides.len() + chosen + } + + pub(super) fn validate(&self, check_ref: &dyn Fn(&str, &str) -> Result<(), String>) -> Result<(), String> { + self.validate_groups()?; + for p in &self.patterns { + validate_pattern(&p.pattern)?; + check_ref(&p.icon, "patterns")?; + if let Some(group) = &p.group + && !self.groups.iter().any(|g| &g.id == group) + { + return Err(format!("pattern {:?}: unknown group {group:?}", p.pattern)); + } + if let Some(open) = &p.open_icon { + if p.target != Target::Folder { + return Err(format!("pattern {:?}: openIcon is only for folder patterns", p.pattern)); + } + check_ref(open, "patterns")?; + } + } + check_overrides(&self.overrides, check_ref, "overrides")?; + for choice in &self.choices { + for option in &choice.options { + check_overrides(&option.overrides, check_ref, "choices")?; + } + } + Ok(()) + } + + fn validate_groups(&self) -> Result<(), String> { + if self.groups.len() > MAX_GROUPS || self.choices.len() > MAX_CHOICES { + return Err(format!("too many groups or choices (max {MAX_GROUPS} and {MAX_CHOICES})")); + } + for g in &self.groups { + check_named(&g.id, &g.name, "group")?; + } + check_unique(self.groups.iter().map(|g| &g.id), "group")?; + check_unique(self.choices.iter().map(|c| &c.id), "choice")?; + for choice in &self.choices { + check_named(&choice.id, &choice.name, "choice")?; + if choice.options.is_empty() || choice.options.len() > MAX_CHOICE_OPTIONS { + return Err(format!("choice {:?}: expected 1 to {MAX_CHOICE_OPTIONS} options", choice.id)); + } + for option in &choice.options { + check_named(&option.id, &option.name, "choice option")?; + } + check_unique(choice.options.iter().map(|o| &o.id), "choice option")?; + if !choice.options.iter().any(|o| o.id == choice.default) { + return Err(format!("choice {:?}: default {:?} is not one of its options", choice.id, choice.default)); + } + } + Ok(()) + } +} + +fn check_overrides( + overrides: &BTreeMap, + check_ref: &dyn Fn(&str, &str) -> Result<(), String>, + what: &str, +) -> Result<(), String> { + for (name, icon) in overrides { + validate_item_id(name, "overridden icon name")?; + check_ref(icon, what)?; + } + Ok(()) +} diff --git a/crates/ide-host/src/plugins/icon_patterns.rs b/crates/ide-host/src/plugins/icon_patterns.rs new file mode 100644 index 00000000..ef631e99 --- /dev/null +++ b/crates/ide-host/src/plugins/icon_patterns.rs @@ -0,0 +1,105 @@ +use regex_syntax::hir::{Hir, HirKind}; + +pub(crate) const MAX_PATTERN_LEN: usize = 512; +const NAME_LEN: u64 = 256; +const MAX_COST: u64 = 4 * NAME_LEN * NAME_LEN; + +const UNSUPPORTED: &[(&str, &str)] = &[ + ("\\A", "\\A"), + ("\\z", "\\z"), + ("\\Z", "\\Z"), + ("\\p", "\\p"), + ("\\P", "\\P"), + ("[[:", "a POSIX class"), + ("&&", "class intersection"), + ("~~", "class symmetric difference"), +]; + +pub(crate) fn validate_pattern(pattern: &str) -> Result<(), String> { + if pattern.len() > MAX_PATTERN_LEN { + return Err(format!("pattern {pattern:?} is too long (max {MAX_PATTERN_LEN})")); + } + if let Some(construct) = unsupported_construct(pattern) { + return Err(format!("pattern {pattern:?}: {construct} is not supported")); + } + let hir = regex_syntax::Parser::new().parse(pattern).map_err(|e| format!("invalid pattern {pattern:?}: {e}"))?; + match cost(&hir) { + Err(why) => Err(format!("pattern {pattern:?}: {why} is not supported")), + Ok(c) if c > MAX_COST => Err(format!("pattern {pattern:?} is too complex: too many ways to match one name")), + Ok(_) => Ok(()), + } +} + +fn unsupported_construct(pattern: &str) -> Option<&'static str> { + if pattern.match_indices("(?").any(|(at, _)| !pattern[at + 2..].starts_with(':')) { + return Some("an inline flag, lookaround or named group"); + } + UNSUPPORTED.iter().find(|(needle, _)| pattern.contains(needle)).map(|(_, what)| *what) +} + +fn cost(hir: &Hir) -> Result { + match hir.kind() { + HirKind::Empty | HirKind::Literal(_) | HirKind::Class(_) | HirKind::Look(_) => Ok(1), + HirKind::Capture(c) => cost(&c.sub), + HirKind::Concat(items) => items.iter().try_fold(1u64, |acc, h| Ok(acc.saturating_mul(cost(h)?))), + HirKind::Alternation(items) => items.iter().try_fold(0u64, |acc, h| Ok(acc.saturating_add(cost(h)?))), + HirKind::Repetition(r) => match r.max { + None if matches!(r.sub.kind(), HirKind::Literal(_) | HirKind::Class(_)) => Ok(NAME_LEN), + None => Err("a repeated group (`(…)*`, `(…)+`)"), + Some(max) => Ok(cost(&r.sub)?.saturating_add(1).saturating_pow(max)), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn err(pattern: &str) -> String { + validate_pattern(pattern).unwrap_err() + } + + #[test] + fn accepts_typical_file_name_patterns() { + for pattern in [ + r"^(architecture|ARCHITECTURE)(\.(md|rst|txt))?$", + r"(main|workflow|ci|release|build|config)\.ya?ml$", + r"^[a-z]+\.d\.ts$", + r"^\.?eslintrc(\.(js|cjs|json|ya?ml))?$", + r"^.*\.config\.[cm]?[jt]s$", + r"^.*-(test|spec)\..*$", + r"^[0-9]{4}-[0-9]{2}\.log$", + ] { + assert!(validate_pattern(pattern).is_ok(), "{pattern}: {:?}", validate_pattern(pattern)); + } + } + + #[test] + fn rejects_patterns_that_backtrack_catastrophically() { + assert!(err(r"(a+)+$").contains("repeated group")); + assert!(err(r"(?:a*)*").contains("repeated group")); + assert!(err(r"^(\w|\w\w)*!$").contains("repeated group")); + assert!(err(r"^(a|aa)+$").contains("repeated group")); + assert!(err(r"^.*.*.*x$").contains("too complex")); + assert!(err(r"^(a?){30}a{30}$").contains("too complex")); + assert!(err(&format!("^{}b$", "(a|aa)".repeat(20))).contains("too complex")); + assert!(err(r"x{0,40}").contains("too complex")); + } + + #[test] + fn rejects_constructs_javascript_reads_differently() { + assert!(err(r"(?i)readme").contains("not supported")); + assert!(err(r"(?Pa)").contains("not supported")); + assert!(err(r"\Areadme").contains("not supported")); + assert!(err(r"readme\z").contains("not supported")); + assert!(err(r"\p{L}").contains("not supported")); + assert!(err(r"[[:alpha:]]").contains("not supported")); + assert!(err(r"[a-z&&[^x]]").contains("not supported")); + } + + #[test] + fn rejects_invalid_and_oversized_patterns() { + assert!(err("(").contains("invalid pattern")); + assert!(err(&"a".repeat(MAX_PATTERN_LEN + 1)).contains("too long")); + } +} diff --git a/crates/ide-host/src/plugins/icon_settings.rs b/crates/ide-host/src/plugins/icon_settings.rs new file mode 100644 index 00000000..b1cbbf45 --- /dev/null +++ b/crates/ide-host/src/plugins/icon_settings.rs @@ -0,0 +1,185 @@ +use super::icon_extensions::Extensions; +use serde::{Deserialize, Serialize}; + +const MAX_SECTIONS: usize = 16; +const MAX_ITEMS: usize = 32; +const MAX_TEXT_LEN: usize = 255; +const MAX_HINT_LEN: usize = 512; +const MAX_COLUMNS: u8 = 4; + +#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum SettingOption { + HideFiles, + HideFolders, + HideUiIcons, + HidePsiIcons, + HollowFolders, + Groups, + Choice, + Accent, + Monochrome, + Saturation, + Opacity, + IconSize, + RowHeight, +} + +#[derive(Deserialize, Serialize)] +#[serde(untagged)] +pub(super) enum SettingDefault { + Number(u8), + Color(String), +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct SettingItem { + option: SettingOption, + #[serde(default)] + label: Option, + #[serde(default)] + hint: Option, + #[serde(default)] + invert: bool, + #[serde(default)] + choice: Option, + #[serde(default)] + columns: Option, + #[serde(default)] + default: Option, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct SettingsSection { + title: String, + items: Vec, +} + +#[derive(Serialize)] +struct ChoiceSummary<'a> { + id: &'a str, + name: &'a str, + default: &'a str, + options: Vec>, +} + +#[derive(Serialize)] +struct OptionSummary<'a> { + id: &'a str, + name: &'a str, +} + +fn check_text(text: &str, max: usize, what: &str) -> Result<(), String> { + if text.trim().is_empty() || text.len() > max { + return Err(format!("settings: {what} must have 1 to {max} characters")); + } + Ok(()) +} + +fn range(option: SettingOption) -> Option<(u8, u8)> { + match option { + SettingOption::Saturation => Some((0, 20)), + SettingOption::Opacity => Some((10, 100)), + SettingOption::IconSize => Some((12, 32)), + SettingOption::RowHeight => Some((16, 48)), + _ => None, + } +} + +fn is_color(text: &str) -> bool { + text.len() == 7 && text.starts_with('#') && text[1..].chars().all(|c| c.is_ascii_hexdigit()) +} + +fn is_flag(option: SettingOption) -> bool { + matches!( + option, + SettingOption::HideFiles + | SettingOption::HideFolders + | SettingOption::HideUiIcons + | SettingOption::HidePsiIcons + | SettingOption::HollowFolders + ) +} + +impl SettingItem { + fn validate(&self, ext: &Extensions) -> Result<(), String> { + let named = !matches!(self.option, SettingOption::Groups | SettingOption::Choice); + match &self.label { + Some(label) => check_text(label, MAX_TEXT_LEN, "label")?, + None if named => return Err("settings: label is required".into()), + None => {} + } + if let Some(hint) = &self.hint { + check_text(hint, MAX_HINT_LEN, "hint")?; + } + if self.invert && !is_flag(self.option) { + return Err("settings: invert is only for on/off options".into()); + } + if self.columns.is_some_and(|c| c == 0 || c > MAX_COLUMNS) + || (self.columns.is_some() && self.option != SettingOption::Groups) + { + return Err(format!("settings: columns are 1 to {MAX_COLUMNS}, for groups only")); + } + match (self.option, &self.choice) { + (SettingOption::Choice, Some(id)) if !ext.choices.iter().any(|c| &c.id == id) => { + return Err(format!("settings: unknown choice {id:?}")); + } + (SettingOption::Choice, None) => return Err("settings: choice items name their choice".into()), + (SettingOption::Choice, Some(_)) | (_, None) => {} + (_, Some(_)) => return Err("settings: only choice items name a choice".into()), + } + if self.option == SettingOption::Accent && !ext.accent { + return Err("settings: the accent option needs \"accent\": true".into()); + } + self.validate_default() + } + + fn validate_default(&self) -> Result<(), String> { + let colored = matches!(self.option, SettingOption::Accent | SettingOption::Monochrome); + match (range(self.option), &self.default) { + (Some((min, max)), Some(SettingDefault::Number(n))) if (min..=max).contains(n) => Ok(()), + (Some((min, max)), _) => Err(format!("settings: a default from {min} to {max} is required")), + (None, Some(SettingDefault::Color(c))) if colored && is_color(c) => Ok(()), + (None, _) if colored => Err("settings: a #rrggbb default is required".into()), + (None, None) => Ok(()), + (None, Some(_)) => Err("settings: this option takes no default".into()), + } + } +} + +pub(super) fn validate(sections: &[SettingsSection], ext: &Extensions) -> Result<(), String> { + if sections.len() > MAX_SECTIONS { + return Err(format!("settings: more than {MAX_SECTIONS} sections")); + } + for section in sections { + check_text(§ion.title, MAX_TEXT_LEN, "title")?; + if section.items.len() > MAX_ITEMS { + return Err(format!("settings: more than {MAX_ITEMS} items in {:?}", section.title)); + } + for item in §ion.items { + item.validate(ext)?; + } + } + Ok(()) +} + +pub(super) fn summary(sections: &[SettingsSection], ext: &Extensions) -> serde_json::Value { + let choices: Vec = ext + .choices + .iter() + .map(|c| ChoiceSummary { + id: &c.id, + name: &c.name, + default: &c.default, + options: c.options.iter().map(|o| OptionSummary { id: &o.id, name: &o.name }).collect(), + }) + .collect(); + serde_json::json!({ + "groups": ext.groups, + "choices": choices, + "accent": ext.accent, + "settings": sections, + }) +} diff --git a/crates/ide-host/src/plugins/icons.rs b/crates/ide-host/src/plugins/icons.rs index 729bf626..60b1ac2e 100644 --- a/crates/ide-host/src/plugins/icons.rs +++ b/crates/ide-host/src/plugins/icons.rs @@ -24,6 +24,8 @@ //! asset; the client fetches the mapping and the icons only when the theme is //! selected, so installed icon themes cost nothing at start-up. +use super::icon_extensions::{Choice, DARK_SUFFIX, Extensions, Group, IconDefinition, Pattern, check_icon_id}; +use super::icon_settings::{self, SettingsSection}; use ide_extension::{Package, UiDecl, check_relative, decode, validate_item_id}; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; @@ -45,7 +47,7 @@ struct Decl { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct ThemeFile { - icon_definitions: BTreeMap, + icon_definitions: BTreeMap, #[serde(default)] file: Option, #[serde(default)] @@ -53,11 +55,25 @@ struct ThemeFile { #[serde(default)] folder_expanded: Option, #[serde(default)] + folder_open: Option, + #[serde(default)] file_extensions: BTreeMap, #[serde(default)] file_names: BTreeMap, #[serde(default)] folder_names: BTreeMap, + #[serde(default)] + patterns: Vec, + #[serde(default)] + overrides: BTreeMap, + #[serde(default)] + groups: Vec, + #[serde(default)] + choices: Vec, + #[serde(default)] + accent: bool, + #[serde(default)] + settings: Vec, } /// Asset id of a theme's mapping (JSON). @@ -65,6 +81,10 @@ pub fn map_asset(theme: &str) -> String { format!("iconTheme/{theme}/map") } +pub fn settings_asset(theme: &str) -> String { + format!("iconTheme/{theme}/settings") +} + /// Asset id prefix of a theme's SVGs (`…/svg/`). pub fn svg_prefix(theme: &str) -> String { format!("iconTheme/{theme}/svg/") @@ -87,12 +107,20 @@ pub(super) fn icon_theme(pkg: &Package, decl: toml::Value) -> Result MAX_ICONS { return Err(format!("{}: more than {MAX_ICONS} icons", d.path.display())); } - let mappings = theme.file_extensions.len() + theme.file_names.len() + theme.folder_names.len(); + let mappings = + theme.file_extensions.len() + theme.file_names.len() + theme.folder_names.len() + extensions.mappings(); if mappings > MAX_MAPPINGS { return Err(format!("{}: more than {MAX_MAPPINGS} mappings", d.path.display())); } @@ -100,7 +128,17 @@ pub(super) fn icon_theme(pkg: &Package, decl: toml::Value) -> Result Result<(), String> { if defined.contains(icon) { Ok(()) } else { Err(format!("{}: unknown icon {icon:?}", at(what))) } }; - for (what, icon) in [("file", &theme.file), ("folder", &theme.folder), ("folderExpanded", &theme.folder_expanded)] { + for icon in theme.icon_definitions.keys() { + check_icon_id(icon)?; + } + extensions.validate(&check_ref)?; + icon_settings::validate(&theme.settings, &extensions).map_err(|e| format!("{}: {e}", d.path.display()))?; + for (what, icon) in [ + ("file", &theme.file), + ("folder", &theme.folder), + ("folderExpanded", &theme.folder_expanded), + ("folderOpen", &theme.folder_open), + ] { if let Some(icon) = icon { check_ref(icon, what)?; } @@ -125,20 +163,31 @@ pub(super) fn icon_theme(pkg: &Package, decl: toml::Value) -> Result"#; + + fn theme_package(theme: &str) -> (tempfile::TempDir, Package) { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + std::fs::create_dir_all(dir.join("icons")).unwrap(); + for icon in ["file", "folder", "folder-src", "folder-src-open", "rust", "rust-dark", "tests", "arrow"] { + std::fs::write(dir.join(format!("icons/{icon}.svg")), SVG).unwrap(); + } + std::fs::write(dir.join("theme.json"), theme).unwrap(); + let manifest = ide_extension::Manifest::parse( + "id = \"p\"\nname = \"P\"\nversion = \"1.0.0\"\nengines.openidea = \"^1.0\"", + ) + .unwrap(); + let pkg = Package::new(manifest, dir.to_path_buf()); + (tmp, pkg) + } + + fn load(theme: &str) -> Result { + let (_tmp, pkg) = theme_package(theme); + let mut t = toml::Table::new(); + t.insert("id".into(), "m".into()); + t.insert("label".into(), "M".into()); + t.insert("path".into(), "theme.json".into()); + icon_theme(&pkg, toml::Value::Table(t)) + } + + fn with(extra: &str) -> String { + format!( + r#"{{"iconDefinitions":{{"file":"icons/file.svg","folder":"icons/folder.svg", + "folder-src":"icons/folder-src.svg","folder-src-open":"icons/folder-src-open.svg", + "rust":{{"light":"icons/rust.svg","dark":"icons/rust-dark.svg"}}, + "tests":"icons/tests.svg","arrow":"icons/arrow.svg"}}, + "file":"file","folder":"folder"{extra}}}"# + ) + } + + const EXTENSIONS: &str = r#","patterns":[ + {"pattern":"^.*\\.rs$","icon":"rust","ignoreCase":true}, + {"pattern":"/tests?/","icon":"tests","path":true,"group":"tests"}, + {"pattern":"^src$","icon":"folder-src","target":"folder","openIcon":"folder-src-open"}], + "overrides":{"chevron-right":"arrow","symbol-class":"rust"}, + "groups":[{"id":"tests","name":"Tests","enabled":false}], + "choices":[{"id":"arrows","name":"Arrows","default":"plain","options":[ + {"id":"plain","name":"Default"},{"id":"material","name":"Material","overrides":{"chevron-down":"arrow"}}]}], + "accent":true,"folderOpen":"folder-src-open""#; + + fn map_of(d: &UiDecl) -> serde_json::Value { + serde_json::from_str(&d.assets.iter().find(|(k, _)| *k == map_asset("m")).unwrap().1).unwrap() + } + + #[test] + fn theme_extensions_reach_the_mapping() { + let d = load(&with(EXTENSIONS)).unwrap(); + let map = map_of(&d); + assert_eq!( + map["patterns"], + serde_json::json!([ + {"pattern":"^.*\\.rs$","icon":"rust","target":"file","path":false,"ignoreCase":true,"group":null,"openIcon":null}, + {"pattern":"/tests?/","icon":"tests","target":"file","path":true,"ignoreCase":false,"group":"tests","openIcon":null}, + {"pattern":"^src$","icon":"folder-src","target":"folder","path":false,"ignoreCase":false,"group":null,"openIcon":"folder-src-open"} + ]) + ); + assert_eq!(map["overrides"], serde_json::json!({"chevron-right":"arrow","symbol-class":"rust"})); + assert_eq!(map["groups"], serde_json::json!([{"id":"tests","name":"Tests","enabled":false}])); + assert_eq!(map["choices"][0]["default"], "plain"); + assert_eq!(map["choices"][0]["options"][1]["overrides"], serde_json::json!({"chevron-down":"arrow"})); + assert_eq!(map["choices"][0]["options"][0]["overrides"], serde_json::json!({})); + assert_eq!(map["accent"], true); + assert_eq!(map["folderOpen"], "folder-src-open"); + assert_eq!(d.json["icons"], 7); + assert!(d.assets.iter().any(|(k, _)| *k == format!("{}rust", svg_prefix("m")))); + assert!(d.assets.iter().any(|(k, _)| *k == format!("{}rust_dark", svg_prefix("m")))); + assert!(d.files.iter().any(|f| f.ends_with("icons/rust-dark.svg"))); + } + + #[test] + fn themes_without_extensions_keep_their_mapping() { + let map = map_of(&load(&with("")).unwrap()); + assert_eq!(map["patterns"], serde_json::json!([])); + assert_eq!(map["overrides"], serde_json::json!({})); + assert_eq!(map["groups"], serde_json::json!([])); + assert_eq!(map["choices"], serde_json::json!([])); + assert_eq!(map["accent"], false); + } + + #[test] + fn theme_extensions_are_validated() { + let cases = [ + (r#","patterns":[{"pattern":"x","icon":"nope"}]"#, "unknown icon"), + (r#","patterns":[{"pattern":"(a+)+$","icon":"rust"}]"#, "repeated group"), + (r#","patterns":[{"pattern":"(","icon":"rust"}]"#, "invalid pattern"), + (r#","patterns":[{"pattern":"x","icon":"rust","group":"nope"}]"#, "unknown group"), + (r#","patterns":[{"pattern":"x","icon":"rust","openIcon":"tests"}]"#, "openIcon"), + (r#","patterns":[{"pattern":"x","icon":"folder","target":"folder","openIcon":"nope"}]"#, "unknown icon"), + (r#","patterns":[{"pattern":"x","icon":"rust","priority":1}]"#, "unknown field"), + (r#","overrides":{"chevron-right":"nope"}"#, "unknown icon"), + (r#","folderOpen":"nope""#, "unknown icon"), + (r#","overrides":{"Chevron":"rust"}"#, "invalid overridden icon"), + (r#","groups":[{"id":"a","name":" "}]"#, "name"), + (r#","groups":[{"id":"a","name":"A"},{"id":"a","name":"B"}]"#, "duplicate"), + (r#","choices":[{"id":"c","name":"C","default":"x","options":[{"id":"y","name":"Y"}]}]"#, "default"), + (r#","choices":[{"id":"c","name":"C","default":"y","options":[]}]"#, "options"), + ( + r#","choices":[{"id":"c","name":"C","default":"y","options":[{"id":"y","name":"Y","overrides":{"a":"nope"}}]}]"#, + "unknown icon", + ), + ]; + for (extra, needle) in cases { + let err = load(&with(extra)).err().unwrap_or_else(|| panic!("accepted: {extra}")); + assert!(err.contains(needle), "{extra}: {err}"); + } + let dark_id = r#"{"iconDefinitions":{"x_dark":"icons/rust.svg"}}"#; + assert!(load(dark_id).unwrap_err().contains("_dark")); + } + + const SETTINGS: &str = r##","settings":[ + {"title":"Toggle","items":[ + {"option":"hideFiles","label":"Enable file icons","hint":"Files","invert":true}, + {"option":"hollowFolders","label":"Hollow"}]}, + {"title":"Packs","items":[{"option":"groups","columns":2}]}, + {"title":"Other","items":[ + {"option":"choice","choice":"arrows"}, + {"option":"accent","label":"Accent","default":"#009688"}, + {"option":"saturation","label":"Saturation","default":10}]}]"##; + + fn settings_of(d: &UiDecl) -> Option { + d.assets.iter().find(|(k, _)| *k == settings_asset("m")).map(|(_, v)| serde_json::from_str(v).unwrap()) + } + + #[test] + fn theme_settings_get_their_own_asset() { + let d = load(&with(&format!("{EXTENSIONS}{SETTINGS}"))).unwrap(); + let s = settings_of(&d).unwrap(); + assert_eq!(s["accent"], true); + assert_eq!(s["groups"], serde_json::json!([{"id":"tests","name":"Tests","enabled":false}])); + assert_eq!( + s["choices"], + serde_json::json!([{"id":"arrows","name":"Arrows","default":"plain","options":[ + {"id":"plain","name":"Default"},{"id":"material","name":"Material"}]}]) + ); + assert_eq!(s["settings"][0]["title"], "Toggle"); + assert_eq!( + s["settings"][0]["items"][0], + serde_json::json!({"option":"hideFiles","label":"Enable file icons","hint":"Files","invert":true, + "choice":null,"columns":null,"default":null}) + ); + assert_eq!(s["settings"][2]["items"][1]["default"], "#009688"); + assert_eq!(s["settings"][2]["items"][2]["default"], 10); + assert!(settings_of(&load(&with(EXTENSIONS)).unwrap()).is_none()); + } + + #[test] + fn theme_settings_are_validated() { + let section = |items: &str| with(&format!(r#"{EXTENSIONS},"settings":[{{"title":"T","items":[{items}]}}]"#)); + let cases = [ + (section(r#"{"option":"nope","label":"X"}"#), "unknown variant"), + (section(r#"{"option":"hideFiles"}"#), "label"), + (section(r#"{"option":"hideFiles","label":" "}"#), "label"), + (section(r#"{"option":"choice","choice":"nope"}"#), "unknown choice"), + (section(r#"{"option":"choice"}"#), "choice"), + (section(r#"{"option":"groups","columns":9}"#), "columns"), + (section(r#"{"option":"saturation","label":"S"}"#), "default"), + (section(r#"{"option":"saturation","label":"S","default":50}"#), "default"), + (section(r##"{"option":"opacity","label":"O","default":"#000000"}"##), "default"), + (section(r#"{"option":"monochrome","label":"M","default":"red"}"#), "default"), + (section(r#"{"option":"saturation","label":"S","default":5,"invert":true}"#), "invert"), + (section(r#"{"option":"hideFiles","label":"F","extra":1}"#), "unknown field"), + (with(&format!(r#"{EXTENSIONS},"settings":[{{"title":"","items":[]}}]"#)), "title"), + ( + with(r##","settings":[{"title":"T","items":[{"option":"accent","label":"A","default":"#000000"}]}]"##), + "accent", + ), + ]; + for (theme, needle) in cases { + let err = load(&theme).err().unwrap_or_else(|| panic!("accepted: {theme}")); + assert!(err.contains(needle), "{theme}: {err}"); + } + } } diff --git a/crates/ide-host/src/plugins/intellij.rs b/crates/ide-host/src/plugins/intellij.rs new file mode 100644 index 00000000..3ccc62fa --- /dev/null +++ b/crates/ide-host/src/plugins/intellij.rs @@ -0,0 +1,874 @@ +//! IntelliJ theme plugins, read for import as OpenIDEA theme plugins: a +//! Marketplace download (a `.jar`, or a `.zip` holding the plugin's +//! `lib/*.jar`), an installed plugin's folder or a loose `.theme.json`; and the +//! theme plugins of the JetBrains IDEs on this machine, with the theme each IDE +//! is set to (`options/laf.xml`). +//! +//! Nothing is converted here: the UI maps IntelliJ keys onto design tokens +//! (`src/features/plugins/intellijTheme.ts`, shared with `pnpm +//! theme:import-intellij`) and [`super::PluginHost::stage_package`] writes the +//! result for the install page, which validates it like any package. + +use ide_core::{IdeError, IdeResult, IpcPath}; +use quick_xml::{Reader, XmlVersion, events::Event}; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::ffi::OsString; +use std::io::{Cursor, Read, Seek}; +use std::path::{Path, PathBuf}; + +/// A descriptor, theme or editor scheme larger than this is not one. +const MAX_FILE: u64 = 4 << 20; +/// A plugin jar read into memory from a Marketplace zip. +const MAX_JAR: u64 = 64 << 20; +/// Jars of one plugin looked into. +const MAX_JARS: usize = 64; +/// All the jars read into memory from one Marketplace zip. +const MAX_NESTED: u64 = 256 << 20; +/// Descriptors read for one plugin: `plugin.xml` and those it includes. +const MAX_DESCRIPTORS: usize = 64; +const PLUGIN_XML: &str = "META-INF/plugin.xml"; + +/// A theme plugin's themes, as read from its files. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct IntellijThemeSource { + /// The plugin's ``, or the theme's for a loose file. + pub name: String, + pub plugin_id: Option, + pub version: Option, + pub vendor: Option, + pub themes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct IntellijThemeFile { + /// The `themeProvider` id (what `laf.xml` names), or the file name without `.theme.json`. + pub id: String, + /// The `.theme.json`, comments removed. + pub theme: String, + /// The editor scheme (`.icls` XML) its `editorScheme` names, when the plugin has it. + pub editor_scheme: Option, +} + +/// A theme of a plugin installed in a JetBrains IDE. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct InstalledIntellijTheme { + /// The IDE and its version: `IntelliJ IDEA 2025.2`. + pub ide: String, + /// The plugin's name. + pub plugin: String, + /// What to import: the plugin's folder or jar. + pub path: IpcPath, + pub theme_id: String, + pub theme_name: String, + pub dark: bool, + /// The IDE is set to this theme. + pub active: bool, +} + +/// Reads the themes of an IntelliJ theme plugin (a `.jar`, a Marketplace +/// `.zip`, an installed plugin's folder) or of a `.theme.json`. +pub fn read_source(path: &Path) -> IdeResult { + read(path).map_err(IdeError::invalid) +} + +/// The theme plugins of the JetBrains IDEs (and Android Studio) of this user, +/// the themes their IDEs are set to first. +pub fn installed_themes() -> Vec { + let Some(home) = dirs::home_dir() else { return Vec::new() }; + let layout = if cfg!(target_os = "macos") { + Layout::Mac + } else if cfg!(windows) { + Layout::Windows + } else { + Layout::Xdg + }; + find_installed(&ide_dirs(layout, &home, |key| std::env::var_os(key))) +} + +fn read(path: &Path) -> Result { + if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("json")) { + return loose_theme(path); + } + let mut jars = plugin_jars(path)?; + let plugin = descriptor(&mut jars)?; + let shown = || path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + if plugin.themes.is_empty() { + return Err(format!("{} is not a theme plugin: its plugin.xml declares no themeProvider", shown())); + } + let mut themes = Vec::new(); + let mut problem = None; + for (id, theme_path) in &plugin.themes { + match plugin_theme(&mut jars, id, theme_path) { + Ok(theme) => themes.push(theme), + Err(e) => problem = problem.or(Some(e)), + } + } + if themes.is_empty() { + return Err(problem.unwrap_or_else(|| format!("{} has no readable theme", shown()))); + } + Ok(IntellijThemeSource { + name: plugin.name.unwrap_or_else(shown), + plugin_id: plugin.id, + version: plugin.version, + vendor: plugin.vendor, + themes, + }) +} + +fn plugin_theme(jars: &mut [Jar], id: &str, path: &str) -> Result { + let bytes = find(jars, entry_name(path), MAX_FILE)?.ok_or_else(|| format!("{path} is missing from the plugin"))?; + let (theme, fields) = theme_json(&bytes).map_err(|e| format!("{path}: {e}"))?; + let editor_scheme = match string_field(&fields, "editorScheme") { + Some(scheme) => find(jars, entry_name(scheme), MAX_FILE)?.map(|b| text(&b)), + None => None, + }; + Ok(IntellijThemeFile { id: id.to_owned(), theme, editor_scheme }) +} + +/// A `.theme.json` with the editor scheme beside it (its `editorScheme` names a path in the plugin's jar). +fn loose_theme(path: &Path) -> Result { + let (theme, fields) = theme_json(&read_file(path)?).map_err(|e| format!("{}: {e}", path.display()))?; + // Any JSON object parses: an IntelliJ theme has UI colours or an editor scheme. + let scheme_ref = string_field(&fields, "editorScheme"); + if !fields.get("ui").is_some_and(serde_json::Value::is_object) && scheme_ref.is_none() { + return Err(format!("{} is not an IntelliJ theme: it has no \"ui\" or \"editorScheme\"", path.display())); + } + let editor_scheme = scheme_ref + .and_then(|p| Path::new(p).file_name().map(|n| path.with_file_name(n))) + .filter(|p| p.is_file()) + .map(|p| read_file(&p).map(|b| text(&b))) + .transpose()?; + let file = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let id = file.strip_suffix(".theme.json").or_else(|| file.strip_suffix(".json")).unwrap_or(&file).to_owned(); + Ok(IntellijThemeSource { + name: string_field(&fields, "name").map_or_else(|| id.clone(), str::to_owned), + plugin_id: None, + version: None, + vendor: string_field(&fields, "author").map(str::to_owned), + themes: vec![IntellijThemeFile { id, theme, editor_scheme }], + }) +} + +type JsonObject = serde_json::Map; + +/// The theme JSON, comments removed, and its fields, if it is an object. +fn theme_json(bytes: &[u8]) -> Result<(String, JsonObject), String> { + let json = super::theme::strip_json_comments(&text(bytes)); + match serde_json::from_str::(&json) { + Ok(serde_json::Value::Object(fields)) => Ok((json, fields)), + Ok(_) => Err("not a theme: expected a JSON object".into()), + Err(e) => Err(format!("invalid theme JSON: {e}")), + } +} + +fn string_field<'a>(fields: &'a JsonObject, key: &str) -> Option<&'a str> { + fields.get(key)?.as_str().filter(|s| !s.trim().is_empty()) +} + +/// `/themes/x.theme.json` → `themes/x.theme.json`. +fn entry_name(path: &str) -> &str { + path.trim().trim_start_matches('/') +} + +fn text(bytes: &[u8]) -> String { + let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes); + String::from_utf8_lossy(bytes).into_owned() +} + +fn read_file(path: &Path) -> Result, String> { + let shown = |e: std::io::Error| format!("{}: {e}", path.display()); + let file = std::fs::File::open(path).map_err(shown)?; + let mut buf = Vec::new(); + file.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(shown)?; + if buf.len() as u64 > MAX_FILE { + return Err(format!("{} is too large for a theme", path.display())); + } + Ok(buf) +} + +trait ReadSeek: Read + Seek {} +impl ReadSeek for T {} +type Jar = zip::ZipArchive>; + +fn open_jar(path: &Path) -> Result { + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let reader: Box = Box::new(std::io::BufReader::new(file)); + zip::ZipArchive::new(reader).map_err(|e| format!("{} is not a jar or zip file ({e})", path.display())) +} + +/// The jars of the plugin at `path`: a jar, a Marketplace zip (`/lib/*.jar`) +/// or an installed plugin's folder (`lib/*.jar`). +fn plugin_jars(path: &Path) -> Result, String> { + let not_plugin = || format!("{} is not an IntelliJ plugin: no {PLUGIN_XML} and no lib/*.jar", path.display()); + if path.is_dir() { + let mut found: Vec = [path.join("lib"), path.join("lib").join("modules")] + .iter() + .flat_map(|dir| std::fs::read_dir(dir).into_iter().flatten().flatten()) + .map(|e| e.path()) + .filter(|p| p.is_file() && is_jar(p)) + .collect(); + found.sort(); + let jars: Vec = found.iter().take(MAX_JARS).filter_map(|p| open_jar(p).ok()).collect(); + return if jars.is_empty() { Err(not_plugin()) } else { Ok(jars) }; + } + let mut archive = open_jar(path)?; + if archive.index_for_name(PLUGIN_XML).is_some() { + return Ok(vec![archive]); + } + let jars = nested_jars(&mut archive, MAX_JAR, MAX_NESTED); + if jars.is_empty() { Err(not_plugin()) } else { Ok(jars) } +} + +/// The `lib` jars of a Marketplace zip, smallest first (a theme's own jar is small) +/// while they fit `max_total`; library jars too large or not jars at all are skipped. +fn nested_jars(archive: &mut Jar, max_jar: u64, max_total: u64) -> Vec { + let mut nested: Vec<(u64, String)> = entry_names(archive) + .into_iter() + .filter(|n| is_lib_jar(n)) + .filter_map(|n| Some((archive.by_name(&n).ok()?.size(), n))) + .filter(|(size, _)| *size <= max_jar) + .collect(); + nested.sort(); + let (mut jars, mut loaded) = (Vec::new(), 0u64); + for (size, name) in nested.into_iter().take(MAX_JARS) { + if loaded.saturating_add(size) > max_total { + break; + } + let Ok(Some(bytes)) = read_entry(archive, &name, max_jar) else { continue }; + loaded += size; + let reader: Box = Box::new(Cursor::new(bytes)); + if let Ok(jar) = zip::ZipArchive::new(reader) { + jars.push(jar); + } + } + jars +} + +fn entry_names(jar: &Jar) -> Vec { + jar.file_names().filter_map(Result::ok).map(|n| n.into_owned()).collect() +} + +fn is_jar(path: &Path) -> bool { + path.extension().is_some_and(|e| e.eq_ignore_ascii_case("jar")) +} + +/// `lib/x.jar`, `/lib/x.jar` or `/lib/modules/x.jar` in a Marketplace zip. +fn is_lib_jar(name: &str) -> bool { + let parts: Vec<&str> = name.split('/').collect(); + let n = parts.len(); + name.to_ascii_lowercase().ends_with(".jar") + && (2..=4).contains(&n) + && (parts[n - 2] == "lib" || (n >= 3 && parts[n - 2] == "modules" && parts[n - 3] == "lib")) +} + +fn read_entry(jar: &mut Jar, name: &str, max: u64) -> Result>, String> { + let Ok(file) = jar.by_name(name) else { return Ok(None) }; + if file.is_dir() { + return Ok(None); + } + if file.size() > max { + return Err(format!("{name} is too large")); + } + let mut buf = Vec::with_capacity(file.size() as usize); + file.take(max.saturating_add(1)).read_to_end(&mut buf).map_err(|e| format!("{name}: {e}"))?; + if buf.len() as u64 > max { + return Err(format!("{name} is too large")); + } + Ok(Some(buf)) +} + +fn find(jars: &mut [Jar], name: &str, max: u64) -> Result>, String> { + for jar in jars.iter_mut() { + if let Some(bytes) = read_entry(jar, name, max)? { + return Ok(Some(bytes)); + } + } + Ok(None) +} + +/// What the plugin's descriptors say: ``'s `id`, `name`, `version`, +/// `vendor`, and its `themeProvider`s (id, path in the jar). +#[derive(Debug, Default, PartialEq)] +struct Descriptor { + id: Option, + name: Option, + version: Option, + vendor: Option, + themes: Vec<(String, String)>, +} + +/// Reads `META-INF/plugin.xml` and the descriptors it includes (``), in any jar. +fn descriptor(jars: &mut [Jar]) -> Result { + let main = find(jars, PLUGIN_XML, MAX_FILE) + .ok() + .flatten() + .ok_or_else(|| format!("not an IntelliJ plugin: no {PLUGIN_XML}"))?; + let mut out = Descriptor::default(); + let mut includes = Vec::new(); + parse_descriptor(&text(&main), true, &mut out, &mut includes).map_err(|e| format!("{PLUGIN_XML}: {e}"))?; + let mut read = std::collections::HashSet::from([PLUGIN_XML.to_owned()]); + while let Some(href) = includes.pop() { + let name = include_entry(&href); + if read.len() >= MAX_DESCRIPTORS || !read.insert(name.clone()) { + continue; + } + let Ok(Some(bytes)) = find(jars, &name, MAX_FILE) else { continue }; + let _ = parse_descriptor(&text(&bytes), false, &mut out, &mut includes); + } + let mut seen = std::collections::HashSet::new(); + out.themes.retain(|(id, _)| seen.insert(id.clone())); + Ok(out) +} + +/// An `` as a jar entry: `/META-INF/x.xml` from the jar's root, `x.xml` beside plugin.xml. +fn include_entry(href: &str) -> String { + let href = href.trim(); + href.strip_prefix('/').map_or_else(|| format!("META-INF/{href}"), str::to_owned) +} + +fn parse_descriptor(xml: &str, main: bool, out: &mut Descriptor, includes: &mut Vec) -> Result<(), String> { + let mut reader = Reader::from_str(xml); + let mut stack: Vec = Vec::new(); + let mut text = String::new(); + loop { + let event = reader.read_event().map_err(|e| e.to_string())?; + match event { + Event::Start(ref tag) | Event::Empty(ref tag) => { + let name = tag.local_name().as_ref().to_owned(); + if name == "include" { + let href = tag.attributes().flatten().find(|a| a.key.local_name().as_ref() == "href"); + if let Some(href) = href.and_then(|a| a.normalized_value(XmlVersion::Implicit1_0).ok()) { + includes.push(href.into_owned()); + } + } + if name == "themeProvider" { + let (mut id, mut path) = (None, None); + for a in tag.attributes() { + let a = a.map_err(|e| e.to_string())?; + let value = a.normalized_value(XmlVersion::Implicit1_0).map_err(|e| e.to_string())?; + match a.key.local_name().as_ref() { + "id" => id = Some(value.trim().to_owned()), + "path" => path = Some(value.trim().to_owned()), + _ => {} + } + } + if let (Some(id), Some(path)) = (id, path) + && !id.is_empty() + && !path.is_empty() + { + out.themes.push((id, path)); + } + } + if matches!(event, Event::Start(_)) { + stack.push(name); + text.clear(); + } + } + Event::Text(t) => text.push_str(&t.xml_content(XmlVersion::Implicit1_0)), + Event::CData(t) => text.push_str(t.as_ref()), + Event::GeneralRef(r) => { + let name = r.as_ref(); + let escaped = format!("&{name};"); + text.push_str(&quick_xml::escape::unescape(&escaped).map_err(|e| e.to_string())?); + } + Event::End(_) => { + let name = stack.pop().ok_or("unbalanced element")?; + // Direct children of . + if main && stack.len() == 1 && stack[0] == "idea-plugin" { + let value = text.trim(); + let slot = match name.as_str() { + "id" => Some(&mut out.id), + "name" => Some(&mut out.name), + "version" => Some(&mut out.version), + "vendor" => Some(&mut out.vendor), + _ => None, + }; + if let Some(slot) = slot + && slot.is_none() + && !value.is_empty() + { + *slot = Some(value.to_owned()); + } + } + text.clear(); + } + Event::Eof => return if stack.is_empty() { Ok(()) } else { Err("unclosed element".into()) }, + _ => {} + } + } +} + +/// Where an OS keeps the IDEs' settings and plugins. +#[derive(Clone, Copy, Debug)] +enum Layout { + /// `~/Library/Application Support//`, plugins in its `plugins/`. + Mac, + /// `%APPDATA%\\`, plugins in its `plugins\`. + Windows, + /// `$XDG_CONFIG_HOME//`, plugins in `$XDG_DATA_HOME//`. + Xdg, +} + +#[derive(Debug, PartialEq)] +struct IdeDir { + label: String, + product: String, + version: (u32, u32), + config: PathBuf, + plugins: PathBuf, +} + +/// The settings folders of JetBrains IDEs and Android Studio (2020.1 and later), newest first. +fn ide_dirs(layout: Layout, home: &Path, env: impl Fn(&str) -> Option) -> Vec { + let var = |key: &str| env(key).filter(|v| !v.is_empty()).map(PathBuf::from); + let (config_root, data_root) = match layout { + Layout::Mac => { + let root = home.join("Library").join("Application Support"); + (root.clone(), root) + } + Layout::Windows => { + let root = var("APPDATA").unwrap_or_else(|| home.join("AppData").join("Roaming")); + (root.clone(), root) + } + Layout::Xdg => ( + var("XDG_CONFIG_HOME").unwrap_or_else(|| home.join(".config")), + var("XDG_DATA_HOME").unwrap_or_else(|| home.join(".local").join("share")), + ), + }; + let mut out = Vec::new(); + for vendor in ["JetBrains", "Google"] { + for entry in std::fs::read_dir(config_root.join(vendor)).into_iter().flatten().flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + let Some((product, version, shown)) = product_dir(&name) else { continue }; + let config = entry.path(); + if !config.is_dir() { + continue; + } + let plugins = match layout { + Layout::Xdg => data_root.join(vendor).join(&name), + Layout::Mac | Layout::Windows => config.join("plugins"), + }; + out.push(IdeDir { + label: format!("{} {shown}", product_label(product)), + product: product.to_owned(), + version, + config, + plugins, + }); + } + } + out.sort_by(|a, b| b.version.cmp(&a.version).then_with(|| a.label.cmp(&b.label))); + out +} + +/// `IntelliJIdea2025.2` → (`IntelliJIdea`, (2025, 2), `2025.2`). +fn product_dir(name: &str) -> Option<(&str, (u32, u32), &str)> { + let at = name.find(|c: char| c.is_ascii_digit())?; + let (product, version) = name.split_at(at); + if product.is_empty() || !product.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + let (year, minor) = version.split_once('.')?; + let ok = |s: &str| !s.is_empty() && s.len() <= 4 && s.chars().all(|c| c.is_ascii_digit()); + if year.len() != 4 || !ok(year) || !ok(minor) { + return None; + } + Some((product, (year.parse().ok()?, minor.parse().ok()?), version)) +} + +fn product_label(product: &str) -> &str { + match product { + "IntelliJIdea" => "IntelliJ IDEA", + "IdeaIC" => "IntelliJ IDEA Community Edition", + "PyCharmCE" => "PyCharm Community Edition", + "AndroidStudio" => "Android Studio", + "AndroidStudioPreview" => "Android Studio Preview", + other => other, + } +} + +/// The theme ids `options/laf.xml` selects: the current one, and with "Sync with OS" the preferred light and dark. +fn active_theme_ids(config: &Path) -> Vec { + let Ok(bytes) = read_file(&config.join("options").join("laf.xml")) else { return Vec::new() }; + let xml = text(&bytes); + let mut reader = Reader::from_str(&xml); + let (mut ids, mut preferred, mut autodetect) = (Vec::new(), Vec::new(), false); + loop { + match reader.read_event() { + Ok(Event::Start(tag) | Event::Empty(tag)) => { + let attr = |key: &str| { + tag.attributes() + .flatten() + .find(|a| a.key.as_ref() == key) + .and_then(|a| a.normalized_value(XmlVersion::Implicit1_0).ok().map(|v| v.trim().to_owned())) + }; + match tag.local_name().as_ref() { + "component" if attr("name").as_deref() == Some("LafManager") => { + autodetect = attr("autodetect").as_deref() == Some("true"); + } + "laf" => ids.extend(attr("themeId")), + "preferred-light-laf" | "preferred-dark-laf" => preferred.extend(attr("themeId")), + _ => {} + } + } + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + } + if autodetect { + ids.extend(preferred); + } + ids.retain(|id| !id.is_empty()); + ids +} + +fn find_installed(ides: &[IdeDir]) -> Vec { + let mut out = Vec::new(); + for ide in ides { + let active = active_theme_ids(&ide.config); + let mut plugins: Vec = std::fs::read_dir(&ide.plugins) + .into_iter() + .flatten() + .flatten() + .map(|e| e.path()) + .filter(|p| p.is_dir() || (p.is_file() && is_jar(p))) + .collect(); + plugins.sort(); + for plugin in plugins { + let Ok(mut jars) = plugin_jars(&plugin) else { continue }; + let Ok(found) = descriptor(&mut jars) else { continue }; + for (id, path) in &found.themes { + let Ok(Some(bytes)) = find(&mut jars, entry_name(path), MAX_FILE) else { continue }; + let Ok((_, fields)) = theme_json(&bytes) else { continue }; + let file = || plugin.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + out.push(InstalledIntellijTheme { + ide: ide.label.clone(), + plugin: found.name.clone().unwrap_or_else(file), + path: plugin.clone().into(), + theme_id: id.clone(), + theme_name: string_field(&fields, "name").unwrap_or(id).to_owned(), + dark: fields.get("dark").and_then(serde_json::Value::as_bool).unwrap_or(true), + active: active.contains(id), + }); + } + } + } + // Stable: the IDEs' order (newest first) within each group. + out.sort_by_key(|t| !t.active); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + const PLUGIN: &str = r#" + com.example.deepsea + Deep Sea & Friends + 2.3 + Someone + Dark themes]]> + com.intellij.modules.platform + + + + +"#; + + const DEEP: &str = r##"{ + // comment + "name": "Deep Sea", + "dark": true, + "editorScheme": "/themes/DeepSea.xml", + "ui": { "*": { "background": "#101820" } } +}"##; + const SHALLOW: &str = r#"{ "name": "Shallow", "dark": false, "ui": {} }"#; + const SCHEME: &str = r#""#; + + fn jar_bytes(entries: &[(&str, &[u8])]) -> Vec { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, data) in entries { + zip.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(data).unwrap(); + } + zip.finish().unwrap().into_inner() + } + + fn theme_jar() -> Vec { + jar_bytes(&[ + ("META-INF/MANIFEST.MF", b"Manifest-Version: 1.0\n"), + (PLUGIN_XML, PLUGIN.as_bytes()), + ("themes/DeepSea.theme.json", DEEP.as_bytes()), + ("themes/DeepSea.xml", SCHEME.as_bytes()), + ("themes/Shallow.theme.json", SHALLOW.as_bytes()), + ]) + } + + fn write(path: &Path, bytes: &[u8]) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); + } + + fn expected_source() -> IntellijThemeSource { + IntellijThemeSource { + name: "Deep Sea & Friends".into(), + plugin_id: Some("com.example.deepsea".into()), + version: Some("2.3".into()), + vendor: Some("Someone".into()), + themes: vec![ + IntellijThemeFile { + id: "deep-sea-id".into(), + theme: super::super::theme::strip_json_comments(DEEP), + editor_scheme: Some(SCHEME.into()), + }, + IntellijThemeFile { id: "shallow-id".into(), theme: SHALLOW.into(), editor_scheme: None }, + ], + } + } + + #[test] + fn reads_a_theme_jar_with_its_editor_scheme() { + let tmp = tempfile::tempdir().unwrap(); + let jar = tmp.path().join("deep-sea.jar"); + write(&jar, &theme_jar()); + let source = read_source(&jar).unwrap(); + assert_eq!(source, expected_source()); + assert!(!source.themes[0].theme.contains("comment")); + } + + #[test] + fn reads_a_marketplace_zip_and_an_installed_folder() { + let tmp = tempfile::tempdir().unwrap(); + let other = jar_bytes(&[("META-INF/MANIFEST.MF", b"x"), ("a/B.class", b"\xCA\xFE")]); + let zip = tmp.path().join("DeepSea-2.3.zip"); + write(&zip, &jar_bytes(&[("DeepSea/lib/a-util.jar", &other), ("DeepSea/lib/deep-sea.jar", &theme_jar())])); + assert_eq!(read_source(&zip).unwrap(), expected_source()); + + let dir = tmp.path().join("plugins").join("DeepSea"); + write(&dir.join("lib").join("deep-sea.jar"), &theme_jar()); + write(&dir.join("lib").join("a-util.jar"), &other); + assert_eq!(read_source(&dir).unwrap(), expected_source()); + } + + #[test] + fn skips_library_jars_it_cannot_or_need_not_read() { + let tmp = tempfile::tempdir().unwrap(); + // Stored, not deflated: as large in the zip as its content. + let mut big = zip::ZipWriter::new(Cursor::new(Vec::new())); + let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + big.start_file("lib/Data.class", stored).unwrap(); + big.write_all(&[b'x'; 8192]).unwrap(); + let big = big.finish().unwrap().into_inner(); + let zip = tmp.path().join("DeepSea.zip"); + write( + &zip, + &jar_bytes(&[ + ("DeepSea/lib/a-broken.jar", b"not a jar"), + ("DeepSea/lib/b-big.jar", &big), + ("DeepSea/lib/deep-sea.jar", &theme_jar()), + ]), + ); + assert_eq!(read_source(&zip).unwrap(), expected_source()); + + // The broken jar is skipped; a jar over the size limit, or past the total + // (smallest first), is not read, while the theme's small jar still is. + let mut archive = open_jar(&zip).unwrap(); + let theme_len = theme_jar().len() as u64; + assert_eq!(nested_jars(&mut archive, u64::MAX, u64::MAX).len(), 2); + let one = nested_jars(&mut archive, theme_len + 64, u64::MAX); + assert_eq!(one.len(), 1); + assert!(one[0].index_for_name(PLUGIN_XML).is_some()); + assert_eq!(nested_jars(&mut archive, u64::MAX, theme_len + 64).len(), 1); + assert!(nested_jars(&mut archive, 8, u64::MAX).is_empty()); + } + + #[test] + fn reads_a_loose_theme_json_with_the_scheme_beside_it() { + let tmp = tempfile::tempdir().unwrap(); + let theme = tmp.path().join("DeepSea.theme.json"); + write(&theme, DEEP.as_bytes()); + write(&tmp.path().join("DeepSea.xml"), SCHEME.as_bytes()); + let source = read_source(&theme).unwrap(); + assert_eq!(source.name, "Deep Sea"); + assert_eq!(source.themes.len(), 1); + assert_eq!(source.themes[0].id, "DeepSea"); + assert_eq!(source.themes[0].editor_scheme.as_deref(), Some(SCHEME)); + + std::fs::remove_file(tmp.path().join("DeepSea.xml")).unwrap(); + assert_eq!(read_source(&theme).unwrap().themes[0].editor_scheme, None); + } + + #[test] + fn says_what_is_wrong_with_other_files() { + let tmp = tempfile::tempdir().unwrap(); + let not_zip = tmp.path().join("notes.jar"); + write(¬_zip, b"hello"); + assert!(read_source(¬_zip).unwrap_err().message.contains("not a jar or zip")); + + let other_plugin = tmp.path().join("tools.jar"); + let xml = "Tools"; + write(&other_plugin, &jar_bytes(&[(PLUGIN_XML, xml.as_bytes())])); + assert!(read_source(&other_plugin).unwrap_err().message.contains("not a theme plugin")); + + let package = tmp.path().join("package.zip"); + write(&package, &jar_bytes(&[("plugin.toml", b"id = \"x\"")])); + assert!(read_source(&package).unwrap_err().message.contains("not an IntelliJ plugin")); + + let other = tmp.path().join("package.json"); + write(&other, br#"{ "name": "app", "version": "1.0.0" }"#); + assert!(read_source(&other).unwrap_err().message.contains("is not an IntelliJ theme")); + + let broken = tmp.path().join("broken.theme.json"); + write(&broken, b"{ \"name\": "); + assert!(read_source(&broken).unwrap_err().message.contains("invalid theme JSON")); + + let missing = tmp.path().join("missing.jar"); + let xml = r#"M"#; + write(&missing, &jar_bytes(&[(PLUGIN_XML, xml.as_bytes())])); + assert!(read_source(&missing).unwrap_err().message.contains("/m.theme.json is missing")); + } + + #[test] + fn finds_theme_providers_in_included_descriptors() { + let tmp = tempfile::tempdir().unwrap(); + let jar = tmp.path().join("split.jar"); + let main = r#"Split + "#; + let part = r#" + Not this + "#; + let more = r#""#; + let unrelated = r#""#; + write( + &jar, + &jar_bytes(&[ + ("META-INF/themes.xml", part.as_bytes()), + ("META-INF/more.xml", more.as_bytes()), + ("META-INF/unrelated.xml", unrelated.as_bytes()), + (PLUGIN_XML, main.as_bytes()), + ("s.theme.json", br#"{"name":"S"}"#), + ("m.theme.json", br#"{"name":"M"}"#), + ("x.theme.json", br#"{"name":"X"}"#), + ]), + ); + let source = read_source(&jar).unwrap(); + assert_eq!(source.name, "Split"); + // Only what plugin.xml includes (relative or from the root, once each) counts. + assert_eq!(source.themes.iter().map(|t| t.id.as_str()).collect::>(), ["s", "m"]); + } + + #[test] + fn parses_ide_folder_names() { + assert_eq!(product_dir("IntelliJIdea2025.2"), Some(("IntelliJIdea", (2025, 2), "2025.2"))); + assert_eq!(product_dir("PyCharmCE2023.10"), Some(("PyCharmCE", (2023, 10), "2023.10"))); + assert_eq!(product_dir("AndroidStudio2024.3"), Some(("AndroidStudio", (2024, 3), "2024.3"))); + for name in + ["Toolbox", "consentOptions", "IntelliJIdea2025", "Idea-2025.1", "2025.1", "Rider25.1", "Go Land2025.1"] + { + assert_eq!(product_dir(name), None, "{name}"); + } + } + + #[test] + fn locates_the_settings_and_plugins_of_each_os() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path(); + let mk = |p: PathBuf| { + std::fs::create_dir_all(&p).unwrap(); + p + }; + let mac = mk(home.join("Library/Application Support/JetBrains/IntelliJIdea2025.2")); + mk(home.join("Library/Application Support/JetBrains/Toolbox")); + let dirs = ide_dirs(Layout::Mac, home, |_| None); + assert_eq!(dirs.len(), 1); + assert_eq!(dirs[0].label, "IntelliJ IDEA 2025.2"); + assert_eq!((dirs[0].config.clone(), dirs[0].plugins.clone()), (mac.clone(), mac.join("plugins"))); + + let appdata = mk(home.join("Roaming")); + let win = mk(appdata.join("JetBrains/PyCharm2024.3")); + let studio = mk(appdata.join("Google/AndroidStudio2025.1")); + let env = |k: &str| (k == "APPDATA").then(|| appdata.clone().into_os_string()); + let dirs = ide_dirs(Layout::Windows, home, env); + assert_eq!( + dirs.iter().map(|d| d.label.as_str()).collect::>(), + ["Android Studio 2025.1", "PyCharm 2024.3"] + ); + assert_eq!(dirs[1].plugins, win.join("plugins")); + assert_eq!(dirs[0].config, studio); + + let config = mk(home.join(".config/JetBrains/IdeaIC2024.1")); + mk(home.join(".config/JetBrains/GoLand2025.2")); + let dirs = ide_dirs(Layout::Xdg, home, |_| None); + assert_eq!(dirs.iter().map(|d| d.product.as_str()).collect::>(), ["GoLand", "IdeaIC"]); + assert_eq!(dirs[1].label, "IntelliJ IDEA Community Edition 2024.1"); + assert_eq!(dirs[1].config, config); + assert_eq!(dirs[1].plugins, home.join(".local/share/JetBrains/IdeaIC2024.1")); + + let xdg_data = home.join("data"); + let env = |k: &str| (k == "XDG_DATA_HOME").then(|| xdg_data.clone().into_os_string()); + assert_eq!(ide_dirs(Layout::Xdg, home, env)[1].plugins, xdg_data.join("JetBrains/IdeaIC2024.1")); + } + + #[test] + fn reads_the_selected_theme_from_laf_xml() { + let tmp = tempfile::tempdir().unwrap(); + let laf = tmp.path().join("options/laf.xml"); + write( + &laf, + br#" + + + + +"#, + ); + assert_eq!(active_theme_ids(tmp.path()), ["deep-sea-id"]); + let synced = std::fs::read_to_string(&laf) + .unwrap() + .replace(r#"name="LafManager""#, r#"name="LafManager" autodetect="true""#); + write(&laf, synced.as_bytes()); + assert_eq!(active_theme_ids(tmp.path()), ["deep-sea-id", "shallow-id"]); + assert!(active_theme_ids(&tmp.path().join("missing")).is_empty()); + } + + #[test] + fn lists_installed_theme_plugins_the_ides_current_first() { + let tmp = tempfile::tempdir().unwrap(); + let config = tmp.path().join("IntelliJIdea2025.2"); + let plugins = config.join("plugins"); + write(&plugins.join("DeepSea/lib/deep-sea.jar"), &theme_jar()); + write( + &plugins.join("Tools/lib/tools.jar"), + &jar_bytes(&[(PLUGIN_XML, b"T")]), + ); + write(&plugins.join("broken.jar"), b"not a jar"); + write( + &config.join("options/laf.xml"), + br#""#, + ); + let ide = IdeDir { + label: "IntelliJ IDEA 2025.2".into(), + product: "IntelliJIdea".into(), + version: (2025, 2), + config: config.clone(), + plugins: plugins.clone(), + }; + let found = find_installed(&[ide]); + let theme = |id: &str, name: &str, dark: bool, active: bool| InstalledIntellijTheme { + ide: "IntelliJ IDEA 2025.2".into(), + plugin: "Deep Sea & Friends".into(), + path: plugins.join("DeepSea").into(), + theme_id: id.into(), + theme_name: name.into(), + dark, + active, + }; + assert_eq!(found, [theme("shallow-id", "Shallow", false, true), theme("deep-sea-id", "Deep Sea", true, false)]); + assert!(find_installed(&[]).is_empty()); + } +} diff --git a/crates/ide-host/src/plugins/market.rs b/crates/ide-host/src/plugins/market.rs index a9e045c1..fdf2e514 100644 --- a/crates/ide-host/src/plugins/market.rs +++ b/crates/ide-host/src/plugins/market.rs @@ -3,11 +3,14 @@ //! plugins, and revocations. The host keeps the last index it read //! (`market-index.json`), checks signatures against it, and disables what it //! revokes — at start-up and on every refresh (the desktop refreshes daily). +//! Release builds pin the official publisher to the keys they were built +//! with, so no index can sign `openidea.*` plugins with another key. -use super::{PluginHost, PluginSource, PluginStatus}; +use super::{PackageFile, PluginEvent, PluginHost, PluginSource, PluginStatus, PluginUpdate, install}; use ide_core::{IdeError, IdeResult}; -use ide_extension::Manifest; use ide_extension::package::{self, Index, IndexEntry, PackageSignature, SIGNATURE_FILE}; +use ide_extension::{MANIFEST_FILE, Manifest}; +use semver::Version; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -15,11 +18,18 @@ use std::path::{Path, PathBuf}; /// the app, next to the signed packages of the official plugins. pub const OFFICIAL_INDEX: &str = "https://github.com/l0ng-ai/openidea/releases/latest/download/plugins-index.json"; +/// The publisher of the official plugins. +pub const OFFICIAL_PUBLISHER: &str = "openidea"; + /// The last index read, in the plugins directory. pub const INDEX_FILE: &str = "market-index.json"; const DOWNLOADS: &str = ".downloads"; const MAX_INDEX: usize = 16 << 20; const MAX_PACKAGE: usize = 256 << 20; +const MAX_STAGED_FILES: usize = 256; +const MAX_STAGED_BYTES: usize = 16 << 20; +/// `moved` links followed before giving up (a loop, or a misconfiguration). +const MAX_MOVES: usize = 3; /// A marketplace entry with what is installed. #[derive(Clone, Debug, Serialize, Deserialize, specta::Type)] @@ -29,6 +39,32 @@ pub struct MarketPlugin { pub entry: IndexEntry, /// Installed version, if any. pub installed: Option, + /// Whether this IDE can load `entry` (the newest version it can, else the newest). + pub compatible: bool, +} + +impl MarketPlugin { + /// A version this IDE can load, newer than the one installed. + pub fn is_update(&self) -> bool { + let newer = |installed: &String| match (Version::parse(&self.entry.version), Version::parse(installed)) { + (Ok(offered), Ok(installed)) => offered > installed, + _ => false, + }; + self.compatible && self.installed.as_ref().is_some_and(newer) + } +} + +/// The official publisher's keys this IDE was built with +/// (`OPENIDEA_OFFICIAL_PLUGIN_KEYS`: base64, comma-separated). Development +/// builds have none: the index decides. +pub(super) fn built_in_official_keys() -> Vec { + let keys = option_env!("OPENIDEA_OFFICIAL_PLUGIN_KEYS").unwrap_or_default(); + keys.split(',').map(str::trim).filter(|k| !k.is_empty()).map(str::to_owned).collect() +} + +fn pinned(mut index: Index, official_keys: &[String]) -> Index { + index.pin_publisher(OFFICIAL_PUBLISHER, "OpenIDEA", official_keys); + index } /// Reads `url`: `http(s)://…`, `file://…` or a local path. @@ -56,6 +92,33 @@ pub fn read_url(url: &str, max: usize) -> Result, String> { Ok(bytes) } +/// The index at `url`, following `moved`, with downloads made absolute against +/// where it was read from. +fn read_index(url: &str) -> IdeResult { + let mut url = url.to_owned(); + for _ in 0..=MAX_MOVES { + let bytes = read_url(&url, MAX_INDEX).map_err(IdeError::unavailable)?; + let text = String::from_utf8(bytes).map_err(|_| IdeError::invalid("the index is not UTF-8"))?; + let mut index = Index::parse(&text).map_err(IdeError::invalid)?; + match index.moved.take().filter(|m| !m.trim().is_empty()) { + Some(next) => { + let next = resolve(&url, &next); + if url.starts_with("https://") && !next.starts_with("https://") { + return Err(IdeError::invalid(format!("the marketplace moved to {next}, which isn't https"))); + } + url = next; + } + None => { + for e in &mut index.plugins { + e.download = resolve(&url, &e.download); + } + return Ok(index); + } + } + } + Err(IdeError::unavailable(format!("the marketplace index moved more than {MAX_MOVES} times"))) +} + /// `href` relative to the index at `base`: a URL, or a local path (with the /// platform's separators, e.g. `C:\\market\\index.json`). pub fn resolve(base: &str, href: &str) -> String { @@ -75,9 +138,16 @@ pub fn resolve(base: &str, href: &str) -> String { } impl PluginHost { - pub(super) fn load_market_index(dir: &Path) -> Option { + pub(super) fn load_market_index(dir: &Path, official_keys: &[String]) -> Option { let text = std::fs::read_to_string(dir.join(INDEX_FILE)).ok()?; - Index::parse(&text).map_err(|e| tracing::warn!("{INDEX_FILE}: {e}")).ok() + let index = Index::parse(&text).map_err(|e| tracing::warn!("{INDEX_FILE}: {e}")).ok()?; + Some(pinned(index, official_keys)) + } + + /// Pins the official publisher to `keys` (base64) instead of the built-in + /// ones, from the next index read. + pub fn set_official_keys(&self, keys: Vec) { + *self.official_keys.write() = keys; } /// The marketplace index URL: the user's, else the default (none: no marketplace). @@ -95,14 +165,26 @@ impl PluginHost { self.save_state(); } - /// Reads the index again, keeps it, and applies its revocations. + /// The marketplace's plugins: read again when `refresh`, else the last index + /// read. Reads it when none has been read yet (first run, nothing cached). + pub fn market_listing(&self, refresh: bool) -> IdeResult> { + if refresh || (self.market.read().is_none() && self.market_url().is_some()) { + self.refresh_market() + } else { + Ok(self.market_plugins()) + } + } + + /// Reads the index again, keeps it, and applies its revocations. Announces + /// updates of installed plugins it hasn't announced yet. pub fn refresh_market(&self) -> IdeResult> { let url = self.market_url().ok_or_else(|| IdeError::unavailable("no plugin marketplace is configured"))?; - let bytes = read_url(&url, MAX_INDEX).map_err(IdeError::unavailable)?; - let text = String::from_utf8(bytes).map_err(|_| IdeError::invalid("the index is not UTF-8"))?; - let index = Index::parse(&text).map_err(IdeError::invalid)?; - let _ = std::fs::write(self.plugins_dir.join(INDEX_FILE), &text); - *self.market.write() = Some(index); + let index = read_index(&url)?; + if let Ok(json) = serde_json::to_string(&index) { + let _ = std::fs::write(self.plugins_dir.join(INDEX_FILE), json); + } + let newer = index.is_newer().then_some(index.format); + *self.market.write() = Some(pinned(index, &self.official_keys.read())); // Trust may have changed: who signed what, what is revoked. let installed: Vec<_> = self.plugins.read().values().filter(|p| p.source == PluginSource::Installed).cloned().collect(); @@ -111,7 +193,41 @@ impl PluginHost { plugin.meta.lock().publisher = publisher; } self.apply_revocations(); - Ok(self.market_plugins()) + if let Some(format) = newer { + return Err(IdeError::unavailable(format!( + "the marketplace needs a newer OpenIDEA (index format {format}); its revocations still apply" + ))); + } + let plugins = self.market_plugins(); + self.announce_updates(&plugins); + Ok(plugins) + } + + /// Tells the windows about updates once per set of versions: a later + /// refresh offering the same ones says nothing. + fn announce_updates(&self, plugins: &[MarketPlugin]) { + let updates: Vec = plugins + .iter() + .filter(|p| p.is_update()) + .map(|p| PluginUpdate { + id: p.entry.id.clone(), + name: p.entry.name.clone(), + version: p.entry.version.clone(), + }) + .collect(); + let key: Vec = updates.iter().map(|u| format!("{}@{}", u.id, u.version)).collect(); + if std::mem::replace(&mut *self.announced_updates.lock(), key.clone()) != key && !updates.is_empty() { + self.emit(None, PluginEvent::UpdatesAvailable { updates }); + } + } + + /// Whether the package in `src` would replace the installed `id`'s copy + /// with one signed by another key than that copy, without that key's + /// endorsement: the user confirms the change of signer. + pub(super) fn signer_changed(&self, id: &str, src: &Path) -> bool { + let Some(current) = install::active_dir(&self.plugins_dir.join(id)) else { return false }; + let market = self.market.read(); + !package::signer_continues(market.as_ref(), read_signature(¤t).as_ref(), read_signature(src).as_ref()) } /// The plugins of the last index read. @@ -119,16 +235,14 @@ impl PluginHost { let Some(index) = self.market.read().clone() else { return Vec::new() }; let plugins = self.plugins.read(); index - .plugins - .iter() - .filter(|e| index.revocation(&e.id, &e.version).is_none()) - .cloned() - .map(|entry| { + .offered() + .into_iter() + .map(|(entry, compatible)| { let installed = plugins .get(&entry.id) .filter(|p| p.source == PluginSource::Installed) .map(|p| p.manifest().version.clone()); - MarketPlugin { entry, installed } + MarketPlugin { entry: entry.clone(), installed, compatible } }) .collect() } @@ -139,12 +253,18 @@ impl PluginHost { pub fn download_market_plugin(&self, id: &str) -> IdeResult { let url = self.market_url().ok_or_else(|| IdeError::unavailable("no plugin marketplace is configured"))?; let index = self.market.read().clone().ok_or_else(|| IdeError::unavailable("refresh the marketplace first"))?; - let entry = index - .plugins - .iter() - .find(|e| e.id == id) - .cloned() + let (entry, compatible) = index + .offered() + .into_iter() + .find(|(e, _)| e.id == id) + .map(|(e, ok)| (e.clone(), ok)) .ok_or_else(|| IdeError::not_found(format!("{id} is not in the marketplace")))?; + if !compatible { + return Err(IdeError::unavailable(format!( + "{} {} needs plugin API {}: update OpenIDEA", + entry.name, entry.version, entry.engine + ))); + } let bytes = read_url(&resolve(&url, &entry.download), MAX_PACKAGE).map_err(IdeError::unavailable)?; let downloads = self.plugins_dir.join(DOWNLOADS); std::fs::create_dir_all(&downloads)?; @@ -182,6 +302,45 @@ impl PluginHost { Ok(dir) } + /// Writes a package the UI generated (an imported IntelliJ theme) into a + /// folder next to the downloads, named after `name` (its plugin id); returns the folder. + /// The install page validates it like any other package. + pub fn stage_package(&self, name: &str, files: &[PackageFile]) -> IdeResult { + ide_extension::validate_item_id(name, "package name").map_err(IdeError::invalid)?; + if !files.iter().any(|f| f.path == MANIFEST_FILE) { + return Err(IdeError::invalid(format!("a package needs a {MANIFEST_FILE}"))); + } + let total: usize = files.iter().map(|f| f.text.len()).sum(); + if files.len() > MAX_STAGED_FILES || total > MAX_STAGED_BYTES { + return Err(IdeError::invalid("the package is too large")); + } + let mut seen = std::collections::HashSet::new(); + for f in files { + let valid = !f.path.is_empty() + && f.path.len() <= 255 + && f.path.split('/').all(|part| { + !matches!(part, "" | "." | "..") && !part.contains(['\\', ':', '\0']) && !part.starts_with('.') + }); + if !valid { + return Err(IdeError::invalid(format!("invalid package path {:?}", f.path))); + } + if !seen.insert(f.path.to_ascii_lowercase()) { + return Err(IdeError::invalid(format!("{} is in the package twice", f.path))); + } + } + // One folder per package, replaced by the next import of it: they don't pile up. + let dir = self.plugins_dir.join(DOWNLOADS).join(format!("{name}-generated")); + let _ = std::fs::remove_dir_all(&dir); + for f in files { + let path = dir.join(&f.path); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &f.text)?; + } + Ok(dir) + } + /// Files of the package in `dir` (what its declarations reference, and /// the component): what a package file holds besides the manifest. pub fn package_files_of(&self, dir: &Path) -> IdeResult> { diff --git a/crates/ide-host/src/plugins/mod.rs b/crates/ide-host/src/plugins/mod.rs index bc0beba1..0b82a935 100644 --- a/crates/ide-host/src/plugins/mod.rs +++ b/crates/ide-host/src/plugins/mod.rs @@ -16,8 +16,12 @@ //! projects) and folders linked with `openidea-plugin link` (`links.json`). mod api; +mod icon_extensions; +mod icon_patterns; +mod icon_settings; mod icons; mod install; +pub mod intellij; mod market; mod points; mod providers; @@ -32,7 +36,10 @@ pub mod web; pub use api::{FileSecretStore, HostWorkspace, PluginWorkspace, SecretStore}; pub use icons::{map_asset as icon_theme_map_asset, sanitize_svg, svg_prefix as icon_theme_svg_prefix}; -pub use market::{INDEX_FILE as MARKET_INDEX_FILE, MarketPlugin, OFFICIAL_INDEX as OFFICIAL_MARKET_INDEX}; +pub use market::{ + INDEX_FILE as MARKET_INDEX_FILE, MarketPlugin, OFFICIAL_INDEX as OFFICIAL_MARKET_INDEX, OFFICIAL_PUBLISHER, + read_url as read_market_url, resolve as resolve_market_href, +}; pub use points::{COMMANDS, CommandDecl, register_builtin_points}; pub(crate) use providers::editor_providers as editor_providers_for; pub use providers::{ProviderKind, set_editor_providers}; @@ -238,12 +245,16 @@ pub struct Consent { /// Optional capabilities the user accepted (`net:crates.io`, …). Required /// ones are implied by consenting at all. pub optional: Vec, + /// The user accepted a package signed by another key than the installed + /// copy ([`InstallPreview::signer_changed`]). + #[serde(default)] + pub signer_change: bool, } impl Consent { /// Accept everything the plugin asks for (tests, trusted official plugins). pub fn all() -> Self { - Self { optional: vec!["*".into()] } + Self { optional: vec!["*".into()], signer_change: true } } fn accepts(&self, cap: &Capability) -> bool { @@ -270,6 +281,14 @@ pub struct InstallPreview { pub unsigned: bool, /// The verified publisher. pub publisher: Option, + /// Replaces an installed copy signed with another key, which didn't + /// endorse this package's (or this one is unsigned): the user confirms it + /// ([`Consent::signer_change`]). + pub signer_changed: bool, + /// Something to confirm: a first install, an unsigned package, another + /// signer, or capabilities the installed version wasn't granted or + /// refused. Without, an update installs with the grants it has. + pub needs_consent: bool, } type ProjectPolicy = Arc bool + Send + Sync>; @@ -316,6 +335,10 @@ pub struct PluginHost { pins: Arc>>, /// The last marketplace index read: trust root for signatures, revocations. market: RwLock>, + /// The official publisher's keys the index is pinned to (none: as listed). + official_keys: RwLock>, + /// `id@version` of the updates last announced. + announced_updates: Mutex>, /// The marketplace used while the user has set none (the app's official one). default_market_url: RwLock>, /// Updates not confirmed yet (the new version hasn't started once): what @@ -354,9 +377,12 @@ impl PluginHost { policy_pins.read().get(&(plugin.to_owned(), cap.to_string())).cloned() })); let shared = api::Shared::new(sink.clone(), plugins_dir.join(".data"), pins.clone(), ui.clone()); - let market = RwLock::new(Self::load_market_index(&plugins_dir)); + let official_keys = market::built_in_official_keys(); + let market = RwLock::new(Self::load_market_index(&plugins_dir, &official_keys)); let host = Self { market, + official_keys: RwLock::new(official_keys), + announced_updates: Mutex::default(), default_market_url: RwLock::new(None), updates: Mutex::default(), project_policy: RwLock::new(Arc::new(|_| false)), @@ -699,14 +725,27 @@ impl PluginHost { } let publisher = self.verified_publisher(src); let m = &pkg.manifest; + let replaces = self.plugins.read().get(&m.id).map(|p| p.manifest().version.clone()); + // An update shows what the installed version was granted. + let grant = replaces + .as_ref() + .and_then(|_| self.ledger.lock().get(&Subject::Installed(m.id.clone())).cloned()) + .filter(|g| g.confirmed); + let granted = |c: &Capability| grant.as_ref().is_some_and(|g| g.has(c)); let mut capabilities: Vec = - m.capabilities.all().map(|c| CapabilityInfo::new(c, m.capabilities.is_optional(c), false)).collect(); + m.capabilities.all().map(|c| CapabilityInfo::new(c, m.capabilities.is_optional(c), granted(c))).collect(); for extra in &set.capabilities { if !m.declares(extra) { - capabilities.push(CapabilityInfo::new(extra, true, false)); + capabilities.push(CapabilityInfo::new(extra, true, granted(extra))); } } capabilities.sort_by(|a, b| b.risk.cmp(&a.risk).then(a.id.cmp(&b.id))); + let signer_changed = self.signer_changed(&m.id, src); + let answered = |c: &CapabilityInfo| { + c.granted || (c.optional && grant.as_ref().is_some_and(|g| g.declined.contains(&c.id))) + }; + let needs_consent = + grant.is_none() || publisher.is_none() || signer_changed || !capabilities.iter().all(answered); Ok(InstallPreview { id: m.id.clone(), name: m.name.clone(), @@ -717,9 +756,11 @@ impl PluginHost { capabilities, contributions: counts(m), warnings: set.warnings.clone(), - replaces: self.plugins.read().get(&m.id).map(|p| p.manifest().version.clone()), + replaces, unsigned: publisher.is_none(), publisher, + signer_changed, + needs_consent, }) } @@ -742,6 +783,11 @@ impl PluginHost { if let Some(problem) = self.signature_problem(src) { return Err(IdeError::invalid(format!("refusing {id}: {problem}"))); } + if !consent.signer_change && self.signer_changed(&id, src) { + return Err(IdeError::invalid(format!( + "refusing {id}: it is signed by another key than the installed version; confirm the change of signer" + ))); + } let version = pkg.manifest.version.clone(); if let Some(r) = self.market.read().as_ref().and_then(|i| i.revocation(&id, &version)) { return Err(IdeError::invalid(format!("{id} {version} was revoked: {}", r.reason))); @@ -2207,6 +2253,7 @@ fn placeholder_manifest(dir: &Path) -> Manifest { homepage: None, license: None, change_notes: None, + declared_categories: Vec::new(), engine: ide_extension::ENGINE_VERSION.into(), legacy: false, wasm: None, diff --git a/crates/ide-host/src/plugins/providers.rs b/crates/ide-host/src/plugins/providers.rs index d60b806a..b37c12e1 100644 --- a/crates/ide-host/src/plugins/providers.rs +++ b/crates/ide-host/src/plugins/providers.rs @@ -292,6 +292,21 @@ impl PluginHost { .collect() } + /// The edits of an action `plugin_id` returned with `data`, now that the + /// user picked it: a call with the command budget, not the edit path's. + pub fn resolve_code_action( + &self, + plugin_id: &str, + doc: &ProviderDocument, + action: CodeAction, + ) -> ide_core::IdeResult { + let d = Arc::new(doc.clone()); + let timeout = self.limits.timeout; + self.code_call(plugin_id, ide_plugin::Invocation::default(), "resolving the code action", move |rt, _| { + rt.resolve_code_action(&d, &action, timeout) + }) + } + pub fn provide_definition(&self, doc: &ProviderDocument, at: ide_core::Pos) -> Vec { let d = Arc::new(doc.clone()); self.edit_calls(ProviderKind::Definition, &doc.language, move |rt, timeout| { diff --git a/crates/ide-host/src/plugins/remote_host.rs b/crates/ide-host/src/plugins/remote_host.rs index 71be9116..9c869023 100644 --- a/crates/ide-host/src/plugins/remote_host.rs +++ b/crates/ide-host/src/plugins/remote_host.rs @@ -114,6 +114,7 @@ pub fn forwarded(event: &PluginEvent) -> bool { | PluginEvent::ViewChanged { .. } | PluginEvent::ViewMessage { .. } | PluginEvent::OpenTerminal { .. } + | PluginEvent::OpenLocation { .. } ) } @@ -272,7 +273,8 @@ impl RemotePlugins { report.failed.push(format!("{}: invalid plugin id", pkg.id)); continue; } - let consent = Consent { optional: pkg.granted.clone() }; + // The desktop asked the user, change of signer included. + let consent = Consent { optional: pkg.granted.clone(), signer_change: true }; if self.current_digest(&pkg.id).as_deref() == Some(pkg.digest.as_str()) { if let Err(e) = self.follow_grants(&pkg, &consent) { report.failed.push(format!("{}: {e}", pkg.id)); diff --git a/crates/ide-host/src/plugins/theme.rs b/crates/ide-host/src/plugins/theme.rs index e4f38ff5..7b4fd88b 100644 --- a/crates/ide-host/src/plugins/theme.rs +++ b/crates/ide-host/src/plugins/theme.rs @@ -134,7 +134,7 @@ impl ExtensionPoint for ThemesPoint { /// Blanks out `//` and `/* */` comments outside strings, keeping newlines so /// parse errors still point at the right line. -fn strip_json_comments(src: &str) -> String { +pub(super) fn strip_json_comments(src: &str) -> String { let mut out = String::with_capacity(src.len()); let mut chars = src.chars().peekable(); let (mut in_str, mut escaped) = (false, false); diff --git a/crates/ide-host/src/plugins/types.rs b/crates/ide-host/src/plugins/types.rs index c9cac430..bd5e197c 100644 --- a/crates/ide-host/src/plugins/types.rs +++ b/crates/ide-host/src/plugins/types.rs @@ -138,12 +138,24 @@ pub struct PluginsSnapshot { pub ui: Vec, } +/// A newer version of an installed plugin in the marketplace. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct PluginUpdate { + pub id: String, + pub name: String, + pub version: String, +} + /// Backend → UI notifications (Tauri event `ide://plugin`). #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)] #[serde(tag = "type", rename_all = "camelCase")] pub enum PluginEvent { /// Plugins, their status or their contributions changed: re-fetch `plugin_list`. CommandsChanged, + /// The marketplace offers newer versions of installed plugins (announced + /// once per set of versions). + UpdatesAvailable { updates: Vec }, #[serde(rename_all = "camelCase")] Notification { plugin_id: String, level: NotificationLevel, title: String, body: String }, /// A plugin's status bar item was set (`item`) or removed. @@ -180,6 +192,8 @@ pub enum PluginEvent { args: Vec, cwd: ide_core::IpcPath, }, + #[serde(rename_all = "camelCase")] + OpenLocation { plugin_id: String, path: ide_core::IpcPath, line: u32, column: u32 }, /// Plugin languages, or the servers they may start, changed: re-fetch /// `languages` (the host re-detects open documents: [`crate::Host::plugin_languages_changed`]). LanguagesChanged, @@ -212,6 +226,14 @@ impl PluginEditor { } } +/// A text file of a package the UI generated, at its `/`-separated path in the package. +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct PackageFile { + pub path: String, + pub text: String, +} + /// A plugin's status bar item. #[derive(Clone, Debug, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] diff --git a/crates/ide-host/src/plugins/ui_points.rs b/crates/ide-host/src/plugins/ui_points.rs index 3b5e3d14..135ddd7d 100644 --- a/crates/ide-host/src/plugins/ui_points.rs +++ b/crates/ide-host/src/plugins/ui_points.rs @@ -378,9 +378,19 @@ pub(super) fn view(pkg: &Package, decl: toml::Value) -> Result { } let mut out = UiDecl::new(d.id.clone(), d.title.clone(), json(&d)?); out.files = files; + if let Some(icon) = d.icon.as_deref().filter(|i| i.ends_with(".svg")) { + let path = std::path::PathBuf::from(icon); + let svg = super::icons::read_icon(pkg, &path)?; + out.assets.push((view_icon_asset(&d.id), svg)); + out.files.push(path); + } Ok(out) } +pub fn view_icon_asset(id: &str) -> String { + format!("viewIcon/{id}") +} + #[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] struct CustomEditorDecl { diff --git a/crates/ide-host/src/project.rs b/crates/ide-host/src/project.rs index e63e8db8..0c5e9648 100644 --- a/crates/ide-host/src/project.rs +++ b/crates/ide-host/src/project.rs @@ -34,6 +34,8 @@ pub struct Project { pub lsp: Arc, /// Git repositories: the one containing the project root and those nested below it. pub repos: Arc>, + /// Explicitly opened external logs remain read-only and outside project indexing. + pub(crate) external_log_repos: RwLock>>, /// Incrementally maintained syntax trees of the open documents. pub syntax: Arc, /// Tree-sitter structure queries (LSP fallback) over `syntax`. @@ -56,6 +58,9 @@ pub struct Project { pub(crate) watcher: Watcher, /// Watchers of the Project Structure content roots outside the project folder. pub(crate) content_watchers: Mutex>, + /// Watchers of folders outside the project that language servers registered file + /// watchers for (a virtualenv elsewhere, a path dependency). + lsp_watchers: Mutex>, pub(crate) _scratch_watcher: Option, /// A LightEdit window's loose files ([`Host::open_light`]): global settings only, /// nothing indexed, no language servers, repositories or project services. @@ -83,6 +88,7 @@ impl Host { return Ok(current); } self.close_project(window); + self.wait_detached_closes(); let sink: SharedSink = (self.sink_for)(window); // Before anything the project starts: its children get the project's environment. let trusted = Arc::new(AtomicBool::new(self.trust.trusted(&root)?)); @@ -204,6 +210,7 @@ impl Host { cfg.launch_error = Some(e); None }); + ide_sdk::apply::configure_java_compiler(cfg, &data_dir, &effective.inspections); ide_sdk::java::configure_jdtls( cfg, server_root, @@ -218,9 +225,16 @@ impl Host { ide_sdk::jdtls::launch_directly(cfg); } ide_sdk::apply::server_protocol_overrides(cfg, &effective); + // After the overrides: the PHP version they may set decides the storage. + ide_sdk::php::configure_intelephense_storage(cfg, &data_dir); if cfg.name == "jdtls" { let (excluded_poms, build_files) = jdtls_maven_import(root, &effective); ide_sdk::apply::configure_maven_import(cfg, root, &excluded_poms, build_files.as_deref()); + ide_sdk::apply::configure_gradle_import( + cfg, + effective.languages.java.gradle.linked_projects.as_deref(), + build_files.as_deref(), + ); } })); } @@ -302,12 +316,16 @@ impl Host { sink.clone(), Box::new(move |batch| { let changed: Vec = batch.created.iter().chain(&batch.modified).cloned().collect(); - index.update(&changed, &batch.removed); + let (ignored, ignored_gone) = ignored_changes(batch); + index.update(&[&changed[..], &ignored].concat(), &[&batch.removed[..], &ignored_gone].concat()); kotlin_bridge.files_changed(changed.iter().chain(&batch.removed)); lsp.project_files_changed(batch.created.iter().chain(&batch.removed)); - lsp.watched_files_changed(&batch.created, &batch.modified, &batch.removed); + lsp.watched_files_changed(&batch.created, &batch.modified, &batch.removed, &batch.excluded_dirs); + if composer_changed(&root, changed.iter().chain(&batch.removed)) { + lsp.reconfigure_soon(); + } // Atomic saves by external tools surface as `created`. - documents.on_disk_changed(&changed); + documents.on_disk_changed(&[&changed[..], &ignored].concat()); // A framework's root marker (`angular.json`) created or deleted: its // templates gain or lose their mode. Markers in `node_modules` only // count here when the folder isn't excluded (it is by default). @@ -411,7 +429,16 @@ impl Host { let content_watchers = extra_roots .iter() .filter_map(|r| { - let w = content_root_watcher(r, &excludes, &sink, &index, &documents, &history, Some(&kotlin_bridge))?; + let w = content_root_watcher( + r, + &excludes, + &sink, + &index, + &documents, + &history, + &lsp, + Some(&kotlin_bridge), + )?; Some((r.clone(), w)) }) .collect(); @@ -444,6 +471,7 @@ impl Host { index, lsp, repos, + external_log_repos: RwLock::default(), structure: Arc::new(StructureService::new(syntax.clone())), syntax, detected_roots, @@ -458,6 +486,7 @@ impl Host { kotlin_bridge: Some(kotlin_bridge), watcher, content_watchers: Mutex::new(content_watchers), + lsp_watchers: Mutex::new(Vec::new()), _scratch_watcher: scratch_watcher, light: false, }); @@ -467,6 +496,14 @@ impl Host { weak.upgrade().map(|p| p.reference_candidates(q, cancel)).unwrap_or_default() })); } + { + let weak = Arc::downgrade(&project); + project.lsp.set_file_watch_roots_hook(Box::new(move |roots| { + if let Some(p) = weak.upgrade() { + p.watch_for_language_servers(roots); + } + })); + } { let weak = Arc::downgrade(&project); project.lsp.set_project_excludes_hook(Box::new(move || { @@ -489,12 +526,15 @@ impl Host { root: &Path, project_key: &str, ) -> (Arc, Arc, Arc) { - let db = Arc::new(ide_db::DbManager::new( - root.to_path_buf(), - self.data_dir.join("db-passwords.json"), - self.data_dir.join("db-history.json"), - self.ssh.clone(), - )); + let db = Arc::new( + ide_db::DbManager::new( + root.to_path_buf(), + self.data_dir.join("db-passwords.json"), + self.data_dir.join("db-history.json"), + self.ssh.clone(), + ) + .with_global_sources(self.data_dir.clone()), + ); let http_history = Arc::new(ide_http::History::new(self.data_dir.join("http-history").join(project_key))); let agents = Arc::new(ide_agent::AgentManager::new( ide_agent::AgentPaths { @@ -536,6 +576,7 @@ impl Host { return Ok(current); } self.close_project(window); + self.wait_detached_closes(); let sink: SharedSink = (self.sink_for)(window); let settings = Arc::new(SettingsStore::load(&self.config_dir, None)); let effective = settings.effective(); @@ -568,6 +609,7 @@ impl Host { index, lsp, repos: Arc::default(), + external_log_repos: RwLock::default(), structure: Arc::new(StructureService::new(syntax.clone())), syntax, detected_roots: Arc::default(), @@ -582,6 +624,7 @@ impl Host { kotlin_bridge: None, watcher, content_watchers: Mutex::new(Vec::new()), + lsp_watchers: Mutex::new(Vec::new()), _scratch_watcher: None, light: true, root: dir, @@ -664,7 +707,14 @@ fn scan_repositories( build_cache: &Path, ) { let ignored = settings.effective().ignored_names(); - let skip = |p: &Path| p.file_name().is_some_and(|n| ignored.is_ignored_name(&n.to_string_lossy())); + let skip = |p: &Path| { + p.file_name().is_some_and(|n| { + let name = n.to_string_lossy(); + ignored.is_ignored_name(&name) + || (ide_workspace::unity::GENERATED_DIRS.contains(&name.as_ref()) + && p.parent().is_some_and(ide_workspace::unity::is_project)) + }) + }; let mut roots = vec![root.to_path_buf()]; roots.extend(content_roots(&settings.effective(), root, build_cache)); match Repos::discover_roots(&roots, &skip) { @@ -711,6 +761,51 @@ impl Project { } } + pub fn open_external_log(&self, path: &Path) -> IdeResult { + let root = dunce::canonicalize(path)?; + let repo = Repo::discover(&root)?.ok_or_else(|| IdeError::unavailable("folder is not a Git worktree"))?; + let root = repo.root().to_path_buf(); + repo.set_execution_allowed(false); + let mut external = self.external_log_repos.write(); + if !external.iter().any(|r| r.root() == root) { + external.push(Arc::new(repo)); + } + Ok(root) + } + + /// Log read operations may also use an explicitly opened external worktree. + /// Mutations continue to use `with_repo_at` and cannot reach these repositories. + pub fn with_log_repo_at(&self, at: Option<&Path>, f: impl FnOnce(&Repo) -> IdeResult) -> IdeResult { + if let Some(path) = at + && self.repos().at_root(path).is_none() + { + let repo = self + .external_log_repos + .read() + .iter() + .find(|r| r.root() == path) + .cloned() + .ok_or_else(|| IdeError::unavailable("external Git log is not open"))?; + return f(&repo); + } + self.with_repo_at(at, f) + } + + pub fn with_log_repo_of(&self, path: &Path, f: impl FnOnce(&Repo) -> IdeResult) -> IdeResult { + if self.repos().for_path(path).is_some() { + return self.with_repo_of(path, f); + } + let repo = self + .external_log_repos + .read() + .iter() + .filter(|r| path.starts_with(r.root())) + .max_by_key(|r| r.root().components().count()) + .cloned() + .ok_or_else(|| IdeError::unavailable("path is not in an open Git log"))?; + f(&repo) + } + /// Runs `f` with the repository `path` (a file or folder) belongs to. pub fn with_repo_of(&self, path: &Path, f: impl FnOnce(&Repo) -> IdeResult) -> IdeResult { match self.repos().for_path(path) { @@ -786,8 +881,14 @@ impl Project { /// Effective source / test / resource / excluded roots (detected + Project Structure marks). pub fn roots(&self) -> ProjectRoots { - let detected = with_gradle_roots(&self.root, &self.build_cache_dir(), &self.detected_roots.read()); - linked_roots(&self.root, &self.settings.effective(), &detected) + let settings = self.settings.effective(); + let detected = with_gradle_roots( + &self.root, + &self.build_cache_dir(), + &settings.languages.java.gradle, + &self.detected_roots.read(), + ); + linked_roots(&self.root, &settings, &detected) } /// Where the Maven / Gradle tool windows cache what the build tools reported. @@ -816,7 +917,8 @@ impl Project { /// Project files of `q.languages` under `q.root` containing `q.word` as a whole /// word (unsaved buffers included; excluded roots, ignored names and - /// `.gitignore`d files skipped): the files a references search opens on its server. + /// `.gitignore`d folders skipped, while `.gitignore`d files elsewhere count, as + /// in IDEA's project scope): the files a references search opens on its server. pub fn reference_candidates( &self, q: &ide_lsp::CandidateQuery, @@ -894,6 +996,7 @@ impl Project { &self.index, &self.documents, &self.history, + &self.lsp, self.kotlin_bridge.as_ref(), ) { watchers.push((r, w)); @@ -925,7 +1028,7 @@ impl Project { return encoding_rs::Encoding::for_label(loaded.encoding.as_bytes()) .ok_or_else(|| ide_core::IdeError::invalid("unknown file encoding")); } - let bytes = self.with_repo_of(path, |r| r.file_bytes_at(path, rev))?.unwrap_or_default(); + let bytes = self.with_log_repo_of(path, |r| r.file_bytes_at(path, rev))?.unwrap_or_default(); Ok(ide_vfs::detect_encoding(&bytes, fallback)) } @@ -948,6 +1051,11 @@ const HISTORY_BATCH_LIMIT: usize = 500; /// Local History for changes made outside the IDE (other tools, VCS, deletions). /// Contents equal to the last snapshot (e.g. the IDE's own saves) are skipped. +/// The batch's ignored files (`.env`), split into those still on disk and those removed. +fn ignored_changes(batch: &ide_vfs::FsBatch) -> (Vec, Vec) { + batch.ignored.iter().cloned().partition(|p| p.symlink_metadata().is_ok()) +} + fn record_external_changes(history: &LocalHistory, changed: &[PathBuf], removed: &[PathBuf]) { for path in changed.iter().take(HISTORY_BATCH_LIMIT) { if let Err(e) = history.record_file(path, "External change", false) { @@ -981,7 +1089,7 @@ fn content_roots(settings: &ide_core::settings::Settings, root: &Path, build_cac } let mut out: Vec = Vec::new(); let user = settings.project_structure.content_roots.iter().map(|r| PathBuf::from(r.trim())); - for p in user.chain(ide_build::gradle_external_roots(root, build_cache)) { + for p in user.chain(ide_build::gradle_external_roots(root, build_cache, &settings.languages.java.gradle)) { if p.is_absolute() && p.is_dir() && !p.starts_with(root) && !out.contains(&p) { out.push(p); } @@ -991,6 +1099,31 @@ fn content_roots(settings: &ide_core::settings::Settings, root: &Path, build_cac /// Watches a content root outside the project: the index, open documents and /// Local History follow its changes like the project folder's. +impl Project { + /// Watches `roots` (outside the project) for the language servers that asked to. + fn watch_for_language_servers(&self, roots: Vec) { + let mut watchers = self.lsp_watchers.lock(); + watchers.retain(|(r, _)| roots.contains(r)); + for root in roots { + if watchers.iter().any(|(r, _)| *r == root) || !root.is_dir() { + continue; + } + let lsp = Arc::downgrade(&self.lsp); + let on_change = Box::new(move |batch: &ide_vfs::FsBatch| { + if let Some(lsp) = lsp.upgrade() { + lsp.watched_files_changed(&batch.created, &batch.modified, &batch.removed, &batch.excluded_dirs); + } + }); + // Not project files: nothing else hears of them. + match Watcher::start(&root, Vec::new(), Arc::new(ide_core::NullSink), on_change) { + Ok(w) => watchers.push((root, w)), + Err(e) => tracing::warn!("language server watcher {}: {e}", root.display()), + } + } + } +} + +#[allow(clippy::too_many_arguments)] fn content_root_watcher( dir: &Path, excludes: &[String], @@ -998,21 +1131,28 @@ fn content_root_watcher( index: &Arc, documents: &Arc, history: &Arc, + lsp: &Arc, kotlin_bridge: Option<&Arc>, ) -> Option { let kotlin_bridge = kotlin_bridge.cloned(); - let (index, documents, history) = (index.clone(), documents.clone(), history.clone()); + let (index, documents, history, lsp) = (index.clone(), documents.clone(), history.clone(), lsp.clone()); + let root = dir.to_path_buf(); Watcher::start( dir, excludes.to_vec(), sink.clone(), Box::new(move |batch| { let changed: Vec = batch.created.iter().chain(&batch.modified).cloned().collect(); - index.update(&changed, &batch.removed); + let (ignored, ignored_gone) = ignored_changes(batch); + index.update(&[&changed[..], &ignored].concat(), &[&batch.removed[..], &ignored_gone].concat()); + // A workspace member's PHP language level follows its own composer.json. + if composer_changed(&root, changed.iter().chain(&batch.removed)) { + lsp.reconfigure_soon(); + } if let Some(bridge) = &kotlin_bridge { bridge.files_changed(changed.iter().chain(&batch.removed)); } - documents.on_disk_changed(&changed); + documents.on_disk_changed(&[&changed[..], &ignored].concat()); record_external_changes(&history, &changed, &batch.removed); }), ) @@ -1020,6 +1160,13 @@ fn content_root_watcher( .ok() } +/// Whether the `composer.json` of `root` is among `paths`: the PHP language level follows it +/// (`ide_sdk::php`). +fn composer_changed<'a>(root: &Path, mut paths: impl Iterator) -> bool { + let composer = root.join("composer.json"); + paths.any(|p| *p == composer) +} + /// Where agent tasks create their worktrees: next to the project, `/.worktrees`, /// so they sit on the project's drive and outside its tree (not indexed, watched or shown in /// its git status); `fallback` for a project at a filesystem root. @@ -1046,31 +1193,93 @@ pub(crate) fn jdtls_maven_import( settings: &ide_core::settings::Settings, ) -> (Vec, Option>) { let maven = &settings.languages.java.maven; - if maven.unlinked_projects.is_empty() { + let gradle = &settings.languages.java.gradle; + if maven.unlinked_projects.is_empty() && gradle.linked_projects.is_none() { return (Vec::new(), None); } let model = ide_build::maven::load(root); let unlinked = ide_build::maven::UnlinkedProjects::from_model(&model, maven); let excluded = unlinked.poms().map(PathBuf::from).collect(); - let build_files = unlinked.import_poms(&model).map(|poms| { - let gradle = ide_build::gradle::load(root).and_then(|g| { + let poms = unlinked.import_poms(&model).or_else(|| { + gradle.linked_projects.as_ref().map(|_| { + model + .projects + .iter() + .filter(|p| !p.aggregated && !unlinked.contains(&p.pom)) + .map(|p| p.pom.clone()) + .collect() + }) + }); + let build_files = poms.map(|poms| { + let gradle = ide_build::gradle::load_linked(root, gradle).into_iter().filter_map(|g| { let dir = root.join(&g.dir); let names = ide_build::gradle::SETTINGS_FILES.iter().chain(ide_build::gradle::BUILD_FILES); names.map(|n| dir.join(n)).find(|f| f.is_file()) }); - poms.iter().map(|pom| root.join(pom)).chain(gradle).map(|f| ide_lsp::path_to_uri_string(&f)).collect() + let eclipse = if settings.languages.java.gradle.linked_projects.is_some() { + independent_eclipse_projects(root) + } else { + Vec::new() + }; + poms.iter() + .map(|pom| root.join(pom)) + .chain(gradle) + .chain(eclipse) + .map(|f| ide_lsp::path_to_uri_string(&f)) + .collect() }); (excluded, build_files) } +/// Explicit Gradle imports must retain Eclipse projects without re-importing generated build metadata. +fn independent_eclipse_projects(root: &Path) -> Vec { + let managed = |dir: &Path| { + std::iter::once("pom.xml") + .chain(ide_build::gradle::SETTINGS_FILES.iter().copied()) + .chain(ide_build::gradle::BUILD_FILES.iter().copied()) + .any(|name| dir.join(name).is_file()) + }; + if managed(root) { + return Vec::new(); + } + let walker = ignore::WalkBuilder::new(root) + .standard_filters(false) + .filter_entry(move |entry| { + if !entry.file_type().is_some_and(|kind| kind.is_dir()) { + return false; + } + if entry.depth() > 0 + && (entry.file_name().to_string_lossy().starts_with('.') + || matches!( + entry.file_name().to_str(), + Some("node_modules" | "target" | "build" | "dist" | "out" | "bin" | "vendor") + ) + || entry.path().parent().is_some_and(|parent| parent.join(".project").is_file())) + { + return false; + } + !managed(entry.path()) + }) + .build(); + let mut projects: Vec<_> = + walker.flatten().map(|entry| entry.path().join(".project")).filter(|path| path.is_file()).collect(); + projects.sort(); + projects +} + /// The detected roots with the last Gradle sync's source sets first: Gradle evaluated what /// static detection guesses. Folders outside the project keep their absolute path, which /// `normalize` turns into the same form the UI compares (`Users/me/svc/src/main/java`). -fn with_gradle_roots(root: &Path, build_cache: &Path, detected: &[SourceFolder]) -> Vec { +fn with_gradle_roots( + root: &Path, + build_cache: &Path, + settings: &ide_core::settings::GradleSettings, + detected: &[SourceFolder], +) -> Vec { use ide_build::gradle::ReportedRoot; use ide_workspace::roots::RootKind; let mut out: Vec = Vec::new(); - for (dir, kind) in ide_build::gradle_source_roots(build_cache) { + for (dir, kind) in ide_build::gradle_source_roots(root, build_cache, settings) { if !dir.is_dir() { continue; } @@ -1240,6 +1449,37 @@ pub fn apply_git_executable(settings: &ide_core::settings::Settings) { #[cfg(test)] mod maven_linkage_tests { + #[test] + fn explicit_gradle_links_preserve_independent_eclipse_projects() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + for dir in + ["linked", "unlinked", "eclipse", "nested/eclipse", "unlinked/subproject", "maven", "build/generated"] + { + std::fs::create_dir_all(root.join(dir)).unwrap(); + std::fs::write(root.join(dir).join(".project"), "").unwrap(); + } + for dir in ["linked", "unlinked"] { + std::fs::write(root.join(dir).join("settings.gradle"), "").unwrap(); + } + std::fs::write(root.join("maven/pom.xml"), "").unwrap(); + let mut settings = ide_core::settings::Settings::default(); + settings.languages.java.gradle.linked_projects = Some(vec!["linked".into()]); + let (_, files) = super::jdtls_maven_import(root, &settings); + let files = files.unwrap(); + let uri = |path: &str| ide_lsp::path_to_uri_string(&root.join(path)); + assert!(files.contains(&uri("linked/settings.gradle"))); + assert!(files.contains(&uri("maven/pom.xml"))); + assert!(files.contains(&uri("eclipse/.project"))); + assert!(files.contains(&uri("nested/eclipse/.project"))); + assert!(!files.iter().any(|path| path.contains("unlinked") || path.contains("build/generated"))); + assert!(!files.contains(&uri("linked/.project"))); + assert!(!files.contains(&uri("maven/.project"))); + settings.languages.java.gradle.linked_projects = Some(vec![]); + let (_, files) = super::jdtls_maven_import(root, &settings); + assert_eq!(files.unwrap(), [uri("maven/pom.xml"), uri("eclipse/.project"), uri("nested/eclipse/.project")]); + } + #[test] fn jdtls_imports_the_linked_modules_and_the_gradle_build_once_a_reactor_module_is_unlinked() { let tmp = tempfile::tempdir().unwrap(); @@ -1337,7 +1577,8 @@ mod gradle_roots_tests { std::fs::write(cache.join("gradle.json"), json.to_string()).unwrap(); let guessed = SourceFolder { path: "src/it/java".into(), kind: RootKind::Source, detected_from: None }; let other = SourceFolder { path: "web".into(), kind: RootKind::Source, detected_from: None }; - let out = with_gradle_roots(&root, &cache, &[guessed, other.clone()]); + std::fs::write(root.join("build.gradle"), "").unwrap(); + let out = with_gradle_roots(&root, &cache, &Default::default(), &[guessed, other.clone()]); let abs = |p: PathBuf| ide_workspace::roots::normalize(&s(p)); let got: Vec<(String, RootKind)> = out.iter().map(|f| (f.path.clone(), f.kind)).collect(); assert_eq!( @@ -1436,3 +1677,21 @@ mod agent_worktree_tests { assert_eq!(agent_worktrees_dir(Path::new("/"), fallback), fallback()); } } + +#[cfg(test)] +mod composer_tests { + use super::composer_changed; + use std::path::{Path, PathBuf}; + + #[test] + fn only_the_roots_own_composer_json_counts() { + let root = Path::new("/work/app"); + let changed = |paths: &[&str]| { + let paths: Vec = paths.iter().map(|p| root.join(p)).collect(); + composer_changed(root, paths.iter()) + }; + assert!(changed(&["src/a.php", "composer.json"])); + assert!(!changed(&["composer.lock", "vendor/acme/lib/composer.json", "packages/b/composer.json"])); + assert!(!changed(&[])); + } +} diff --git a/crates/ide-host/src/proxy.rs b/crates/ide-host/src/proxy.rs index 1ce2ed02..6644d250 100644 --- a/crates/ide-host/src/proxy.rs +++ b/crates/ide-host/src/proxy.rs @@ -1,6 +1,7 @@ //! System Settings › HTTP Proxy on the host: the global settings made current for //! connections and child processes, the password kept in the saved passwords -//! (owner-only `proxy-password.json`) instead of `settings.toml`. +//! (the OS credential store) instead of `settings.toml`, or for the session +//! when there is no credential store. use crate::Host; use ide_core::IdeResult; @@ -12,7 +13,9 @@ impl Host { /// `settings` with the saved proxy password filled in, as the Settings dialog edits it. pub fn with_proxy_password(&self, mut settings: Settings) -> Settings { // A password typed into settings.toml by hand counts too, unless one was saved. - if let Some(password) = self.proxy_password.get(PASSWORD_KEY) { + let session = || self.proxy_session_password.lock().clone(); + // The session copy is the latest write that could not be saved. + if let Some(password) = session().or_else(|| self.proxy_password.get(PASSWORD_KEY)) { settings.proxy.password = password; } settings @@ -40,11 +43,70 @@ impl Host { _ => return Ok(Some(value)), }; if let Some(password) = password { - match password.as_str().filter(|p| !p.is_empty()) { - Some(p) => self.proxy_password.set(PASSWORD_KEY, p)?, + let password = password.as_str().filter(|p| !p.is_empty()); + let mut session = self.proxy_session_password.lock(); + *session = None; + match password { + Some(p) => { + if let Err(e) = self.proxy_password.set(PASSWORD_KEY, p) { + tracing::warn!("the HTTP proxy password is kept for this session only: {}", e.message); + *session = Some(p.to_owned()); + } + } None => self.proxy_password.delete(PASSWORD_KEY), } } Ok((key != "proxy.password").then_some(value)) } } + +#[cfg(test)] +mod tests { + use super::*; + use ide_ssh::secrets::{SecretFile, StoreError, StoreResult, Vault}; + use std::sync::Arc; + + /// A machine without a credential store (a headless Linux host). + struct NoStore; + + impl Vault for NoStore { + fn available(&self) -> bool { + false + } + fn get(&self, _: &str, _: &str) -> StoreResult { + Err(StoreError::NoDefaultStore) + } + fn set(&self, _: &str, _: &str, _: &str) -> StoreResult<()> { + Err(StoreError::NoDefaultStore) + } + fn delete(&self, _: &str, _: &str) -> StoreResult<()> { + Err(StoreError::NoDefaultStore) + } + } + + #[tokio::test] + async fn without_a_credential_store_the_password_lasts_the_session() { + let tmp = tempfile::tempdir().unwrap(); + let mut host = Host::new(crate::HostConfig { + config_dir: tmp.path().join("config"), + data_dir: tmp.path().join("data"), + local_data_dir: tmp.path().join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(ide_core::NullSink)), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + let path = tmp.path().join("data/proxy-password.json"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, r#"{"httpProxy": "old"}"#).unwrap(); + host.proxy_password = SecretFile::with_vault(path, |v| serde_json::from_value(v).ok(), Arc::new(NoStore)); + + let left = host.take_proxy_password("proxy.password", serde_json::json!("s3cret")).unwrap(); + assert_eq!(left, None); + assert_eq!(host.with_proxy_password(Settings::default()).proxy.password, "s3cret"); + host.take_proxy_password("proxy.password", serde_json::json!("")).unwrap(); + assert_eq!(host.with_proxy_password(Settings::default()).proxy.password, ""); + } +} diff --git a/crates/ide-host/src/rpc.rs b/crates/ide-host/src/rpc.rs index 034c40fb..fafe557a 100644 --- a/crates/ide-host/src/rpc.rs +++ b/crates/ide-host/src/rpc.rs @@ -188,7 +188,7 @@ impl Dispatcher { ops::fs::create_file_bytes(&c.host, c.arg("path")?, c.arg("data")?).await }) .add("fs_create_dir", |c| async move { ide_vfs::create_dir(&c.arg::("path")?) }) - .add("fs_rename", |c| async move { ide_vfs::rename(&c.arg::("from")?, &c.arg::("to")?) }) + .add("fs_rename", |c| async move { ops::fs::rename(&c.host, c.arg("from")?, c.arg("to")?).await }) .add("fs_set_writable", |c| async move { ops::fs::set_writable(&c.host, &c.window, c.arg("path")?, c.arg("writable")?) }) diff --git a/crates/ide-host/src/shell_env.rs b/crates/ide-host/src/shell_env.rs index 6d2aa62c..d9f7e3b1 100644 --- a/crates/ide-host/src/shell_env.rs +++ b/crates/ide-host/src/shell_env.rs @@ -177,6 +177,10 @@ pub(crate) fn read_login_env(shell: &str, timeout: std::time::Duration) -> Optio } } } + // Nor what the last read found that this process lacks: children get it, the shell must not. + for (k, _) in ide_core::process::shell_env() { + cmd.env_remove(k); + } ide_core::process::prepare(&mut cmd); let mut child = cmd .args(["-ilc", &format!("printf '{MARK}'; /usr/bin/env -0; printf '{MARK}'")]) @@ -257,6 +261,14 @@ mod tests { std::fs::write(&rc, "export OPENIDEA_TEST_RC='two words'\n").unwrap(); let second = read_login_env(&shell, Duration::from_secs(10)).unwrap(); assert_eq!(get(&second, "OPENIDEA_TEST_RC").as_deref(), Some("two words")); + + // What a read found goes to children, not into the next read: an rc file that no + // longer exports a variable drops it. + ide_core::process::set_shell_env(second); + std::fs::write(&rc, "export OPENIDEA_TEST_SEEN=\"${OPENIDEA_TEST_RC:-unset}\"\n").unwrap(); + let third = read_login_env(&shell, Duration::from_secs(10)).unwrap(); + assert_eq!(get(&third, "OPENIDEA_TEST_RC"), None); + assert_eq!(get(&third, "OPENIDEA_TEST_SEEN").as_deref(), Some("unset")); } #[test] diff --git a/crates/ide-host/src/trust.rs b/crates/ide-host/src/trust.rs index ea9b9604..2ddd6859 100644 --- a/crates/ide-host/src/trust.rs +++ b/crates/ide-host/src/trust.rs @@ -222,10 +222,12 @@ pub fn restricted_command_allowed(cmd: &str) -> bool { | "project_current" | "project_roots" | "index_status" + | "index_file_extensions" | "jvm_file_presentations" | "deploy_coverage" | "deploy_test_connection" | "find_files" + | "find_dirs" | "find_symbols" | "mark_recent" | "remove_recent" @@ -274,6 +276,15 @@ pub fn restricted_command_allowed(cmd: &str) -> bool { | "plugin_set_theme_frame" | "plugin_catalog" | "plugin_assets" + | "plugin_market" + | "plugin_market_url" + | "plugin_set_market_url" + | "plugin_market_download" + | "plugin_unpack_package" + | "plugin_stage_package" + | "plugin_preview_install" + | "plugin_read_intellij_theme" + | "plugin_installed_intellij_themes" | "notebook_open" | "notebook_get" | "notebook_edit" @@ -295,7 +306,16 @@ pub fn restricted_command_allowed(cmd: &str) -> bool { | "vcs_line_hunks" | "vcs_file_at" | "vcs_blame" + | "vcs_block_history" | "vcs_log" + | "vcs_log_refs" + | "vcs_log_users" + | "vcs_log_details" + | "vcs_log_details_batch" + | "vcs_log_index" + | "vcs_revision_children" + | "vcs_revision_files" + | "vcs_open_external_log" | "vcs_commit_files" | "vcs_commits_changes" | "vcs_branches" @@ -319,12 +339,31 @@ pub fn restricted_command_allowed(cmd: &str) -> bool { ) || cmd.starts_with("plugin:") } +/// Commands that change which plugin code runs: allowed where no project is open (the +/// welcome screen), otherwise only in a trusted project. +pub fn projectless_command_allowed(cmd: &str) -> bool { + matches!( + cmd, + "plugin_install_dir" + | "plugin_uninstall" + | "plugin_set_enabled" + | "plugin_grant" + | "plugin_set_capability" + | "plugin_roll_back" + | "plugin_reload" + ) +} + impl crate::Host { pub fn authorize_command(&self, window: &str, command: &str) -> IdeResult<()> { if restricted_command_allowed(command) { return Ok(()); } - self.project(window)?.require_trust() + match self.project(window) { + Ok(project) => project.require_trust(), + Err(_) if projectless_command_allowed(command) => Ok(()), + Err(error) => Err(error), + } } } diff --git a/crates/ide-host/tests/close_project.rs b/crates/ide-host/tests/close_project.rs index e0b07cd1..3da5acbd 100644 --- a/crates/ide-host/tests/close_project.rs +++ b/crates/ide-host/tests/close_project.rs @@ -5,6 +5,7 @@ use ide_core::NullSink; use ide_host::{Host, HostConfig, ops}; use std::sync::Arc; +use std::time::{Duration, Instant}; fn options(root: &std::path::Path) -> ide_terminal::TerminalOptions { ide_terminal::TerminalOptions { @@ -52,3 +53,53 @@ fn closing_a_project_closes_its_terminals_and_keeps_other_windows() { host.terminals.kill_key("other"); }); } + +/// The desktop closes a project on the event-loop thread while runtime workers may be +/// blocked waiting for that thread (IPC responses, window queries): closing must not +/// wait for workers that need the runtime, and exit still waits for them once free. +#[test] +fn detached_close_returns_while_runtime_workers_wait_for_the_closing_thread() { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap().join("project"); + std::fs::create_dir_all(&root).unwrap(); + let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(1).enable_all().build().unwrap(); + let _guard = runtime.enter(); + let host = Arc::new(Host::new(HostConfig { + config_dir: tmp.path().join("config"), + data_dir: tmp.path().join("data"), + local_data_dir: tmp.path().join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |w| w, + runtime: runtime.handle().clone(), + project_environment: false, + })); + host.trust.decide(&root, true, false).unwrap(); + host.open_project("main", &root).unwrap(); + + // Occupy the only runtime worker until "the event loop" answers. + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + runtime.spawn(async move { + started_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + started_rx.recv().unwrap(); + + let (closed_tx, closed_rx) = std::sync::mpsc::channel(); + let closer = host.clone(); + std::thread::spawn(move || { + closer.close_project_detached("main"); + let _ = closed_tx.send(()); + }); + let returned = closed_rx.recv_timeout(Duration::from_secs(5)); + release_tx.send(()).unwrap(); + assert!(returned.is_ok(), "closing waited for a runtime worker that waits for the closing thread"); + assert!(host.project("main").is_err(), "the window no longer has the project"); + host.open_project("main", &root).expect("the window opens a project once the close has finished"); + + let started = Instant::now(); + host.shutdown_notebooks_on_exit(); + assert!(started.elapsed() < Duration::from_secs(5), "exit joined the finished close instead of timing out"); +} diff --git a/crates/ide-host/tests/distribution.rs b/crates/ide-host/tests/distribution.rs index 540073a5..38002f36 100644 --- a/crates/ide-host/tests/distribution.rs +++ b/crates/ide-host/tests/distribution.rs @@ -131,25 +131,44 @@ fn installs_from_before_versioning_keep_working_and_move_on_update() { /// `hello` as `acme.hello` at `version`, packed and signed with `key`. fn signed_hello(dir: &Path, version: &str, key: &PublisherKey, host: &PluginHost) -> (PathBuf, IndexEntry) { - let src = variant(&repo().join("crates/ide-plugin/tests/fixtures/hello"), dir, &format!("src-{version}"), |t| { - t.replace("id = \"hello\"", "id = \"acme.hello\"") - .replace("version = \"0.3.0\"", &format!("version = \"{version}\"")) - }); + signed(dir, "acme", version, key, host, |t| t.to_owned()) +} + +/// `hello` as `.hello` at `version`, its manifest changed by +/// `edit`, packed and signed with `key`. +fn signed( + dir: &Path, + publisher: &str, + version: &str, + key: &PublisherKey, + host: &PluginHost, + edit: impl Fn(&str) -> String, +) -> (PathBuf, IndexEntry) { + let id = format!("{publisher}.hello"); + let src = + variant(&repo().join("crates/ide-plugin/tests/fixtures/hello"), dir, &format!("src-{id}-{version}"), |t| { + edit( + &t.replace("id = \"hello\"", &format!("id = \"{id}\"")) + .replace("version = \"0.3.0\"", &format!("version = \"{version}\"")), + ) + }); let files = host.package_files_of(&src).unwrap(); let digest = package::package_digest(&src, &files).unwrap(); - let sig = key.sign("acme", "acme.hello", version, &digest); - let out = dir.join(format!("acme.hello-{version}.oiplugin")); + let sig = key.sign(publisher, &id, version, &digest); + let out = dir.join(format!("{id}-{version}.oiplugin")); package::pack(&src, &files, &out, Some(&sig)).unwrap(); let entry = IndexEntry { - id: "acme.hello".into(), + id: id.clone(), name: "Hello".into(), version: version.into(), description: String::new(), authors: vec![], - publisher: "acme".into(), + publisher: publisher.into(), download: out.file_name().unwrap().to_string_lossy().into_owned(), digest, capabilities: vec![], + engine: "^1.0".into(), + categories: vec!["tools".into()], }; (out, entry) } @@ -166,16 +185,39 @@ fn write_index( publishers: vec![Publisher { id: "acme".into(), name: "ACME".into(), - keys: vec![KeyEntry { key: key.public_base64(), revoked: key_revoked }], + keys: vec![KeyEntry { revoked: key_revoked, ..KeyEntry::new(key.public_base64()) }], }], plugins, revoked, + moved: None, }; let path = dir.join("index.json"); std::fs::write(&path, serde_json::to_string(&index).unwrap()).unwrap(); path.to_string_lossy().into_owned() } +/// First run: nothing cached yet, so listing the marketplace reads its index +/// instead of showing an empty one; later listings use the index read. +#[test] +fn the_first_marketplace_listing_reads_the_index() { + let f = fixture(); + assert!(f.host.market_listing(false).unwrap().is_empty(), "no marketplace configured"); + + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let key = PublisherKey::from_seed([3; 32]); + let (_, entry) = signed_hello(&market, "1.0.0", &key, &f.host); + let url = write_index(&market, &key, vec![entry], vec![], false); + f.host.set_market_url(Some(url.clone())); + + let listed = f.host.market_listing(false).unwrap(); + assert_eq!(listed.iter().map(|p| p.entry.id.as_str()).collect::>(), ["acme.hello"]); + + std::fs::remove_file(&url).unwrap(); + assert_eq!(f.host.market_listing(false).unwrap().len(), 1, "the index read is kept"); + assert!(f.host.market_listing(true).is_err(), "refresh reads the index again"); +} + #[test] fn signed_marketplace_plugins_install_and_revocations_disable_them() { let f = fixture(); @@ -260,3 +302,210 @@ fn package_files_install_from_disk_unsigned() { let info = f.host.install_from_dir(&dir, &Consent::all()).unwrap(); assert_eq!((info.status, info.publisher), (PluginStatus::Inactive, None)); } + +#[test] +fn the_marketplace_offers_the_newest_version_this_ide_can_load() { + let f = fixture(); + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let key = PublisherKey::from_seed([3; 32]); + let (_, old) = signed_hello(&market, "1.0.0", &key, &f.host); + let (_, new) = signed_hello(&market, "2.0.0", &key, &f.host); + let next_major = ide_extension::ENGINE_VERSION.split('.').next().unwrap().parse::().unwrap() + 1; + let new = IndexEntry { engine: format!("^{next_major}.0"), ..new }; + f.host.set_market_url(Some(write_index(&market, &key, vec![old.clone(), new.clone()], vec![], false))); + + let listed = f.host.refresh_market().unwrap(); + assert_eq!((listed.len(), listed[0].entry.version.as_str(), listed[0].compatible), (1, "1.0.0", true)); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + assert_eq!(f.host.preview_install(&dir).unwrap().version, "1.0.0"); + + // Only a version for a newer IDE: shown as such, not downloaded. + write_index(&market, &key, vec![new], vec![], false); + let listed = f.host.refresh_market().unwrap(); + assert_eq!((listed[0].entry.version.as_str(), listed[0].compatible), ("2.0.0", false)); + let err = f.host.download_market_plugin("acme.hello").unwrap_err(); + assert!(err.message.contains("update OpenIDEA"), "{}", err.message); +} + +#[test] +fn a_moved_marketplace_is_read_where_it_moved_to() { + let f = fixture(); + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let key = PublisherKey::from_seed([3; 32]); + let (_, entry) = signed_hello(&market, "1.0.0", &key, &f.host); + write_index(&market, &key, vec![entry], vec![], false); + // The old address points at the new one; downloads resolve against where the index is. + let old = f.work.join("old"); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("index.json"), r#"{"format": 1, "moved": "../market/index.json"}"#).unwrap(); + f.host.set_market_url(Some(old.join("index.json").to_string_lossy().into_owned())); + + let listed = f.host.refresh_market().unwrap(); + assert_eq!(listed[0].entry.id, "acme.hello"); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + assert_eq!(f.host.preview_install(&dir).unwrap().version, "1.0.0"); + + // A loop gives up. + std::fs::write(old.join("index.json"), r#"{"format": 1, "moved": "index.json"}"#).unwrap(); + let err = f.host.refresh_market().unwrap_err(); + assert!(err.message.contains("moved more than"), "{}", err.message); +} + +/// `dir/index.json` listing `publishers` and `plugins`; its path. +fn index_of(dir: &Path, publishers: Vec, plugins: Vec) -> String { + let index = Index { format: package::INDEX_FORMAT, publishers, plugins, ..Index::default() }; + let path = dir.join("index.json"); + std::fs::write(&path, serde_json::to_string(&index).unwrap()).unwrap(); + path.to_string_lossy().into_owned() +} + +fn publisher(id: &str, keys: Vec) -> Publisher { + Publisher { id: id.into(), name: id.into(), keys } +} + +fn updates_announced(events: &Events) -> Vec> { + events + .lock() + .iter() + .filter_map(|e| match e { + PluginEvent::UpdatesAvailable { updates } => { + Some(updates.iter().map(|u| format!("{} {}", u.id, u.version)).collect()) + } + _ => None, + }) + .collect() +} + +/// What the install page sends for an update it doesn't need to show: the grants the installed version has. +fn as_granted(preview: &InstallPreview) -> Consent { + let optional = preview.capabilities.iter().filter(|c| c.optional && c.granted).map(|c| c.id.clone()).collect(); + Consent { optional, signer_change: false } +} + +#[test] +fn updates_keep_their_signer_and_need_no_consent_when_nothing_is_new() { + let f = fixture(); + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let (k1, k2, k3) = + (PublisherKey::from_seed([1; 32]), PublisherKey::from_seed([2; 32]), PublisherKey::from_seed([3; 32])); + let same = |t: &str| t.to_owned(); + let (_, v1) = signed(&market, "acme", "1.0.0", &k1, &f.host, same); + f.host.set_market_url(Some(index_of( + &market, + vec![publisher("acme", vec![KeyEntry::new(k1.public_base64())])], + vec![v1.clone()], + ))); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + let first = f.host.preview_install(&dir).unwrap(); + assert!(first.needs_consent && !first.signer_changed, "a first install is confirmed"); + f.host.install_from_dir(&dir, &Consent::default()).unwrap(); + assert!(updates_announced(&f.events).is_empty()); + + // 1.1.0, same key and needs: announced once, installed with the grants it has. + let (_, v11) = signed(&market, "acme", "1.1.0", &k1, &f.host, same); + let keys1 = vec![KeyEntry::new(k1.public_base64())]; + index_of(&market, vec![publisher("acme", keys1.clone())], vec![v1.clone(), v11.clone()]); + f.host.refresh_market().unwrap(); + f.host.refresh_market().unwrap(); + assert_eq!(updates_announced(&f.events), [["acme.hello 1.1.0"]], "once per set of versions"); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + let preview = f.host.preview_install(&dir).unwrap(); + assert_eq!(preview.replaces.as_deref(), Some("1.0.0")); + assert!(!preview.needs_consent && !preview.signer_changed); + assert!(preview.capabilities.iter().all(|c| c.granted), "shows what the installed version was granted"); + f.host.install_from_dir(&dir, &as_granted(&preview)).unwrap(); + assert!(f.host.market_plugins().iter().all(|p| !p.is_update())); + + // 1.2.0 asks for something new: the user is asked. + let (_, v12) = signed(&market, "acme", "1.2.0", &k1, &f.host, |t| { + t.replace( + "required = [\"editor.read\", \"editor.write\"]", + "required = [\"editor.read\", \"editor.write\"]\noptional = [\"net:example.com\"]", + ) + }); + index_of(&market, vec![publisher("acme", keys1.clone())], vec![v12.clone()]); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + let preview = f.host.preview_install(&dir).unwrap(); + assert!(preview.needs_consent && !preview.signer_changed); + f.host.install_from_dir(&dir, &Consent::default()).unwrap(); + + // 1.3.0 signed with a key the installed one didn't endorse: another signer, the user's call. + let (_, v13) = signed(&market, "acme", "1.3.0", &k2, &f.host, same); + let keys12 = vec![KeyEntry::new(k1.public_base64()), KeyEntry::new(k2.public_base64())]; + index_of(&market, vec![publisher("acme", keys12)], vec![v13]); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + let preview = f.host.preview_install(&dir).unwrap(); + assert!(preview.signer_changed && preview.needs_consent); + let err = f.host.install_from_dir(&dir, &as_granted(&preview)).unwrap_err(); + assert!(err.message.contains("another key"), "{}", err.message); + assert_eq!(f.host.info_of("acme.hello").unwrap().version, "1.2.0", "nothing changed"); + f.host.install_from_dir(&dir, &Consent { signer_change: true, ..as_granted(&preview) }).unwrap(); + + // 1.4.0 signed with k3, which k2 endorsed: a rotation the installed signer vouched for. + let (_, v14) = signed(&market, "acme", "1.4.0", &k3, &f.host, same); + let keys = vec![ + KeyEntry { endorsement: Some(k2.endorse("acme", &k3.public_base64())), ..KeyEntry::new(k3.public_base64()) }, + KeyEntry::new(k2.public_base64()), + ]; + index_of(&market, vec![publisher("acme", keys)], vec![v14]); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + let preview = f.host.preview_install(&dir).unwrap(); + assert!(!preview.signer_changed && !preview.needs_consent); + f.host.install_from_dir(&dir, &as_granted(&preview)).unwrap(); + assert_eq!(f.host.info_of("acme.hello").unwrap().version, "1.4.0"); +} + +#[test] +fn an_index_in_a_newer_format_still_revokes() { + let f = fixture(); + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let key = PublisherKey::from_seed([3; 32]); + let (_, entry) = signed_hello(&market, "1.0.0", &key, &f.host); + f.host.set_market_url(Some(write_index(&market, &key, vec![entry], vec![], false))); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("acme.hello").unwrap(); + f.host.install_from_dir(&dir, &Consent::all()).unwrap(); + + let newer = format!( + r#"{{"format": {}, "plugins": "a shape this IDE can't read", + "revoked": [{{"id": "acme.hello", "reason": "steals tokens"}}]}}"#, + package::INDEX_FORMAT + 1 + ); + std::fs::write(market.join("index.json"), newer).unwrap(); + let err = f.host.refresh_market().unwrap_err(); + assert!(err.message.contains("newer OpenIDEA"), "{}", err.message); + let info = f.host.info_of("acme.hello").unwrap(); + assert_eq!((info.status, info.error.as_deref()), (PluginStatus::Failed, Some("revoked: steals tokens"))); + assert!(f.host.market_plugins().is_empty()); +} + +#[test] +fn official_plugins_need_the_key_this_ide_was_built_with() { + let f = fixture(); + let market = f.work.join("market"); + std::fs::create_dir_all(&market).unwrap(); + let (root, intruder) = (PublisherKey::from_seed([1; 32]), PublisherKey::from_seed([6; 32])); + f.host.set_official_keys(vec![root.public_base64()]); + // The index lists the intruder's key for the official publisher. + let (_, forged) = signed(&market, OFFICIAL_PUBLISHER, "1.0.0", &intruder, &f.host, |t| t.to_owned()); + let keys = vec![KeyEntry::new(intruder.public_base64())]; + f.host.set_market_url(Some(index_of(&market, vec![publisher(OFFICIAL_PUBLISHER, keys.clone())], vec![forged]))); + f.host.refresh_market().unwrap(); + let err = f.host.download_market_plugin("openidea.hello").unwrap_err(); + assert!(err.message.contains("trusted publisher"), "{}", err.message); + + // Signed with the built-in key it installs, whatever keys the index lists. + let (_, genuine) = signed(&market, OFFICIAL_PUBLISHER, "1.0.1", &root, &f.host, |t| t.to_owned()); + index_of(&market, vec![publisher(OFFICIAL_PUBLISHER, keys)], vec![genuine]); + f.host.refresh_market().unwrap(); + let dir = f.host.download_market_plugin("openidea.hello").unwrap(); + assert_eq!(f.host.preview_install(&dir).unwrap().publisher.as_deref(), Some(OFFICIAL_PUBLISHER)); +} diff --git a/crates/ide-host/tests/external_log.rs b/crates/ide-host/tests/external_log.rs new file mode 100644 index 00000000..83d58f04 --- /dev/null +++ b/crates/ide-host/tests/external_log.rs @@ -0,0 +1,115 @@ +//! Explicit external logs allow history reads without adding a writable project root. +use ide_core::NullSink; +use ide_host::{Host, HostConfig, ops::vcs}; +use std::{path::Path, process::Command, sync::Arc}; + +fn git(root: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .current_dir(root) + .args(["-c", "user.name=Test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false"]) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn external_history_is_readable_and_never_becomes_a_write_target() { + let temp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(temp.path()).unwrap(); + let root = base.join("project"); + let external = base.join("external"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&external).unwrap(); + git(&external, &["init", "-b", "main"]); + let deleted = external.join("deleted.txt"); + std::fs::write(&deleted, "historical content\n").unwrap(); + git(&external, &["add", "."]); + git(&external, &["commit", "-m", "initial"]); + let first = git(&external, &["rev-parse", "HEAD"]); + git(&external, &["rm", "deleted.txt"]); + git(&external, &["commit", "-m", "delete"]); + let head = git(&external, &["rev-parse", "HEAD"]); + let host = Host::new(HostConfig { + config_dir: base.join("config"), + data_dir: base.join("data"), + local_data_dir: base.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + // A standalone editor needs no project, indexer, or writable VCS root. + vcs::vcs_open_external_log(&host, "standalone", external.clone()).await.unwrap(); + let standalone = vcs::vcs_log(&host, "standalone", Some(external.clone()), Default::default()).await.unwrap(); + assert_eq!(standalone[0].id, head); + let details = vcs::vcs_log_details_batch( + &host, + "standalone", + Some(external.clone()), + vec![head.clone(), first.clone()], + ide_core::RequestId(987), + ) + .await + .unwrap(); + assert!(details.iter().all(|d| d.signature == "N")); + assert_eq!( + vcs::vcs_revision_children(&host, "standalone", Some(external.clone()), first.clone(), String::new()) + .await + .unwrap(), + vec!["deleted.txt"] + ); + assert!(host.project("standalone").is_err()); + host.trust.decide(&external, false, false).unwrap(); + let restricted = host.open_project("restricted", &external).unwrap(); + assert!(!restricted.is_trusted()); + vcs::vcs_open_external_log(&host, "restricted", external.clone()).await.unwrap(); + let restricted_log = vcs::vcs_log( + &host, + "restricted", + Some(external.clone()), + ide_vcs::LogQuery { request: Some(ide_core::RequestId(988)), summary_only: true, ..Default::default() }, + ) + .await + .unwrap(); + assert_eq!(restricted_log[0].id, head); + assert_eq!( + vcs::vcs_log_details_batch( + &host, + "restricted", + Some(external.clone()), + vec![head.clone()], + ide_core::RequestId(989) + ) + .await + .unwrap()[0] + .signature, + "N" + ); + host.close_project("restricted"); + let payload = + serde_json::to_vec(&serde_json::json!({ "repo": external, "revision": first, "path": "deleted.txt" })).unwrap(); + let hex = payload.iter().map(|b| format!("{b:02x}")).collect::(); + let document = + vcs::revision_virtual_document(&host, "standalone", &format!("git-revision:{hex}/deleted.txt")).await.unwrap(); + assert_eq!(document.text, "historical content\n"); + host.trust.decide(&root, true, false).unwrap(); + let project = host.open_project("main", &root).unwrap(); + assert!(vcs::vcs_log(&host, "main", Some(external.clone()), Default::default()).await.is_err()); + vcs::vcs_open_external_log(&host, "main", external.clone()).await.unwrap(); + let log = vcs::vcs_log(&host, "main", Some(external.clone()), Default::default()).await.unwrap(); + assert_eq!(log.iter().map(|c| c.id.as_str()).collect::>(), vec![head.as_str(), first.as_str()]); + assert_eq!( + vcs::vcs_file_at(&host, "main", deleted.clone(), Some(first.clone())).await.unwrap().as_deref(), + Some("historical content\n") + ); + assert!(project.repos().at_root(&external).is_none()); + assert!(vcs::vcs_checkout(&host, "main", Some(external.clone()), first.clone(), false, None).await.is_err()); + assert!(vcs::vcs_apply_patch(&host, "main", Some(external.clone()), String::new()).await.is_err()); + assert_eq!(git(&external, &["rev-parse", "HEAD"]), head); + assert!(!deleted.exists()); + host.close_project("main"); +} diff --git a/crates/ide-host/tests/gradle_linkage.rs b/crates/ide-host/tests/gradle_linkage.rs new file mode 100644 index 00000000..d53c41f4 --- /dev/null +++ b/crates/ide-host/tests/gradle_linkage.rs @@ -0,0 +1,49 @@ +use ide_core::NullSink; +use ide_host::{Host, HostConfig, ops}; +use std::sync::Arc; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn links_are_validated_persisted_and_unlinked_without_removing_files() { + let temp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(temp.path()).unwrap(); + let root = base.join("workspace"); + for dir in [root.join("a"), root.join("b"), base.join("external")] { + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("settings.gradle"), "rootProject.name = 'same-name'").unwrap(); + } + let config = base.join("config"); + let host = Host::new(HostConfig { + config_dir: config.clone(), + data_dir: base.join("data"), + local_data_dir: base.join("local"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |window| window, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + host.open_project("main", &root).unwrap(); + assert!(ops::build::gradle_link_projects(&host, "main", vec!["b".into()]).await.is_err()); + host.set_project_trust(&root, true, false).await.unwrap(); + let model = ops::build::gradle_link_projects(&host, "main", vec!["b/settings.gradle".into()]).await.unwrap(); + assert_eq!(model.gradle.iter().map(|g| g.dir.as_str()).collect::>(), ["a", "b"]); + let saved = + || ide_workspace::SettingsStore::load(&config, Some(&root)).effective().languages.java.gradle.linked_projects; + assert_eq!(saved(), Some(vec!["a".into(), "b".into()])); + assert!(ops::build::gradle_link_projects(&host, "main", vec!["missing".into()]).await.is_err()); + assert_eq!(saved(), Some(vec!["a".into(), "b".into()])); + let outside = base.join("external").to_string_lossy().into_owned(); + let model = ops::build::gradle_link_projects(&host, "main", vec![outside.clone()]).await.unwrap(); + assert_eq!(model.gradle.len(), 3); + let model = ops::build::gradle_unlink_projects(&host, "main", vec!["a".into(), "b".into(), outside]).await.unwrap(); + assert!(model.gradle.is_empty()); + assert_eq!(saved(), Some(vec![])); + for dir in [root.join("a"), root.join("b"), base.join("external")] { + assert!(dir.join("settings.gradle").is_file()); + } + let model = ops::build::gradle_link_projects(&host, "main", vec!["b".into()]).await.unwrap(); + assert_eq!(model.gradle.len(), 1); + assert_eq!(model.gradle[0].dir, "b"); + host.shutdown().await; +} diff --git a/crates/ide-host/tests/language_smoke.rs b/crates/ide-host/tests/language_smoke.rs new file mode 100644 index 00000000..8e32b39e --- /dev/null +++ b/crates/ide-host/tests/language_smoke.rs @@ -0,0 +1,994 @@ +//! Language smoke tests: the everyday code intelligence of each language — completion, go to +//! definition, Find Usages, hover, diagnostics with a quick fix, an intention, rename, +//! formatting, parameter info, the Structure view, auto-import, Optimize Imports and moving a +//! file — on its `fixtures/` project, through the `ops` the app's commands call and the +//! language server the app starts by default. +//! +//! `#[ignore]`d because they need the servers installed. CI's `Language smoke` job installs +//! pinned versions and runs one language per job: +//! `cargo test -p ide-host --test language_smoke -- --ignored --exact `. + +use ide_core::{ChangeSet, DocId, IdeEvent, Pos, Range, RequestId, TextChange}; +use ide_host::{Host, HostConfig, ops}; +use ide_lsp::types::{FormatOptions, GotoKind}; +use std::collections::BTreeSet; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const WINDOW: &str = "main"; + +/// `word` on the first line of `file` that contains `line`. +#[derive(Clone, Copy, Debug)] +struct At { + file: &'static str, + line: &'static str, + word: &'static str, +} + +const fn at(file: &'static str, line: &'static str, word: &'static str) -> At { + At { file, line, word } +} + +/// A line typed after the first line of `file` containing `after`. +#[derive(Clone, Copy, Debug)] +struct Typed { + file: &'static str, + after: &'static str, + text: &'static str, +} + +const fn typed(file: &'static str, after: &'static str, text: &'static str) -> Typed { + Typed { file, after, text } +} + +/// One language's checklist. Every check names what it looks for, so a failure says which +/// operation broke. +struct Checks { + fixture: &'static str, + /// Files written into the project copy before it opens. + files: &'static [(&'static str, &'static str)], + /// How long the first answer may take (project import and indexing). + warm_up: Duration, + /// Go to definition from the first spot lands on the second. + definition: (At, At), + /// Find Usages of the symbol lists usages in these files. + references: (At, &'static [&'static str]), + /// Hover shows this text. + hover: (At, &'static str), + /// Completion at the end of the typed line offers this item. + completion: (Typed, &'static str), + /// The typed line is reported with this diagnostic; the quick fix with this title makes the + /// file contain the last text. + quick_fix: QuickFix, + /// With the word selected, this intention is offered with an edit or command. + intention: (At, &'static str), + /// Formatting turns the typed line into the expected one. + format: (Typed, &'static str), + indent: Indent, + /// Renaming the symbol to the new name edits these files. + rename: (At, &'static str, &'static [&'static str]), + /// Parameter info in a call being typed: a signature naming this parameter. + signature: (Typed, &'static str), + /// The Structure view of a file: these symbols, at any depth. + outline: (&'static str, &'static [&'static str]), + /// Completing a name the file doesn't import yet also imports it. + auto_import: AutoImport, + /// Optimize Imports, with an import typed after a line. + organize_imports: Option<(Typed, Imports)>, + /// Moving a file updates its references: (file, new path, a file that then contains this). + file_move: Option<(&'static str, &'static str, &'static str, &'static str)>, +} + +#[derive(Clone, Copy, Debug)] +struct AutoImport { + /// The line typed, ending in the start of the name. + typed: Typed, + /// The completion item. + label: &'static str, + /// What the item's import adds to the file. + result: &'static str, +} + +/// What Optimize Imports does with the typed import. +#[derive(Clone, Copy, Debug)] +enum Imports { + /// Removes it (unused). + Removed, + /// Sorts it before this line. + Before(&'static str), +} + +#[derive(Clone, Copy, Debug)] +struct QuickFix { + typed: Typed, + diagnostic: &'static str, + title: &'static str, + result: &'static str, +} + +/// The editor's indentation, sent with a format request. +#[derive(Clone, Copy, Debug)] +enum Indent { + Spaces(u32), + Tabs, +} + +type Events = Arc>>; + +struct Smoke { + language: &'static str, + host: Arc, + root: PathBuf, + events: Events, + warm_up: Duration, + _tmp: tempfile::TempDir, +} + +fn fixture_dir(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../fixtures").join(name) +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name(); + if matches!(name.to_str(), Some("node_modules" | "target" | "build" | ".gradle" | ".venv")) { + continue; + } + let target = to.join(&name); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +} + +fn utf16_len(s: &str) -> u32 { + s.encode_utf16().count() as u32 +} + +/// The 0-based line index and text of the first line containing `needle`. +fn find_line<'a>(text: &'a str, needle: &str, what: &str) -> (u32, &'a str) { + text.lines() + .enumerate() + .find(|(_, l)| l.contains(needle)) + .map(|(i, l)| (i as u32, l)) + .unwrap_or_else(|| panic!("{what}: no line containing {needle:?}")) +} + +/// The byte offset of `word` in `line`, not as part of a longer identifier (`parse` in `parser::parse`). +fn find_word(line: &str, word: &str) -> Option { + let ident = |c: char| c.is_alphanumeric() || c == '_'; + line.match_indices(word).map(|(i, _)| i).find(|&i| { + let before = line[..i].chars().next_back(); + let after = line[i + word.len()..].chars().next(); + !(word.starts_with(ident) && before.is_some_and(ident)) && !(word.ends_with(ident) && after.is_some_and(ident)) + }) +} + +impl Smoke { + async fn start(language: &'static str, checks: &Checks) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap().join(checks.fixture); + copy_dir(&fixture_dir(checks.fixture), &root); + for (path, text) in checks.files { + let path = root.join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); + } + let events: Events = Arc::default(); + let app = tmp.path().join("app"); + let host = Arc::new(Host::new(HostConfig { + config_dir: app.join("config"), + data_dir: app.join("data"), + local_data_dir: app.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new({ + let events = events.clone(); + move |_| { + let events = events.clone(); + Arc::new(move |e: IdeEvent| events.lock().unwrap().push(e)) + } + }), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + })); + host.trust.decide(&root, true, false).unwrap(); + host.open_project(WINDOW, &root).unwrap(); + Smoke { language, host, root, events, warm_up: checks.warm_up, _tmp: tmp } + } + + fn fail(&self, check: &str, message: impl std::fmt::Display) -> ! { + let servers = ops::lsp::lsp_servers(&self.host, WINDOW).unwrap_or_default(); + let servers: Vec = servers.iter().map(|s| format!("{} ({:?})", s.name, s.status)).collect(); + panic!("{} — {check}: {message}\nservers: {servers:?}", self.language) + } + + /// Retries `f` until it succeeds (servers answer emptily while importing and indexing); on + /// timeout, fails with its last answer. + async fn retry>>( + &self, + check: &str, + timeout: Duration, + mut f: impl FnMut() -> F, + ) -> T { + let deadline = Instant::now() + timeout; + loop { + let last = match f().await { + Ok(v) => return v, + Err(last) => last, + }; + if Instant::now() > deadline { + self.fail(check, format_args!("timed out after {timeout:?}; last answer: {last:?}")); + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + } + + async fn doc(&self, file: &str) -> (DocId, String) { + let info = ops::doc::open(&self.host, WINDOW, self.root.join(file)).await.unwrap(); + let info = ops::doc::snapshot(&self.host, WINDOW, info.id).await.unwrap(); + (info.id, info.text) + } + + async fn locate(&self, at: At) -> (DocId, Pos) { + let (doc, text) = self.doc(at.file).await; + let (line, content) = find_line(&text, at.line, at.file); + let col = find_word(content, at.word).unwrap_or_else(|| panic!("{at:?}: word not on the line")); + (doc, Pos::new(line, utf16_len(&content[..col]))) + } + + async fn word_range(&self, at: At) -> (DocId, Range) { + let (doc, start) = self.locate(at).await; + (doc, Range::new(start, Pos::new(start.line, start.col + utf16_len(at.word)))) + } + + /// Types a new line; returns the position at its end and the text before it. + async fn type_line(&self, t: Typed) -> (DocId, Pos, String) { + let (doc, text) = self.doc(t.file).await; + let (line, content) = find_line(&text, t.after, t.file); + let line_start: usize = text.lines().take(line as usize).map(|l| l.len() + 1).sum(); + let end = utf16_len(&text[..line_start + content.len()]); + self.replace(doc, end, end, &format!("\n{}", t.text)).await; + let col = utf16_len(t.text); + (doc, Pos::new(line + 1, col), text) + } + + async fn replace(&self, doc: DocId, from: u32, to: u32, insert: &str) { + let version = ops::doc::snapshot(&self.host, WINDOW, doc).await.unwrap().version; + let changes = ChangeSet { changes: vec![TextChange { from, to, insert: insert.into() }] }; + ops::doc::edit(&self.host, WINDOW, doc, version, changes).unwrap(); + } + + async fn restore(&self, doc: DocId, text: &str) { + let current = ops::doc::snapshot(&self.host, WINDOW, doc).await.unwrap().text; + self.replace(doc, 0, utf16_len(¤t), text).await; + } + + fn path(&self, file: &str) -> PathBuf { + self.root.join(file) + } + + fn files>(&self, paths: impl IntoIterator) -> BTreeSet { + paths + .into_iter() + .filter_map(|p| { + let p = p.as_ref().strip_prefix(&self.root).ok()?; + Some(p.to_string_lossy().replace('\\', "/")) + }) + .collect() + } + + async fn definition(&self, from: At, to: At) { + let (doc, pos) = self.locate(from).await; + let (_, target) = self.locate(to).await; + let want = self.path(to.file); + self.retry("go to definition", self.warm_up, || async { + let found = ops::lsp::lsp_goto(&self.host, WINDOW, doc, pos, GotoKind::Definition).await; + match &found { + Ok(locs) if locs.iter().any(|l| *l.path == *want && l.range.start.line == target.line) => Ok(()), + _ => Err(found), + } + }) + .await; + } + + async fn references(&self, of: At, files: &[&str]) { + let (doc, pos) = self.locate(of).await; + let want: BTreeSet = files.iter().map(|f| f.to_string()).collect(); + self.retry("find usages", self.warm_up, || async { + let found = ops::lsp::lsp_references(&self.host, WINDOW, RequestId(1), doc, pos, true, true).await; + match &found { + Ok(r) if want.is_subset(&self.files(r.locations.iter().map(|l| l.path.clone()))) => Ok(()), + _ => Err(found.map(|r| self.files(r.locations.into_iter().map(|l| l.path)))), + } + }) + .await; + } + + async fn hover(&self, over: At, shows: &str) { + let (doc, pos) = self.locate(over).await; + self.retry("hover", self.warm_up, || async { + let found = ops::lsp::lsp_hover(&self.host, WINDOW, RequestId(2), doc, pos).await; + match &found { + Ok(Some(h)) if h.contents.value.contains(shows) => Ok(()), + _ => Err(found.map(|h| h.map(|h| h.contents.value))), + } + }) + .await; + } + + async fn completion(&self, line: Typed, offers: &str) { + let (doc, pos, before) = self.type_line(line).await; + let trigger = line.text.ends_with('.').then(|| ".".to_string()); + self.retry("completion", self.warm_up, || async { + let found = ops::lsp::lsp_completion(&self.host, WINDOW, RequestId(3), doc, pos, trigger.clone()).await; + match &found { + Ok(list) + if list.items.iter().any(|i| i.label == offers || i.filter_text.as_deref() == Some(offers)) => + { + Ok(()) + } + _ => Err(found.map(|l| l.items.into_iter().take(30).map(|i| i.label).collect::>())), + } + }) + .await; + self.restore(doc, &before).await; + } + + async fn diagnostic(&self, file: &str, message: &str) -> Range { + let path = self.path(file); + self.retry("diagnostics", self.warm_up, || async { + let events = self.events.lock().unwrap(); + let latest = events.iter().rev().filter_map(|e| match e { + IdeEvent::Diagnostics { path: p, items, source } if **p == *path => Some((source, items)), + _ => None, + }); + let mut seen = Vec::new(); + for (source, items) in latest { + if let Some(d) = items.iter().find(|d| d.message.contains(message)) { + return Ok(d.range); + } + seen.push(format!("{source}: {:?}", items.iter().map(|d| &d.message).collect::>())); + } + Err(seen.into_iter().take(4).collect::>()) + }) + .await + } + + /// The code actions over `range` (all kinds, as the ⌥⏎ popup asks), the one titled `title`. + async fn action(&self, check: &str, doc: DocId, range: Range, title: &str) -> ide_lsp::types::CodeAction { + let action = self + .retry(check, self.warm_up, || async { + let found = ops::lsp::lsp_code_actions(&self.host, WINDOW, doc, range, vec![]).await; + match found.as_ref().ok().and_then(|a| a.iter().find(|a| a.title.contains(title))) { + Some(action) => Ok(action.clone()), + None => Err(found.map(|a| a.into_iter().map(|a| a.title).collect::>())), + } + }) + .await; + if action.edit.is_some() || action.command.is_some() { + return action; + } + ops::lsp::lsp_resolve_code_action(&self.host, WINDOW, doc, action) + .await + .unwrap_or_else(|e| self.fail(check, format_args!("resolving {title:?}: {e}"))) + } + + async fn quick_fix(&self, fix: QuickFix) { + let (doc, _, before) = self.type_line(fix.typed).await; + assert!(!before.contains(fix.result), "{fix:?}: the file already contains the result"); + let range = self.diagnostic(fix.typed.file, fix.diagnostic).await; + let action = self.action("quick fix", doc, range, fix.title).await; + ops::lsp::lsp_execute_code_action(&self.host, WINDOW, RequestId(4), doc, action, vec![]) + .await + .unwrap_or_else(|e| self.fail("quick fix", format_args!("applying {:?}: {e}", fix.title))); + self.retry("quick fix", Duration::from_secs(10), || async { + let text = self.doc(fix.typed.file).await.1; + if text.contains(fix.result) { Ok(()) } else { Err(text) } + }) + .await; + self.restore(doc, &before).await; + } + + async fn intention(&self, on: At, title: &str) { + let (doc, range) = self.word_range(on).await; + let action = self.action("intention", doc, range, title).await; + let edits = action.edit.as_ref().is_some_and(|e| !e.files.is_empty()); + if !edits && action.command.is_none() { + self.fail("intention", format_args!("{title:?} has neither an edit nor a command: {action:?}")); + } + } + + async fn format(&self, line: Typed, indent: Indent, formatted: &str) { + let (doc, _, before) = self.type_line(line).await; + assert!(!before.lines().any(|l| l == formatted), "{line:?}: the file already has {formatted:?}"); + let (tab_size, insert_spaces) = match indent { + Indent::Spaces(n) => (n, true), + Indent::Tabs => (4, false), + }; + let options = FormatOptions { tab_size, insert_spaces, on_save: false, tool: None }; + let text = self + .retry("format", self.warm_up, || async { + let edits = ops::lsp::lsp_format(&self.host, WINDOW, doc, None, options.clone()).await; + let text = self.doc(line.file).await.1; + let mut document = ide_text::Document::new(&text); + match &edits { + Ok(edits) if !edits.is_empty() && document.apply_edits(edits).is_ok() => Ok(document.text()), + _ => Err(edits), + } + }) + .await; + if !text.lines().any(|l| l == formatted) { + self.fail("format", format_args!("no line {formatted:?} in:\n{text}")); + } + self.restore(doc, &before).await; + } + + async fn rename(&self, symbol: At, to: &str, files: &[&str]) { + let (doc, pos) = self.locate(symbol).await; + let prepared = ops::lsp::lsp_prepare_rename(&self.host, WINDOW, doc, pos).await; + if !matches!(prepared, Ok(Some(_))) { + self.fail("rename", format_args!("prepare rename: {prepared:?}")); + } + let renamed = ops::lsp::lsp_rename(&self.host, WINDOW, RequestId(5), doc, pos, to.into()) + .await + .unwrap_or_else(|e| self.fail("rename", e)); + let edited = self.files(renamed.edit.files.iter().filter_map(|f| match f { + ide_core::FileEdit::Edit { path, .. } => Some(path.clone()), + _ => None, + })); + let want: BTreeSet = files.iter().map(|f| f.to_string()).collect(); + if !want.is_subset(&edited) { + self.fail("rename", format_args!("edits {edited:?}, expected at least {want:?}: {:?}", renamed.edit)); + } + let mut before = Vec::new(); + for file in files { + before.push(self.doc(file).await.1.matches(to).count()); + } + ops::lsp::apply_workspace_edit(&self.host, WINDOW, renamed.edit) + .await + .unwrap_or_else(|e| self.fail("rename", format_args!("applying the edit: {e}"))); + for (file, before) in files.iter().zip(before) { + if self.doc(file).await.1.matches(to).count() <= before { + self.fail("rename", format_args!("{file} has no new {to:?} after the rename")); + } + } + } + + async fn signature(&self, call: Typed, parameter: &str) { + let (doc, pos, before) = self.type_line(call).await; + self.retry("parameter info", self.warm_up, || async { + let found = ops::lsp::lsp_signature_help(&self.host, WINDOW, doc, pos).await; + match &found { + Ok(Some(h)) if h.signatures.iter().any(|s| s.label.contains(parameter)) => Ok(()), + _ => Err(found.map(|h| h.map(|h| h.signatures.into_iter().map(|s| s.label).collect::>()))), + } + }) + .await; + self.restore(doc, &before).await; + } + + async fn outline(&self, file: &str, symbols: &[&str]) { + let (doc, _) = self.doc(file).await; + fn names(items: &[ide_core::OutlineItem], out: &mut BTreeSet) { + for item in items { + out.insert(item.name.clone()); + names(&item.children, out); + } + } + self.retry("structure", self.warm_up, || async { + let found = ops::doc::doc_outline(&self.host, WINDOW, doc).await; + let mut shown = BTreeSet::new(); + if let Ok(items) = &found { + names(items, &mut shown); + } + if symbols.iter().all(|s| shown.contains(*s)) { Ok(()) } else { Err(shown) } + }) + .await; + } + + async fn auto_import(&self, a: AutoImport) { + let (doc, pos, before) = self.type_line(a.typed).await; + assert!(!before.contains(a.result), "{a:?}: the file already contains the result"); + self.retry("auto-import completion", self.warm_up, || async { + let list = ops::lsp::lsp_completion(&self.host, WINDOW, RequestId(6), doc, pos, None).await; + let items = match &list { + Ok(l) => l.items.iter().filter(|i| i.label == a.label).cloned().collect::>(), + Err(e) => return Err(format!("{e}")), + }; + let text = self.doc(a.typed.file).await.1; + let mut imports = Vec::new(); + for item in items { + // Servers compute the import when the item is resolved (on selection). + let item = if item.additional_edits.is_empty() { + ops::lsp::lsp_resolve_completion(&self.host, WINDOW, doc, item.clone()).await.unwrap_or(item) + } else { + item + }; + let mut document = ide_text::Document::new(&text); + if document.apply_edits(&item.additional_edits).is_ok() && document.text().contains(a.result) { + return Ok(()); + } + // Or accepting it runs a server command that edits the file (kotlin-lsp). + if let Some(command) = item.command.clone() { + let ran = ops::lsp::lsp_execute_command(&self.host, WINDOW, doc, command).await; + let text = self.doc(a.typed.file).await.1; + if ran.is_ok() && text.contains(a.result) { + return Ok(()); + } + } + imports.push(format!("{:?} {:?}", item.additional_edits, item.command)); + } + let labels: Vec = list.map(|l| l.items.into_iter().take(20).map(|i| i.label).collect()).unwrap(); + Err(format!("{:?} items' imports: {imports:?}; items: {labels:?}", a.label)) + }) + .await; + self.restore(doc, &before).await; + } + + async fn organize_imports(&self, import: Typed, expect: Imports) { + let (doc, _, before) = self.type_line(import).await; + let typed = import.text.trim(); + let organized = |text: &str| match expect { + Imports::Removed => !text.lines().any(|l| l.trim() == typed), + Imports::Before(line) => { + let at = |needle: &str| text.lines().position(|l| l.trim() == needle); + matches!((at(typed), at(line)), (Some(a), Some(b)) if a < b) + } + }; + self.retry("optimize imports", self.warm_up, || async { + let text = self.doc(import.file).await.1; + // The whole file, as Optimize Imports asks (positions past the end fail on jdtls). + let last = text.split('\n').count() as u32 - 1; + let whole = Range::new(Pos::new(0, 0), Pos::new(last, utf16_len(text.rsplit('\n').next().unwrap_or("")))); + let kind = "source.organizeImports".to_string(); + let found = ops::lsp::lsp_code_actions(&self.host, WINDOW, doc, whole, vec![kind.clone()]).await; + let actions: Vec<_> = match &found { + Ok(a) => a.iter().filter(|a| a.kind.as_ref().is_some_and(|k| k.starts_with(&kind))).cloned().collect(), + Err(e) => return Err(format!("{e}")), + }; + let Some(action) = actions.iter().find(|a| a.is_preferred).or(actions.first()).cloned() else { + return Err(format!( + "no {kind} action: {:?}", + found.map(|a| a.into_iter().map(|a| a.title).collect::>()) + )); + }; + ops::lsp::lsp_apply_code_action(&self.host, WINDOW, doc, action).await.map_err(|e| e.to_string())?; + let text = self.doc(import.file).await.1; + if organized(&text) { Ok(()) } else { Err(text) } + }) + .await; + self.restore(doc, &before).await; + } + + async fn file_move(&self, from: &str, to: &str, referencing: &str, result: &str) { + // The servers know the referencing file (opened, as in an editor). + let _ = self.doc(referencing).await; + let moves = vec![ide_workspace::FileMove { from: self.path(from).into(), to: self.path(to).into() }]; + let edit = self + .retry("move file", self.warm_up, || async { + let found = + ops::refactor::refactor_file_move_edits(&self.host, WINDOW, RequestId(7), moves.clone()).await; + let want = self.path(referencing); + match &found { + Ok(e) + if e.edit + .files + .iter() + .any(|f| matches!(f, ide_core::FileEdit::Edit { path, .. } if **path == *want)) => + { + Ok(e.edit.clone()) + } + _ => Err(found), + } + }) + .await; + ops::refactor::refactor_move_files(&self.host, WINDOW, moves, edit, "Move".into()) + .await + .unwrap_or_else(|e| self.fail("move file", e)); + if !self.path(to).is_file() { + self.fail("move file", format_args!("{to} doesn't exist after the move")); + } + let text = self.doc(referencing).await.1; + if !text.contains(result) { + self.fail("move file", format_args!("{referencing} has no {result:?} after the move:\n{text}")); + } + } + + async fn run(&self, c: &Checks) { + self.definition(c.definition.0, c.definition.1).await; + self.references(c.references.0, c.references.1).await; + self.hover(c.hover.0, c.hover.1).await; + self.completion(c.completion.0, c.completion.1).await; + self.quick_fix(c.quick_fix).await; + self.intention(c.intention.0, c.intention.1).await; + self.format(c.format.0, c.indent, c.format.1).await; + self.signature(c.signature.0, c.signature.1).await; + self.outline(c.outline.0, c.outline.1).await; + self.auto_import(c.auto_import).await; + if let Some((import, expect)) = c.organize_imports { + self.organize_imports(import, expect).await; + } + self.rename(c.rename.0, c.rename.1, c.rename.2).await; + // Last: it moves a file the other checks use. + if let Some((from, to, referencing, result)) = c.file_move { + self.file_move(from, to, referencing, result).await; + } + let shutdown = self.host.project(WINDOW).unwrap().lsp.clone(); + let _ = tokio::time::timeout(Duration::from_secs(20), shutdown.shutdown_all()).await; + } +} + +async fn smoke(language: &'static str, checks: Checks) { + let smoke = Smoke::start(language, &checks).await; + smoke.run(&checks).await; +} + +const TYPESCRIPT: Checks = Checks { + fixture: "ts-app", + files: &[], + warm_up: Duration::from_secs(120), + definition: ( + at("src/index.ts", "Total: ${formatMoney(total)}", "formatMoney"), + at("src/money.ts", "export function formatMoney", "formatMoney"), + ), + references: (at("src/money.ts", "export function formatMoney", "formatMoney"), &["src/index.ts", "src/money.ts"]), + hover: (at("src/index.ts", "price: cents(8999)", "cents"), "cents(value: number): Cents"), + completion: (typed("src/index.ts", "const total = cart.total();", " cart."), "applyDiscount"), + quick_fix: QuickFix { + typed: typed("src/index.ts", "const total = cart.total();", " console.log(addMoney(total, total));"), + diagnostic: "Cannot find name 'addMoney'", + title: "import from \"./money.ts\"", + result: "import { addMoney, cents, formatMoney } from \"./money.ts\";", + }, + intention: (at("src/index.ts", "const total = cart.total();", "cart.total()"), "Extract to constant"), + format: ( + typed("src/index.ts", "const total = cart.total();", " const shown={total,count:cart.count}"), + " const shown = { total, count: cart.count }", + ), + indent: Indent::Spaces(2), + rename: ( + // From a usage in another file: the declaration is renamed too, not aliased in the import. + at("src/index.ts", "Total: ${formatMoney(total)}", "formatMoney"), + "formatPrice", + &["src/index.ts", "src/money.ts"], + ), + signature: (typed("src/index.ts", "const total = cart.total();", " cart.add("), "product"), + outline: ("src/cart.ts", &["Cart", "add", "total"]), + auto_import: AutoImport { + typed: typed("src/index.ts", "const total = cart.total();", " addMon"), + label: "addMoney", + result: "addMoney", + }, + organize_imports: Some(( + typed( + "src/index.ts", + "import { cents, formatMoney } from \"./money.ts\";", + "import { readFileSync } from \"node:fs\";", + ), + Imports::Removed, + )), + file_move: Some(("src/money.ts", "src/lib/money.ts", "src/index.ts", "\"./lib/money.ts\"")), +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs vtsls"] +async fn typescript() { + smoke("typescript", TYPESCRIPT).await; +} + +const PYTHON: Checks = Checks { + fixture: "python-app", + // Ruff (lint, quick fixes, formatting) starts in projects configured for it. + files: &[("ruff.toml", "")], + warm_up: Duration::from_secs(120), + definition: ( + at("src/tasks/cli.py", "tasks = TaskList()", "TaskList"), + at("src/tasks/models.py", "class TaskList:", "TaskList"), + ), + references: ( + at("src/tasks/models.py", "class TaskList:", "TaskList"), + &["src/tasks/cli.py", "src/tasks/models.py"], + ), + hover: (at("src/tasks/cli.py", "tasks.add(\"Write release notes\"", "add"), "def add("), + completion: (typed("src/tasks/cli.py", "tasks = build_demo()", " tasks."), "pending"), + quick_fix: QuickFix { + typed: typed("src/tasks/cli.py", "tasks = build_demo()", " print(Task)"), + diagnostic: "\"Task\" is not defined", + title: "from .models import Task", + result: "from .models import Priority, Task, TaskList", + }, + // basedpyright has no refactorings; Ruff's fixes are the Python intentions. + intention: (at("src/tasks/models.py", "import os", "os"), "Remove unused import"), + format: ( + typed("src/tasks/cli.py", "tasks = build_demo()", " shown2 = {'a':1,'b': 2}"), + " shown2 = {\"a\": 1, \"b\": 2}", + ), + indent: Indent::Spaces(4), + rename: ( + at("src/tasks/models.py", "class TaskList:", "TaskList"), + "TaskBoard", + &["src/tasks/cli.py", "src/tasks/models.py"], + ), + signature: (typed("src/tasks/cli.py", "tasks = build_demo()", " tasks.add("), "title"), + outline: ("src/tasks/models.py", &["Priority", "Task", "TaskList", "pending"]), + auto_import: AutoImport { + typed: typed("src/tasks/cli.py", "tasks = build_demo()", " stamp = datetim"), + label: "datetime", + result: "import datetime", + }, + organize_imports: Some(( + typed("src/tasks/models.py", "from enum import IntEnum", "import abc"), + Imports::Before("import os"), + )), + // basedpyright has no `workspace/willRenameFiles`. + file_move: None, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs basedpyright and ruff"] +async fn python() { + smoke("python", PYTHON).await; +} + +const RUST: Checks = Checks { + fixture: "rust-app", + files: &[], + warm_up: Duration::from_secs(180), + definition: ( + at("src/main.rs", "match parser::parse(SAMPLE)", "parse"), + at("src/parser.rs", "pub fn parse(text: &str)", "parse"), + ), + references: (at("src/parser.rs", "pub fn parse(text: &str)", "parse"), &["src/main.rs", "src/parser.rs"]), + hover: (at("src/main.rs", "inventory.take(&Sku", "take"), "pub fn take("), + completion: (typed("src/main.rs", "let mut inventory = Inventory::new();", " inventory."), "low_stock"), + quick_fix: QuickFix { + // rust-analyzer's own analysis, pulled as the file changes (no save, no `cargo check`). + typed: typed( + "src/main.rs", + "let mut inventory = Inventory::new();", + " let _blank = parser::ParseError { line: 1 };", + ), + diagnostic: "missing structure fields", + title: "Fill struct fields", + result: "ParseError { line: 1, message", + }, + intention: (at("src/main.rs", "match parser::parse(SAMPLE)", "parser::parse(SAMPLE)"), "Extract into variable"), + format: ( + typed("src/main.rs", "let mut inventory = Inventory::new();", " let _spaced=( 1,2 );"), + " let _spaced = (1, 2);", + ), + indent: Indent::Spaces(4), + rename: (at("src/parser.rs", "pub fn parse(text: &str)", "parse"), "parse_all", &["src/main.rs", "src/parser.rs"]), + signature: (typed("src/main.rs", "let mut inventory = Inventory::new();", " inventory.take("), "sku"), + outline: ("src/lib.rs", &["Inventory", "Item", "take"]), + auto_import: AutoImport { + typed: typed("src/main.rs", "let mut inventory = Inventory::new();", " let _seen = HashSe"), + label: "HashSet", + result: "use std::collections::HashSet;", + }, + // rust-analyzer has no `source.organizeImports`. + organize_imports: None, + file_move: Some(("src/parser.rs", "src/csv.rs", "src/lib.rs", "pub mod csv;")), +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs rust-analyzer"] +async fn rust() { + smoke("rust", RUST).await; +} + +const GO: Checks = Checks { + fixture: "go-app", + files: &[], + warm_up: Duration::from_secs(180), + definition: ( + at("cmd/server/main.go", "s := store.New()", "New"), + at("internal/store/store.go", "func New() *Store", "New"), + ), + references: ( + at("internal/store/store.go", "func New() *Store", "New"), + &["cmd/server/main.go", "internal/store/store.go"], + ), + hover: ( + at("cmd/server/main.go", "writeJSON(w, s.Search(", "Search"), + "func (s *store.Store) Search(q string) []store.Note", + ), + completion: (typed("cmd/server/main.go", "s := store.New()", "\ts."), "Create"), + quick_fix: QuickFix { + typed: typed("cmd/server/main.go", "s := store.New()", "\tfmt.Println(s)"), + diagnostic: "undefined: fmt", + title: "\"fmt\"", + result: "\t\"fmt\"", + }, + intention: (at("cmd/server/main.go", "s := store.New()", "store.New()"), "Extract"), + format: (typed("cmd/server/main.go", "s := store.New()", "\t_ = []int{1,2}"), "\t_ = []int{1, 2}"), + indent: Indent::Tabs, + rename: ( + at("internal/store/store.go", "func New() *Store", "New"), + "Open", + &["cmd/server/main.go", "internal/store/store.go"], + ), + signature: (typed("cmd/server/main.go", "s := store.New()", "\ts.Create("), "title"), + outline: ("internal/store/store.go", &["Store", "New", "(*Store).Create"]), + auto_import: AutoImport { + typed: typed("cmd/server/main.go", "s := store.New()", "\t_ = strings.ToUpp"), + label: "ToUpper", + result: "\t\"strings\"", + }, + organize_imports: Some((typed("cmd/server/main.go", "\"strconv\"", "\t\"sort\""), Imports::Removed)), + // gopls renames packages, not files: moving a Go file changes no references. + file_move: None, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs gopls"] +async fn go() { + smoke("go", GO).await; +} + +const JAVA: Checks = Checks { + fixture: "java-app", + files: &[], + warm_up: Duration::from_secs(300), + definition: ( + at("src/main/java/com/example/orders/App.java", "book.place(\"alice\", \"Mechanical", "place"), + at("src/main/java/com/example/orders/OrderBook.java", "public Order place(", "place"), + ), + references: ( + at("src/main/java/com/example/orders/OrderBook.java", "public Order place(", "place"), + &["src/main/java/com/example/orders/App.java", "src/test/java/com/example/orders/OrderBookTest.java"], + ), + hover: ( + at("src/main/java/com/example/orders/App.java", "long revenue = book.revenueCents()", "revenueCents"), + "long com.example.orders.OrderBook.revenueCents()", + ), + completion: ( + typed("src/main/java/com/example/orders/App.java", "long revenue = book.revenueCents();", " book."), + "largest", + ), + quick_fix: QuickFix { + typed: typed( + "src/main/java/com/example/orders/App.java", + "long revenue = book.revenueCents();", + " List all = book.byCustomer(\"alice\");", + ), + diagnostic: "List cannot be resolved to a type", + title: "Import 'List' (java.util)", + result: "import java.util.List;", + }, + intention: ( + at("src/main/java/com/example/orders/App.java", "long revenue = book.revenueCents();", "book.revenueCents()"), + "Extract to local variable", + ), + format: ( + typed( + "src/main/java/com/example/orders/App.java", + "long revenue = book.revenueCents();", + " long doubled=revenue*2;", + ), + " long doubled = revenue * 2;", + ), + indent: Indent::Spaces(4), + rename: ( + at("src/main/java/com/example/orders/OrderBook.java", "public Order place(", "place"), + "submit", + &["src/main/java/com/example/orders/App.java", "src/main/java/com/example/orders/OrderBook.java"], + ), + signature: ( + typed( + "src/main/java/com/example/orders/App.java", + "long revenue = book.revenueCents();", + " book.place(", + ), + "customer", + ), + outline: ( + "src/main/java/com/example/orders/OrderBook.java", + &["OrderBook", "place(String, String, int, long)", "revenueCents()"], + ), + auto_import: AutoImport { + typed: typed( + "src/main/java/com/example/orders/App.java", + "long revenue = book.revenueCents();", + " ArrayLis", + ), + label: "ArrayList", + result: "import java.util.ArrayList;", + }, + organize_imports: Some(( + typed("src/main/java/com/example/orders/OrderBook.java", "import java.util.Optional;", "import java.util.Map;"), + Imports::Removed, + )), + file_move: Some(( + "src/main/java/com/example/orders/OrderBook.java", + "src/main/java/com/example/orders/Ledger.java", + "src/main/java/com/example/orders/App.java", + "Ledger book", + )), +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs jdtls and JDK 21"] +async fn java() { + smoke("java", JAVA).await; +} + +const KOTLIN: Checks = Checks { + fixture: "kotlin-gradle", + files: &[], + warm_up: Duration::from_secs(420), + definition: ( + at("src/main/kotlin/com/example/library/App.kt", "val library = Library()", "Library"), + at("src/main/kotlin/com/example/library/Library.kt", "class Library(", "Library"), + ), + references: ( + at("src/main/kotlin/com/example/library/Library.kt", "fun search(query: String)", "search"), + &["src/main/kotlin/com/example/library/App.kt", "src/main/kotlin/com/example/library/Library.kt"], + ), + hover: ( + at("src/main/kotlin/com/example/library/App.kt", "library.search(\"java\")", "search"), + "fun search(query: String): List", + ), + completion: ( + typed("src/main/kotlin/com/example/library/App.kt", "val library = Library()", " library."), + "lend", + ), + quick_fix: QuickFix { + typed: typed( + "src/main/kotlin/com/example/library/App.kt", + "val library = Library()", + " val stamp = LocalDateTime.now()", + ), + diagnostic: "Unresolved reference", + title: "Import", + result: "import java.time.LocalDateTime", + }, + intention: ( + at("src/main/kotlin/com/example/library/Library.kt", "fun giveBack(isbn: String): Boolean", "giveBack"), + "block body", + ), + format: ( + typed("src/main/kotlin/com/example/library/App.kt", "val library = Library()", " val pair=listOf(1,2)"), + " val pair = listOf(1, 2)", + ), + indent: Indent::Spaces(4), + rename: ( + at("src/main/kotlin/com/example/library/Library.kt", "fun search(query: String)", "search"), + "find", + &["src/main/kotlin/com/example/library/App.kt", "src/main/kotlin/com/example/library/Library.kt"], + ), + signature: ( + typed("src/main/kotlin/com/example/library/App.kt", "val library = Library()", " library.lend("), + "isbn", + ), + outline: ("src/main/kotlin/com/example/library/Library.kt", &["Library", "lend", "search"]), + auto_import: AutoImport { + typed: typed( + "src/main/kotlin/com/example/library/App.kt", + "val library = Library()", + " val stamp = LocalDateTi", + ), + label: "LocalDateTime", + result: "import java.time.LocalDateTime", + }, + organize_imports: Some(( + typed("src/main/kotlin/com/example/library/App.kt", "import java.time.LocalDate", "import java.util.UUID"), + Imports::Removed, + )), + // kotlin-lsp has no `workspace/willRenameFiles`. + file_move: None, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "language smoke: needs kotlin-lsp, a JDK and Gradle"] +async fn kotlin() { + smoke("kotlin", KOTLIN).await; +} diff --git a/crates/ide-host/tests/plugins.rs b/crates/ide-host/tests/plugins.rs index b052cd72..299ece22 100644 --- a/crates/ide-host/tests/plugins.rs +++ b/crates/ide-host/tests/plugins.rs @@ -123,6 +123,19 @@ fn misbehave_pkg() -> PathBuf { repo().join("crates/ide-plugin/tests/fixtures/misbehave") } +/// Whether `f` holds within 5 s. A call a slow runner keeps waiting behind +/// another (or behind a new instance starting) holds once it caught up. +fn eventually(mut f: impl FnMut() -> bool) -> bool { + let until = Instant::now() + Duration::from_secs(5); + while !f() { + if Instant::now() > until { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + true +} + /// A theme package with a light and a dark theme, plus licence files, in `dir`. fn write_theme_pair(dir: &Path) { std::fs::create_dir_all(dir.join("t")).unwrap(); @@ -397,11 +410,11 @@ fn crashes_restart_with_backoff_until_the_circuit_opens() { let info = f.host.info_of("misbehave").unwrap(); assert_eq!((info.status, info.stats.activations, info.error), (PluginStatus::Active, 2, None)); - // Three in a row: stopped for the session, with growing waits between. + // Three in a row: stopped for the session, backing off between them (the wait + // doubles: `Limits::backoff`, tested where it is defined; wall-clock waits on CI + // are too noisy to compare). f.take_events(); - let mut waits = Vec::new(); - for _ in 0..3 { - let started = Instant::now(); + for crash in 0..3 { let err = loop { match f.run("misbehave", "panic", None) { Err(e) if e.contains("restarts in") => std::thread::sleep(Duration::from_millis(20)), @@ -409,9 +422,10 @@ fn crashes_restart_with_backoff_until_the_circuit_opens() { } }; assert!(err.contains("crashed"), "{err}"); - waits.push(started.elapsed()); + if crash < 2 { + assert!(f.run("misbehave", "echo", None).unwrap_err().contains("restarts in"), "backing off"); + } } - assert!(waits[2] >= backoff * 2 && waits[2] > waits[1], "{waits:?}"); let info = f.host.info_of("misbehave").unwrap(); assert_eq!(info.status, PluginStatus::Failed); assert!(info.error.as_deref().is_some_and(|e| e.contains("3 times in a row")), "{:?}", info.error); @@ -479,12 +493,15 @@ fn running_calls_can_be_cancelled() { let f = f.clone(); std::thread::spawn(move || f.run("misbehave", "spin", None)) }; - std::thread::sleep(Duration::from_millis(100)); - let started = Instant::now(); - f.host.cancel("misbehave"); + // A cancel before the call runs (the instance may still be starting) does nothing: repeat it. + let until = Instant::now() + Duration::from_secs(5); + while !spinning.is_finished() { + assert!(Instant::now() < until, "the running call wasn't interrupted"); + f.host.cancel("misbehave"); + std::thread::sleep(Duration::from_millis(20)); + } let err = spinning.join().unwrap().unwrap_err(); assert!(err.contains("was cancelled"), "{err}"); - assert!(started.elapsed() < Duration::from_secs(1)); assert_eq!(f.run("misbehave", "echo", Some("x")).unwrap().as_deref(), Some("x"), "no backoff"); let stats = f.host.info_of("misbehave").unwrap().stats; assert_eq!((stats.crashes, stats.activations), (0, 2)); @@ -526,16 +543,22 @@ fn edit_path_calls_have_soft_and_hard_deadlines() { let provider = host.syntax_provider(); let mis = LanguageId::new("mis"); - let tokens = provider.highlights(&mis, "fast").unwrap(); + // The first call starts the instance, which may take longer than the soft deadline. + let mut tokens = None; + assert!(eventually(|| { + tokens = provider.highlights(&mis, "fast"); + tokens.is_some() + })); + let tokens = tokens.unwrap(); assert_eq!((tokens[0].start, tokens[0].end, tokens[0].kind.as_str()), (0, 4, "keyword")); // Slower than the soft deadline: dropped (however late this thread gets // to look), but the instance lives on. assert!(provider.highlights(&mis, "sleep:150").is_none()); - std::thread::sleep(Duration::from_millis(150)); - assert!(provider.highlights(&mis, "fast").is_some()); + assert!(eventually(|| provider.highlights(&mis, "fast").is_some())); let info = host.info_of("misbehave").unwrap(); - assert_eq!((info.stats.slow_calls, info.stats.crashes, info.stats.activations), (1, 0, 1)); + assert!(info.stats.slow_calls >= 1, "{:?}", info.stats); + assert_eq!((info.stats.crashes, info.stats.activations), (0, 1)); // Past the hard deadline: interrupted; the next call gets a new instance. assert!(provider.highlights(&mis, "spin").is_none()); @@ -544,7 +567,8 @@ fn edit_path_calls_have_soft_and_hard_deadlines() { let info = host.info_of("misbehave").unwrap(); assert_eq!(info.stats.crashes, 1, "{:?}", info.error); assert!(info.error.as_deref().is_some_and(|e| e.contains("timed out after 400ms")), "{:?}", info.error); - assert!(provider.highlights(&mis, "fast").is_some()); + // The new instance may take longer to start than the soft deadline on a slow runner. + assert!(eventually(|| provider.highlights(&mis, "fast").is_some())); assert_eq!(host.info_of("misbehave").unwrap().stats.activations, 2); } @@ -553,9 +577,10 @@ fn edit_path_calls_have_soft_and_hard_deadlines() { #[test] fn providers_of_several_plugins_run_concurrently() { let tmp = tempfile::tempdir().unwrap(); + // A budget no runner misses, so only the overlap is measured. let limits = Limits { - edit_soft: Duration::from_millis(300), - edit_hard: Duration::from_secs(2), + edit_soft: Duration::from_secs(5), + edit_hard: Duration::from_secs(10), fuel: u64::MAX, ..Limits::default() }; @@ -584,16 +609,17 @@ fn providers_of_several_plugins_run_concurrently() { text: text.into(), }; let at = ide_core::Pos::new(0, 0); - // Activation is not what is measured. - assert_eq!(host.provide_completion(&doc("fast"), at, None).len(), 2); + // Activation is not what is measured: a first call that starts an instance + // can run past the edit-path budget, and its answer is dropped. + assert!(eventually(|| host.provide_completion(&doc("fast"), at, None).len() == 2)); let started = Instant::now(); - let items = host.provide_completion(&doc("sleep:200"), at, None); + let items = host.provide_completion(&doc("sleep:500"), at, None); let elapsed = started.elapsed(); let mut from: Vec<&str> = items.iter().map(|(p, _)| p.as_str()).collect(); from.sort(); assert_eq!(from, ["slow-a", "slow-b"], "both within the budget"); - assert!(elapsed < Duration::from_millis(350), "one after the other would take 400ms: {elapsed:?}"); + assert!(elapsed < Duration::from_millis(950), "one after the other would take 1000ms: {elapsed:?}"); assert_eq!(host.info_of("slow-b").unwrap().stats.slow_calls, 0); } @@ -613,7 +639,7 @@ fn idle_instances_are_reclaimed_and_schedules_reactivate() { assert_eq!(f.host.instances(), 0); let info = f.host.info_of("misbehave").unwrap(); assert_eq!((info.status, info.stats.reclaims), (PluginStatus::Inactive, 1)); - assert_eq!(f.command_titles("misbehave").len(), 8, "contributions stay"); + assert_eq!(f.command_titles("misbehave").len(), 10, "contributions stay"); assert_eq!(f.run("misbehave", "echo", Some("y")).unwrap().as_deref(), Some("y")); assert_eq!(f.host.info_of("misbehave").unwrap().stats.activations, 2); @@ -1651,22 +1677,21 @@ fn official_ai_completion_cancels_superseded_requests() { }; let at = ide_core::Pos::new(1, 4); let host = Arc::new(f); - // A slow request, superseded 200 ms later by the next keystroke's. + // A slow request, superseded by the next keystroke's once the server has it + // (starting the instance comes first, and takes longer on a slow runner). let first = ide_core::CancellationToken::new(); let slow = { let (host, doc, first) = (host.clone(), doc.clone(), first.clone()); - std::thread::spawn(move || { - let started = Instant::now(); - let r = host.host.provide_inline_completion(&doc, at, &first); - (r, started.elapsed()) - }) + std::thread::spawn(move || (host.host.provide_inline_completion(&doc, at, &first), Instant::now())) }; - std::thread::sleep(Duration::from_millis(200)); + assert!(eventually(|| seen.lock().len() == 1), "the first request reaches the server"); + let cancelled = Instant::now(); first.cancel(); let started = Instant::now(); let second = host.host.provide_inline_completion(&doc, at, &ide_core::CancellationToken::new()); let took = started.elapsed(); - let (old, old_took) = slow.join().unwrap(); + let (old, old_done) = slow.join().unwrap(); + let old_took = old_done.saturating_duration_since(cancelled); assert!(old.is_none(), "the superseded request yields nothing"); assert!(old_took < Duration::from_secs(1), "cancelled, not waited out: {old_took:?}"); assert_eq!(second.map(|(_, s)| s.insert_text).as_deref(), Some("a + b")); @@ -1844,7 +1869,12 @@ fn official_mybatis_navigates_between_interface_and_xml() { assert_eq!((target.path, target.range.start), (xml_path.clone(), ide_core::Pos::new(1, 14))); let xdoc = doc(&xml_path, "xml", xml); - let to_java = f.host.provide_definition(&xdoc, ide_core::Pos::new(1, 16)); + // Reads the interface from disk: on a slow machine that too can run past the budget. + let mut to_java = Vec::new(); + assert!(eventually(|| { + to_java = f.host.provide_definition(&xdoc, ide_core::Pos::new(1, 16)); + !to_java.is_empty() + })); assert_eq!( to_java.iter().map(|l| (l.path.clone(), l.range.start)).collect::>(), [(java_path.clone(), ide_core::Pos::new(3, 9))] @@ -2015,6 +2045,83 @@ fn pinned_programs_fill_views_and_open_terminals() { assert_eq!(args, ["follow", "build", "--lines=5"]); } +#[test] +fn views_open_project_files_at_a_line() { + let f = fixture(Limits::default()); + let info = f.install(&sample("jobs")); + assert_eq!(info.status, PluginStatus::Inactive, "{:?}", info.error); + std::fs::write(f.project.join("jobs.log"), "build\ndeploy\nfailed\n").unwrap(); + let press = |button: &str| { + f.take_events(); + let result = f.host.view_event("jobs", "jobs", ide_plugin::ViewEvent::Button { id: button.into() }, f.inv()); + let opened = f.take_events().into_iter().find_map(|(client, e)| match e { + PluginEvent::OpenLocation { path, line, column, .. } => Some((client, path.0, line, column)), + _ => None, + }); + (result, opened) + }; + + let (result, opened) = press("log"); + assert!(result.is_ok(), "{result:?}"); + let (client, path, line, column) = opened.expect("a location"); + assert_eq!((client.as_deref(), line, column), (Some("main"), 2, 4)); + assert_eq!(path.canonicalize().unwrap(), f.project.join("jobs.log").canonicalize().unwrap()); + + let (result, opened) = press("outside"); + assert!(result.is_err(), "a path outside the project is refused"); + assert!(opened.is_none()); + + f.host.set_capability("jobs", "project.read", false).unwrap(); + let (result, opened) = press("log"); + assert!(result.unwrap_err().message.contains("project.read"), "needs project.read"); + assert!(opened.is_none()); +} + +#[test] +fn views_can_use_an_svg_icon_from_the_package() { + let f = fixture(Limits::default()); + let tmp = tempfile::tempdir().unwrap(); + let clean = r##""##; + let package = |dir: &str, icon: &str, svg: &str| { + let pkg = tmp.path().join(dir); + for (path, text) in [("panel/index.html", "

panel

"), ("icons/panel.svg", svg)] { + std::fs::create_dir_all(pkg.join(path).parent().unwrap()).unwrap(); + std::fs::write(pkg.join(path), text).unwrap(); + } + std::fs::write( + pkg.join("plugin.toml"), + format!( + r#"id = "panel-icon" +name = "Panel Icon" +version = "1.0.0" +engines.openidea = "^1.0" + +[[contributes.views]] +id = "panel" +title = "Panel" +icon = "{icon}" +location = "bottom" +web = "panel/index.html" +"# + ), + ) + .unwrap(); + pkg + }; + + assert!(f.host.install_from_dir(&package("missing", "icons/missing.svg", clean), &Consent::all()).is_err()); + let scripted = clean.replace("alert(1)>()); } + +/// Plugin API 1.0 leaves room to grow: event kinds are names, and the editor +/// providers carry snippets, imports, lazily computed quick fixes and tagged +/// problems. +#[test] +fn completion_quick_fixes_problems_and_event_kinds_of_api_1() { + let f = fixture(Limits::default()); + let pkg = f._tmp.path().join("misbehave-providers"); + copy_dir(&misbehave_pkg(), &pkg); + let manifest = std::fs::read_to_string(pkg.join("plugin.toml")).unwrap(); + std::fs::write( + pkg.join("plugin.toml"), + format!("{manifest}\n[[contributes.providers]]\nkinds = [\"completion\", \"code-actions\"]\nlanguages = [\"text\"]\n"), + ) + .unwrap(); + f.install(&pkg); + let doc = |text: &str| ide_plugin::ProviderDocument { + path: f.project.join("a.txt"), + language: "text".into(), + version: 1, + text: text.into(), + }; + let at = ide_core::Pos::new(0, 0); + + // The first call starts the instance, which may take longer than the edit-path budget. + let mut items = Vec::new(); + assert!(eventually(|| { + items = f.host.provide_completion(&doc("rich"), at, None); + !items.is_empty() + })); + let item = &items[0].1; + assert_eq!( + (item.insert_text.as_str(), item.snippet, item.filter_text.as_deref()), + ("f(${1:x})$0", true, Some("fun")) + ); + assert_eq!(item.additional_edits[0].new_text, "use f;\n"); + + // Offered without edits; computed when picked. + let actions = f.host.provide_code_actions(&doc("lazy"), ide_core::Range::default()); + let (plugin, action) = actions.into_iter().next().expect("offered"); + assert!(action.edits.is_empty() && action.data.as_deref() == Some("fix")); + let resolved = f.host.resolve_code_action(&plugin, &doc("lazy"), action.clone()).unwrap(); + assert_eq!(resolved.edits[0].1[0].new_text, "fixed "); + let foreign = ide_plugin::CodeAction { data: Some("other".into()), ..action }; + assert!(f.host.resolve_code_action(&plugin, &doc("lazy"), foreign).is_err()); + + // A kind this IDE doesn't know is refused, not a trap. + let err = f.host.run_command("misbehave", "subscribe", Some("file-deleted"), f.inv()).unwrap_err(); + assert!(err.message.contains("no event kind"), "{}", err.message); + let err = f.host.run_command("misbehave", "subscribe", Some("file-saved"), f.inv()).unwrap_err(); + assert!(err.message.contains("events"), "known, but not granted: {}", err.message); + + // Tags the IDE knows, the tool that found it. + let path = f.project.join("a.txt"); + f.host.run_command("misbehave", "diagnose", Some(&path.to_string_lossy()), f.inv()).unwrap(); + let d = f.workspace.diagnostics.lock().get(&("misbehave".into(), path)).cloned().unwrap(); + assert_eq!((d[0].unnecessary, d[0].deprecated, d[0].source.as_deref()), (true, false, Some("lint"))); +} diff --git a/crates/ide-host/tests/project_smoke.rs b/crates/ide-host/tests/project_smoke.rs new file mode 100644 index 00000000..9ed4cd4f --- /dev/null +++ b/crates/ide-host/tests/project_smoke.rs @@ -0,0 +1,289 @@ +//! Opening each `fixtures/` project the way the app does, through the `ops` its commands call: +//! the source and test roots detected from its build file, the index (files and symbols), +//! Search Everywhere / Go to Class / Go to Symbol answered from it and kept current as files +//! come and go, and the Maven / Gradle model. +//! +//! The `#[ignore]`d tests run Maven, Gradle, python3 and a JDK 21 (CI's `Language smoke` +//! java, kotlin and python jobs): +//! `cargo test -p ide-host --test project_smoke -- --ignored`. + +use ide_host::{Host, HostConfig, ops}; +use ide_index::SymbolFilter; +use ide_workspace::roots::RootKind; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const WINDOW: &str = "main"; + +struct Fixture { + dir: &'static str, + /// Detected roots: (project-relative path, kind). + roots: &'static [(&'static str, RootKind)], + /// Search Everywhere › Files: (query, file found). + file: (&'static str, &'static str), + /// Go to Class: (name, file declaring it). + class: (&'static str, &'static str), + /// Go to Symbol: (name, file declaring it). + symbol: (&'static str, &'static str), + /// A file created while the project is open: (path, text declaring the class `FreshWidget`). + added: (&'static str, &'static str), +} + +struct Opened { + host: Arc, + root: PathBuf, + _tmp: tempfile::TempDir, +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap().flatten() { + let (src, dst) = (entry.path(), to.join(entry.file_name())); + if entry.file_type().unwrap().is_dir() { + copy_dir(&src, &dst); + } else { + std::fs::copy(&src, &dst).unwrap(); + } + } +} + +async fn open(fixture: &str) -> Opened { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap().join(fixture); + copy_dir(&Path::new(env!("CARGO_MANIFEST_DIR")).join("../../fixtures").join(fixture), &root); + let app = tmp.path().join("app"); + let host = Arc::new(Host::new(HostConfig { + config_dir: app.join("config"), + data_dir: app.join("data"), + local_data_dir: app.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(|_| {})), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + })); + host.trust.decide(&root, true, false).unwrap(); + ops::project::open(&host, WINDOW, &root).await.unwrap(); + Opened { host, root, _tmp: tmp } +} + +/// Polls `check` until it holds; fails with its last answer. +async fn until(what: &str, timeout: Duration, mut check: impl FnMut() -> Result<(), T>) { + let deadline = Instant::now() + timeout; + loop { + match check() { + Ok(()) => return, + Err(last) if Instant::now() > deadline => panic!("{what}: timed out after {timeout:?}; last: {last:?}"), + Err(_) => tokio::time::sleep(Duration::from_millis(100)).await, + } + } +} + +impl Opened { + fn files(&self, query: &str) -> Vec { + let found = ops::nav::find_files(&self.host, WINDOW, query.into(), 20).unwrap(); + found.into_iter().map(|f| f.rel_path).collect() + } + + fn symbols(&self, query: &str, filter: SymbolFilter) -> Vec<(String, String)> { + let found = ops::nav::find_symbols(&self.host, WINDOW, query.into(), filter, 20).unwrap(); + found.into_iter().map(|s| (s.name, s.rel_path)).collect() + } + + fn has_symbol(&self, name: &str, file: &str, filter: SymbolFilter) -> Result<(), Vec<(String, String)>> { + let found = self.symbols(name, filter); + if found.iter().any(|(n, f)| n == name && f == file) { Ok(()) } else { Err(found) } + } +} + +async fn smoke(f: Fixture) { + let p = open(f.dir).await; + until("indexing", Duration::from_secs(30), || { + let s = ops::nav::index_status(&p.host, WINDOW).unwrap(); + if !s.indexing && s.files > 0 && s.symbols > 0 { Ok(()) } else { Err(s) } + }) + .await; + + let roots = ops::project::roots(&p.host, WINDOW).unwrap().roots; + for (path, kind) in f.roots { + assert!( + roots.iter().any(|r| r.path == *path && r.kind == *kind), + "{}: no {kind:?} root {path:?} in {roots:?}", + f.dir + ); + } + + let files = p.files(f.file.0); + assert!(files.iter().any(|r| r == f.file.1), "{}: files for {:?}: {files:?}", f.dir, f.file.0); + if let Err(found) = p.has_symbol(f.class.0, f.class.1, SymbolFilter::Types) { + panic!("{}: Go to Class {:?}: {found:?}", f.dir, f.class.0); + } + if let Err(found) = p.has_symbol(f.symbol.0, f.symbol.1, SymbolFilter::All) { + panic!("{}: Go to Symbol {:?}: {found:?}", f.dir, f.symbol.0); + } + + // The index follows the file system while the project is open. + let (added, text) = f.added; + std::fs::write(p.root.join(added), text).unwrap(); + until("a new file's class", Duration::from_secs(20), || p.has_symbol("FreshWidget", added, SymbolFilter::Types)) + .await; + std::fs::remove_file(p.root.join(added)).unwrap(); + until("a deleted file's class", Duration::from_secs(20), || { + let found = p.symbols("FreshWidget", SymbolFilter::Types); + if found.is_empty() { Ok(()) } else { Err(found) } + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn rust_app() { + smoke(Fixture { + dir: "rust-app", + roots: &[("src", RootKind::Source), ("tests", RootKind::Test)], + file: ("pars", "src/parser.rs"), + class: ("Inventory", "src/lib.rs"), + symbol: ("low_stock", "src/lib.rs"), + added: ("src/fresh.rs", "pub struct FreshWidget;\n"), + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ts_app() { + smoke(Fixture { + dir: "ts-app", + roots: &[("src", RootKind::Source), ("tests", RootKind::Test)], + file: ("cart", "src/cart.ts"), + class: ("Cart", "src/cart.ts"), + symbol: ("formatMoney", "src/money.ts"), + added: ("src/fresh.ts", "export class FreshWidget {}\n"), + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn python_app() { + smoke(Fixture { + dir: "python-app", + roots: &[("src", RootKind::Source), ("tests", RootKind::Test)], + file: ("models", "src/tasks/models.py"), + class: ("TaskList", "src/tasks/models.py"), + symbol: ("build_demo", "src/tasks/cli.py"), + added: ("src/tasks/fresh.py", "class FreshWidget:\n pass\n"), + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn go_app() { + smoke(Fixture { + dir: "go-app", + roots: &[("", RootKind::Source)], + file: ("store", "internal/store/store.go"), + class: ("Store", "internal/store/store.go"), + symbol: ("writeJSON", "cmd/server/main.go"), + added: ("internal/store/fresh.go", "package store\n\ntype FreshWidget struct{}\n"), + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn java_app() { + smoke(Fixture { + dir: "java-app", + roots: &[("src/main/java", RootKind::Source), ("src/test/java", RootKind::Test)], + file: ("OrderBook", "src/main/java/com/example/orders/OrderBook.java"), + class: ("OrderBook", "src/main/java/com/example/orders/OrderBook.java"), + symbol: ("revenueCents", "src/main/java/com/example/orders/OrderBook.java"), + added: ( + "src/main/java/com/example/orders/FreshWidget.java", + "package com.example.orders;\n\npublic class FreshWidget {}\n", + ), + }) + .await; + let p = open("java-app").await; + let model = ops::build::build_model(&p.host, WINDOW).await.unwrap(); + let maven = model.maven.expect("a Maven model"); + assert_eq!(maven.projects.iter().map(|m| m.artifact_id.as_str()).collect::>(), ["openidea-fixture-orders"]); + assert!(maven.projects[0].dependencies.iter().any(|d| d.artifact_id == "junit-jupiter"), "{maven:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn kotlin_gradle() { + smoke(Fixture { + dir: "kotlin-gradle", + roots: &[("src/main/kotlin", RootKind::Source), ("src/test/kotlin", RootKind::Test)], + file: ("Library", "src/main/kotlin/com/example/library/Library.kt"), + class: ("Library", "src/main/kotlin/com/example/library/Library.kt"), + symbol: ("giveBack", "src/main/kotlin/com/example/library/Library.kt"), + added: ("src/main/kotlin/com/example/library/Fresh.kt", "package com.example.library\n\nclass FreshWidget\n"), + }) + .await; + let p = open("kotlin-gradle").await; + let model = ops::build::build_model(&p.host, WINDOW).await.unwrap(); + assert_eq!(model.gradle.len(), 1, "one independent Gradle build"); + let gradle = &model.gradle[0]; + assert_eq!(gradle.projects.iter().map(|g| g.name.as_str()).collect::>(), ["openidea-fixture-library"]); +} + +/// Maven / Gradle sync (Reload in the build tool window): the build's own model. +async fn sync(fixture: &str, tool: ide_build::BuildTool) -> ide_build::BuildModel { + let p = open(fixture).await; + let progress = ops::Channel::new(|_: ide_build::SyncProgress| {}); + let model = ops::build::build_sync(&p.host, WINDOW, tool, ide_core::RequestId(1), false, vec![], None, progress) + .await + .unwrap(); + // The cached sync is what the next open shows. + let reloaded = ops::build::build_model(&p.host, WINDOW).await.unwrap(); + assert_eq!(reloaded, model); + model +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "project smoke: needs Maven and a JDK"] +async fn maven_sync() { + let maven = sync("java-app", ide_build::BuildTool::Maven).await.maven.unwrap(); + assert_eq!(maven.sync_error, None); + assert!(maven.synced_at.is_some()); + let junit = maven.projects[0].dependencies.iter().find(|d| d.artifact_id == "junit-jupiter").unwrap(); + // The resolved tree, not only the declaration. + assert!(!junit.children.is_empty(), "{junit:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "project smoke: needs Gradle and a JDK"] +async fn gradle_sync() { + let model = sync("kotlin-gradle", ide_build::BuildTool::Gradle).await; + assert_eq!(model.gradle.len(), 1, "one independent Gradle build"); + let gradle = &model.gradle[0]; + assert_eq!(gradle.sync_error, None); + assert!(gradle.synced_at.is_some()); + let tasks: Vec<&str> = gradle.projects[0].tasks.iter().map(|t| t.name.as_str()).collect(); + // The build's own tasks: the application and Kotlin plugins' too. + for task in ["run", "compileKotlin", "test"] { + assert!(tasks.contains(&task), "no {task} in {tasks:?}"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "project smoke: needs python3"] +async fn python_venv() { + let p = open("python-app").await; + let made = std::process::Command::new("python3").args(["-m", "venv", ".venv"]).current_dir(&p.root).status(); + assert!(made.unwrap().success()); + // The project's virtualenv: listed first, and the interpreter the project uses. + let found = ops::sdk::sdk_detect(&p.host, WINDOW, ide_sdk::SdkKind::Python).await.unwrap(); + assert_eq!(found.first().map(|s| s.source.as_str()), Some("venv"), "{found:?}"); + let effective = ops::sdk::sdk_python_effective(&p.host, WINDOW).unwrap().expect("an interpreter"); + assert!(effective.starts_with(p.root.join(".venv")), "{effective:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "project smoke: needs JDK 21"] +async fn jdk() { + let p = open("java-app").await; + let found = ops::sdk::sdk_detect(&p.host, WINDOW, ide_sdk::SdkKind::Jdk).await.unwrap(); + assert!(found.iter().any(|s| s.version.as_deref().is_some_and(|v| v.starts_with("21"))), "{found:?}"); +} diff --git a/crates/ide-host/tests/proxy.rs b/crates/ide-host/tests/proxy.rs index cf74d5dd..91fbb42d 100644 --- a/crates/ide-host/tests/proxy.rs +++ b/crates/ide-host/tests/proxy.rs @@ -43,6 +43,7 @@ fn walk(dir: &std::path::Path) -> Vec { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn manual_proxy_applies_and_the_password_is_saved_apart() { + ide_ssh::secrets::testing::use_memory_store(); let tmp = tempfile::tempdir().unwrap(); let host = host(tmp.path()); assert_eq!(proxy().mode, ProxyMode::Auto); @@ -61,7 +62,7 @@ async fn manual_proxy_applies_and_the_password_is_saved_apart() { assert!(!settings_files(tmp.path()).contains("s3cret"), "no password in settings.toml"); let saved = std::fs::read_to_string(tmp.path().join("data/proxy-password.json")).unwrap(); - assert!(saved.contains("s3cret")); + assert!(!saved.contains("s3cret"), "the password is in the credential store, not on disk: {saved}"); assert_eq!(ops::project::settings(&host, "main").proxy.password, "s3cret"); // A whole-table write keeps the password out of the file too; an empty one forgets it. diff --git a/crates/ide-host/tests/rpc.rs b/crates/ide-host/tests/rpc.rs index fa055108..6e546095 100644 --- a/crates/ide-host/tests/rpc.rs +++ b/crates/ide-host/tests/rpc.rs @@ -228,7 +228,9 @@ async fn serves_projects_files_documents_search_and_runs() { // Real project-host ignore writes travel through the remote command dispatcher. assert!(Dispatcher::standard().commands().contains(&"vcs_add_to_ignore")); let ignore_root = dunce::canonicalize(&project).unwrap(); - for (name, destination) in [("shared.tmp", "gitignore"), ("private.tmp", "localExclude")] { + // Not `.tmp`: a machine's global excludes file (often `*.tmp`) already + // ignores those, so the backend never writes the entry anywhere. + for (name, destination) in [("shared.ignore-me", "gitignore"), ("private.ignore-me", "localExclude")] { let path = ignore_root.join(name); std::fs::write(&path, "temporary").unwrap(); invoke(&peer, "vcs_add_to_ignore", json!({ "paths": [path], "destination": destination })).await.unwrap(); @@ -237,9 +239,9 @@ async fn serves_projects_files_documents_search_and_runs() { assert_eq!(entry["ignored"], true); assert_eq!(entry["excluded"], false); } - assert!(std::fs::read_to_string(project.join(".gitignore")).unwrap().contains("/shared.tmp")); - assert!(std::fs::read_to_string(project.join(".git/info/exclude")).unwrap().contains("/private.tmp")); - let outside = tmp.path().join("outside.tmp"); + assert!(std::fs::read_to_string(project.join(".gitignore")).unwrap().contains("/shared.ignore-me")); + assert!(std::fs::read_to_string(project.join(".git/info/exclude")).unwrap().contains("/private.ignore-me")); + let outside = tmp.path().join("outside.ignore-me"); std::fs::write(&outside, "keep").unwrap(); assert!( invoke(&peer, "vcs_add_to_ignore", json!({ "paths": [outside], "destination": "gitignore" })).await.is_err() diff --git a/crates/ide-host/tests/run_configs.rs b/crates/ide-host/tests/run_configs.rs index c45bccda..dbec2bc7 100644 --- a/crates/ide-host/tests/run_configs.rs +++ b/crates/ide-host/tests/run_configs.rs @@ -141,3 +141,19 @@ async fn workspace_cmake_before_launch_tasks_resolve_in_the_member() { assert!(configs.iter().any(|c| c.id == task.target && c.cwd == build.cwd)); } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gradle_init_scripts_are_written_by_the_host_that_runs_the_build() { + let tmp = tempfile::tempdir().unwrap(); + let host = host(tmp.path()); + let text = "allprojects { }\n".to_string(); + let path = + ops::run::run_gradle_init_script(&host, "main", "openidea-gradle-tests".into(), text.clone()).await.unwrap(); + assert!(path.contains("openidea-gradle-tests-"), "{path}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), text); + let again = ops::run::run_gradle_init_script(&host, "main", "openidea-gradle-tests".into(), text).await.unwrap(); + assert_eq!(again, path); + // A name is not a path. + let escaped = ops::run::run_gradle_init_script(&host, "main", "../escape".into(), String::new()).await; + assert!(escaped.is_err(), "{escaped:?}"); +} diff --git a/crates/ide-host/tests/trust.rs b/crates/ide-host/tests/trust.rs index eb2868ad..b50d5abc 100644 --- a/crates/ide-host/tests/trust.rs +++ b/crates/ide-host/tests/trust.rs @@ -58,6 +58,11 @@ async fn application_commands_work_in_restricted_projects_and_without_a_project( "vcs_write_patch", "remote_bootstrap_list", "deploy_test_connection", + "plugin_market", + "plugin_market_url", + "plugin_set_market_url", + "plugin_market_download", + "plugin_preview_install", ] { assert!(ide_host::trust::restricted_command_allowed(command), "{command}"); for window in ["main", "welcome"] { @@ -72,8 +77,6 @@ async fn application_commands_work_in_restricted_projects_and_without_a_project( "tools_list", "tools_install", "tools_uninstall", - "plugin_install_dir", - "plugin_reload", "vcs_git_version", "docker_compose_op", "db_execute", @@ -93,6 +96,12 @@ async fn application_commands_work_in_restricted_projects_and_without_a_project( assert!(host.authorize_command(window, command).is_err(), "{window}: {command}"); } } + // Plugin installs run plugin code: from the welcome screen, but not in an untrusted project. + for command in ["plugin_install_dir", "plugin_uninstall", "plugin_set_enabled", "plugin_reload"] { + assert!(!ide_host::trust::restricted_command_allowed(command), "{command}"); + assert!(host.authorize_command("main", command).is_err(), "{command}"); + assert!(host.authorize_command("welcome", command).is_ok(), "{command}"); + } // These requests validate the vendor before any network access, independently of project trust. for window in ["main", "welcome"] { let error = ide_host::ops::tools::jdk_download_releases( @@ -240,6 +249,7 @@ async fn malicious_project_stays_editable_then_starts_in_place_after_trust() { ide_core::RequestId(472), false, vec![], + None, ops::Channel::new(|_| {}), ) }; diff --git a/crates/ide-host/tests/unity.rs b/crates/ide-host/tests/unity.rs new file mode 100644 index 00000000..f38fdbdd --- /dev/null +++ b/crates/ide-host/tests/unity.rs @@ -0,0 +1,100 @@ +use ide_core::{NullSink, RequestId}; +use ide_host::{Host, HostConfig, ops}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn opens_unity_without_generated_solutions_and_excludes_caches_from_the_index() { + let temp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(temp.path()).unwrap(); + let root = base.join("Game"); + for name in ["Assets/Scripts", "Packages", "ProjectSettings", "Library/PackageCache", "Temp", "Logs"] { + std::fs::create_dir_all(root.join(name)).unwrap(); + } + std::fs::write(root.join("ProjectSettings/ProjectVersion.txt"), "m_EditorVersion: 6000.0.62f1\n").unwrap(); + std::fs::write(root.join("Assets/Scripts/Player.cs"), "class Player {}\n").unwrap(); + std::fs::write(root.join("Packages/manifest.json"), "{}\n").unwrap(); + std::fs::write(root.join("Library/PackageCache/Cached.cs"), "class Cached {}\n").unwrap(); + std::fs::write(root.join("Temp/Temporary.cs"), "class Temporary {}\n").unwrap(); + std::fs::write(root.join("Logs/Editor.log"), "log\n").unwrap(); + let host = Host::new(HostConfig { + config_dir: base.join("config"), + data_dir: base.join("data"), + local_data_dir: base.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + let project = host.open_project("main", &root).unwrap(); + let info = ops::project::info(&project); + assert_eq!(info.name, "Game"); + let unity = info.unity.unwrap(); + assert_eq!(unity.editor_version.as_deref(), Some("6000.0.62f1")); + assert!(unity.projects.is_empty() && unity.solutions.is_empty()); + assert!(project.index.wait_idle(Duration::from_secs(20))); + assert_eq!(project.index.find_files("Player.cs", 100).len(), 1); + for name in ["Cached.cs", "Temporary.cs", "Editor.log"] { + assert!(project.index.find_files(name, 100).is_empty(), "indexed generated file {name}"); + } + let hits = Arc::new(std::sync::Mutex::new(Vec::new())); + let out = hits.clone(); + let query = serde_json::from_value(serde_json::json!({ + "pattern": "class", "isRegex": false, "caseSensitive": false, "wholeWord": false, + "include": [], "exclude": [], "scope": { "kind": "project" }, "maxMatches": 100, + })) + .unwrap(); + ops::search::start( + &host, + "main", + RequestId::next(), + query, + Arc::new(move |event| { + if let ops::search::SearchEvent::File { file } = event { + out.lock().unwrap().push(file.rel_path); + } + }), + ) + .await + .unwrap(); + assert_eq!(*hits.lock().unwrap(), ["Assets/Scripts/Player.cs"]); + let listing = ops::fs::read_dir(&host, "main", root.clone()).await.unwrap(); + assert!(listing.iter().any(|e| e.name == "Library" && e.excluded)); + assert!(project.exclude_globs().contains(&"/Obj".to_owned())); + + // Project metadata refreshes without reopening when Unity generates its solution. + std::fs::write(root.join("Game.sln"), "").unwrap(); + std::fs::write(root.join("Assembly-CSharp.csproj"), "").unwrap(); + let unity = ops::project::info(&project).unity.unwrap(); + assert_eq!(unity.solutions, ["Game.sln"]); + assert_eq!(unity.projects, ["Assembly-CSharp.csproj"]); + + // Desktop commands and remote RPC share these host operations. + let original = root.join("Assets/Scripts/Player.cs"); + let renamed = root.join("Assets/Scripts/Hero.cs"); + let copied = root.join("Assets/Scripts/HeroCopy.cs"); + let meta = |path: &std::path::Path| -> std::path::PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".meta"); + name.into() + }; + let original_metadata = "fileFormatVersion: 2\nguid: 11111111111111111111111111111111\n"; + std::fs::write(meta(&original), original_metadata).unwrap(); + ops::fs::rename(&host, original.clone(), renamed.clone()).await.unwrap(); + assert!(!original.exists() && !meta(&original).exists()); + assert_eq!(std::fs::read_to_string(meta(&renamed)).unwrap(), original_metadata); + ops::fs::copy(&host, renamed.clone(), copied.clone()).await.unwrap(); + assert!(copied.exists()); + assert_ne!(std::fs::read_to_string(meta(&copied)).unwrap(), original_metadata); + assert_eq!(std::fs::read_to_string(meta(&renamed)).unwrap(), original_metadata); + host.close_project("main"); + + let ordinary = base.join("ConsoleApp"); + std::fs::create_dir(&ordinary).unwrap(); + std::fs::write(ordinary.join("App.csproj"), "").unwrap(); + let project = host.open_project("main", &ordinary).unwrap(); + assert!(ops::project::info(&project).unity.is_none()); + assert!(!project.exclude_globs().contains(&"/Library".to_owned())); + host.close_project("main"); +} diff --git a/crates/ide-host/tests/vcs_flow.rs b/crates/ide-host/tests/vcs_flow.rs new file mode 100644 index 00000000..031b628d --- /dev/null +++ b/crates/ide-host/tests/vcs_flow.rs @@ -0,0 +1,209 @@ +//! A day with Git on a `fixtures/` project, through the `ops` the app's commands call: an edit +//! marked in the gutter, Local Changes and its diff, a commit in the log, a branch, a stash, and +//! a merge that conflicts, resolved and committed. Open documents follow every step. + +use ide_core::DocId; +use ide_host::{Host, HostConfig, ops}; +use ide_vcs::{ChangeKind, CommitOptions, FileStatus, HunkKind, LogQuery}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::time::Duration; + +const WINDOW: &str = "main"; +/// The line every branch changes, in `src/lib.rs`. +const LINE: &str = "pub fn format_price(cents: u64) -> String {"; + +fn git(root: &Path, args: &[&str]) -> String { + let out = Command::new("git").current_dir(root).args(args).output().unwrap(); + assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).unwrap() +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap().flatten() { + let (src, dst) = (entry.path(), to.join(entry.file_name())); + if entry.file_type().unwrap().is_dir() { + copy_dir(&src, &dst); + } else { + std::fs::copy(&src, &dst).unwrap(); + } + } +} + +struct Flow { + host: Arc, + root: PathBuf, + _tmp: tempfile::TempDir, +} + +impl Flow { + async fn new(fixture: &str) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(tmp.path()).unwrap(); + let root = base.join(fixture); + copy_dir(&Path::new(env!("CARGO_MANIFEST_DIR")).join("../../fixtures").join(fixture), &root); + git(&root, &["init", "-q", "-b", "main"]); + for (key, value) in [ + ("user.name", "Test"), + ("user.email", "test@example.com"), + ("commit.gpgsign", "false"), + ("core.autocrlf", "false"), + ] { + git(&root, &["config", key, value]); + } + git(&root, &["add", "-A"]); + git(&root, &["commit", "-q", "-m", "Initial commit"]); + let host = Arc::new(Host::new(HostConfig { + config_dir: base.join("config"), + data_dir: base.join("data"), + local_data_dir: base.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(|_| {})), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + })); + host.trust.decide(&root, true, false).unwrap(); + let project = ops::project::open(&host, WINDOW, &root).await.unwrap(); + for _ in 0..500 { + if !project.repos().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + Flow { host, root, _tmp: tmp } + } + + async fn open(&self, file: &str) -> DocId { + ops::doc::open(&self.host, WINDOW, self.root.join(file)).await.unwrap().id + } + + async fn text(&self, doc: DocId) -> String { + ops::doc::snapshot(&self.host, WINDOW, doc).await.unwrap().text + } + + /// Types `with` over `LINE` in the editor (unsaved). + async fn edit_line(&self, doc: DocId, with: &str) { + let text = self.text(doc).await; + let current = text.lines().find(|l| l.starts_with("pub fn format_price")).unwrap().to_owned(); + let project = self.host.project(WINDOW).unwrap(); + project.documents.replace_all(doc, &text.replacen(¤t, with, 1)).unwrap(); + } + + async fn save(&self, doc: DocId) { + ops::doc::save(&self.host, WINDOW, doc).await.unwrap(); + } + + async fn status(&self) -> Vec { + ops::vcs::status(&self.host, WINDOW).await.unwrap() + } + + async fn commit(&self, files: &[&str], message: &str) -> String { + let paths = files.iter().map(|f| self.root.join(f)).collect(); + ops::vcs::vcs_stage(&self.host, WINDOW, paths).await.unwrap(); + let options = CommitOptions { run_hooks: true, ..Default::default() }; + ops::vcs::vcs_commit(&self.host, WINDOW, None, message.into(), options).await.unwrap() + } + + async fn head(&self) -> ide_vcs::Commit { + let log = ops::vcs::vcs_log(&self.host, WINDOW, None, LogQuery::default()).await.unwrap(); + log.into_iter().next().expect("a commit") + } +} + +fn changed<'a>(status: &'a [FileStatus], file: &str) -> Option<&'a FileStatus> { + status.iter().find(|s| s.rel_path == file) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn change_commit_branch_stash_and_merge_a_conflict() { + let f = Flow::new("rust-app").await; + let lib_path = f.root.join("src/lib.rs"); + let lib = f.open("src/lib.rs").await; + let original = f.text(lib).await; + let line = original.lines().position(|l| l == LINE).expect("the fixture's line") as u32; + + // An edit in the editor: a modified line in the gutter, against HEAD, before saving. + let base = "pub fn format_price(cents: u64) -> String { // base"; + f.edit_line(lib, base).await; + let hunks = ops::vcs::vcs_line_hunks(&f.host, WINDOW, lib_path.clone()).await.unwrap(); + assert!( + matches!(&hunks[..], [h] if h.kind == HunkKind::Modified && h.start == line && h.old_text.contains(LINE)), + "{hunks:?}" + ); + assert!(f.status().await.is_empty(), "nothing saved yet"); + + // Saved: Local Changes lists it, and its diff shows the edit. + f.save(lib).await; + let status = f.status().await; + assert_eq!(changed(&status, "src/lib.rs").and_then(|s| s.unstaged), Some(ChangeKind::Modified), "{status:?}"); + let diff = ops::vcs::vcs_diff_text(&f.host, WINDOW, lib_path.clone(), false).await.unwrap(); + assert!(diff.contains(&format!("-{LINE}")) && diff.contains(&format!("+{base}")), "{diff}"); + + // Commit: the log's head; nothing left to commit; no gutter marks. + let id = f.commit(&["src/lib.rs"], "Mark the base").await; + let head = f.head().await; + assert_eq!((head.id.as_str(), head.summary.as_str()), (id.as_str(), "Mark the base")); + assert!(f.status().await.is_empty()); + assert!(ops::vcs::vcs_line_hunks(&f.host, WINDOW, lib_path.clone()).await.unwrap().is_empty()); + + // A branch changes the line its way. + ops::vcs::vcs_create_branch(&f.host, WINDOW, None, "feature".into(), None, true).await.unwrap(); + let theirs = "pub fn format_price(cents: u64) -> String { // feature"; + f.edit_line(lib, theirs).await; + f.save(lib).await; + f.commit(&["src/lib.rs"], "Feature's change").await; + let branches = ops::vcs::vcs_branches(&f.host, WINDOW, None).await.unwrap(); + assert!(branches.iter().any(|b| b.name == "feature" && b.current), "{branches:?}"); + + // Back on main, the open file shows main's text. + ops::vcs::vcs_checkout(&f.host, WINDOW, None, "main".into(), false, None).await.unwrap(); + assert!(f.text(lib).await.contains(base)); + + // Stash: work in progress put aside and brought back. + let main_rs = f.open("src/main.rs").await; + let wip = format!("{}// work in progress\n", f.text(main_rs).await); + f.host.project(WINDOW).unwrap().documents.replace_all(main_rs, &wip).unwrap(); + f.save(main_rs).await; + ops::vcs::vcs_stash_push(&f.host, WINDOW, None, Some("wip".into()), false, false).await.unwrap(); + assert!(f.status().await.is_empty()); + assert!(!f.text(main_rs).await.contains("work in progress")); + let stashes = ops::vcs::vcs_stashes(&f.host, WINDOW, None).await.unwrap(); + assert!(matches!(&stashes[..], [s] if s.message.contains("wip")), "{stashes:?}"); + ops::vcs::vcs_stash_apply(&f.host, WINDOW, None, 0, true, false, None).await.unwrap(); + assert_eq!(f.text(main_rs).await, wip); + assert!(ops::vcs::vcs_stashes(&f.host, WINDOW, None).await.unwrap().is_empty()); + ops::vcs::vcs_discard(&f.host, WINDOW, vec![f.root.join("src/main.rs")]).await.unwrap(); + assert!(f.status().await.is_empty()); + + // Main changes the same line: merging the branch conflicts. + let ours = "pub fn format_price(cents: u64) -> String { // main"; + f.edit_line(lib, ours).await; + f.save(lib).await; + f.commit(&["src/lib.rs"], "Main's change").await; + let merged = ops::vcs::vcs_merge(&f.host, WINDOW, None, "feature".into()).await; + let status = f.status().await; + let conflicted = changed(&status, "src/lib.rs").map(|s| (s.staged, s.unstaged)); + assert!( + matches!(conflicted, Some((Some(ChangeKind::Conflicted), _) | (_, Some(ChangeKind::Conflicted)))), + "merge: {merged:?}; status: {status:?}" + ); + // The open file shows the conflict as Git left it. + assert!(f.text(lib).await.contains("<<<<<<<"), "{}", f.text(lib).await); + let versions = ops::vcs::vcs_conflict_versions(&f.host, WINDOW, lib_path.clone()).await.unwrap(); + let has = |side: &Option, line: &str| side.as_deref().is_some_and(|t| t.contains(line)); + assert!(has(&versions.base, base) && has(&versions.ours, ours) && has(&versions.theirs, theirs), "{versions:?}"); + + // Resolved (Accept Theirs) and committed: a merge commit, nothing left. + let resolved = versions.theirs.clone().unwrap(); + f.host.project(WINDOW).unwrap().documents.replace_all(lib, &resolved).unwrap(); + f.save(lib).await; + f.commit(&["src/lib.rs"], "Merge feature").await; + let head = f.head().await; + assert_eq!(head.parents.len(), 2, "{head:?}"); + assert!(f.status().await.is_empty()); + assert_eq!(std::fs::read_to_string(&lib_path).unwrap(), resolved); +} diff --git a/crates/ide-host/tests/workspace_edits.rs b/crates/ide-host/tests/workspace_edits.rs index 7bdc5c57..2994603d 100644 --- a/crates/ide-host/tests/workspace_edits.rs +++ b/crates/ide-host/tests/workspace_edits.rs @@ -175,3 +175,66 @@ async fn contended_version_reads_do_not_block_the_async_worker() { assert!(watchdog.join().unwrap(), "request {request_kind} blocked the only async worker"); } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_actions_skip_a_command_no_server_provides_after_their_edit() { + use ide_lsp::types::{CodeAction, Command}; + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.path().join("imports.txt"); + std::fs::write(&file, "unused\nused\n").unwrap(); + let host = Host::new(HostConfig { + config_dir: tmp.path().join("config"), + data_dir: tmp.path().join("data"), + local_data_dir: tmp.path().join("local-data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(|_| {})), + owner: |window| window, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + host.open_light("main", &file).unwrap(); + let info = ops::doc::open(&host, "main", file.clone()).await.unwrap(); + // Optimize Imports as vtsls offers it: the edit, then a command its client runs itself. + let action = |edit: bool| CodeAction { + title: "Organize Imports".into(), + kind: Some("source.organizeImports".into()), + is_preferred: false, + disabled_reason: None, + edit: edit.then(|| { + serde_json::from_value(json!({ "files": [{ "kind": "edit", "path": file, "edits": [ + { "range": { "start": { "line": 0, "col": 0 }, "end": { "line": 1, "col": 0 } }, "new_text": "" } + ] }] })) + .unwrap() + }), + command: Some(Command { + title: String::new(), + command: "_typescript.didOrganizeImports".into(), + arguments: vec![], + }), + data: None, + server: ide_core::ServerId(1), + }; + let follow_up = action(true).command.unwrap(); + let asked = ops::lsp::lsp_resolve_command( + &host, + "main", + ide_core::RequestId(1), + info.id, + ide_core::ServerId(1), + follow_up, + vec![], + ) + .await; + assert!(matches!(asked, Ok(None)), "{asked:?}"); + ops::lsp::lsp_execute_code_action(&host, "main", ide_core::RequestId(2), info.id, action(true), vec![]) + .await + .unwrap(); + assert_eq!(ops::doc::snapshot(&host, "main", info.id).await.unwrap().text, "used\n"); + ops::lsp::lsp_apply_code_action(&host, "main", info.id, action(true)).await.unwrap(); + assert_eq!(ops::doc::snapshot(&host, "main", info.id).await.unwrap().text, ""); + // Without an edit the command is the action: it still can't run. + let ran = + ops::lsp::lsp_execute_code_action(&host, "main", ide_core::RequestId(3), info.id, action(false), vec![]).await; + assert!(ran.is_err(), "{ran:?}"); +} diff --git a/crates/ide-host/tests/workspaces.rs b/crates/ide-host/tests/workspaces.rs index cfa22542..1b86eeca 100644 --- a/crates/ide-host/tests/workspaces.rs +++ b/crates/ide-host/tests/workspaces.rs @@ -114,3 +114,54 @@ async fn linked_projects_share_navigation_but_keep_files_repositories_and_run_di assert_eq!(automatic.index.find_files("same.txt", 100).len(), 1); host.close_project("main"); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn php_language_level_follows_the_member_of_the_file() { + use ide_sdk::php::{PhpLanguageLevel, PhpLevelSource}; + let temp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(temp.path()).unwrap(); + let (api, legacy) = (base.join("api"), base.join("legacy")); + for (member, composer) in [ + (&api, r#"{"require": {"php": "^7.4"}}"#), + (&legacy, r#"{"require": {"php": "^7.4"}, "config": {"platform": {"php": "8.0.5"}}}"#), + ] { + std::fs::create_dir(member).unwrap(); + std::fs::write(member.join("composer.json"), composer).unwrap(); + } + let root = base.join("workspace"); + Workspace::save(&root, "PHP", vec![api.clone(), legacy.clone()], true).unwrap(); + std::fs::write(root.join("composer.json"), r#"{"require": {"php": "^8.2"}}"#).unwrap(); + let config_dir = base.join("config"); + ide_workspace::SettingsStore::load(&config_dir, Some(&legacy)) + .set(ide_workspace::SettingsScope::Project, "languages.php.languageLevel", "8.3".into()) + .unwrap(); + let host = Host::new(HostConfig { + config_dir, + data_dir: base.join("data"), + local_data_dir: base.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + host.trust.decide(&root, true, false).unwrap(); + host.open_project("main", &root).unwrap(); + let found = |version: &str, source| Some(PhpLanguageLevel { version: version.into(), source }); + let level = async |path: std::path::PathBuf| ops::sdk::sdk_php_language_level(&host, "main", path).await.unwrap(); + + let level_of_api = level(api.join("src/Invoice.php")).await; + assert_eq!(level_of_api.effective, found("7.4.0", PhpLevelSource::Require)); + assert_eq!(level_of_api.auto, level_of_api.effective); + assert_eq!(level_of_api.member.as_deref(), Some(api.as_path())); + // The member's own settings, not the workspace's. + let level_of_legacy = level(legacy.join("index.php")).await; + assert_eq!(level_of_legacy.effective, found("8.3.0", PhpLevelSource::Settings)); + assert_eq!(level_of_legacy.auto, found("8.0.5", PhpLevelSource::Platform)); + // Outside every member: the workspace's own. + let level_elsewhere = level(base.join("elsewhere.php")).await; + assert_eq!(level_elsewhere.effective, found("8.2.0", PhpLevelSource::Require)); + assert_eq!(level_elsewhere.member, None); + host.close_project("main"); +} diff --git a/crates/ide-index/benches/index.rs b/crates/ide-index/benches/index.rs index d888e2c4..efe17337 100644 --- a/crates/ide-index/benches/index.rs +++ b/crates/ide-index/benches/index.rs @@ -14,6 +14,9 @@ fn bench(c: &mut Criterion) { for q in ["ordcon", "src/user", "PaymentService", "zzzz", "a"] { g.bench_function(format!("find_files/{q}"), |b| b.iter(|| black_box(index.find_files(q, 50).len()))); } + for q in ["util", "src/ui", "a"] { + g.bench_function(format!("find_dirs/{q}"), |b| b.iter(|| black_box(index.find_dirs(q, 50).len()))); + } for q in ["OCo", "invoiceRepo", "renderView", "q"] { g.bench_function(format!("find_symbols/{q}"), |b| { b.iter(|| black_box(index.find_symbols(q, SymbolFilter::All, 50).len())) diff --git a/crates/ide-index/src/data.rs b/crates/ide-index/src/data.rs index 59115430..1bda8c24 100644 --- a/crates/ide-index/src/data.rs +++ b/crates/ide-index/src/data.rs @@ -40,6 +40,8 @@ pub(crate) struct FileRecord { pub size: u64, pub hash: [u8; 32], pub symbols: Vec, + /// Matched by an ignore file: listed by name only (no hash, no symbols). + pub ignored: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -72,6 +74,7 @@ pub(crate) struct FileEntry { pub size: u64, pub hash: [u8; 32], pub symbols: Box<[Sym]>, + pub ignored: bool, } impl FileEntry { @@ -100,6 +103,7 @@ impl FileEntry { size: record.size, hash: record.hash, symbols, + ignored: record.ignored, } } @@ -126,14 +130,23 @@ impl FileEntry { range: s.range, }) .collect(), + ignored: self.ignored, } } + + /// Whether the entry is out of date for `found` on disk. + pub(crate) fn stale_for(&self, found: &crate::filter::Found) -> bool { + self.mtime != found.mtime || self.size != found.size || self.ignored != found.ignored + } } #[derive(Default)] pub(crate) struct IndexData { pub files: Vec>, by_path: HashMap, + /// Every folder holding an indexed file at any depth, keyed by its `rel` with a + /// trailing `/`: its absolute path and the number of files below it. + pub dirs: HashMap, (PathBuf, u32)>, free: Vec, pub file_count: u32, pub symbol_count: u32, @@ -167,6 +180,11 @@ impl IndexData { } }; let path = self.files[slot as usize].as_ref().map(|e| e.path.clone()).unwrap_or_default(); + if let Some(e) = &self.files[slot as usize] { + for (rel, abs) in ancestors(e) { + self.dirs.entry(rel.into()).or_insert_with(|| (abs.to_owned(), 0)).1 += 1; + } + } self.by_path.insert(path, slot); self.file_count += 1; self.symbol_count += symbols; @@ -176,6 +194,14 @@ impl IndexData { let slot = self.by_path.remove(path)?; let old = self.files[slot as usize].take()?; self.free.push(slot); + for (rel, _) in ancestors(&old) { + if let Some(d) = self.dirs.get_mut(rel) { + d.1 -= 1; + if d.1 == 0 { + self.dirs.remove(rel); + } + } + } self.file_count -= 1; self.symbol_count -= old.symbols.len() as u32; Some(old) @@ -197,6 +223,7 @@ impl IndexData { self.files.shrink_to_fit(); self.free.shrink_to_fit(); self.by_path.shrink_to_fit(); + self.dirs.shrink_to_fit(); } pub(crate) fn paths(&self) -> impl Iterator { @@ -204,6 +231,19 @@ impl IndexData { } } +/// The folders holding `e` (`rel` with a trailing `/`, absolute path), innermost first. +fn ancestors(e: &FileEntry) -> impl Iterator { + let mut rel = e.dir(); + let mut abs = e.path.parent(); + std::iter::from_fn(move || { + let path = abs.filter(|_| !rel.is_empty())?; + let here = rel; + rel = rel[..rel.len() - 1].rfind('/').map_or("", |i| &rel[..=i]); + abs = path.parent(); + Some((here, path)) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -223,6 +263,23 @@ mod tests { FileEntry::new(PathBuf::from("/r").join(path), path.to_owned(), record) } + #[test] + fn dirs_follow_inserts_and_removals() { + let mut d = IndexData::default(); + d.insert(entry("a/b/c.rs", 0)); + d.insert(entry("a/d.rs", 0)); + d.insert(entry("a/d.rs", 0)); + d.insert(entry("top.rs", 0)); + let mut dirs: Vec<_> = d.dirs.iter().map(|(k, v)| (k.to_string(), v.1)).collect(); + dirs.sort(); + assert_eq!(dirs, [("a/".into(), 2), ("a/b/".into(), 1)]); + assert_eq!(d.dirs["a/b/"].0, PathBuf::from("/r/a/b")); + d.remove(Path::new("/r/a/b/c.rs")); + assert!(!d.dirs.contains_key("a/b/")); + d.remove_tree(Path::new("/r/a")); + assert!(d.dirs.is_empty()); + } + #[test] fn insert_replace_remove_keep_counts() { let mut d = IndexData::default(); diff --git a/crates/ide-index/src/filter.rs b/crates/ide-index/src/filter.rs index b826df87..bae11d65 100644 --- a/crates/ide-index/src/filter.rs +++ b/crates/ide-index/src/filter.rs @@ -1,10 +1,16 @@ //! Which paths belong in the index: `.gitignore`/`.ignore` rules (via the //! `ignore` crate) plus the user's `files.exclude` globs, below the project root //! and the extra content roots (Project Structure). +//! +//! Like IDEA, ignored directories are left out entirely, while ignored files in +//! the remaining directories (`.env`) are listed by name only, so Go to File finds +//! them but their symbols stay out of the index. use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use ignore::gitignore::{Gitignore, GitignoreBuilder}; use ignore::{WalkBuilder, WalkState}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; @@ -57,11 +63,25 @@ fn rel_string(rel: &Path) -> String { out } +/// Whether `path` below `root` matches the exclude globs. +fn glob_excluded(excludes: &GlobSet, root: &Path, path: &Path) -> bool { + match path.strip_prefix(root) { + Ok(rel) if !rel.as_os_str().is_empty() => { + let rel = rel.to_string_lossy(); + let rel = if std::path::MAIN_SEPARATOR == '/' { rel } else { rel.replace('\\', "/").into() }; + excludes.is_match(rel.as_ref()) + } + _ => false, + } +} + #[derive(Clone, Debug)] pub(crate) struct Found { pub path: PathBuf, pub mtime: u64, pub size: u64, + /// Matched by an ignore file: listed by name only. + pub ignored: bool, } pub(crate) fn mtime_of(meta: &std::fs::Metadata) -> u64 { @@ -177,8 +197,9 @@ impl PathFilter { ALWAYS_SKIP.contains(&name) } - /// Whether a path reported by the watcher should be ignored (outside the - /// root, inside an excluded/ignored directory, or ignored itself). + /// Whether a path reported by the watcher should be left out (outside the + /// root, excluded, or an ignored directory or inside one). Ignored files + /// elsewhere stay in, listed by name only. pub(crate) fn is_excluded(&self, path: &Path, is_dir: bool) -> bool { let Some((base, _, gitignore)) = self.base_of(path) else { return true }; let rel = path.strip_prefix(base).map(rel_string).unwrap_or_default(); @@ -198,16 +219,27 @@ impl PathFilter { return true; } } - gitignore.matched_path_or_any_parents(path, is_dir).is_ignore() + let dir = if is_dir { Some(path) } else { path.parent().filter(|p| *p != base) }; + dir.is_some_and(|d| gitignore.matched_path_or_any_parents(d, true).is_ignore()) } /// All indexable files at or below `dir`, walked in parallel. /// Stops early (returning what was found so far) once `cancel` is set. pub(crate) fn walk(&self, dir: &Path, cancel: &AtomicBool) -> Vec { + self.walk_to(dir, None, cancel) + } + + /// The indexable files directly in `dir`, classified by the same rules as [`Self::walk`]. + pub(crate) fn list_dir(&self, dir: &Path) -> Vec { + self.walk_to(dir, Some(1), &AtomicBool::new(false)) + } + + fn walk_to(&self, dir: &Path, max_depth: Option, cancel: &AtomicBool) -> Vec { let (tx, rx) = mpsc::channel::>(); let Some((root, _, _)) = self.base_of(dir) else { return Vec::new() }; let root = root.to_path_buf(); let excludes = self.excludes.clone(); + let (walk_root, walk_excludes) = (root.clone(), excludes.clone()); let walker = WalkBuilder::new(dir) .hidden(false) .parents(true) @@ -215,43 +247,65 @@ impl PathFilter { .git_ignore(true) .git_global(true) .git_exclude(true) - .require_git(false) + .require_git(ide_core::ignored::inside_git_repo(dir)) .follow_links(false) + .max_depth(max_depth) .filter_entry(move |e| { let name = e.file_name().to_string_lossy(); - if Self::skip_name(&name) { - return false; - } - match e.path().strip_prefix(&root) { - Ok(rel) if !rel.as_os_str().is_empty() => { - let rel = rel.to_string_lossy(); - let rel = if std::path::MAIN_SEPARATOR == '/' { rel } else { rel.replace('\\', "/").into() }; - !excludes.is_match(rel.as_ref()) - } - _ => true, - } + !Self::skip_name(&name) && !glob_excluded(&walk_excludes, &walk_root, e.path()) }) .build_parallel(); walker.run(|| { + let (root, excludes) = (&root, &excludes); let mut batch = Batch { tx: tx.clone(), items: Vec::with_capacity(256) }; Box::new(move |entry| { if cancel.load(Ordering::Relaxed) { return WalkState::Quit; } let Ok(entry) = entry else { return WalkState::Continue }; - if entry.file_type().is_some_and(|t| t.is_file()) + let Some(file_type) = entry.file_type() else { return WalkState::Continue }; + if file_type.is_dir() && max_depth.is_none_or(|max| entry.depth() < max) { + // The walker drops ignored files unseen: list every file of a + // walked directory; those the walker doesn't report are ignored. + let Ok(children) = std::fs::read_dir(entry.path()) else { return WalkState::Continue }; + for child in children.flatten() { + let path = child.path(); + if child.file_type().is_ok_and(|t| t.is_file()) + && !Self::skip_name(&child.file_name().to_string_lossy()) + && !glob_excluded(excludes, root, &path) + && let Ok(meta) = child.metadata() + { + batch.items.push(Found { path, mtime: mtime_of(&meta), size: meta.len(), ignored: true }); + } + } + } else if file_type.is_file() && let Ok(meta) = entry.metadata() { - batch.items.push(Found { path: entry.into_path(), mtime: mtime_of(&meta), size: meta.len() }); - if batch.items.len() >= 256 && batch.tx.send(std::mem::take(&mut batch.items)).is_err() { - return WalkState::Quit; - } + batch.items.push(Found { + path: entry.into_path(), + mtime: mtime_of(&meta), + size: meta.len(), + ignored: false, + }); + } + if batch.items.len() >= 256 && batch.tx.send(std::mem::take(&mut batch.items)).is_err() { + return WalkState::Quit; } WalkState::Continue }) }); drop(tx); - rx.into_iter().flatten().collect() + // Each file comes once from its directory listing and, unless ignored, once from the walker. + let mut by_path: HashMap = HashMap::new(); + for f in rx.into_iter().flatten() { + match by_path.entry(f.path.clone()) { + Entry::Occupied(mut e) => e.get_mut().ignored &= f.ignored, + Entry::Vacant(e) => { + e.insert(f); + } + } + } + by_path.into_values().collect() } } @@ -311,6 +365,103 @@ mod tests { assert!(!filter.is_excluded(&root.join("src/new.rs"), false)); } + #[test] + fn ignored_files_are_listed_but_ignored_directories_are_not() { + let dir = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(dir.path()).unwrap(); + for f in [ + ".env", + "src/a.rs", + "src/debug.log", + "src/local/secrets.json", + "pkg/.gitignore", + "pkg/gen.rs", + "pkg/keep.rs", + "build/out.rs", + "build/nested/deep.rs", + "logs/app.log", + "node_modules/x/.env", + "src/.DS_Store", + "src/.git", + ] { + touch(&root, f); + } + std::fs::write( + root.join(".gitignore"), + ".env +*.log +build/ +logs/ +src/local/ +", + ) + .unwrap(); + std::fs::write( + root.join("pkg/.gitignore"), + "gen.rs +", + ) + .unwrap(); + let filter = PathFilter::new(&root, &vec![].into(), &["**/node_modules".into(), "**/.DS_Store".into()]); + let mut found: Vec<(String, bool)> = filter + .walk(&root, &AtomicBool::new(false)) + .iter() + .filter_map(|f| Some((filter.rel(&f.path)?, f.ignored))) + .collect(); + found.sort(); + let expected = [ + (".env", true), + (".gitignore", false), + ("pkg/.gitignore", false), + ("pkg/gen.rs", true), + ("pkg/keep.rs", false), + ("src/a.rs", false), + ("src/debug.log", true), + ]; + assert_eq!(found, expected.map(|(p, i)| (p.to_owned(), i))); + + // One directory at a time (watcher updates): same classification, no descent. + let mut listed: Vec<(String, bool)> = + filter.list_dir(&root.join("pkg")).iter().filter_map(|f| Some((filter.rel(&f.path)?, f.ignored))).collect(); + listed.sort(); + let expected = [("pkg/.gitignore", false), ("pkg/gen.rs", true), ("pkg/keep.rs", false)]; + assert_eq!(listed, expected.map(|(p, i)| (p.to_owned(), i))); + let top: Vec = filter.list_dir(&root).iter().filter_map(|f| filter.rel(&f.path)).collect(); + assert!(top.iter().all(|p| !p.contains('/')), "{top:?}"); + + for (rel, excluded) in [ + (".env", false), + ("src/new.log", false), + ("src/new.rs", false), + ("build/x.rs", true), + ("logs/app.log", true), + ("src/local/x.json", true), + ] { + assert_eq!(filter.is_excluded(&root.join(rel), false), excluded, "{rel}"); + } + assert!(filter.is_excluded(&root.join("build"), true)); + assert!(!filter.is_excluded(&root.join("src"), true)); + } + + #[test] + fn a_nested_repository_is_not_hidden_by_the_outer_repositorys_gitignore() { + let dir = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(dir.path()).unwrap(); + for f in ["projects/nested/src/a.rs", "projects/nested/out/x.rs"] { + touch(&base, f); + } + std::fs::create_dir(base.join(".git")).unwrap(); + std::fs::create_dir(base.join("projects/nested/.git")).unwrap(); + std::fs::write(base.join("projects/.gitignore"), "*\n!.gitignore\n").unwrap(); + std::fs::write(base.join("projects/nested/.gitignore"), "out/\n").unwrap(); + let root = base.join("projects/nested"); + let filter = PathFilter::new(&root, &vec![].into(), &[]); + let mut found: Vec = + filter.walk(&root, &AtomicBool::new(false)).iter().filter_map(|f| filter.rel(&f.path)).collect(); + found.sort(); + assert_eq!(found, [".gitignore", "src/a.rs"]); + } + #[test] fn extra_content_roots_are_walked_and_prefixed() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/ide-index/src/lib.rs b/crates/ide-index/src/lib.rs index 88923dec..711373fe 100644 --- a/crates/ide-index/src/lib.rs +++ b/crates/ide-index/src/lib.rs @@ -8,6 +8,7 @@ use ide_language::{LanguageRegistry, Syntax}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use specta::Type; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -64,6 +65,14 @@ pub struct IndexStatus { pub symbols: u32, } +/// How many indexed files share an extension (lower case, without the dot). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct ExtensionCount { + pub extension: String, + pub files: u32, +} + pub struct ProjectIndex { shared: Arc, worker: Mutex>>, @@ -159,6 +168,27 @@ impl ProjectIndex { } } + /// The extensions of the indexed files, most common first: the project's file + /// types, for suggestions such as Find in Files' file masks. Dotfiles + /// (`.gitignore`) have no extension. + pub fn file_extensions(&self) -> Vec { + let d = self.shared.data.read(); + let mut counts: HashMap = HashMap::new(); + for file in d.files.iter().flatten() { + let name = &file.rel[file.name_start as usize..]; + if let Some((stem, extension)) = name.rsplit_once('.') + && !stem.is_empty() + && !extension.is_empty() + { + *counts.entry(extension.to_ascii_lowercase()).or_default() += 1; + } + } + let mut out: Vec = + counts.into_iter().map(|(extension, files)| ExtensionCount { extension, files }).collect(); + out.sort_by(|a, b| b.files.cmp(&a.files).then_with(|| a.extension.cmp(&b.extension))); + out + } + /// Fuzzy file search. IDEA-like ranking: file-name matches beat directory /// matches; `/` in the query matches path segments; recently opened files boost. pub fn find_files(&self, query: &str, limit: usize) -> Vec { @@ -167,6 +197,11 @@ impl ProjectIndex { matching::find_files(&d, &matching::Recents { paths: &recent }, query, limit) } + /// Fuzzy search for folders that hold indexed files; `rel_path` ends in `/`. + pub fn find_dirs(&self, query: &str, limit: usize) -> Vec { + matching::find_dirs(&self.shared.data.read(), query, limit) + } + /// Fuzzy symbol search; CamelHump aware (`OCo` → `OrderController`). pub fn find_symbols(&self, query: &str, filter: SymbolFilter, limit: usize) -> Vec { let recent = self.shared.recent.lock().clone(); @@ -366,6 +401,10 @@ pub mod bench_support { matching::find_files(&self.data, &matching::Recents { paths: &self.recent }, query, limit) } + pub fn find_dirs(&self, query: &str, limit: usize) -> Vec { + matching::find_dirs(&self.data, query, limit) + } + pub fn find_symbols(&self, query: &str, filter: SymbolFilter, limit: usize) -> Vec { matching::find_symbols(&self.data, &matching::Recents { paths: &self.recent }, query, filter, limit) } diff --git a/crates/ide-index/src/matching.rs b/crates/ide-index/src/matching.rs index bea3f3c5..dcb9b383 100644 --- a/crates/ide-index/src/matching.rs +++ b/crates/ide-index/src/matching.rs @@ -12,7 +12,7 @@ use nucleo_matcher::{Config, Matcher, Utf32Str, Utf32String}; use rayon::prelude::*; use std::cell::RefCell; use std::cmp::Ordering; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; const TIER_NAME: u32 = 1 << 20; const EXACT: u32 = 4000; @@ -251,6 +251,76 @@ pub(crate) fn find_files(data: &IndexData, recents: &Recents, query: &str, limit .collect() } +/// Fuzzy folder search over the folders holding indexed files: folder-name matches beat +/// path matches; `/` in the query matches the whole path. `rel_path` ends in `/`. +pub(crate) fn find_dirs(data: &IndexData, query: &str, limit: usize) -> Vec { + let q = normalize_file_query(query); + if limit == 0 || q.is_empty() { + return Vec::new(); + } + let by_path = q.contains('/'); + let needle = Needle::new(q.trim_end_matches('/')); + // (score, rel without the trailing `/`, char offset of the name when it matched by name) + let score = |m: &mut Matcher, buf: &mut Vec, rel: &str| -> Option<(u32, Option)> { + let trimmed = &rel[..rel.len() - 1]; + let start = trimmed.rfind('/').map_or(0, |i| i + 1); + let name = &trimmed[start..]; + if !by_path + && needle.mask & char_mask(name) == needle.mask + && let Some(s) = needle.score(m, name, buf) + { + return Some((TIER_NAME + s * 4 + needle.name_boost(name), Some(trimmed[..start].chars().count() as u32))); + } + if needle.mask & char_mask(trimmed) != needle.mask { + return None; + } + needle.score(m, trimmed, buf).map(|s| (s, None)) + }; + let mut best: Vec<(u32, &str, &Path, Option)> = data + .dirs + .par_iter() + .filter_map(|(rel, (path, _))| { + let mut buf = Vec::new(); + with_matcher(|m| score(m, &mut buf, rel)).map(|(s, at)| (s, &**rel, path.as_path(), at)) + }) + .collect(); + let order = |a: &(u32, &str, &Path, Option), b: &(u32, &str, &Path, Option)| { + b.0.cmp(&a.0).then_with(|| a.1.len().cmp(&b.1.len())).then_with(|| a.1.cmp(b.1)) + }; + if best.len() > limit { + best.select_nth_unstable_by(limit, order); + best.truncate(limit); + } + // Match indices and the CamelHump bonus `find_files` gives, so equal matches rank alike. + let mut out: Vec = with_matcher(|m| { + let (mut buf, mut idx) = (Vec::new(), Vec::new()); + best.into_iter() + .map(|(score, rel, path, name_at)| { + let trimmed = &rel[..rel.len() - 1]; + let (bonus, indices) = match name_at { + Some(at) => { + let name = &trimmed[trimmed.rfind('/').map_or(0, |i| i + 1)..]; + needle.indices(m, name, &mut buf, &mut idx); + (hump_bonus(name, &idx), idx.iter().map(|i| i + at).collect()) + } + None => { + needle.indices(m, trimmed, &mut buf, &mut idx); + (hump_bonus(trimmed, &idx), idx.clone()) + } + }; + FileMatch { path: path.to_path_buf().into(), rel_path: rel.to_owned(), score: score + bonus, indices } + }) + .collect() + }); + out.sort_by(|a, b| { + b.score + .cmp(&a.score) + .then_with(|| a.rel_path.len().cmp(&b.rel_path.len())) + .then_with(|| a.rel_path.cmp(&b.rel_path)) + }); + out +} + fn file(data: &IndexData, slot: u32) -> &FileEntry { data.files[slot as usize].as_ref().expect("candidate slots are live") } diff --git a/crates/ide-index/src/store.rs b/crates/ide-index/src/store.rs index 7419c709..f32dac55 100644 --- a/crates/ide-index/src/store.rs +++ b/crates/ide-index/src/store.rs @@ -8,7 +8,7 @@ use std::path::Path; const FILES: TableDefinition<&str, &[u8]> = TableDefinition::new("files"); const META: TableDefinition<&str, &[u8]> = TableDefinition::new("meta"); /// Bump when [`FileRecord`] or its encoding changes. -const SCHEMA: &str = "openidea-index-2"; +const SCHEMA: &str = "openidea-index-3"; type Result = std::result::Result; diff --git a/crates/ide-index/src/tests.rs b/crates/ide-index/src/tests.rs index 97ba3c7c..d8dadee2 100644 --- a/crates/ide-index/src/tests.rs +++ b/crates/ide-index/src/tests.rs @@ -111,6 +111,28 @@ fn indexes_files_and_symbols_honouring_excludes() { assert_eq!(m.path, fx.root.join("src/main.rs")); } +#[test] +fn counts_file_extensions_most_common_first() { + let fx = Fixture::new(&[ + ("src/a.rs", "fn a() {}\n"), + ("src/b.RS", "fn b() {}\n"), + ("src/lib.rs", "fn c() {}\n"), + ("Cargo.toml", "[package]\n"), + ("README.md", "# r\n"), + ("docs/guide.md", "# g\n"), + (".gitignore", "target\n"), + ("Makefile", "all:\n"), + ("target/debug/gen.rs", "fn generated() {}\n"), + ]); + let idx = fx.open(); + let counts: Vec<(String, u32)> = idx.file_extensions().into_iter().map(|c| (c.extension, c.files)).collect(); + assert_eq!( + counts, + [("rs".into(), 3), ("md".into(), 2), ("toml".into(), 1)], + "case-folded, excluded folders and extension-less files left out" + ); +} + #[test] fn file_ranking_is_idea_like() { let fx = sample(); @@ -138,6 +160,23 @@ fn file_ranking_is_idea_like() { assert_eq!(shown.to_lowercase(), "uibtn"); } +#[test] +fn find_dirs_ranks_folder_names() { + let f = sample(); + let idx = f.open(); + let dirs = |q: &str| idx.find_dirs(q, 20).into_iter().map(|m| m.rel_path).collect::>(); + assert_eq!(dirs("order"), ["src/order/"]); + assert_eq!(dirs("src")[0], "src/"); + assert!(dirs("src").contains(&"src/ui/".to_owned()), "{:?}", dirs("src")); + assert_eq!(dirs("src/ui/"), ["src/ui/"]); + assert!(dirs("").is_empty()); + assert!(dirs("main").is_empty()); + + let m = &idx.find_dirs("ui", 1)[0]; + assert_eq!(m.path.0, f.root.join("src/ui")); + assert_eq!(m.indices, vec![4, 5]); +} + #[test] fn shorter_paths_win_ties_and_recent_files_boost() { let fx = Fixture::new(&[("a/b/c/util.rs", ""), ("x/util.rs", ""), ("y/util.rs", "")]); @@ -276,6 +315,56 @@ fn binary_and_huge_files_are_listed_without_symbols() { assert_eq!(idx.status().symbols, 0); } +#[test] +fn go_to_file_finds_ignored_files_but_not_their_symbols() { + let fx = Fixture::new(&[ + (".gitignore", ".env\n*.local.ts\ndist/\n"), + (".env", "SECRET=1\n"), + ("src/app.ts", "export function app() {}\n"), + ("src/config.local.ts", "export function localOnly() {}\n"), + ("dist/bundle.js", "function bundled() {}\n"), + ]); + let idx = fx.open(); + assert_eq!(files(&idx, ".env"), vec![".env"], "the reported case: a git-ignored root file"); + assert_eq!(files(&idx, "config.local"), vec!["src/config.local.ts"]); + assert!(files(&idx, "bundle").is_empty(), "ignored directories stay out"); + assert!(symbols(&idx, "localOnly", SymbolFilter::All).is_empty(), "ignored files are listed by name only"); + assert_eq!(symbols(&idx, "app", SymbolFilter::All), vec!["app"]); + + // Watcher updates keep the distinction. + let fresh = fx.write("src/fresh.local.ts", "export function freshLocal() {}\n"); + let bundled = fx.write("dist/more.js", "function more() {}\n"); + idx.update(&[fresh, bundled], &[]); + assert!(idx.wait_idle(WAIT)); + assert_eq!(files(&idx, "fresh.local"), vec!["src/fresh.local.ts"]); + assert!(symbols(&idx, "freshLocal", SymbolFilter::All).is_empty()); + assert!(files(&idx, "more.js").is_empty()); + + // Nested ignore files apply to watcher updates as they do to the scan. + fx.write("pkg/.gitignore", "gen.ts\n"); + let generated = fx.write("pkg/gen.ts", "export function generatedThing() {}\n"); + let kept = fx.write("pkg/kept.ts", "export function keptThing() {}\n"); + idx.update(&[generated, kept], &[]); + assert!(idx.wait_idle(WAIT)); + assert_eq!(files(&idx, "pkg/gen"), vec!["pkg/gen.ts"]); + assert!(symbols(&idx, "generatedThing", SymbolFilter::All).is_empty()); + assert_eq!(symbols(&idx, "keptThing", SymbolFilter::All), vec!["keptThing"]); + drop(idx); + + // No longer ignored: the next scan parses it although its mtime is unchanged. + std::fs::write(fx.root.join(".gitignore"), ".env\ndist/\n").unwrap(); + let idx = fx.open(); + assert_eq!(symbols(&idx, "localOnly", SymbolFilter::All), vec!["localOnly"]); + assert_eq!(files(&idx, ".env"), vec![".env"]); + drop(idx); + + // Ignored again: its symbols are dropped. + std::fs::write(fx.root.join(".gitignore"), "*.local.ts\n").unwrap(); + let idx = fx.open(); + assert!(symbols(&idx, "localOnly", SymbolFilter::All).is_empty()); + assert_eq!(files(&idx, "bundle"), vec!["dist/bundle.js"]); +} + #[test] fn second_window_on_same_project_works_in_memory() { let fx = sample(); diff --git a/crates/ide-index/src/worker.rs b/crates/ide-index/src/worker.rs index a6947271..3928edd4 100644 --- a/crates/ide-index/src/worker.rs +++ b/crates/ide-index/src/worker.rs @@ -2,13 +2,13 @@ //! updates queued by the watcher. use crate::data::{FileEntry, FileRecord, IndexData, StoredSymbol}; -use crate::filter::{Found, PathFilter, mtime_of}; +use crate::filter::{Found, PathFilter}; use crate::store::Store; use ide_core::{IdeEvent, SharedSink, TaskId}; use ide_language::{LanguageRegistry, Syntax}; use parking_lot::{Condvar, Mutex, RwLock}; use rayon::prelude::*; -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -233,10 +233,7 @@ impl Shared { let seen: HashSet<&Path> = found.iter().map(|f| f.path.as_path()).collect(); let (todo, stale): (Vec<&Found>, Vec) = { let d = self.data.read(); - let todo = found - .iter() - .filter(|f| d.get(&f.path).is_none_or(|e| e.mtime != f.mtime || e.size != f.size)) - .collect(); + let todo = found.iter().filter(|f| d.get(&f.path).is_none_or(|e| e.stale_for(f))).collect(); let stale = d.paths().filter(|p| !seen.contains(p.as_path())).cloned().collect(); (todo, stale) }; @@ -311,6 +308,7 @@ impl Shared { let mut rec = d.get(&f.path)?.record(); rec.mtime = f.mtime; rec.size = f.size; + rec.ignored = f.ignored; Some((rel, rec, f.path.clone())) } Extracted::New(rec) => Some((rel, rec, f.path.clone())), @@ -342,8 +340,8 @@ impl Shared { } fn extract(&self, f: &Found, prev_hash: Option<[u8; 32]>) -> Extracted { - let mut rec = FileRecord { mtime: f.mtime, size: f.size, ..Default::default() }; - if f.size > MAX_SYMBOL_FILE { + let mut rec = FileRecord { mtime: f.mtime, size: f.size, ignored: f.ignored, ..Default::default() }; + if f.ignored || f.size > MAX_SYMBOL_FILE { return Extracted::New(rec); } let mut bytes = Vec::with_capacity(f.size as usize); @@ -420,6 +418,7 @@ impl Shared { let mut files: Vec = Vec::new(); let mut seen = HashSet::new(); + let mut by_dir: HashMap> = HashMap::new(); for path in changed { if !seen.insert(path.clone()) { continue; @@ -430,16 +429,19 @@ impl Shared { } if meta.is_dir() { files.extend(filter.walk(&path, &self.cancelled)); - } else if meta.is_file() { - files.push(Found { mtime: mtime_of(&meta), size: meta.len(), path }); + } else if meta.is_file() + && let Some(dir) = path.parent() + { + by_dir.entry(dir.to_owned()).or_default().insert(path); } } + // Listing the folder applies every ignore file (nested, global) to tell ignored files apart. + for (dir, wanted) in by_dir { + files.extend(filter.list_dir(&dir).into_iter().filter(|f| wanted.contains(&f.path))); + } let files: Vec = { let d = self.data.read(); - files - .into_iter() - .filter(|f| d.get(&f.path).is_none_or(|e| e.mtime != f.mtime || e.size != f.size)) - .collect() + files.into_iter().filter(|f| d.get(&f.path).is_none_or(|e| e.stale_for(f))).collect() }; if !files.is_empty() { touched = true; @@ -457,7 +459,7 @@ impl Shared { .filter(|p| before.detect(p, None) != self.languages.detect(p, None)) .filter_map(|p| { let e = d.get(p)?; - Some(Found { path: p.clone(), mtime: e.mtime, size: e.size }) + Some(Found { path: p.clone(), mtime: e.mtime, size: e.size, ignored: e.ignored }) }) .collect() }; diff --git a/crates/ide-jsonrpc/Cargo.toml b/crates/ide-jsonrpc/Cargo.toml index 615d1f81..4161989a 100644 --- a/crates/ide-jsonrpc/Cargo.toml +++ b/crates/ide-jsonrpc/Cargo.toml @@ -16,3 +16,6 @@ parking_lot.workspace = true thiserror.workspace = true tracing.workspace = true encoding_rs.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/ide-jsonrpc/src/lib.rs b/crates/ide-jsonrpc/src/lib.rs index c04331ad..f78646ce 100644 --- a/crates/ide-jsonrpc/src/lib.rs +++ b/crates/ide-jsonrpc/src/lib.rs @@ -216,6 +216,9 @@ pub trait Handler: Send + Sync + 'static { fn on_request_with_data(&self, method: &str, params: Value) -> Result)> { self.on_request(method, params).map_err(|(code, message)| (code, message, None)) } + + /// Called once the response to a `method` request is queued: what it sends follows the response. + fn after_response(&self, _method: &str) {} } /// A [`Handler`] that ignores notifications and rejects requests. @@ -243,6 +246,9 @@ struct Inner { pid: Option, /// Fired when the reader task has stopped (EOF, error or kill). reader_done: CancellationToken, + /// Fired once the child process has exited and been reaped (after [`Peer::kill`], + /// also what it started); at once when there is no child. + exited: CancellationToken, } impl Inner { @@ -432,6 +438,7 @@ impl Peer { (child.wait().await.ok(), true) } }; + inner.exited.cancel(); if !killed { // Bounded: a grandchild may keep the pipes open. let drained = async { @@ -510,7 +517,11 @@ impl Peer { kill: CancellationToken::new(), pid, reader_done: CancellationToken::new(), + exited: CancellationToken::new(), }); + if pid.is_none() { + inner.exited.cancel(); + } // Writer: a single task drains the queue so messages never interleave. { @@ -704,6 +715,13 @@ impl Peer { self.inner.close_transport(); self.inner.kill.cancel(); } + + /// Resolves once the child process is gone: exited on its own, or killed with what it + /// started and reaped. [`closed`](Self::closed) can come earlier, at EOF. Until then the + /// child may still hold files open, which on Windows can't be deleted. + pub async fn exited(&self) { + self.inner.exited.cancelled().await; + } } /// Removes the pending entry and tells the remote side to stop when a request @@ -740,7 +758,8 @@ enum Incoming { async fn handle_request(inner: Arc, handler: Arc, id: Value, method: String, params: Value) { let m = method.clone(); - let result = tokio::task::spawn_blocking(move || handler.on_request_with_data(&m, params)).await; + let h = handler.clone(); + let result = tokio::task::spawn_blocking(move || h.on_request_with_data(&m, params)).await; let reply = match result { Ok(Ok(result)) => json!({ "jsonrpc": "2.0", "id": id, "result": result }), Ok(Err((code, message, data))) => { @@ -755,7 +774,9 @@ async fn handle_request(inner: Arc, handler: Arc, id: Value, "error": { "code": codes::INTERNAL_ERROR, "message": format!("handler for {method} panicked") } }), }; - let _ = inner.send_value(&reply); + if inner.send_value(&reply).is_ok() { + handler.after_response(&method); + } } fn dispatch(inner: &Arc, notif_tx: &std::sync::mpsc::Sender, body: &[u8]) { diff --git a/crates/ide-jsonrpc/src/tests.rs b/crates/ide-jsonrpc/src/tests.rs index 7db0823f..9efee601 100644 --- a/crates/ide-jsonrpc/src/tests.rs +++ b/crates/ide-jsonrpc/src/tests.rs @@ -251,6 +251,34 @@ async fn server_to_client_request() { assert!(matches!(err, RpcError::Server { code: codes::METHOD_NOT_FOUND, .. })); } +#[tokio::test(flavor = "multi_thread")] +async fn after_response_messages_follow_the_response() { + #[derive(Default)] + struct Client(std::sync::OnceLock); + impl Handler for Client { + fn on_notification(&self, _: &str, _: Value) {} + fn on_request(&self, _: &str, _: Value) -> Result { + Ok(json!("answer")) + } + fn after_response(&self, method: &str) { + self.0.get().unwrap().notify("after", method).unwrap(); + } + } + let client = Arc::new(Client::default()); + let (client_io, server_io) = duplex(64 * 1024); + let (cr, cw) = tokio::io::split(client_io); + assert!(client.0.set(Peer::connect(cr, cw, client.clone())).is_ok()); + let (sr, mut sw) = tokio::io::split(server_io); + let request = br#"{"jsonrpc":"2.0","id":1,"method":"workspace/configuration","params":{}}"#; + framing::write_message(&mut sw, request).await.unwrap(); + let mut reader = tokio::io::BufReader::new(sr); + let mut next = async || -> Value { + serde_json::from_slice(&framing::read_message(&mut reader).await.unwrap().unwrap()).unwrap() + }; + assert_eq!(next().await, json!({ "jsonrpc": "2.0", "id": 1, "result": "answer" })); + assert_eq!(next().await, json!({ "jsonrpc": "2.0", "method": "after", "params": "workspace/configuration" })); +} + #[tokio::test(flavor = "multi_thread")] async fn requests_start_after_the_notifications_before_them() { // `seen` answers with the notifications handled so far; handling one is slow. @@ -413,6 +441,65 @@ async fn kill_terminates_process() { tokio::time::timeout(Duration::from_secs(5), peer.closed()).await.unwrap(); } +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread")] +async fn exited_resolves_once_the_killed_process_is_reaped() { + let peer = + Peer::spawn("sleep", &["30".into()], &std::env::temp_dir(), &[], Arc::new(NoopHandler), Box::new(|_| {})) + .unwrap(); + let pid = peer.pid().unwrap().to_string(); + let exited = tokio::time::timeout(Duration::from_millis(200), peer.exited()).await; + assert!(exited.is_err(), "running until killed"); + peer.kill(); + tokio::time::timeout(Duration::from_secs(5), peer.exited()).await.unwrap(); + let alive = std::process::Command::new("kill").args(["-0", &pid]).status().unwrap().success(); + assert!(!alive, "reaped, not even a zombie"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn exited_resolves_at_once_without_a_process() { + let (client, _server) = pair(Arc::new(NoopHandler), Arc::new(Recorder::default())); + tokio::time::timeout(Duration::from_secs(1), client.exited()).await.unwrap(); +} + +#[cfg(windows)] +#[tokio::test(flavor = "multi_thread")] +async fn exited_waits_for_a_windows_descendant_to_release_its_file() { + use std::os::windows::fs::OpenOptionsExt; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("hold.ps1"), + "$f=[IO.File]::Open('locked.txt','Create','ReadWrite','None'); [IO.File]::WriteAllText('ready.txt','ready'); Start-Sleep 30", + ) + .unwrap(); + let peer = Peer::spawn( + "cmd", + &[ + "/D".into(), + "/C".into(), + "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File hold.ps1".into(), + ], + tmp.path(), + &[], + Arc::new(NoopHandler), + Box::new(|_| {}), + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while !tmp.path().join("ready.txt").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let open = || std::fs::OpenOptions::new().read(true).share_mode(0).open(tmp.path().join("locked.txt")); + assert_eq!(open().unwrap_err().raw_os_error(), Some(32)); + assert!(tokio::time::timeout(Duration::from_millis(200), peer.exited()).await.is_err()); + peer.kill(); + tokio::time::timeout(Duration::from_secs(5), peer.exited()).await.unwrap(); + drop(open().expect("exited must wait for the descendant's file handle to close")); +} + /// A launcher that prints in the system code page and exits before speaking the /// protocol: `jdtls.bat` on a Chinese Windows when its Python step fails, then /// `pause` prints `请按任意键继续. . .` in GBK on stdout. diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rs index 7054d781..8d606d3d 100644 --- a/crates/ide-language/src/lib.rs +++ b/crates/ide-language/src/lib.rs @@ -60,6 +60,9 @@ pub struct LanguageConfig { /// External formatter used when no language server can format, or selected in settings. #[serde(default)] pub formatter: Option, + /// Content of a new scratch file; `$END$` marks where the caret goes. + #[serde(default)] + pub scratch_template: Option, } /// An external formatting tool and its language-specific parser. @@ -152,6 +155,11 @@ pub struct LanguageServerConfig { /// Register source files with tsserver when no project configuration exists. #[serde(default)] pub tsserver_project: bool, + /// Re-sends the open documents (unchanged) once `workspace/configuration` is answered: + /// the server (Intelephense) applies some settings, such as the PHP version, only when + /// it parses a document. + #[serde(default)] + pub resync_on_configuration: bool, /// Set on plugin languages: start exactly this approved program. #[serde(skip)] pub pinned: Option, @@ -242,6 +250,10 @@ pub struct LanguageInfo { #[serde(default)] #[specta(optional)] pub first_line: Option, + /// Content of a new scratch file; `$END$` marks where the caret goes. + #[serde(default)] + #[specta(optional)] + pub scratch_template: Option, } impl From<&LanguageConfig> for LanguageInfo { @@ -258,6 +270,7 @@ impl From<&LanguageConfig> for LanguageInfo { language_servers: c.language_servers.iter().map(|s| s.name.clone()).collect(), filenames: c.filenames.clone(), first_line: c.first_line.clone(), + scratch_template: c.scratch_template.clone(), } } } diff --git a/crates/ide-language/src/registry.rs b/crates/ide-language/src/registry.rs index 83264595..aff9d7de 100644 --- a/crates/ide-language/src/registry.rs +++ b/crates/ide-language/src/registry.rs @@ -772,6 +772,10 @@ mod tests { assert!(addr.parse::().is_ok(), "{name}: bad connect address {addr}"); } } + if let Some(t) = &cfg.scratch_template { + assert_eq!(t.matches("$END$").count(), 1, "{name}: scratch-template needs one $END$"); + assert!(!cfg.extensions.is_empty(), "{name}: scratch-template without an extension"); + } for ext in &cfg.extensions { assert_eq!(ext, &ext.to_ascii_lowercase(), "{name}: extensions must be lower case"); assert!(!ext.starts_with('.'), "{name}: extension {ext} must not start with a dot"); diff --git a/crates/ide-lsp/src/caps.rs b/crates/ide-lsp/src/caps.rs index a20296ef..ad84f59e 100644 --- a/crates/ide-lsp/src/caps.rs +++ b/crates/ide-lsp/src/caps.rs @@ -37,7 +37,7 @@ pub fn client_capabilities() -> Value { "fileOperations": { "dynamicRegistration": true, "willRename": true, "didRename": true }, "configuration": true, "didChangeConfiguration": { "dynamicRegistration": false }, - "didChangeWatchedFiles": { "dynamicRegistration": false, "relativePatternSupport": false }, + "didChangeWatchedFiles": { "dynamicRegistration": true, "relativePatternSupport": true }, "symbol": { "dynamicRegistration": false, "symbolKind": { "valueSet": symbol_kinds }, @@ -48,7 +48,7 @@ pub fn client_capabilities() -> Value { "semanticTokens": { "refreshSupport": true }, "codeLens": { "refreshSupport": true }, "inlayHint": { "refreshSupport": true }, - "diagnostics": { "refreshSupport": false }, + "diagnostics": { "refreshSupport": true }, "textDocumentContent": { "dynamicRegistration": true } }, "textDocument": { @@ -130,6 +130,7 @@ pub fn client_capabilities() -> Value { "prepareSupportDefaultBehavior": 1, "honorsChangeAnnotations": false }, + "diagnostic": { "dynamicRegistration": false, "relatedDocumentSupport": false }, "publishDiagnostics": { "relatedInformation": true, "tagSupport": { "valueSet": [1, 2] }, diff --git a/crates/ide-lsp/src/external_project.rs b/crates/ide-lsp/src/external_project.rs index cb9393c9..b00900a4 100644 --- a/crates/ide-lsp/src/external_project.rs +++ b/crates/ide-lsp/src/external_project.rs @@ -59,7 +59,7 @@ fn source_files(root: &Path, exclude_root: &Path, excludes: Vec, limit: } let Ok(ignores) = ignores.build() else { return Vec::new() }; let mut walk = ignore::WalkBuilder::new(root); - walk.require_git(false).follow_links(false).filter_entry(move |entry| { + walk.require_git(ide_core::ignored::inside_git_repo(root)).follow_links(false).filter_entry(move |entry| { if entry.depth() == 0 { return true; } diff --git a/crates/ide-lsp/src/file_watch.rs b/crates/ide-lsp/src/file_watch.rs new file mode 100644 index 00000000..6ec5290e --- /dev/null +++ b/crates/ide-lsp/src/file_watch.rs @@ -0,0 +1,150 @@ +//! `workspace/didChangeWatchedFiles` registrations: which changes on disk a +//! server asked to hear of (its globs, relative patterns and change kinds). + +use crate::uri; +use globset::{GlobBuilder, GlobMatcher}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +/// `FileChangeType`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Change { + Created = 1, + Changed = 2, + Deleted = 3, +} + +impl Change { + /// The `WatchKind` bit. + fn kind_bit(self) -> u64 { + match self { + Self::Created => 1, + Self::Changed => 2, + Self::Deleted => 4, + } + } +} + +struct Watcher { + /// Relative pattern's base; `None` for a plain glob, matched below the server's root. + base: Option, + /// A plain glob that is an absolute path pattern. + absolute: bool, + glob: GlobMatcher, + kind: u64, +} + +/// A server's registrations, by registration id. +#[derive(Default)] +pub struct FileWatchers(BTreeMap>); + +impl FileWatchers { + pub fn register(&mut self, id: &str, options: Option<&Value>) { + let watchers = options + .and_then(|o| o.get("watchers")) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(parse) + .collect(); + self.0.insert(id.to_owned(), watchers); + } + + pub fn unregister(&mut self, id: &str) { + self.0.remove(id); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Whether a server rooted at `root` wants to hear of `change` to `path`. + pub fn wants(&self, root: &Path, path: &Path, change: Change) -> bool { + self.0.values().flatten().any(|w| { + if w.kind & change.kind_bit() == 0 { + return false; + } + match &w.base { + Some(base) => path.strip_prefix(base).is_ok_and(|rel| w.glob.is_match(rel)), + None if w.absolute => w.glob.is_match(path), + None => path.strip_prefix(root).is_ok_and(|rel| w.glob.is_match(rel)), + } + }) + } + + /// Bases of relative patterns, which may lie outside the project (a virtualenv's + /// `site-packages`, a path dependency): the client watches them for the server. + pub fn bases(&self) -> impl Iterator { + self.0.values().flatten().filter_map(|w| w.base.as_deref()) + } +} + +fn parse(watcher: &Value) -> Option { + let kind = watcher.get("kind").and_then(Value::as_u64).unwrap_or(7); + let (base, pattern) = match watcher.get("globPattern")? { + Value::String(p) => (None, p.as_str()), + relative => { + let base = relative.get("baseUri")?; + // A `WorkspaceFolder` or a URI. + let base = base.get("uri").unwrap_or(base).as_str()?; + let base = uri::uri_to_path(base)?; + // Changes are reported for canonical paths (`/tmp` is `/private/tmp` on macOS). + (Some(dunce::canonicalize(&base).unwrap_or(base)), relative.get("pattern")?.as_str()?) + } + }; + let glob = GlobBuilder::new(pattern).literal_separator(true).build().ok()?.compile_matcher(); + let absolute = base.is_none() && Path::new(pattern).is_absolute(); + Some(Watcher { base, absolute, glob, kind }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn watchers(list: Value) -> FileWatchers { + let mut w = FileWatchers::default(); + w.register("1", Some(&json!({ "watchers": list }))); + w + } + + #[test] + fn matches_globs_relative_patterns_and_kinds() { + let root = Path::new("/p"); + let venv = uri::path_to_uri(Path::new("/opt/venv/lib/site-packages")); + let w = watchers(json!([ + { "globPattern": "**/*.{go,mod,sum}" }, + { "globPattern": "**/pyrightconfig.json", "kind": 2 }, + { "globPattern": { "baseUri": venv, "pattern": "**" } }, + { "globPattern": { "baseUri": { "uri": uri::path_to_uri(Path::new("/shared")), "name": "s" }, "pattern": "*.toml" } }, + ])); + let cases = [ + ("/p/go.mod", Change::Changed, true), + ("/p/sub/main.go", Change::Created, true), + ("/elsewhere/main.go", Change::Created, false), + ("/p/pyrightconfig.json", Change::Changed, true), + ("/p/pyrightconfig.json", Change::Created, false), + ("/opt/venv/lib/site-packages/idna/__init__.py", Change::Created, true), + ("/opt/venv/lib/site-packages", Change::Deleted, true), + ("/shared/Cargo.toml", Change::Changed, true), + ("/shared/nested/Cargo.toml", Change::Changed, false), + ("/p/readme.md", Change::Changed, false), + ]; + for (path, change, expected) in cases { + assert_eq!(w.wants(root, Path::new(path), change), expected, "{path} {change:?}"); + } + let mut bases: Vec<_> = w.bases().collect(); + bases.sort(); + assert_eq!(bases, [Path::new("/opt/venv/lib/site-packages"), Path::new("/shared")]); + } + + #[test] + fn unregistering_drops_the_watchers() { + let mut w = watchers(json!([{ "globPattern": "**" }])); + assert!(w.wants(Path::new("/p"), Path::new("/p/a"), Change::Created)); + w.unregister("1"); + assert!(w.is_empty()); + assert!(!w.wants(Path::new("/p"), Path::new("/p/a"), Change::Created)); + } +} diff --git a/crates/ide-lsp/src/handler.rs b/crates/ide-lsp/src/handler.rs index 006e4d09..1c89fd9c 100644 --- a/crates/ide-lsp/src/handler.rs +++ b/crates/ide-lsp/src/handler.rs @@ -130,15 +130,8 @@ impl ClientHandler { { return; } - let conv = mgr.conv(self.encoding()); - let items = params - .diagnostics - .iter() - .cloned() - .map(|d| convert::diagnostic(&conv, &path, d, &self.server.name)) - .collect(); - self.server.set_diagnostics(path.clone(), params.diagnostics); - mgr.sink().emit(IdeEvent::Diagnostics { path: path.into(), source: self.server.name.clone(), items }); + let shown = self.server.set_diagnostics(path.clone(), params.diagnostics); + mgr.emit_diagnostics(&self.server, path, shown, self.encoding()); } fn show_message(&self, mgr: &LspManager, params: &Value) { @@ -206,19 +199,33 @@ impl ClientHandler { Ok(reply) } - fn registration(&self, params: &Value, register: bool) { - let Some(run) = self.server.running_gen(self.generation) else { return }; + fn registration(&self, mgr: &LspManager, params: &Value, register: bool) { + let Some(run) = self.server.initialized_gen(self.generation) else { return }; let key = if register { "registrations" } else { "unregisterations" }; let list = params.get(key).or_else(|| params.get("unregistrations")).and_then(Value::as_array); let mut caps = run.caps.write(); + let mut watchers_changed = false; for reg in list.into_iter().flatten() { let Some(method) = reg.get("method").and_then(Value::as_str) else { continue }; - if register { + if method == "workspace/didChangeWatchedFiles" { + let id = reg.get("id").and_then(Value::as_str).unwrap_or_default(); + let mut watchers = run.watchers.write(); + if register { + watchers.register(id, reg.get("registerOptions")); + } else { + watchers.unregister(id); + } + watchers_changed = true; + } else if register { caps.register(method, reg.get("registerOptions")); } else { caps.unregister(method); } } + drop(caps); + if watchers_changed { + mgr.file_watch_roots_changed(); + } } } @@ -278,11 +285,11 @@ impl Handler for ClientHandler { "workspace/configuration" => Ok(self.configuration(¶ms)), "workspace/workspaceFolders" => Ok(mgr.workspace_folders(&self.server)), "client/registerCapability" => { - self.registration(¶ms, true); + self.registration(&mgr, ¶ms, true); Ok(Value::Null) } "client/unregisterCapability" => { - self.registration(¶ms, false); + self.registration(&mgr, ¶ms, false); Ok(Value::Null) } "workspace/applyEdit" => self.apply_edit(&mgr, params), @@ -295,10 +302,23 @@ impl Handler for ClientHandler { mgr.sink().emit(IdeEvent::LspRefresh { server: self.server.id, kind }); Ok(Value::Null) } - "workspace/diagnostic/refresh" | "workspace/inlineValue/refresh" => Ok(Value::Null), + "workspace/diagnostic/refresh" => { + mgr.refresh_pulled_diagnostics(&self.server); + Ok(Value::Null) + } + "workspace/inlineValue/refresh" => Ok(Value::Null), other => Err((codes::METHOD_NOT_FOUND, format!("unsupported method {other}"))), } } + + fn after_response(&self, method: &str) { + if method == "workspace/configuration" + && self.server.config.resync_on_configuration + && let Some(mgr) = self.mgr.upgrade() + { + mgr.resync_documents(&self.server, self.generation); + } + } } /// What `workspace/configuration` sections are looked up in: the server's diff --git a/crates/ide-lsp/src/inspect.rs b/crates/ide-lsp/src/inspect.rs index 19c4e580..e8509425 100644 --- a/crates/ide-lsp/src/inspect.rs +++ b/crates/ide-lsp/src/inspect.rs @@ -6,8 +6,6 @@ //! (`textDocument/diagnostic`). The others are waited on: the first //! `publishDiagnostics` for the file after it was opened, then a short quiet //! period so follow-up publishes (syntax first, semantics later) are caught. -//! The client never advertises pull support, so servers keep pushing for the -//! editor as before. use crate::convert; use crate::server::Server; diff --git a/crates/ide-lsp/src/jdtls.rs b/crates/ide-lsp/src/jdtls.rs index 5852f53e..360dfa96 100644 --- a/crates/ide-lsp/src/jdtls.rs +++ b/crates/ide-lsp/src/jdtls.rs @@ -20,6 +20,7 @@ const SERVER: &str = "jdtls"; pub(crate) mod configuration; mod formatter; +mod hierarchy; pub(crate) mod reserved_names; pub use formatter::configure_formatter; /// First start imports the Maven / Gradle model, which can take minutes. diff --git a/crates/ide-lsp/src/jdtls/hierarchy.rs b/crates/ide-lsp/src/jdtls/hierarchy.rs new file mode 100644 index 00000000..55430784 --- /dev/null +++ b/crates/ide-lsp/src/jdtls/hierarchy.rs @@ -0,0 +1,101 @@ +//! Pull Members Up / Push Members Down through the app-owned JDT extension +//! (`java/java-usages`). The server checks the move and computes the edit; the UI +//! shows its problems and applies the edit itself. +use super::COMMAND_TIMEOUT; +use crate::types::{HierarchyChoice, HierarchyMembers, HierarchyRefactored, HierarchyRefactoring, RefactoringProblem}; +use crate::{DocSnap, LspManager, Target, convert}; +use ide_core::{CancellationToken, DocId, IdeError, IdeResult, Pos}; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; + +const MEMBERS: &str = "openidea.java.hierarchy.members"; +const REFACTOR: &str = "openidea.java.hierarchy.refactor"; + +/// The extension's answer: a result, or why the position can't be refactored. +#[derive(Deserialize)] +#[serde(untagged)] +enum Answer { + Refused { refused: String }, + Ok(T), +} + +impl Answer { + fn into_result(self) -> IdeResult { + match self { + Answer::Refused { refused } => Err(IdeError::invalid(refused)), + Answer::Ok(value) => Ok(value), + } + } +} + +#[derive(Deserialize)] +struct Refactored { + problems: Vec, + edit: Option, +} + +fn feature(kind: HierarchyRefactoring) -> &'static str { + match kind { + HierarchyRefactoring::PullUp => "Pull Members Up", + HierarchyRefactoring::PushDown => "Push Members Down", + } +} + +impl LspManager { + async fn hierarchy_command( + &self, + doc: DocId, + pos: Pos, + kind: HierarchyRefactoring, + command: &str, + extra: &[Value], + cancel: &CancellationToken, + ) -> IdeResult<(R, DocSnap, Target)> { + let (snap, mut targets) = + self.doc_targets(doc, feature(kind), |c| c.commands().iter().any(|c| c == command)).await?; + let t = targets.swap_remove(0); + let at = snap.position(&t, pos); + let mut arguments = vec![json!(snap.uri), json!(at.line), json!(at.character), json!(kind)]; + arguments.extend_from_slice(extra); + let params = json!({ "command": command, "arguments": arguments }); + let answer: Answer = + self.call(&t, "workspace/executeCommand", params, Some(cancel), COMMAND_TIMEOUT).await?; + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + Ok((answer.into_result()?, snap, t)) + } + + /// The class at `pos` and its movable members; an invalid-input error says why there are none. + pub async fn hierarchy_members( + &self, + doc: DocId, + pos: Pos, + kind: HierarchyRefactoring, + cancel: CancellationToken, + ) -> IdeResult { + let (members, _, _) = self.hierarchy_command(doc, pos, kind, MEMBERS, &[], &cancel).await?; + Ok(members) + } + + /// Checks moving `choices` (to `target`, for pull up) and computes the edit without applying it. + pub async fn hierarchy_refactor( + &self, + doc: DocId, + pos: Pos, + kind: HierarchyRefactoring, + target: Option, + choices: Vec, + cancel: CancellationToken, + ) -> IdeResult { + let choices: Vec = choices.iter().map(|c| json!({ "id": c.id, "abstract": c.make_abstract })).collect(); + let (refactored, _, t): (Refactored, _, _) = + self.hierarchy_command(doc, pos, kind, REFACTOR, &[json!(target), json!(choices)], &cancel).await?; + let conv = self.conv(t.enc()); + Ok(HierarchyRefactored { + problems: refactored.problems, + edit: refactored.edit.map(|e| convert::workspace_edit(&conv, e)), + }) + } +} diff --git a/crates/ide-lsp/src/lib.rs b/crates/ide-lsp/src/lib.rs index 252b66bf..2ace34ad 100644 --- a/crates/ide-lsp/src/lib.rs +++ b/crates/ide-lsp/src/lib.rs @@ -12,15 +12,18 @@ mod clang_tidy; mod convert; mod external_project; mod file_ops; +mod file_watch; mod handler; mod inspect; pub mod jdtls; mod launch; mod library_source; +mod pull; mod scope; mod server; mod tcp; mod text; +mod ts_file_moves; mod tsserver_bridge; pub mod types; mod uri; @@ -160,6 +163,9 @@ impl DocSnap { } } +/// Receives the folders outside the project that servers want watched. +pub type WatchRootsFn = dyn Fn(Vec) + Send + Sync; + pub struct LspManager { me: Weak, project_root: PathBuf, @@ -177,6 +183,7 @@ pub struct LspManager { servers_hook: RwLock>>, candidates_hook: RwLock>>, excludes_hook: RwLock>>, + watch_roots_hook: RwLock>>, shutting_down: AtomicBool, execution_allowed: AtomicBool, java_usages_warned: AtomicBool, @@ -336,6 +343,7 @@ impl LspManager { servers_hook: RwLock::new(None), candidates_hook: RwLock::new(None), excludes_hook: RwLock::new(None), + watch_roots_hook: RwLock::new(None), shutting_down: AtomicBool::new(false), execution_allowed: AtomicBool::new(true), java_usages_warned: AtomicBool::new(false), @@ -426,6 +434,7 @@ impl LspManager { let wanted: Vec<(DocId, Vec>)> = open.into_iter().map(|(id, language, path)| (id, self.servers_for(&language, &path))).collect(); let mut start = Vec::new(); + let mut opened = Vec::new(); { let mut docs = self.docs.lock(); for (id, servers) in wanted { @@ -440,6 +449,7 @@ impl LspManager { user_did_open(server, &run, &d.path, &d.uri, d.did_open_params()); } attached.opened_gen = Some(run.generation); + opened.push((id, server.clone())); } d.servers.push(attached); if !start.iter().any(|s: &Arc| s.id == server.id) { @@ -450,6 +460,9 @@ impl LspManager { d.servers.sort_by_key(|a| servers.iter().position(|s| s.id == a.server.id).unwrap_or(usize::MAX)); } } + for (doc, server) in opened { + self.schedule_pull(doc, Some(&server), Duration::ZERO); + } for server in &start { self.ensure_started(server); } @@ -587,6 +600,9 @@ impl LspManager { return; } }; + let run = + Running { generation, peer: peer.clone(), caps: Arc::new(RwLock::new(caps)), watchers: Default::default() }; + server.set_initializing(Some(run.clone())); if let Err(e) = peer.notify("initialized", json!({})) { tracing::warn!("{}: {e}", server.name); } @@ -598,29 +614,34 @@ impl LspManager { { tracing::debug!("{}: didChangeConfiguration: {e}", server.name); } - let run = Running { generation, peer: peer.clone(), caps: Arc::new(RwLock::new(caps)) }; self.refresh_external_project(&server, &run).await; // Publish readiness and (re)open attached documents atomically w.r.t. did_change. + let mut reopened = Vec::new(); { let mut docs = self.docs.lock(); if !server.mark_ready(generation, run.clone()) { drop(docs); + server.set_initializing(None); peer.kill(); return; } let open_close = run.caps.read().open_close; - for d in docs.values_mut() { + for (id, d) in docs.iter_mut() { let params = d.did_open_params(); for a in d.servers.iter_mut().filter(|a| a.server.id == server.id && a.opened_gen != Some(generation)) { if open_close && let Err(e) = peer.notify("textDocument/didOpen", params.clone()) { tracing::debug!("{}: didOpen: {e}", server.name); } a.opened_gen = Some(generation); + reopened.push(*id); } } } + for doc in reopened { + self.schedule_pull(doc, Some(&server), Duration::ZERO); + } self.check_clang_tidy(&server).await; server.refresh_busy_status(&self.sink); server.announce_ready(generation); @@ -644,6 +665,7 @@ impl LspManager { None => return, // superseded instance }; self.clear_server_state(&server); + self.file_watch_roots_changed(); if intentional { server.announce_down(); return; @@ -843,6 +865,7 @@ impl LspManager { } docs.insert(doc, state); } + self.schedule_pull(doc, None, Duration::ZERO); for s in &servers { self.ensure_started(s); if s.running().is_some() @@ -920,6 +943,30 @@ impl LspManager { tracing::debug!("{}: didChange: {e}", server.name); } } + drop(docs); + self.schedule_pull(doc, None, pull::AFTER_EDIT); + } + + /// Sends the documents open on instance `generation` of `server` again, as a `didChange` + /// with their unchanged text, so the server parses them with the settings it just read. + pub(crate) fn resync_documents(&self, server: &Server, generation: u64) { + let Some(run) = server.running_gen(generation) else { return }; + if run.caps.read().sync == SyncKind::None { + return; + } + let docs = self.docs.lock(); + for d in docs.values() { + if !d.servers.iter().any(|a| a.server.id == server.id && a.opened_gen == Some(generation)) { + continue; + } + let params = json!({ + "textDocument": { "uri": d.uri, "version": d.version }, + "contentChanges": [{ "text": d.text.as_str() }], + }); + if let Err(e) = run.peer.notify("textDocument/didChange", params) { + tracing::debug!("{}: didChange: {e}", server.name); + } + } } pub fn did_save(&self, doc: DocId, text: &str) { @@ -942,11 +989,15 @@ impl LspManager { tracing::debug!("{}: didSave: {e}", a.server.name); } } + drop(docs); + self.schedule_pull(doc, None, Duration::ZERO); + self.pull_dependents(doc); } pub fn did_close(&self, doc: DocId) { let Some(d) = self.docs.lock().remove(&doc) else { return }; for a in &d.servers { + self.forget_pulled(&a.server, &d.path, a.server.running().as_ref()); let Some(run) = a.server.running() else { continue }; if a.opened_gen == Some(run.generation) && run.caps.read().open_close @@ -1642,6 +1693,20 @@ impl LspManager { Ok(()) } + /// Whether a server of `doc` (`server`, when given) provides the command `name`. Code actions + /// can carry commands other editors' extensions run themselves after the edit (vtsls' + /// `_typescript.didOrganizeImports`): no server provides those. + pub async fn provides_command(&self, doc: DocId, server: Option, name: &str) -> bool { + let servers = match self.snapshot(doc) { + Ok(snap) => snap.servers, + Err(_) => self.servers.lock().clone(), + }; + self.ready_targets(&servers, |c| c.commands().iter().any(|c| c == name)) + .await + .iter() + .any(|t| server.is_none_or(|s| t.server.id == s)) + } + async fn command_target(&self, doc: DocId, server: ide_core::ServerId, name: &str) -> IdeResult { let snap = self.snapshot(doc)?; self.ready_targets(&snap.servers, |c| c.commands().iter().any(|c| c == name)) @@ -1660,6 +1725,10 @@ impl LspManager { answers: Vec, cancel: CancellationToken, ) -> IdeResult> { + // Nothing to ask for a command its server doesn't provide (executing it decides). + if !self.provides_command(doc, Some(server), &command.command).await { + return Ok(None); + } let t = self.command_target(doc, server, &command.command).await?; let interactive = t.server.name == "gopls" && t.caps() @@ -1942,7 +2011,30 @@ impl LspManager { servers.iter().zip(ready).filter_map(|(s, run)| Some(Target { server: s.clone(), run: run? })).collect(); let mut out = FileRenameEdits::default(); for t in targets { - let Some(params) = Self::file_op_params(&t, "willRename", &moves) else { continue }; + let Some(params) = Self::file_op_params(&t, "willRename", &moves) else { + if t.server.config.tsserver_project && t.caps().commands().iter().any(|c| c == ts_file_moves::COMMAND) { + out.servers.push(t.server.name.clone()); + let opened = self.open_for_file_op(&t, &moves); + for (from, to, _) in &moves { + let r: IdeResult = self + .call( + &t, + "workspace/executeCommand", + ts_file_moves::request(from, to), + Some(&cancel), + LONG_TIMEOUT, + ) + .await; + match r { + Ok(result) => out.edit.files.extend(ts_file_moves::edit(&result).files), + Err(e) if e.kind == ErrorKind::Cancelled && cancel.is_cancelled() => return Err(e), + Err(e) => out.errors.push(e.message), + } + } + drop(opened); + } + continue; + }; out.servers.push(t.server.name.clone()); let opened = self.open_for_file_op(&t, &moves); let r: IdeResult> = @@ -1973,29 +2065,78 @@ impl LspManager { } } - /// Files that changed on disk, outside the editor too (a checkout, another tool): each running - /// server hears of those of its languages under its root through `workspace/didChangeWatchedFiles`. - /// Without it jdtls's incremental build never sees them, so a launch runs stale classes. - pub fn watched_files_changed(&self, created: &[PathBuf], modified: &[PathBuf], removed: &[PathBuf]) { + /// Files that changed on disk, outside the editor too (a checkout, another tool), sent to + /// running servers through `workspace/didChangeWatchedFiles`. A server that registered file + /// watchers hears of what they match, including `excluded_dirs` (folders in excluded or + /// ignored trees whose entries came or went, reported as created): a package installed + /// into `.venv` or a `go.mod` rewritten by `go get` is picked up without a restart. Other + /// servers hear of their languages' files under their root; without it jdtls's incremental + /// build never sees them, so a launch runs stale classes. + pub fn watched_files_changed( + &self, + created: &[PathBuf], + modified: &[PathBuf], + removed: &[PathBuf], + excluded_dirs: &[PathBuf], + ) { + use file_watch::Change; let servers: Vec> = self.servers.lock().clone(); for s in servers { let Some(run) = s.running() else { continue }; - let languages = s.languages(); - let changes: Vec = [(created, 1), (modified, 2), (removed, 3)] + let all = [(created, Change::Created), (modified, Change::Changed), (removed, Change::Deleted)] .into_iter() - .flat_map(|(paths, kind)| paths.iter().map(move |p| (p, kind))) - .filter(|(p, _)| p.starts_with(&s.root) && languages.contains(&self.languages.detect(p, None))) - .map(|(p, kind)| json!({ "uri": uri::path_to_uri(p), "type": kind })) - .collect(); + .flat_map(|(paths, change)| paths.iter().map(move |p| (p, change))); + let watchers = run.watchers.read(); + let changes: Vec<(&PathBuf, Change)> = if watchers.is_empty() { + let languages = s.languages(); + all.filter(|(p, _)| p.starts_with(&s.root) && languages.contains(&self.languages.detect(p, None))) + .collect() + } else { + all.chain(excluded_dirs.iter().map(|p| (p, Change::Created))) + .filter(|(p, change)| watchers.wants(&s.root, p, *change)) + .collect() + }; + drop(watchers); if changes.is_empty() { continue; } + let changes: Vec = + changes.iter().map(|(p, c)| json!({ "uri": uri::path_to_uri(p), "type": *c as u8 })).collect(); if let Err(e) = run.peer.notify("workspace/didChangeWatchedFiles", json!({ "changes": changes })) { tracing::debug!("{}: didChangeWatchedFiles: {e}", s.name); } } } + /// Called with the folders outside the project that servers registered file watchers + /// for (a virtualenv elsewhere, a path dependency), whenever they change. Changes + /// below them go to [`Self::watched_files_changed`]. + pub fn set_file_watch_roots_hook(&self, hook: Box) { + *self.watch_roots_hook.write() = Some(Arc::from(hook)); + self.file_watch_roots_changed(); + } + + pub(crate) fn file_watch_roots_changed(&self) { + let Some(hook) = self.watch_roots_hook.read().clone() else { return }; + let servers: Vec> = self.servers.lock().clone(); + let mut roots: Vec = Vec::new(); + for run in servers.iter().filter_map(|s| s.instance()) { + roots.extend( + run.watchers.read().bases().filter(|b| !b.starts_with(&self.project_root)).map(Path::to_path_buf), + ); + } + roots.sort(); + roots.dedup(); + // Sorted, so a folder's subfolders follow it. + let mut outer: Vec = Vec::new(); + for r in roots { + if !outer.last().is_some_and(|o| r.starts_with(o)) { + outer.push(r); + } + } + hook(outer); + } + /// Checks initialized capabilities, including dynamic registrations, before choosing a formatter. pub async fn can_format(&self, doc: DocId) -> IdeResult { let snap = self.snapshot(doc)?; @@ -2451,6 +2592,12 @@ impl LspManager { } } + /// [`Self::reconfigure`] on the manager's runtime, for callers outside one (file watchers). + pub fn reconfigure_soon(self: &Arc) { + let me = self.clone(); + self.runtime.spawn(async move { me.reconfigure().await }); + } + pub async fn reconfigure(&self) { if !self.execution_allowed.load(Ordering::Acquire) { let servers = self.servers.lock().clone(); diff --git a/crates/ide-lsp/src/pull.rs b/crates/ide-lsp/src/pull.rs new file mode 100644 index 00000000..63016c0e --- /dev/null +++ b/crates/ide-lsp/src/pull.rs @@ -0,0 +1,192 @@ +//! Pull diagnostics (`textDocument/diagnostic`). Servers that provide them may stop pushing +//! their own analysis to a client that can pull (rust-analyzer pushes only `cargo check` +//! then): the editor pulls an open file's diagnostics when it opens, after edits and saves, +//! and when the server asks for a refresh. Pulled diagnostics are kept apart from pushed ones +//! and shown together. + +use crate::server::{Running, Server}; +use crate::{LspManager, convert}; +use ide_core::{DocId, IdeEvent}; +use ide_jsonrpc::RpcError; +use lsp_types as lsp; +use serde_json::{Value, json}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +/// How long typing must pause before a changed file is pulled again. +pub(crate) const AFTER_EDIT: Duration = Duration::from_millis(300); +const TIMEOUT: Duration = Duration::from_secs(30); +/// Retries of a pull the server cancelled (`ServerCancelled`, e.g. while indexing). +const RETRIES: u32 = 20; + +struct PullTarget { + server: Arc, + run: Running, + path: PathBuf, + uri: lsp::Uri, +} + +fn provides_pull(run: &Running) -> bool { + let caps = run.caps.read(); + caps.restriction.diagnostics() && caps.has("diagnosticProvider") +} + +fn inter_file(run: &Running) -> bool { + run.caps.read().raw.pointer("/diagnosticProvider/interFileDependencies").and_then(Value::as_bool) == Some(true) +} + +impl LspManager { + /// Pulls `doc`'s diagnostics from its servers that provide them (`only`: from that one) + /// after `delay`; a later pull of the same file supersedes it. + pub(crate) fn schedule_pull(&self, doc: DocId, only: Option<&Server>, delay: Duration) { + let targets: Vec = { + let docs = self.docs.lock(); + let Some(d) = docs.get(&doc) else { return }; + d.servers + .iter() + .filter(|a| only.is_none_or(|s| s.id == a.server.id)) + .filter_map(|a| { + let run = a.server.running()?; + (a.opened_gen == Some(run.generation) && provides_pull(&run)).then(|| PullTarget { + server: a.server.clone(), + run, + path: d.path.clone(), + uri: d.uri.clone(), + }) + }) + .collect() + }; + for t in targets { + if t.server.suppresses_diagnostics(&t.path) { + continue; + } + let pull = t.server.next_pull(&t.path); + let me = self.me.clone(); + self.runtime.spawn(async move { + tokio::time::sleep(delay).await; + if let Some(me) = me.upgrade() { + me.pull(t, pull).await; + } + }); + } + } + + /// Pulls every open file of `server` again (`workspace/diagnostic/refresh`). + pub(crate) fn refresh_pulled_diagnostics(&self, server: &Server) { + let docs: Vec = self + .docs + .lock() + .iter() + .filter(|(_, d)| d.servers.iter().any(|a| a.server.id == server.id)) + .map(|(id, _)| *id) + .collect(); + for doc in docs { + self.schedule_pull(doc, Some(server), Duration::ZERO); + } + } + + /// After `doc` was saved: servers whose diagnostics depend on other files pull those + /// open files too. + pub(crate) fn pull_dependents(&self, doc: DocId) { + let docs = self.docs.lock(); + let Some(d) = docs.get(&doc) else { return }; + let servers: Vec> = d + .servers + .iter() + .filter(|a| a.server.running().is_some_and(|run| provides_pull(&run) && inter_file(&run))) + .map(|a| a.server.clone()) + .collect(); + let others: Vec<(DocId, Vec>)> = docs + .iter() + .filter(|(id, _)| **id != doc) + .map(|(id, o)| { + (*id, servers.iter().filter(|s| o.servers.iter().any(|a| a.server.id == s.id)).cloned().collect()) + }) + .collect(); + drop(docs); + for (other, servers) in others { + for server in servers { + self.schedule_pull(other, Some(&server), Duration::ZERO); + } + } + } + + /// Drops what was pulled for a closed file and shows what remains. + pub(crate) fn forget_pulled(&self, server: &Server, path: &std::path::Path, run: Option<&Running>) { + if let Some(shown) = server.clear_pulled(path) { + let enc = run.map(|r| r.caps.read().encoding).unwrap_or_default(); + self.emit_diagnostics(server, path.to_owned(), shown, enc); + } + } + + async fn pull(&self, t: PullTarget, pull: u64) { + let mut attempt = 0; + loop { + if !t.server.is_latest_pull(&t.path, pull) { + return; + } + let mut params = json!({ "textDocument": { "uri": t.uri } }); + if let Some(id) = t.server.pulled_result_id(&t.path) { + params["previousResultId"] = json!(id); + } + if let Some(identifier) = t.run.caps.read().raw.pointer("/diagnosticProvider/identifier") { + params["identifier"] = identifier.clone(); + } + let request = t.run.peer.request::("textDocument/diagnostic", params, None); + let report = match tokio::time::timeout(TIMEOUT, request).await { + Ok(Ok(report)) => report, + // This request has no cancellation of its own: the server cancelled it + // (`ServerCancelled` while busy, which asks to be asked again). + Ok(Err(RpcError::Cancelled)) if attempt < RETRIES => { + attempt += 1; + tokio::time::sleep(Duration::from_millis(250 * u64::from(attempt.min(8)))).await; + continue; + } + Ok(Err(e)) => { + tracing::debug!("{}: textDocument/diagnostic: {e}", t.server.name); + return; + } + Err(_) => { + tracing::debug!("{}: textDocument/diagnostic timed out", t.server.name); + return; + } + }; + // A newer pull, a closed file or a restarted server makes this report stale. + if !t.server.is_latest_pull(&t.path, pull) + || t.server.running().is_none_or(|r| r.generation != t.run.generation) + || !self.docs.lock().values().any(|d| d.path == t.path) + { + return; + } + let result_id = report.get("resultId").and_then(Value::as_str).map(str::to_owned); + let items = match report.get("kind").and_then(Value::as_str) { + Some("unchanged") => None, + _ => match serde_json::from_value(report.get("items").cloned().unwrap_or(json!([]))) { + Ok(items) => Some(items), + Err(e) => { + tracing::debug!("{}: bad textDocument/diagnostic report: {e}", t.server.name); + return; + } + }, + }; + let shown = t.server.set_pulled(t.path.clone(), result_id, items); + let enc = t.run.caps.read().encoding; + self.emit_diagnostics(&t.server, t.path, shown, enc); + return; + } + } + + /// Shows `items` as `server`'s diagnostics for `path`. + pub(crate) fn emit_diagnostics( + &self, + server: &Server, + path: PathBuf, + items: Vec, + enc: crate::text::Encoding, + ) { + let conv = self.conv(enc); + let items = items.into_iter().map(|d| convert::diagnostic(&conv, &path, d, &server.name)).collect(); + self.sink.emit(IdeEvent::Diagnostics { path: path.into(), source: server.name.clone(), items }); + } +} diff --git a/crates/ide-lsp/src/server.rs b/crates/ide-lsp/src/server.rs index f15a893e..6d57b887 100644 --- a/crates/ide-lsp/src/server.rs +++ b/crates/ide-lsp/src/server.rs @@ -27,6 +27,8 @@ pub struct Running { pub generation: u64, pub peer: Peer, pub caps: Arc>, + /// `workspace/didChangeWatchedFiles` registrations. + pub watchers: Arc>, } pub enum Conn { @@ -82,6 +84,9 @@ pub struct Server { pub resolved_binary: RwLock>, languages: Mutex>, state: Mutex, + /// The instance being initialized, from `initialized` until ready: registrations + /// sent right after `initialized` must not be lost. + initializing: Mutex>, /// Fired when the instance being started is abandoned (stop / restart while starting). start_cancel: Mutex, phase: watch::Sender, @@ -92,7 +97,13 @@ pub struct Server { started: Mutex>, stderr: Mutex>, pub progress: Mutex>, + /// What the editor shows per path: the pushed and the pulled diagnostics together. diagnostics: Mutex>>, + /// `publishDiagnostics` per path. + pushed: Mutex>>, + /// `textDocument/diagnostic` reports per path, and the latest pull asked for each. + pulled: Mutex>, + pulls: Mutex>, /// When diagnostics were last published per path (empty lists included). published: Mutex>, /// Woken on every `publishDiagnostics` (Inspect Code waits on it). @@ -118,6 +129,7 @@ impl Server { resolved_binary: RwLock::new(None), languages: Mutex::new(vec![language]), state: Mutex::new(State { generation: 0, conn: Conn::Idle }), + initializing: Mutex::new(None), start_cancel: Mutex::new(CancellationToken::new()), phase: watch::channel(Phase::Down).0, status: Mutex::new((ServerStatus::Stopped, None)), @@ -126,6 +138,9 @@ impl Server { stderr: Mutex::new(VecDeque::new()), progress: Mutex::new(HashMap::new()), diagnostics: Mutex::new(HashMap::new()), + pushed: Mutex::new(HashMap::new()), + pulled: Mutex::new(HashMap::new()), + pulls: Mutex::new(HashMap::new()), published: Mutex::new(HashMap::new()), published_notify: Notify::new(), idle_notify: Notify::new(), @@ -210,6 +225,8 @@ impl Server { return false; } st.conn = Conn::Ready(run); + drop(st); + self.set_initializing(None); true } @@ -234,6 +251,20 @@ impl Server { } } + /// The instance told `initialized`, ready or not. + pub fn instance(&self) -> Option { + self.running().or_else(|| self.initializing.lock().clone()) + } + + /// Generation `generation` once it was told `initialized`, ready or not. + pub fn initialized_gen(&self, generation: u64) -> Option { + self.instance().filter(|r| r.generation == generation) + } + + pub fn set_initializing(&self, run: Option) { + *self.initializing.lock() = run; + } + /// The running instance if it is still generation `generation`. pub fn running_gen(&self, generation: u64) -> Option { self.running().filter(|r| r.generation == generation) @@ -471,17 +502,71 @@ impl Server { self.progress.lock().drain().map(|(_, p)| (p.task, p.title)).collect() } - pub fn set_diagnostics(&self, path: PathBuf, items: Vec) { + /// Records the diagnostics the server pushed for `path`; returns what the file now shows. + pub fn set_diagnostics(&self, path: PathBuf, items: Vec) -> Vec { + set_or_remove(&mut self.pushed.lock(), path.clone(), items); + self.combine(path) + } + + /// Records a pulled report for `path` (`None` items: unchanged since `result_id`'s + /// predecessor); returns what the file now shows. + pub fn set_pulled( + &self, + path: PathBuf, + result_id: Option, + items: Option>, + ) -> Vec { + { + let mut pulled = self.pulled.lock(); + let entry = pulled.entry(path.clone()).or_default(); + entry.result_id = result_id; + if let Some(items) = items { + entry.items = items; + } + } + self.combine(path) + } + + /// Forgets the pulled report of a closed file; `None` when there was none. + pub fn clear_pulled(&self, path: &Path) -> Option> { + self.pulls.lock().remove(path); + self.pulled.lock().remove(path)?; + Some(self.combine(path.to_owned())) + } + + /// The `resultId` of the last report pulled for `path`. + pub fn pulled_result_id(&self, path: &Path) -> Option { + self.pulled.lock().get(path).and_then(|p| p.result_id.clone()) + } + + /// Starts a pull for `path`, superseding earlier ones; see [`Server::is_latest_pull`]. + pub fn next_pull(&self, path: &Path) -> u64 { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.pulls.lock().insert(path.to_owned(), id); + id + } + + pub fn is_latest_pull(&self, path: &Path, id: u64) -> bool { + self.pulls.lock().get(path) == Some(&id) + } + + fn combine(&self, path: PathBuf) -> Vec { + let mut items = self.pushed.lock().get(&path).cloned().unwrap_or_default(); + if let Some(pulled) = self.pulled.lock().get(&path) { + for d in &pulled.items { + if !items.contains(d) { + items.push(d.clone()); + } + } + } { let mut d = self.diagnostics.lock(); self.published.lock().insert(path.clone(), Instant::now()); - if items.is_empty() { - d.remove(&path); - } else { - d.insert(path, items); - } + set_or_remove(&mut d, path, items.clone()); } self.published_notify.notify_waiters(); + items } /// When diagnostics for `path` were last published by this server. @@ -506,10 +591,28 @@ impl Server { pub fn take_diagnostic_paths(&self) -> Vec { self.published.lock().clear(); + self.pushed.lock().clear(); + self.pulled.lock().clear(); + self.pulls.lock().clear(); self.diagnostics.lock().drain().map(|(p, _)| p).collect() } } +/// A file's last `textDocument/diagnostic` report. +#[derive(Default)] +struct Pulled { + result_id: Option, + items: Vec, +} + +fn set_or_remove(map: &mut HashMap>, path: PathBuf, items: Vec) { + if items.is_empty() { + map.remove(&path); + } else { + map.insert(path, items); + } +} + /// The last `n` stderr lines, led by the exception headline: Node.js / Python / /// JVM crashes print the error first and a stack after it, so a plain tail /// shows frames without saying what went wrong. diff --git a/crates/ide-lsp/src/ts_file_moves.rs b/crates/ide-lsp/src/ts_file_moves.rs new file mode 100644 index 00000000..d46ed7c3 --- /dev/null +++ b/crates/ide-lsp/src/ts_file_moves.rs @@ -0,0 +1,89 @@ +//! Reference updates for moved files from vtsls, which has no `workspace/willRenameFiles`: +//! it updates imports itself after `didRenameFiles`, behind a prompt the client answers. The +//! IDE asks tsserver for the same edits up front (`getEditsForFileRename`, through +//! `typescript.tsserverRequest`), so they join the move's preview and undo, and the server's +//! own update is off (`updateImportsOnFileMove.enabled = "never"` in `languages/*.toml`). + +use ide_core::{FileEdit, Pos, Range, TextEdit, WorkspaceEdit}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::path::PathBuf; + +pub(crate) const COMMAND: &str = "typescript.tsserverRequest"; + +/// The `typescript.tsserverRequest` arguments asking for the edits of moving `from` to `to`. +pub(crate) fn request(from: &std::path::Path, to: &std::path::Path) -> Value { + json!({ + "command": COMMAND, + "arguments": ["getEditsForFileRename", { "oldFilePath": from, "newFilePath": to }], + }) +} + +#[derive(Deserialize)] +struct Location { + line: u32, + offset: u32, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Change { + start: Location, + end: Location, + new_text: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct FileChanges { + file_name: PathBuf, + text_changes: Vec, +} + +/// tsserver's `FileCodeEdits[]` (the command's `body`; 1-based lines and UTF-16 offsets) as an edit. +pub(crate) fn edit(result: &Value) -> WorkspaceEdit { + let body = result.get("body").unwrap_or(result); + let files: Vec = serde_json::from_value(body.clone()).unwrap_or_default(); + let pos = |l: &Location| Pos::new(l.line.saturating_sub(1), l.offset.saturating_sub(1)); + WorkspaceEdit { + files: files + .into_iter() + .filter(|f| !f.text_changes.is_empty()) + .map(|f| FileEdit::Edit { + path: f.file_name.into(), + edits: f + .text_changes + .into_iter() + .map(|c| TextEdit { range: Range::new(pos(&c.start), pos(&c.end)), new_text: c.new_text }) + .collect(), + version: None, + }) + .collect(), + preconditions: Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn converts_tsserver_file_code_edits() { + let result = json!({ "success": true, "body": [ + { "fileName": "/p/src/index.ts", "textChanges": [ + { "start": { "line": 2, "offset": 31 }, "end": { "line": 2, "offset": 41 }, "newText": "./lib/money.ts" } + ] }, + { "fileName": "/p/src/unchanged.ts", "textChanges": [] } + ] }); + let edit = edit(&result); + assert_eq!(edit.files.len(), 1); + let FileEdit::Edit { path, edits, version } = &edit.files[0] else { panic!("{edit:?}") }; + assert_eq!(**path, *std::path::Path::new("/p/src/index.ts")); + assert_eq!( + edits, + &[TextEdit { range: Range::new(Pos::new(1, 30), Pos::new(1, 40)), new_text: "./lib/money.ts".into() }] + ); + assert!(version.is_none()); + assert!(super::edit(&json!({ "success": false })).files.is_empty()); + } +} diff --git a/crates/ide-lsp/src/types.rs b/crates/ide-lsp/src/types.rs index 047aabf4..d2bdd5ff 100644 --- a/crates/ide-lsp/src/types.rs +++ b/crates/ide-lsp/src/types.rs @@ -389,6 +389,87 @@ pub struct Renamed { pub incomplete: Option, } +/// IDEA's class-hierarchy refactorings. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub enum HierarchyRefactoring { + PullUp, + PushDown, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct HierarchyClass { + /// Server-owned identity, passed back as the pull-up destination. + pub id: String, + pub name: String, + pub qualified_name: String, + pub interface: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub enum HierarchyMemberKind { + Field, + Method, + Type, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct HierarchyMember { + pub id: String, + /// `name(Params) : Type`, as the server renders it. + pub label: String, + pub kind: HierarchyMemberKind, + /// The member at the caret. + pub selected: bool, + /// Can be declared abstract in the destination (pull up) or kept abstract (push down). + pub abstractable: bool, +} + +/// The class at a position and what can move: pull-up destinations, or the subclasses +/// members are pushed into. +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct HierarchyMembers { + pub class_name: String, + pub targets: Vec, + pub members: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct HierarchyChoice { + pub id: String, + pub make_abstract: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub enum ProblemSeverity { + Info, + Warning, + Error, + /// The refactoring can't run at all; there is no edit. + Fatal, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct RefactoringProblem { + pub severity: ProblemSeverity, + pub message: String, +} + +/// A checked refactoring: what the server found, and the edit unless a problem is fatal. +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct HierarchyRefactored { + pub problems: Vec, + pub edit: Option, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] pub struct CallHierarchy { diff --git a/crates/ide-lsp/tests/mock_server.rs b/crates/ide-lsp/tests/mock_server.rs index 93d9d370..274e1ba7 100644 --- a/crates/ide-lsp/tests/mock_server.rs +++ b/crates/ide-lsp/tests/mock_server.rs @@ -31,6 +31,8 @@ struct MockState { errors: Mutex>, /// Computes a response from the request and the open documents (before `responses`). responder: Mutex>, + /// Holds the handling of notifications of this method until the gate opens. + held: Mutex)>>, peer: Mutex>, } @@ -74,6 +76,12 @@ struct MockHandler(Arc); impl Handler for MockHandler { fn on_notification(&self, method: &str, params: Value) { let s = &self.0; + let held = s.held.lock().clone(); + if let Some((m, gate)) = held + && m == method + { + gate.hold(); + } // Held throughout: a test that sees the effect (a document, the peer gone) also sees the record. let mut log = s.notifications.lock(); let uri = params["textDocument"]["uri"].as_str().unwrap_or_default().to_owned(); @@ -230,6 +238,7 @@ fn server_cfg(name: &str) -> LanguageServerConfig { max_java: None, launch_error: None, tsserver_project: false, + resync_on_configuration: false, pinned: None, } } @@ -269,6 +278,7 @@ fn harness_configured( ui_mode: None, framework_ui_modes: Vec::new(), formatter: None, + scratch_template: None, tree_sitter: None, language_servers: servers .iter() @@ -419,6 +429,7 @@ async fn files_changed_on_disk_reach_the_servers_of_their_language() { &[created.clone(), other], &[modified.clone(), outside], std::slice::from_ref(&removed), + &[h.root.join(".venv/lib/site-packages")], ); eventually(|| !s.notified("workspace/didChangeWatchedFiles").is_empty()).await; assert_eq!( @@ -430,12 +441,81 @@ async fn files_changed_on_disk_reach_the_servers_of_their_language() { ]) ); // Nothing of its languages: no notification. - h.mgr.watched_files_changed(&[h.root.join("b.txt")], &[], &[]); + h.mgr.watched_files_changed(&[h.root.join("b.txt")], &[], &[], &[]); tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert_eq!(s.notified("workspace/didChangeWatchedFiles").len(), 1); h.mgr.shutdown_all().await; } +/// A server that registers file watchers hears of what they match, inside excluded folders +/// too (`uv add` into `.venv`, `go get` rewriting `go.mod`); folders outside the project +/// that relative patterns name are reported for watching; unregistered, it falls back to +/// its languages' files. +#[tokio::test(flavor = "multi_thread")] +async fn registered_file_watchers_choose_the_changes_a_server_hears_of() { + let h = harness(&[("mock", incremental_caps())]); + let roots = Arc::new(Mutex::new(Vec::>::new())); + { + let roots = roots.clone(); + h.mgr.set_file_watch_roots_hook(Box::new(move |r| roots.lock().push(r))); + } + h.open("a.mk", "x"); + h.wait_status("mock", ServerStatus::Ready).await; + let s = h.launcher.state("mock"); + let outside = tempfile::tempdir().unwrap(); + let outside = dunce::canonicalize(outside.path()).unwrap(); + let external = outside.join("venv/lib/site-packages"); + s.peer() + .request::<_, Value>( + "client/registerCapability", + json!({ "registrations": [{ "id": "w", "method": "workspace/didChangeWatchedFiles", "registerOptions": { "watchers": [ + { "globPattern": "**/*.{mk,mod}" }, + { "globPattern": "**/pyrightconfig.json", "kind": 2 }, + { "globPattern": "**/site-packages/**" }, + { "globPattern": { "baseUri": uri_of(&external), "pattern": "**" } }, + { "globPattern": { "baseUri": uri_of(&h.root.join("sub")), "pattern": "**" } }, + ] } }] }), + None, + ) + .await + .unwrap(); + assert_eq!(roots.lock().last(), Some(&vec![external.clone()]), "only folders outside the project"); + + let sp = h.root.join(".venv/lib/python3.14/site-packages"); + let notified = || s.notified("workspace/didChangeWatchedFiles"); + h.mgr.watched_files_changed( + &[h.root.join("go.mod"), h.root.join("notes.txt"), h.root.join("pyrightconfig.json")], + &[h.root.join("pyrightconfig.json"), external.join("idna/core.py")], + &[h.root.join("old.mk")], + &[sp.join("idna"), h.root.join("node_modules/x")], + ); + eventually(|| !notified().is_empty()).await; + assert_eq!( + notified()[0]["changes"], + json!([ + { "uri": uri_of(&h.root.join("go.mod")), "type": 1 }, + { "uri": uri_of(&h.root.join("pyrightconfig.json")), "type": 2 }, + { "uri": uri_of(&external.join("idna/core.py")), "type": 2 }, + { "uri": uri_of(&h.root.join("old.mk")), "type": 3 }, + { "uri": uri_of(&sp.join("idna")), "type": 1 }, + ]) + ); + + s.peer() + .request::<_, Value>( + "client/unregisterCapability", + json!({ "unregisterations": [{ "id": "w", "method": "workspace/didChangeWatchedFiles" }] }), + None, + ) + .await + .unwrap(); + assert_eq!(roots.lock().last(), Some(&vec![])); + h.mgr.watched_files_changed(&[h.root.join("go.mod"), h.root.join("b.mk")], &[], &[], &[sp.join("idna")]); + eventually(|| notified().len() == 2).await; + assert_eq!(notified()[1]["changes"], json!([{ "uri": uri_of(&h.root.join("b.mk")), "type": 1 }])); + h.mgr.shutdown_all().await; +} + #[tokio::test(flavor = "multi_thread")] async fn initialize_and_incremental_sync() { let h = harness(&[("mock", incremental_caps())]); @@ -697,6 +777,113 @@ async fn diagnostics_become_events_and_feed_code_actions() { assert_eq!(err.kind, ide_core::error::ErrorKind::Unavailable); } +/// The diagnostics shown last for `path` by `source`, as their messages. +fn shown(h: &Harness, path: &Path, source: &str) -> Option> { + h.events.lock().iter().rev().find_map(|e| match e { + IdeEvent::Diagnostics { path: p, source: s, items } if **p == *path && s == source => { + Some(items.iter().map(|d| d.message.clone()).collect()) + } + _ => None, + }) +} + +/// Reports a "bad" diagnostic per line containing `bad`, as `textDocument/diagnostic`; answers +/// `unchanged` when the text is what `previousResultId` was computed from. +fn pull_responder() -> Responder { + Arc::new(|s: &MockState, method: &str, params: &Value| { + if method != "textDocument/diagnostic" { + return None; + } + let text = s.docs.lock().get(params["textDocument"]["uri"].as_str()?).cloned()?; + let id = format!("{}", text.len() * 31 + text.matches("bad").count()); + if params["previousResultId"].as_str() == Some(id.as_str()) { + return Some(json!({ "kind": "unchanged", "resultId": id })); + } + let items: Vec = text + .lines() + .enumerate() + .filter(|(_, l)| l.contains("bad")) + .map(|(i, _)| { + json!({ "range": {"start":{"line":i,"character":0},"end":{"line":i,"character":3}}, + "severity": 1, "message": "bad" }) + }) + .collect(); + Some(json!({ "kind": "full", "resultId": id, "items": items })) + }) +} + +#[tokio::test(flavor = "multi_thread")] +async fn pulled_diagnostics_follow_edits_and_join_pushed_ones() { + let mut pull_caps = incremental_caps(); + pull_caps["diagnosticProvider"] = json!({ "interFileDependencies": false, "workspaceDiagnostics": false }); + let h = harness(&[("mock", pull_caps), ("push", incremental_caps())]); + let s = h.launcher.state("mock"); + *s.responder.lock() = Some(pull_responder()); + let (doc, path) = h.open("a.mk", "ok\nbad\n"); + h.wait_status("mock", ServerStatus::Ready).await; + h.wait_status("push", ServerStatus::Ready).await; + assert!(s.requested("initialize")[0]["capabilities"]["textDocument"]["diagnostic"].is_object()); + eventually(|| shown(&h, &path, "mock") == Some(vec!["bad".into()])).await; + assert!(h.launcher.state("push").requested("textDocument/diagnostic").is_empty(), "it pulls from providers only"); + + // What the server still pushes (rust-analyzer: `cargo check`) shows next to what was pulled. + let pushed = json!({ "range": {"start":{"line":0,"character":0},"end":{"line":0,"character":2}}, + "severity": 2, "message": "pushed" }); + s.peer() + .notify("textDocument/publishDiagnostics", json!({ "uri": uri_of(&path), "diagnostics": [pushed] })) + .unwrap(); + eventually(|| shown(&h, &path, "mock") == Some(vec!["pushed".into(), "bad".into()])).await; + + // Edits are pulled again once typing pauses. + h.mgr.did_change(doc, 2, &[change(rng(0, 0, 0, 0), "bad\n")], &|| unreachable!()); + eventually(|| shown(&h, &path, "mock") == Some(vec!["pushed".into(), "bad".into(), "bad".into()])).await; + // Pulled diagnostics are code actions' context like pushed ones. + let _ = h.mgr.code_actions(doc, rng(2, 1, 2, 1), vec![]).await; + let ctx = &s.requested("textDocument/codeAction")[0]["context"]["diagnostics"]; + assert_eq!(ctx.as_array().unwrap().iter().map(|d| d["message"].as_str().unwrap()).collect::>(), ["bad"]); + + // Saving unchanged text asks with the last result id; `unchanged` keeps what is shown. + let pulls = s.requested("textDocument/diagnostic").len(); + h.mgr.did_save(doc, "bad\nok\nbad\n"); + eventually(|| s.requested("textDocument/diagnostic").len() > pulls).await; + let last = s.requested("textDocument/diagnostic").last().cloned().unwrap(); + assert!(last["previousResultId"].is_string()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(shown(&h, &path, "mock"), Some(vec!["pushed".into(), "bad".into(), "bad".into()])); + + // Closing drops what was pulled; the pushed diagnostics stay the server's to clear. + h.mgr.did_close(doc); + eventually(|| shown(&h, &path, "mock") == Some(vec!["pushed".into()])).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_cancelled_pull_is_retried_and_a_refresh_pulls_again() { + let mut caps = incremental_caps(); + caps["diagnosticProvider"] = json!({ "interFileDependencies": false, "workspaceDiagnostics": false }); + let h = harness(&[("mock", caps)]); + let s = h.launcher.state("mock"); + *s.responder.lock() = Some(pull_responder()); + // Busy indexing: the server cancels and asks to be asked again. + s.errors.lock().insert("textDocument/diagnostic".into(), (-32802, "indexing".into())); + let (_doc, path) = h.open("a.mk", "bad\n"); + h.wait_status("mock", ServerStatus::Ready).await; + eventually(|| s.requested("textDocument/diagnostic").len() >= 2).await; + assert_eq!(shown(&h, &path, "mock"), None); + s.errors.lock().clear(); + eventually(|| shown(&h, &path, "mock") == Some(vec!["bad".into()])).await; + + // `workspace/diagnostic/refresh` (e.g. after a project reload) pulls open files again. + *s.responder.lock() = Some(Arc::new(|_: &MockState, method: &str, _: &Value| { + (method == "textDocument/diagnostic").then(|| { + json!({ "kind": "full", "items": [ + { "range": {"start":{"line":0,"character":0},"end":{"line":0,"character":3}}, "message": "reloaded" } ] }) + }) + })); + let refreshed: Value = s.peer().request("workspace/diagnostic/refresh", Value::Null, None).await.unwrap(); + assert_eq!(refreshed, Value::Null); + eventually(|| shown(&h, &path, "mock") == Some(vec!["reloaded".into()])).await; +} + #[tokio::test(flavor = "multi_thread")] async fn apply_edit_hook_and_server_requests() { let h = harness(&[("mock", incremental_caps())]); @@ -1040,6 +1227,54 @@ async fn preparation_refreshes_config_and_survives_unchanged_settings() { assert_eq!(h.launcher.state("mock").launches.load(Ordering::SeqCst), 1); } +/// Intelephense applies the PHP version only to documents it parses after reading its +/// configuration: once that is answered, servers that opt in get their open documents again. +#[tokio::test(flavor = "multi_thread")] +async fn configuration_answer_resyncs_open_documents_of_opted_in_servers() { + let h = harness_configured("mock", "mk", &[("mock", incremental_caps()), ("extra", incremental_caps())], |cfg| { + cfg.resync_on_configuration = cfg.name == "mock" + }); + let (doc, path) = h.open("a.mk", "enum A {}"); + let uri = uri_of(&path); + let (s, extra) = (h.launcher.state("mock"), h.launcher.state("extra")); + h.wait_status("mock", ServerStatus::Ready).await; + h.wait_status("extra", ServerStatus::Ready).await; + eventually(|| s.docs.lock().contains_key(&uri) && extra.docs.lock().contains_key(&uri)).await; + h.mgr.did_change(doc, 2, &[change(rng(0, 9, 0, 9), "\n")], &|| unreachable!()); + eventually(|| s.notified("textDocument/didChange").len() == 1).await; + + let items = |section: &str| json!({ "items": [{ "section": section }] }); + let options = json!([{ "flag": true, "nested": { "deep": 1 } }]); + let answer: Value = extra.peer().request("workspace/configuration", items("extra"), None).await.unwrap(); + assert_eq!(answer, options); + // The mock handles what arrives in order, answers included: a re-sync sent before the + // answer would hold it back here. + let gate = Arc::new(Gate::default()); + *s.held.lock() = Some(("textDocument/didChange".into(), gate.clone())); + let peer = s.peer(); + let request = peer.request::<_, Value>("workspace/configuration", items("mock"), None); + let answer = tokio::time::timeout(Duration::from_secs(10), request).await.expect("re-sync sent before the answer"); + assert_eq!(answer.unwrap(), options); + eventually(|| gate.entered()).await; + assert_eq!(s.notified("textDocument/didChange").len(), 1); + gate.open(); + eventually(|| s.notified("textDocument/didChange").len() == 2).await; + assert_eq!( + s.notified("textDocument/didChange")[1], + json!({ "textDocument": { "uri": uri, "version": 2 }, "contentChanges": [{ "text": "enum A {}\n" }] }) + ); + assert_eq!(s.docs.lock()[&uri], "enum A {}\n"); + + // A notification sent after both answers reaches the servers after anything they triggered. + h.mgr.did_save(doc, "enum A {}\n"); + eventually(|| extra.notified("textDocument/didSave").len() == 1 && s.notified("textDocument/didSave").len() == 1) + .await; + assert_eq!(s.notified("textDocument/didChange").len(), 2, "one re-sync per answer"); + let methods: Vec = extra.notifications.lock().iter().map(|(m, _)| m.clone()).collect(); + assert_eq!(methods, ["initialized", "textDocument/didOpen", "textDocument/didChange", "textDocument/didSave"]); + assert_eq!(s.launches.load(Ordering::SeqCst), 1); +} + #[tokio::test(flavor = "multi_thread")] async fn settings_changes_reach_servers_without_restarts() { let h = harness(&[("mock", incremental_caps()), ("extra", incremental_caps())]); @@ -2101,6 +2336,41 @@ async fn will_and_did_rename_files_honour_filters() { ); } +#[tokio::test(flavor = "multi_thread")] +async fn will_rename_files_asks_tsserver_when_the_server_has_no_file_operations() { + // vtsls: no `willRename` filters, but tsserver's `getEditsForFileRename` through its command. + let mut caps = incremental_caps(); + caps["executeCommandProvider"] = json!({ "commands": ["typescript.tsserverRequest"] }); + let h = harness_configured("typescript", "ts", &[("vtsls", caps)], |cfg| cfg.tsserver_project = true); + let money = h.root.join("money.ts"); + let index = h.root.join("index.ts"); + std::fs::write(&money, "export const one = 1;\n").unwrap(); + std::fs::write(&index, "import { one } from \"./money\";\n").unwrap(); + let s = h.launcher.state("vtsls"); + *s.responder.lock() = Some(Arc::new(move |_, method, p| { + (method == "workspace/executeCommand" && p["arguments"][0] == "getEditsForFileRename").then(|| { + json!({ "success": true, "body": [{ "fileName": p["arguments"][1]["oldFilePath"].as_str().unwrap().replace("money.ts", "index.ts"), + "textChanges": [{ "start": { "line": 1, "offset": 22 }, "end": { "line": 1, "offset": 29 }, "newText": "./lib/money" }] }] }) + }) + })); + let to = h.root.join("lib/money.ts"); + let result = + h.mgr.will_rename_files(&[(money.clone(), to.clone())], ide_core::CancellationToken::new()).await.unwrap(); + assert_eq!(result.servers, vec!["vtsls".to_owned()]); + assert!(result.errors.is_empty(), "{:?}", result.errors); + match &result.edit.files[..] { + [FileEdit::Edit { path, edits, version: None }] => { + assert_eq!(path, &index); + assert_eq!(edits[0].range, rng(0, 21, 0, 28)); + assert_eq!(edits[0].new_text, "./lib/money"); + } + other => panic!("unexpected edit {other:?}"), + } + let asked = s.requested("workspace/executeCommand"); + let asked = asked.iter().find(|r| r["arguments"][0] == "getEditsForFileRename").unwrap(); + assert_eq!(asked["arguments"][1], json!({ "oldFilePath": money, "newFilePath": to })); +} + // ------------------------------------------------------------------ inspect code struct InspectTestHost { @@ -3338,3 +3608,90 @@ async fn jdtls_import_failure_names_reserved_device_files_on_windows() { } let _ = std::fs::remove_file(&nul); } + +/// Pull Members Up / Push Members Down: the extension's answers, refusals and edits. +#[tokio::test(flavor = "multi_thread")] +async fn java_hierarchy_refactorings_use_the_extension_commands() { + let mut caps = incremental_caps(); + caps["executeCommandProvider"] = + json!({"commands": ["openidea.java.hierarchy.members", "openidea.java.hierarchy.refactor"]}); + let h = harness_for("java", "java", &[("jdtls", caps)], &[]); + let s = h.launcher.state("jdtls"); + *s.responder.lock() = Some(Arc::new(|_, method, params| { + if method != "workspace/executeCommand" { + return None; + } + let args = params["arguments"].as_array().unwrap(); + let uri = args[0].clone(); + Some(match (params["command"].as_str().unwrap(), args[1].as_u64().unwrap()) { + (_, 9) => json!({"refused": "Type has no super types."}), + ("openidea.java.hierarchy.members", _) => json!({ + "className": "Dog", + "targets": [{"id": "=p/src<{Animal.java[Animal", "name": "Animal", "qualifiedName": "Animal", "interface": false}], + "members": [{"id": "=p/src<{Dog.java[Dog~bark", "label": "bark() : String", "kind": "method", "selected": true, "abstractable": true}], + }), + _ => json!({ + "problems": [{"severity": "warning", "message": "The visibility of field 'tail' will be changed to package."}], + "edit": {"changes": {uri.as_str().unwrap(): [ + {"range": {"start": {"line": 0, "character": 0}, "end": {"line": 0, "character": 5}}, "newText": "class"} + ]}}, + }), + }) + })); + let text = + format!("class Dog extends Animal {{ String bark() {{ return \"\"; }} }}{}class Lonely {{}}", "\n".repeat(9)); + let (doc, path) = h.open("Dog.java", &text); + h.wait_status("jdtls", ServerStatus::Ready).await; + eventually(|| s.docs.lock().len() == 1).await; + + let info = + h.mgr.hierarchy_members(doc, Pos::new(0, 35), HierarchyRefactoring::PullUp, Default::default()).await.unwrap(); + assert_eq!(info.class_name, "Dog"); + assert_eq!(info.targets[0].name, "Animal"); + assert!(info.members[0].selected && info.members[0].abstractable); + assert_eq!(info.members[0].kind, HierarchyMemberKind::Method); + let sent = s.requested("workspace/executeCommand"); + assert_eq!(sent[0]["arguments"], json!([uri_of(&path), 0, 35, "pullUp"])); + + let choices = vec![HierarchyChoice { id: info.members[0].id.clone(), make_abstract: true }]; + let done = h + .mgr + .hierarchy_refactor( + doc, + Pos::new(0, 35), + HierarchyRefactoring::PushDown, + Some(info.targets[0].id.clone()), + choices, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(done.problems[0].severity, ProblemSeverity::Warning); + let edit = done.edit.unwrap(); + assert!(matches!(&edit.files[..], [FileEdit::Edit { path: p, edits, .. }] if p == &path && edits.len() == 1)); + let sent = s.requested("workspace/executeCommand"); + assert_eq!( + sent[1]["arguments"], + json!([uri_of(&path), 0, 35, "pushDown", "=p/src<{Animal.java[Animal", + [{"id": "=p/src<{Dog.java[Dog~bark", "abstract": true}]]) + ); + + let refused = h.mgr.hierarchy_members(doc, Pos::new(9, 0), HierarchyRefactoring::PullUp, Default::default()).await; + let e = refused.unwrap_err(); + assert_eq!(e.kind, ide_core::error::ErrorKind::InvalidInput); + assert_eq!(e.message, "Type has no super types."); +} + +#[tokio::test(flavor = "multi_thread")] +async fn hierarchy_refactorings_name_the_servers_without_them() { + let h = harness(&[("plain", incremental_caps())]); + let (doc, _) = h.open("a.mk", "class A {}"); + h.wait_status("plain", ServerStatus::Ready).await; + let e = h + .mgr + .hierarchy_members(doc, Pos::new(0, 6), HierarchyRefactoring::PullUp, Default::default()) + .await + .unwrap_err(); + assert_eq!(e.kind, ide_core::error::ErrorKind::Unavailable); + assert_eq!(e.message, "plain doesn't support Pull Members Up"); +} diff --git a/crates/ide-lsp/tests/real_servers.rs b/crates/ide-lsp/tests/real_servers.rs index 35fa3314..589b7414 100644 --- a/crates/ide-lsp/tests/real_servers.rs +++ b/crates/ide-lsp/tests/real_servers.rs @@ -56,6 +56,7 @@ fn lang(id: &str, ext: &str, server: LanguageServerConfig) -> LanguageConfig { ui_mode: None, framework_ui_modes: Vec::new(), formatter: None, + scratch_template: None, tree_sitter: None, language_servers: vec![server], debugger: None, @@ -134,6 +135,7 @@ async fn missing_binary_is_reported_with_install_hint() { max_java: None, launch_error: None, tsserver_project: false, + resync_on_configuration: false, pinned: None, }; let (mgr, events) = manager(dir.path(), LanguageRegistry::from_configs(vec![lang("nope", "nope", server)])); @@ -403,6 +405,7 @@ async fn typescript_language_server_end_to_end() { max_java: None, launch_error: None, tsserver_project: false, + resync_on_configuration: false, pinned: None, }; let (mgr, events) = manager(&root, LanguageRegistry::from_configs(vec![lang("typescript", "ts", server)])); @@ -452,6 +455,7 @@ fn npm_server(tools: &Path, name: &str, bin: &str, init: Option, cwd: Option<&Path>, ) -> Result<(), HostError>; + fn open_location(&self, client: Option<&str>, path: &Path, line: u32, column: u32) -> Result<(), HostError>; /// A setting of the plugin as JSON (project over global over the schema default). fn setting(&self, root: Option<&Path>, key: &str) -> Option; } @@ -467,7 +468,6 @@ const MAX_SYNTAX_ITEMS: usize = 200_000; const MAX_BATCH: usize = 1000; fn event_to_wit(e: WorkbenchEvent) -> wit_events::Event { - use crate::types::ActionSource as S; match e { WorkbenchEvent::FileOpened(p) => wit_events::Event::FileOpened(p), WorkbenchEvent::FileSaved(p) => wit_events::Event::FileSaved(p), @@ -477,14 +477,7 @@ fn event_to_wit(e: WorkbenchEvent) -> wit_events::Event { WorkbenchEvent::ActionPerformed(a) => wit_events::Event::ActionPerformed(wit_events::ActionPerformed { action: a.action, title: a.title, - source: match a.source { - S::Keyboard => wit_events::ActionSource::Keyboard, - S::Menu => wit_events::ActionSource::Menu, - S::Toolbar => wit_events::ActionSource::Toolbar, - S::ContextMenu => wit_events::ActionSource::ContextMenu, - S::Search => wit_events::ActionSource::Search, - S::Other => wit_events::ActionSource::Other, - }, + source: a.source.name().to_owned(), shortcut: a.shortcut, }), } @@ -670,9 +663,17 @@ impl Runtime { detail: i.detail.map(|d| clip(d, MAX_TITLE)), kind: i.kind.map(|k| clip(k, 32)), insert_text: clip(i.insert_text, 64 << 10), + snippet: i.snippet, replace: i.replace.map(range_from_wit), sort_text: i.sort_text.map(|t| clip(t, 64)), + filter_text: i.filter_text.map(|t| clip(t, MAX_TITLE)), documentation: i.documentation.map(|d| clip(d, 16 << 10)), + additional_edits: i + .additional_edits + .into_iter() + .take(MAX_ADDITIONAL_EDITS) + .map(edit_from_wit) + .collect(), }) .collect()) } @@ -699,27 +700,20 @@ impl Runtime { ) -> Result, CallError> { let d = doc_to_wit(doc); let actions = self.provider_call(timeout, |p, s| p.call_code_actions(s, &d, range_to_wit(selection)))?; - Ok(actions - .into_iter() - .take(MAX_PROVIDER_ITEMS) - .map(|a| CodeAction { - title: clip(a.title, MAX_TITLE), - kind: a.kind.map(|k| clip(k, 64)), - preferred: a.preferred, - edits: a - .edits - .into_iter() - .map(|f| { - let edits = f - .edits - .into_iter() - .map(|e| ide_core::TextEdit { range: range_from_wit(e.range), new_text: e.text }) - .collect(); - (std::path::PathBuf::from(f.path), edits) - }) - .collect(), - }) - .collect()) + Ok(actions.into_iter().take(MAX_PROVIDER_ITEMS).map(code_action_from_wit).collect()) + } + + /// The edits of an action `provide_code_actions` returned with `data`. + pub fn resolve_code_action( + &mut self, + doc: &ProviderDocument, + action: &CodeAction, + timeout: Duration, + ) -> Result { + let d = doc_to_wit(doc); + let a = code_action_to_wit(action); + let resolved = self.provider_call(timeout, |p, s| p.call_resolve_code_action(s, &d, &a))?; + resolved.map(code_action_from_wit).map_err(|e| CallError::Guest(e.into())) } pub fn provide_definition( @@ -968,6 +962,47 @@ const MAX_VIEW_ITEMS: usize = 20_000; const MAX_CELL: usize = 64 << 10; const MAX_HTML: usize = 4 << 20; +/// Most additional edits of a completion item. +const MAX_ADDITIONAL_EDITS: usize = 100; + +fn edit_from_wit(e: wit_types::TextEdit) -> ide_core::TextEdit { + ide_core::TextEdit { range: range_from_wit(e.range), new_text: e.text } +} + +fn code_action_from_wit(a: wit_prov::CodeAction) -> CodeAction { + CodeAction { + title: clip(a.title, MAX_TITLE), + kind: a.kind.map(|k| clip(k, 64)), + preferred: a.preferred, + edits: a + .edits + .into_iter() + .map(|f| (std::path::PathBuf::from(f.path), f.edits.into_iter().map(edit_from_wit).collect())) + .collect(), + data: a.data.map(|d| clip(d, 64 << 10)), + } +} + +fn code_action_to_wit(a: &CodeAction) -> wit_prov::CodeAction { + wit_prov::CodeAction { + title: a.title.clone(), + kind: a.kind.clone(), + preferred: a.preferred, + edits: a + .edits + .iter() + .map(|(path, edits)| wit_prov::FileEdit { + path: path.to_string_lossy().into_owned(), + edits: edits + .iter() + .map(|e| wit_types::TextEdit { range: range_to_wit(e.range), text: e.new_text.clone() }) + .collect(), + }) + .collect(), + data: a.data.clone(), + } +} + fn opt_clip(s: Option, max: usize) -> Option { s.map(|s| clip(s, max)) } @@ -975,6 +1010,9 @@ fn opt_clip(s: Option, max: usize) -> Option { fn view_from_wit(v: wit_vt::View) -> View { let body = match v.body { wit_vt::Body::Message(m) => ViewBody::Message(clip(m, 16 << 10)), + wit_vt::Body::Other(e) => { + ViewBody::Message(format!("This view needs a newer version of OpenIDEA ({}).", clip(e.kind, 64))) + } wit_vt::Body::Tree(items) => ViewBody::Tree( items .into_iter() @@ -1006,10 +1044,8 @@ fn view_from_wit(v: wit_vt::View) -> View { fields .into_iter() .take(MAX_VIEW_ITEMS) - .map(|f| ViewField { - key: clip(f.key, 128), - label: clip(f.label, MAX_TITLE), - value: match f.value { + .filter_map(|f| { + let value = match f.value { wit_vt::FieldValue::Text(t) => FieldValue::Text(clip(t, MAX_CELL)), wit_vt::FieldValue::Multiline(t) => FieldValue::Multiline(clip(t, 1 << 20)), wit_vt::FieldValue::Checkbox(b) => FieldValue::Checkbox(b), @@ -1017,7 +1053,10 @@ fn view_from_wit(v: wit_vt::View) -> View { options: options.into_iter().take(1000).map(|o| clip(o, MAX_TITLE)).collect(), selected: clip(selected, MAX_TITLE), }, - }, + // A field type of a newer API: left out. + wit_vt::FieldValue::Other(_) => return None, + }; + Some(ViewField { key: clip(f.key, 128), label: clip(f.label, MAX_TITLE), value }) }) .collect(), ), @@ -1067,3 +1106,26 @@ fn event_to_wit_view(e: ViewEvent) -> wit_vt::ViewEvent { ViewEvent::Message { text } => W::Message(text), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn ext(kind: &str) -> wit_types::Extension { + wit_types::Extension { kind: kind.into(), data: "{}".into() } + } + + #[test] + fn bodies_and_fields_of_a_newer_api_degrade() { + let view = |body| wit_vt::View { toolbar: Vec::new(), body, status: None }; + let shown = view_from_wit(view(wit_vt::Body::Other(ext("markdown")))); + assert_eq!(shown.body, ViewBody::Message("This view needs a newer version of OpenIDEA (markdown).".into())); + let field = |key: &str, value| wit_vt::Field { key: key.into(), label: key.into(), value }; + let form = view_from_wit(view(wit_vt::Body::Form(vec![ + field("color", wit_vt::FieldValue::Other(ext("color"))), + field("name", wit_vt::FieldValue::Text("n".into())), + ]))); + let ViewBody::Form(fields) = form.body else { panic!("a form") }; + assert_eq!(fields.iter().map(|f| f.key.as_str()).collect::>(), ["name"], "unknown fields are left out"); + } +} diff --git a/crates/ide-plugin/src/types.rs b/crates/ide-plugin/src/types.rs index 9b4dfed0..b1d7b215 100644 --- a/crates/ide-plugin/src/types.rs +++ b/crates/ide-plugin/src/types.rs @@ -126,6 +126,20 @@ pub enum ActionSource { Other, } +impl ActionSource { + /// As plugins see it. + pub fn name(self) -> &'static str { + match self { + Self::Keyboard => "keyboard", + Self::Menu => "menu", + Self::Toolbar => "toolbar", + Self::ContextMenu => "context-menu", + Self::Search => "search", + Self::Other => "other", + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] pub struct ActionPerformed { @@ -303,6 +317,16 @@ pub fn net_host(url: &str) -> Result { #[cfg(test)] mod tests { + #[test] + fn the_restart_backoff_doubles_up_to_a_minute() { + let limits = Limits { restart_backoff: Duration::from_millis(150), ..Limits::default() }; + let waits: Vec = (1..=4).map(|n| limits.backoff(n).as_millis()).collect(); + assert_eq!(waits, [150, 300, 600, 1200]); + assert_eq!(limits.backoff(30), Duration::from_millis(150 * 64), "the factor stops growing"); + let long = Limits { restart_backoff: Duration::from_secs(10), ..Limits::default() }; + assert_eq!(long.backoff(5), Duration::from_secs(60)); + } + use super::*; #[test] @@ -338,9 +362,14 @@ pub struct CompletionItem { /// WIT `completion-kind` name (`method`, `keyword`, …). pub kind: Option, pub insert_text: String, + /// `insert_text` has tab stops and placeholders. + pub snippet: bool, pub replace: Option, pub sort_text: Option, + pub filter_text: Option, pub documentation: Option, + /// Applied with the item elsewhere in the document (an import). + pub additional_edits: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -355,6 +384,8 @@ pub struct CodeAction { pub kind: Option, pub preferred: bool, pub edits: Vec<(PathBuf, Vec)>, + /// The plugin's reference: the edits come from `resolve_code_action`. + pub data: Option, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/crates/ide-plugin/src/wit_host.rs b/crates/ide-plugin/src/wit_host.rs index 265f992a..bcd53cc8 100644 --- a/crates/ide-plugin/src/wit_host.rs +++ b/crates/ide-plugin/src/wit_host.rs @@ -440,20 +440,11 @@ impl vcs::Host for StoreData { } } -fn kind_from_wit(k: events::EventKind) -> EventKind { - match k { - events::EventKind::FileOpened => EventKind::FileOpened, - events::EventKind::FileSaved => EventKind::FileSaved, - events::EventKind::EditorFocused => EventKind::EditorFocused, - events::EventKind::ActionPerformed => EventKind::ActionPerformed, - events::EventKind::VcsChanged => EventKind::VcsChanged, - events::EventKind::Scheduled => EventKind::Scheduled, - } -} - impl events::Host for StoreData { - fn subscribe(&mut self, kind: events::EventKind) -> Fallible<()> { - let kind = kind_from_wit(kind); + fn subscribe(&mut self, kind: String) -> Fallible<()> { + let Some(kind) = EventKind::parse(&kind) else { + return Ok(Err(wit::Error::Invalid(format!("no event kind {kind:?} in this OpenIDEA")))); + }; if let Some(cap) = kind.capability() && let Err(e) = self.require(cap) { @@ -463,8 +454,10 @@ impl events::Host for StoreData { Ok(Ok(())) } - fn unsubscribe(&mut self, kind: events::EventKind) -> wasmtime::Result<()> { - self.host.subscribe(kind_from_wit(kind), false); + fn unsubscribe(&mut self, kind: String) -> wasmtime::Result<()> { + if let Some(kind) = EventKind::parse(&kind) { + self.host.subscribe(kind, false); + } Ok(()) } } @@ -542,10 +535,10 @@ impl diagnostics::Host for StoreData { diagnostics::Severity::Hint => ide_core::Severity::Hint, }, message: clip(d.message, 4096), - source: Some(source.clone()), + source: Some(d.source.map_or_else(|| source.clone(), |s| clip(s, 128))), code: d.code.map(|c| clip(c, 128)), - unnecessary: false, - deprecated: false, + unnecessary: d.tags.iter().any(|t| t == "unnecessary"), + deprecated: d.tags.iter().any(|t| t == "deprecated"), server: Some(source.clone()), }) .collect(); @@ -647,6 +640,21 @@ impl views::Host for StoreData { let title = clip(title, crate::runtime::MAX_TITLE); Ok(self.host.open_terminal(client.as_deref(), title, &program, args, cwd.as_deref()).map_err(to_wit)) } + + fn open_location(&mut self, path: String, line: u32, column: u32) -> Fallible<()> { + if let Err(e) = self.require(Capability::ProjectRead) { + return Ok(Err(e)); + } + let Some(root) = self.invocation.workspace_root.clone() else { + return Ok(Err(wit::Error::Invalid("no project is open in this window".into()))); + }; + let real = match sandbox::resolve_in_root(&root, &path) { + Ok(real) => real, + Err(e) => return Ok(Err(wit::Error::Invalid(e))), + }; + let client = self.invocation.client.clone(); + Ok(self.host.open_location(client.as_deref(), &real, line, column).map_err(to_wit)) + } } impl settings::Host for StoreData { diff --git a/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm b/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm index b0c8caee..368de83b 100644 Binary files a/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm and b/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm differ diff --git a/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm.sources b/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm.sources index 2edfa822..f811c07c 100644 --- a/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm.sources +++ b/crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm.sources @@ -1 +1 @@ -86dfe70a30ab1f7955a16e96c1b32c8be663456b871cc4d3bed7dc6052c062da +1f7402e2b1b4602e4073eb61837c3dc8840f9a4f7275d800dbd6fcf6ce5e40c3 diff --git a/crates/ide-plugin/tests/fixtures/hello/src/lib.rs b/crates/ide-plugin/tests/fixtures/hello/src/lib.rs index 26f75410..414c223e 100644 --- a/crates/ide-plugin/tests/fixtures/hello/src/lib.rs +++ b/crates/ide-plugin/tests/fixtures/hello/src/lib.rs @@ -1,5 +1,5 @@ //! Sample OpenIDEA plugin, written with the `openidea-plugin` SDK. -//! Build with `openidea-plugin build plugins/hello` (see plugins/README.md). +//! Build with `openidea-plugin build crates/ide-plugin/tests/fixtures/hello` (see plugins/README.md). use openidea_plugin::views::{Body, Button, TreeItem, View, ViewEvent, Viewport, Views}; use openidea_plugin::{Context, Error, Invocation, LogLevel, NotificationLevel, Plugin, host, register_plugin}; diff --git a/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm b/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm index aaa324d9..0899f53e 100644 Binary files a/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm and b/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm differ diff --git a/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm.sources b/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm.sources index 61e1857b..3a0de87f 100644 --- a/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm.sources +++ b/crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm.sources @@ -1 +1 @@ -0dacb82ac35c1f132dc66e85dbf0663b729f818279d3b32cd06403e502fbf880 +4f6cd5b5e0c12e7dcbccf66a6ad97276bf968bc3ad26f0fd08e1f7765cbfee14 diff --git a/crates/ide-plugin/tests/fixtures/jobs/plugin.toml b/crates/ide-plugin/tests/fixtures/jobs/plugin.toml index 8a3d2c89..8fc68206 100644 --- a/crates/ide-plugin/tests/fixtures/jobs/plugin.toml +++ b/crates/ide-plugin/tests/fixtures/jobs/plugin.toml @@ -12,6 +12,7 @@ activation = ["on-view:jobs", "on-command:*"] [capabilities] required = ["process:jobs"] +optional = ["project.read"] [[contributes.views]] id = "jobs" diff --git a/crates/ide-plugin/tests/fixtures/jobs/src/lib.rs b/crates/ide-plugin/tests/fixtures/jobs/src/lib.rs index b6686393..fe28d779 100644 --- a/crates/ide-plugin/tests/fixtures/jobs/src/lib.rs +++ b/crates/ide-plugin/tests/fixtures/jobs/src/lib.rs @@ -64,6 +64,8 @@ impl Views for Jobs { return self.follow(); } ViewEvent::Button(b) if b == "follow" => return self.follow(), + ViewEvent::Button(b) if b == "log" => return host::open_location("jobs.log", 2, 4), + ViewEvent::Button(b) if b == "outside" => return host::open_location("../outside.log", 0, 0), _ => return Ok(()), } host::refresh_view(VIEW); diff --git a/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm b/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm index d6018357..37db117e 100644 Binary files a/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm and b/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm differ diff --git a/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm.sources b/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm.sources index 1c5ed64f..f95fc1ae 100644 --- a/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm.sources +++ b/crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm.sources @@ -1 +1 @@ -0ad2e4c6977a576b7eb27fc1e535fd056dd9da64bdfe6cfda9ca9f57537d0384 +3d4dd1b6365c7fbafdd3c1f99f4405e5ee40fc5e56c97da960adfe7af2e7edb8 diff --git a/crates/ide-plugin/tests/fixtures/misbehave/plugin.toml b/crates/ide-plugin/tests/fixtures/misbehave/plugin.toml index 29dba7d6..1eae84d2 100644 --- a/crates/ide-plugin/tests/fixtures/misbehave/plugin.toml +++ b/crates/ide-plugin/tests/fixtures/misbehave/plugin.toml @@ -40,3 +40,11 @@ title = "insert" [[contributes.commands]] id = "cancel" title = "cancel" + +[[contributes.commands]] +id = "subscribe" +title = "subscribe" + +[[contributes.commands]] +id = "diagnose" +title = "diagnose" diff --git a/crates/ide-plugin/tests/fixtures/misbehave/src/lib.rs b/crates/ide-plugin/tests/fixtures/misbehave/src/lib.rs index cd6d6511..890bf845 100644 --- a/crates/ide-plugin/tests/fixtures/misbehave/src/lib.rs +++ b/crates/ide-plugin/tests/fixtures/misbehave/src/lib.rs @@ -14,7 +14,9 @@ wit_bindgen::generate!({ generate_all, }); -use exports::openidea::plugin::providers::{self, CodeAction, CompletionItem, Document, InlineSuggestion, LineMarker}; +use exports::openidea::plugin::providers::{ + self, CodeAction, CompletionItem, Document, FileEdit, InlineSuggestion, LineMarker, +}; use exports::openidea::plugin::syntax_provider::{self, Fold, Symbol, Token}; use exports::openidea::plugin::{commands, event_handler, lifecycle}; use openidea::plugin::{invocation::Invocation, types}; @@ -67,6 +69,23 @@ impl commands::Guest for Misbehave { } // The user gave up: not the plugin failing. "cancel" => Err(types::Error::Cancelled), + // Event kinds are names: one this IDE doesn't know is refused. + "subscribe" => openidea::plugin::events::subscribe(&arg.unwrap_or_default()).map(|()| None), + // Tagged problems from another tool on the line 0 of `arg`. + "diagnose" => { + use openidea::plugin::diagnostics::{self, Diagnostic, Severity}; + let start = types::Position { line: 0, character: 0 }; + let d = Diagnostic { + range: types::Range { start, end: start }, + severity: Severity::Warning, + message: "unused".into(), + code: None, + source: Some("lint".into()), + tags: vec!["unnecessary".into(), "glowing".into()], + related: Vec::new(), + }; + diagnostics::publish(&arg.unwrap_or_default(), &[d]).map(|()| None) + } other => Err(types::Error::NotFound(format!("command {other}"))), } } @@ -113,14 +132,24 @@ impl syntax_provider::Guest for Misbehave { impl providers::Guest for Misbehave { fn completion(doc: Document, _at: types::Position, _trigger: Option) -> Vec { behave(&doc.text); + // `rich`: a snippet that also adds an import. + let rich = doc.text == "rich"; + let start = types::Position { line: 0, character: 0 }; vec![CompletionItem { label: doc.path.clone(), detail: None, kind: Some("keyword".into()), - insert_text: doc.path, + insert_text: if rich { "f(${1:x})$0".into() } else { doc.path }, + snippet: rich, replace: None, sort_text: None, + filter_text: rich.then(|| "fun".into()), documentation: None, + additional_edits: if rich { + vec![types::TextEdit { range: types::Range { start, end: start }, text: "use f;\n".into() }] + } else { + Vec::new() + }, }] } @@ -128,8 +157,27 @@ impl providers::Guest for Misbehave { None } - fn code_actions(_doc: Document, _selection: types::Range) -> Vec { - Vec::new() + /// `lazy`: one action whose edits come from `resolve-code-action`. + fn code_actions(doc: Document, _selection: types::Range) -> Vec { + if doc.text != "lazy" { + return Vec::new(); + } + vec![CodeAction { + title: "Fix later".into(), + kind: Some("quickfix".into()), + preferred: false, + edits: Vec::new(), + data: Some("fix".into()), + }] + } + + fn resolve_code_action(doc: Document, action: CodeAction) -> Result { + if action.data.as_deref() != Some("fix") { + return Err(types::Error::Invalid("not one of mine".into())); + } + let start = types::Position { line: 0, character: 0 }; + let edit = types::TextEdit { range: types::Range { start, end: start }, text: "fixed ".into() }; + Ok(CodeAction { edits: vec![FileEdit { path: doc.path, edits: vec![edit] }], ..action }) } fn definition(_doc: Document, _at: types::Position) -> Vec { diff --git a/crates/ide-remote/src/mapping.rs b/crates/ide-remote/src/mapping.rs index c8a2c36c..ce814b1c 100644 --- a/crates/ide-remote/src/mapping.rs +++ b/crates/ide-remote/src/mapping.rs @@ -143,7 +143,10 @@ impl Mappings { } let (l, rest, _) = best?; // A `..` from the server must not leave the local folder. - if rest.split('/').any(|s| s == ".." || s == ".") { + if rest.split('/').filter(|s| !s.is_empty()).any(|s| { + let mut components = Path::new(s).components(); + !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() + }) { return None; } Some(rest.split('/').filter(|s| !s.is_empty()).fold(l.clone(), |p, s| p.join(s))) @@ -307,6 +310,43 @@ mod tests { assert_eq!(local_folder("/src/", &root), Some(root.join("src"))); } + #[cfg(windows)] + #[test] + fn native_anchors_never_replace_the_project_relative_fallback() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("project"); + let inside = root.join("src"); + let outside = tmp.path().join("outside"); + let missing = tmp.path().join("missing"); + std::fs::create_dir_all(&inside).unwrap(); + std::fs::create_dir(&outside).unwrap(); + for slash in ["\\", "/"] { + let spelling = |path: &Path| path.to_string_lossy().replace('\\', slash); + assert_eq!(local_folder(&spelling(&inside), &root), Some(inside.clone())); + assert_eq!(local_folder(&spelling(&outside), &root), None); + assert_eq!(local_folder(&spelling(&missing), &root), Some(root.join(posix(&missing)))); + assert_eq!(local_folder(&format!("{slash}src{slash}"), &root), Some(inside.clone())); + } + } + + #[test] + fn remote_paths_cannot_replace_or_leave_a_local_mapping() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let m = Mappings::new(&server(vec![]), root, "/srv"); + for remote in ["/srv/../outside", "/srv/./file"] { + assert_eq!(m.local(remote), None, "{remote}"); + } + #[cfg(windows)] + for remote in + ["/srv/..\\outside", "/srv/C:\\outside", "/srv/\\outside", "/srv/dir\\..\\outside", "/srv/C:/outside"] + { + assert_eq!(m.local(remote), None, "{remote}"); + } + assert_eq!(m.local("/srv/dir/file"), Some(root.join("dir/file"))); + assert_eq!(m.local("/srv"), Some(root.to_path_buf())); + } + #[cfg(unix)] #[test] fn absolute_symlink_inside_project_resolves_to_the_project_spelling() { diff --git a/crates/ide-sdk/src/apply.rs b/crates/ide-sdk/src/apply.rs index a496a433..11888554 100644 --- a/crates/ide-sdk/src/apply.rs +++ b/crates/ide-sdk/src/apply.rs @@ -5,7 +5,7 @@ use crate::discover::{project_venv_python, workspace_tsdk}; use crate::exec::{path_with, search_path}; use crate::{angular, godot, pinned, vue}; use crate::{bin_dir, exe, venv_of}; -use ide_core::settings::{LanguageServerSettings, Settings}; +use ide_core::settings::{InspectionLevel, InspectionSettings, LanguageServerSettings, Settings}; use ide_core::version_managers::Tool; use ide_language::{LanguageConfig, LanguageServerConfig}; use ide_run::RunConfig; @@ -123,8 +123,21 @@ fn not_a_jdk(p: &Path) -> String { /// `mvn` of the configured Maven home (`languages.java.maven.home`): the /// executable itself, `/bin/mvn` or `/mvn`; else of the Maven the project -/// pins for mise / asdf when it has no Maven wrapper. +/// pins for mise / asdf when it has no Maven wrapper. With `maven.daemon` on, the +/// Maven Daemon (`mvnd`) replaces all of that — it embeds its own Maven — unless +/// it is not installed or the project JDK is older than the Java 17 its daemon +/// needs, in which case the build stays on mvn (with a logged warning). pub fn maven_command(root: &Path, s: &Settings) -> Option { + if s.languages.java.maven.daemon { + match mvnd_command(root, s) { + // No project JDK at all lets mvnd pick its own Java, which may still be new enough. + Some(mvnd) if project_jdk(root, s).is_none_or(|j| j.major >= 17) => return Some(mvnd), + Some(_) => tracing::warn!( + "Maven daemon: the project JDK is older than the Java 17 its daemon needs; building with mvn" + ), + None => tracing::warn!("Maven daemon is enabled but mvnd was not found; building with mvn"), + } + } let name = if cfg!(windows) { "mvn.cmd" } else { "mvn" }; let Some(p) = setting_path(root, &s.languages.java.maven.home) else { if root.join("mvnw").is_file() || root.join("mvnw.cmd").is_file() { @@ -141,6 +154,41 @@ pub fn maven_command(root: &Path, s: &Settings) -> Option { found } +/// The `mvnd` executable when the Maven Daemon is enabled: the configured home +/// (`languages.java.maven.mvndHome`, the executable itself or `/bin/mvnd`), +/// else `$MVND_HOME`, else `mvnd` on `PATH`, else the common Homebrew / sdkman +/// locations. `None` = not found; the caller falls back to `mvn`. +pub fn mvnd_command(root: &Path, s: &Settings) -> Option { + let name = if cfg!(windows) { "mvnd.cmd" } else { "mvnd" }; + if let Some(p) = setting_path(root, &s.languages.java.maven.mvnd_home) { + let found = [p.clone(), p.join("bin").join(name), p.join(name)].into_iter().find(|c| c.is_file()); + if found.is_none() { + tracing::warn!("Maven Daemon home {} has no {name}", p.display()); + } + return found; + } + if let Some(home) = std::env::var_os("MVND_HOME").map(PathBuf::from) { + let found = [home.join("bin").join(name), home.join(name)].into_iter().find(|c| c.is_file()); + if found.is_none() { + tracing::warn!("MVND_HOME {} has no {name}", home.display()); + } + return found; + } + if let Some(found) = crate::exec::which(name) { + return Some(found); + } + if let Some(home) = crate::home_dir() { + let sdkman = home.join(".sdkman/candidates/mvnd/current"); + let candidates = [ + sdkman.join("bin").join(name), + PathBuf::from("/opt/homebrew/opt/mvnd").join("bin").join(name), + PathBuf::from("/usr/local/opt/mvnd").join("bin").join(name), + ]; + return candidates.into_iter().find(|c| c.is_file()); + } + None +} + /// Project launcher policy for sync and tool-window tasks. pub fn gradle_command(root: &Path, build_dir: &Path, s: &Settings) -> PathBuf { resolve_gradle_command(root, build_dir, s, true) @@ -301,7 +349,7 @@ pub fn tsdk(root: &Path, s: &Settings) -> Option { const PYTHON_SERVERS: [&str; 7] = ["basedpyright", "pyright", "ty", "pyrefly", "pylsp", "jedi-language-server", "ruff"]; -fn merge(base: &mut Value, layer: Value) { +pub(crate) fn merge(base: &mut Value, layer: Value) { match (base, layer) { (Value::Object(base), Value::Object(layer)) => { for (k, v) in layer { @@ -381,6 +429,36 @@ pub fn configure_maven_import( } } +/// Explicit Gradle links constrain discovery and isolate JDT's persisted import state. +pub fn configure_gradle_import( + cfg: &mut LanguageServerConfig, + linked: Option<&[String]>, + build_files: Option<&[String]>, +) { + let Some(linked) = linked.filter(|_| cfg.name == "jdtls") else { return }; + merge_init_options( + cfg, + json!({ + "settings": {"java": {"import": {"gradle": {"enabled": !linked.is_empty()}}}}, + "projectConfigurations": build_files.unwrap_or_default() + }), + ); + if let Some(settings) = &mut cfg.settings { + merge(settings, json!({"java": {"import": {"gradle": {"enabled": !linked.is_empty()}}}})); + } + if let Some(index) = cfg.args.iter().position(|arg| arg == "-data") + && let Some(workspace) = cfg.args.get_mut(index + 1) + { + let mut hash = 0xcbf29ce484222325u64; + for dir in linked { + for byte in dir.bytes().chain([0]) { + hash = (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3); + } + } + *workspace = format!("{workspace}-gradle-{hash:016x}"); + } +} + pub(crate) fn merge_init_options(cfg: &mut LanguageServerConfig, layer: Value) { let mut current = cfg.initialization_options.as_ref().and_then(|v| serde_json::to_value(v).ok()).unwrap_or_else(|| json!({})); @@ -394,15 +472,41 @@ pub(crate) fn merge_init_options(cfg: &mut LanguageServerConfig, layer: Value) { } } +/// Editor › Inspections › Java: "Serializable class without 'serialVersionUID'" (off by default, +/// as in IntelliJ IDEA). ECJ reports it, so it is a compiler option rather than a filter. +pub const SERIAL_VERSION_INSPECTION: &str = "java:missing-serial-version"; + +/// ECJ's level for a missing `serialVersionUID`, from the inspection profile in effect. +fn missing_serial_version(inspections: &InspectionSettings) -> Option<&'static str> { + let o = inspections.active().get(SERIAL_VERSION_INSPECTION)?; + if o.enabled != Some(true) { + return None; + } + Some(match o.severity { + Some(InspectionLevel::Error) => "error", + None | Some(InspectionLevel::Warning) => "warning", + Some(_) => "info", + }) +} + /// Workspace compiler defaults; Eclipse project preferences still take precedence. -pub(crate) fn configure_java_compiler(cfg: &mut LanguageServerConfig, data_dir: &Path) { - let file = data_dir.join("lsp/jdtls/compiler-defaults.prefs"); +/// Each variant has its own file, so changing an inspection restarts the server with it. +pub fn configure_java_compiler(cfg: &mut LanguageServerConfig, data_dir: &Path, inspections: &InspectionSettings) { const DEFAULTS: &str = "org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning\n\ org.eclipse.jdt.core.compiler.problem.discouragedReference=warning\n"; + let serial = missing_serial_version(inspections); + let (name, content) = match serial { + None => ("compiler-defaults.prefs".to_owned(), DEFAULTS.to_owned()), + Some(level) => ( + format!("compiler-defaults-serial-{level}.prefs"), + format!("{DEFAULTS}org.eclipse.jdt.core.compiler.problem.missingSerialVersion={level}\n"), + ), + }; + let file = data_dir.join("lsp/jdtls").join(name); let write = || -> ide_core::IdeResult<()> { std::fs::create_dir_all(file.parent().expect("compiler preferences directory"))?; - if std::fs::read_to_string(&file).ok().as_deref() != Some(DEFAULTS) { - ide_vfs::write_atomic(&file, DEFAULTS.as_bytes())?; + if std::fs::read_to_string(&file).ok().as_deref() != Some(content.as_str()) { + ide_vfs::write_atomic(&file, content.as_bytes())?; } Ok(()) }; @@ -562,6 +666,22 @@ pub fn configure_server(cfg: &mut LanguageServerConfig, root: &Path, server_root configure_vue_typescript(cfg, root, server_root, s); } + // Intelephense reads it as `intelephense.environment.phpVersion` (`workspace/configuration`). + if name == "intelephense" + && let Some(level) = crate::php::language_level(root, s) + { + merge_init_options(cfg, json!({ "environment": { "phpVersion": level.version } })); + } + if name == "intelephense" { + if let Some(stubs) = crate::php::intelephense_stubs(root, s) { + merge_init_options(cfg, json!({ "stubs": stubs })); + } + let include_paths = crate::php::include_paths(root, s); + if !include_paths.is_empty() { + merge_init_options(cfg, json!({ "environment": { "includePaths": include_paths } })); + } + } + // Node-based servers (`#!/usr/bin/env node`) run on the selected Node.js. if let Some(dir) = node_dir(root, s) { path_dirs.push(dir); @@ -628,6 +748,7 @@ fn custom_server(name: &str, ls: &LanguageServerSettings) -> LanguageServerConfi max_java: None, launch_error: None, tsserver_project: false, + resync_on_configuration: false, pinned: None, } } @@ -1030,13 +1151,50 @@ mod java_compiler_tests { serde_json::to_value(cfg.initialization_options.as_ref().unwrap()).unwrap() } + #[test] + fn serial_version_inspection_sets_the_compiler_option() { + let tmp = tempfile::tempdir().unwrap(); + let url = |inspections: &InspectionSettings| { + let mut cfg = custom_server("jdtls", &LanguageServerSettings::default()); + configure_java_compiler(&mut cfg, tmp.path(), inspections); + options(&cfg).pointer("/settings/java/settings/url").unwrap().as_str().unwrap().to_owned() + }; + let set = |o: ide_core::settings::InspectionOverride| { + let mut s = InspectionSettings::default(); + s.default.insert(SERIAL_VERSION_INSPECTION.into(), o); + s + }; + let off = url(&InspectionSettings::default()); + assert!(!std::fs::read_to_string(&off).unwrap().contains("missingSerialVersion")); + let on = url(&set(ide_core::settings::InspectionOverride { enabled: Some(true), ..Default::default() })); + assert_ne!(on, off, "a new file, so the server restarts with it"); + assert!(std::fs::read_to_string(&on).unwrap().ends_with("compiler.problem.missingSerialVersion=warning\n")); + let error = url(&set(ide_core::settings::InspectionOverride { + enabled: Some(true), + severity: Some(InspectionLevel::Error), + ..Default::default() + })); + assert!(std::fs::read_to_string(error).unwrap().ends_with("missingSerialVersion=error\n")); + let weak = url(&set(ide_core::settings::InspectionOverride { + enabled: Some(true), + severity: Some(InspectionLevel::WeakWarning), + ..Default::default() + })); + assert!(std::fs::read_to_string(weak).unwrap().ends_with("missingSerialVersion=info\n")); + let severity_only = url(&set(ide_core::settings::InspectionOverride { + severity: Some(InspectionLevel::Error), + ..Default::default() + })); + assert_eq!(severity_only, off, "off by default"); + } + #[test] fn compiler_defaults_live_in_the_cache_and_preserve_other_options() { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().join("cache with spaces 测试"); let mut cfg = custom_server("jdtls", &LanguageServerSettings::default()); merge_init_options(&mut cfg, json!({"settings": {"java": {"format": {"tabSize": 4}}}})); - configure_java_compiler(&mut cfg, &cache); + configure_java_compiler(&mut cfg, &cache, &InspectionSettings::default()); let options = options(&cfg); let path = PathBuf::from(options.pointer("/settings/java/settings/url").unwrap().as_str().unwrap()); assert!(path.starts_with(&cache)); @@ -1047,7 +1205,7 @@ mod java_compiler_tests { } assert_eq!(options.pointer("/settings/java/format/tabSize"), Some(&json!(4))); let modified = std::fs::metadata(&path).unwrap().modified().unwrap(); - configure_java_compiler(&mut cfg, &cache); + configure_java_compiler(&mut cfg, &cache, &InspectionSettings::default()); assert_eq!(std::fs::metadata(path).unwrap().modified().unwrap(), modified); assert_eq!(self::options(&cfg), options); } @@ -1057,7 +1215,7 @@ mod java_compiler_tests { let tmp = tempfile::tempdir().unwrap(); let mut cfg = custom_server("jdtls", &LanguageServerSettings::default()); merge_init_options(&mut cfg, json!({"settings": {"java": {"settings": {"url": "own.prefs"}}}})); - configure_java_compiler(&mut cfg, tmp.path()); + configure_java_compiler(&mut cfg, tmp.path(), &InspectionSettings::default()); assert_eq!(options(&cfg).pointer("/settings/java/settings/url"), Some(&json!("own.prefs"))); for settings in [ @@ -1090,6 +1248,25 @@ mod java_compiler_tests { mod maven_linkage_tests { use super::*; + #[test] + fn explicit_gradle_links_scope_jdt_import_and_unlink_all_disables_gradle() { + let mut cfg = custom_server("jdtls", &LanguageServerSettings::default()); + cfg.args = vec!["-data".into(), "/cache/jdtls".into()]; + let files = ["file:///p/a/settings.gradle".to_owned(), "file:///p/maven/pom.xml".to_owned()]; + configure_gradle_import(&mut cfg, Some(&["a".into()]), Some(&files)); + let options = serde_json::to_value(&cfg.initialization_options).unwrap(); + assert_eq!(options["projectConfigurations"], json!(files)); + assert_eq!(options.pointer("/settings/java/import/gradle/enabled"), Some(&json!(true))); + assert!(cfg.args[1].starts_with("/cache/jdtls-gradle-")); + let linked_cache = cfg.args[1].clone(); + cfg.args[1] = "/cache/jdtls".into(); + configure_gradle_import(&mut cfg, Some(&[]), Some(&files[1..])); + let options = serde_json::to_value(&cfg.initialization_options).unwrap(); + assert_eq!(options["projectConfigurations"], json!(&files[1..])); + assert_eq!(options.pointer("/settings/java/import/gradle/enabled"), Some(&json!(false))); + assert_ne!(linked_cache, cfg.args[1]); + } + #[test] fn import_excludes_unlinked_modules_and_changes_persistent_workspace() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/ide-sdk/src/discover.rs b/crates/ide-sdk/src/discover.rs index c0de0540..e18b2dae 100644 --- a/crates/ide-sdk/src/discover.rs +++ b/crates/ide-sdk/src/discover.rs @@ -659,6 +659,10 @@ mod maven_path_tests { impl Subst { fn new(target: &Path) -> Self { + // Tests run in parallel and may race for the same free letter; only a drive that + // shows this target's marker is ours. + let marker = format!("{}.subst", target.file_name().unwrap().to_string_lossy()); + std::fs::write(target.join(&marker), "").unwrap(); for letter in (b'D'..=b'Z').rev() { let name = format!("{}:", char::from(letter)); let root = PathBuf::from(format!("{name}\\")); @@ -666,7 +670,7 @@ mod maven_path_tests { continue; } let output = Command::new("subst").arg(&name).arg(target).output().unwrap(); - if output.status.success() { + if output.status.success() && root.join(&marker).exists() { return Self { name, root }; } } diff --git a/crates/ide-sdk/src/java.rs b/crates/ide-sdk/src/java.rs index 05f89fc9..6de7d667 100644 --- a/crates/ide-sdk/src/java.rs +++ b/crates/ide-sdk/src/java.rs @@ -369,7 +369,6 @@ pub fn configure_jdtls( if cfg.name != "jdtls" { return; } - crate::apply::configure_java_compiler(cfg, data_dir); // A Lombok agent from the user's JVM arguments wins. let own_agent = crate::lombok::agent_in_args(cfg.args.iter().map(String::as_str)); let agent = own_agent.as_deref().or(lombok); @@ -1128,7 +1127,7 @@ mod tests { /// The same jar supplies the JDT command and the private Maven output redirector. pub fn kotlin_bridge_bundle(data_dir: &Path) -> Option { let extension = data_dir.join("kotlin-bridge-1.0.0.jar"); - let bytes = include_bytes!("../../../plugins/kotlin-bridge/openidea-kotlin-bridge.jar"); + let bytes = include_bytes!("../../../java/kotlin-bridge/openidea-kotlin-bridge.jar"); if std::fs::read(&extension).is_ok_and(|old| old == bytes) { return Some(extension); } @@ -1170,8 +1169,8 @@ mod kotlin_bridge_bundle_tests { /// Installs the app-owned JDT LS command bundle, falling back to ordinary LSP on IO failure. pub fn usages_bundle(data_dir: &Path) -> Option { - let extension = data_dir.join("java-usages-1.2.1.jar"); - let bytes = include_bytes!("../../../plugins/java-usages/openidea-java-usages.jar"); + let extension = data_dir.join("java-usages-1.3.0.jar"); + let bytes = include_bytes!("../../../java/java-usages/openidea-java-usages.jar"); if std::fs::read(&extension).is_ok_and(|old| old == bytes) { clean_usages_bundles(data_dir, &extension); return Some(extension); @@ -1256,10 +1255,7 @@ mod usages_bundle_tests { fn installs_the_exact_bundle_and_reuses_it() { let dir = tempfile::tempdir().unwrap(); let jar = super::usages_bundle(dir.path()).unwrap(); - assert_eq!( - std::fs::read(&jar).unwrap(), - include_bytes!("../../../plugins/java-usages/openidea-java-usages.jar") - ); + assert_eq!(std::fs::read(&jar).unwrap(), include_bytes!("../../../java/java-usages/openidea-java-usages.jar")); assert_eq!(super::usages_bundle(dir.path()), Some(jar)); assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); } diff --git a/crates/ide-sdk/src/lib.rs b/crates/ide-sdk/src/lib.rs index ac8c34eb..35d52417 100644 --- a/crates/ide-sdk/src/lib.rs +++ b/crates/ide-sdk/src/lib.rs @@ -18,6 +18,7 @@ pub mod locate; pub mod lombok; pub mod maven_profiles; mod npm; +pub mod php; pub mod pinned; pub mod python; pub mod ruby; diff --git a/crates/ide-sdk/src/lombok.rs b/crates/ide-sdk/src/lombok.rs index 8b292d48..3a4b4192 100644 --- a/crates/ide-sdk/src/lombok.rs +++ b/crates/ide-sdk/src/lombok.rs @@ -246,7 +246,7 @@ fn sources_import_lombok(root: &Path) -> bool { // Imports precede the first type, after at most a license header. const HEAD: u64 = 8 * 1024; let mut walker = ignore::WalkBuilder::new(root); - walker.require_git(false).follow_links(false).filter_entry(|e| { + walker.require_git(ide_core::ignored::inside_git_repo(root)).follow_links(false).filter_entry(|e| { e.depth() == 0 || !e.file_type().is_some_and(|t| t.is_dir()) || !OUTPUT_DIRS.contains(&e.file_name().to_string_lossy().as_ref()) diff --git a/crates/ide-sdk/src/php.rs b/crates/ide-sdk/src/php.rs new file mode 100644 index 00000000..4e1bc6c4 --- /dev/null +++ b/crates/ide-sdk/src/php.rs @@ -0,0 +1,376 @@ +//! The PHP language level: the one chosen in Settings › PHP, else the one the root +//! `composer.json` declares (`config.platform.php`, else the lowest version `require.php` allows); +//! the Intelephense storage that goes with it; and the stub sets and include paths Intelephense indexes. + +use crate::apply::{merge, merge_init_options, setting_path}; +use ide_core::settings::Settings; +use ide_language::LanguageServerConfig; +use serde_json::{Value, json}; +use std::path::Path; + +/// A PHP version and where it comes from. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, specta::Type)] +#[serde(rename_all = "camelCase")] +pub struct PhpLanguageLevel { + /// `X.Y.Z`. + pub version: String, + pub source: PhpLevelSource, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, specta::Type)] +#[serde(rename_all = "camelCase")] +pub enum PhpLevelSource { + /// `languages.php.languageLevel`. + Settings, + /// `config.platform.php` of `composer.json`. + Platform, + /// The lowest version `require.php` of `composer.json` allows. + Require, +} + +/// The project's PHP version: the one chosen in the settings, else [`composer_level`]; +/// `None` when neither names one. +pub fn language_level(root: &Path, s: &Settings) -> Option { + let explicit = s.languages.php.language_level.trim(); + if !explicit.is_empty() { + return version(explicit).map(|v| PhpLanguageLevel::new(v, PhpLevelSource::Settings)); + } + composer_level(root) +} + +/// The PHP version the root `composer.json` declares (the setting's Auto): +/// `config.platform.php`, else the lowest version `require.php` allows. +pub fn composer_level(root: &Path) -> Option { + let text = std::fs::read_to_string(root.join("composer.json")).ok()?; + let composer: Value = serde_json::from_str(&text).ok()?; + if let Some(platform) = composer.pointer("/config/platform/php").and_then(Value::as_str).and_then(version) { + return Some(PhpLanguageLevel::new(platform, PhpLevelSource::Platform)); + } + lowest_allowed(composer.pointer("/require/php")?.as_str()?) + .map(|v| PhpLanguageLevel::new(v, PhpLevelSource::Require)) +} + +/// Gives each PHP version its own Intelephense workspace storage (below `data_dir`). +/// Intelephense keeps the built-in functions of the version a workspace was first indexed +/// with in that storage, and reuses them after the version changes. Without a version, or +/// with a `storagePath` of the user's, the server keeps its own storage. +pub fn configure_intelephense_storage(cfg: &mut LanguageServerConfig, data_dir: &Path) { + if cfg.name != "intelephense" { + return; + } + let mut options = + cfg.initialization_options.as_ref().and_then(|v| serde_json::to_value(v).ok()).unwrap_or(Value::Null); + if options.get("storagePath").is_some() { + return; + } + // The version Intelephense reads: `workspace/configuration` is answered from the + // settings merged over the initialization options (`ide_lsp::handler::all_settings`). + if let Some(settings) = &cfg.settings { + merge(&mut options, settings.clone()); + } + let section = options.pointer("/settings/intelephense").unwrap_or(&options); + let version = section.pointer("/environment/phpVersion").and_then(Value::as_str); + let Some(version) = version.filter(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit() || c == '.')) else { + return; + }; + let storage = data_dir.join("lsp").join("intelephense").join(version); + merge_init_options(cfg, json!({ "storagePath": storage.to_string_lossy() })); +} + +/// The stub sets Intelephense loads when `stubs` is not set (Intelephense 1.18.5). Setting +/// `stubs` replaces them, so additions are sent together with these. +#[rustfmt::skip] +const DEFAULT_STUBS: &[&str] = &[ + "apache", "bcmath", "bz2", "calendar", "com_dotnet", "Core", "ctype", "curl", "date", "dba", "dom", "enchant", + "exif", "fileinfo", "filter", "fpm", "ftp", "gd", "hash", "iconv", "imap", "intl", "json", "ldap", "libxml", + "mbstring", "mcrypt", "mssql", "mysqli", "oci8", "odbc", "openssl", "pcntl", "pcre", "PDO", "pgsql", "Phar", + "posix", "pspell", "random", "readline", "Reflection", "regex", "session", "shmop", "SimpleXML", "snmp", "soap", + "sockets", "sodium", "SPL", "sqlite3", "standard", "superglobals", "sybase", "sysvmsg", "sysvsem", "sysvshm", + "tidy", "tokenizer", "uri", "xml", "xmlreader", "xmlrpc", "xmlwriter", "Zend OPcache", "zip", "zlib", +]; + +/// Stub sets outside the defaults whose names are not lower case; the rest are the lower-case +/// extension name. +const MIXED_CASE_STUBS: &[&str] = + &["Ev", "FFI", "LuaSandbox", "Parle", "SQLite", "SaxonC", "SplType", "ZendCache", "ZendDebugger", "ZendUtils"]; + +/// `Zend OPcache`, `zend-opcache` and `zend_opcache` name the same stubs. +fn stub_key(name: &str) -> String { + name.chars().filter(char::is_ascii_alphanumeric).map(|c| c.to_ascii_lowercase()).collect() +} + +/// The `stubs` Intelephense loads: its defaults plus a stub set for each `ext-*` that the root +/// `composer.json` requires (`require`, `require-dev`) and those in Settings › PHP. `None` when +/// that adds nothing to the defaults, so the server keeps its own. +pub fn intelephense_stubs(root: &Path, s: &Settings) -> Option> { + let composer: Option = + std::fs::read_to_string(root.join("composer.json")).ok().and_then(|t| serde_json::from_str(&t).ok()); + let extensions = ["require", "require-dev"] + .into_iter() + .filter_map(|section| composer.as_ref()?.get(section)?.as_object()) + .flat_map(|deps| deps.keys()) + .filter_map(|dep| dep.strip_prefix("ext-")) + .map(|ext| { + let key = stub_key(ext); + MIXED_CASE_STUBS + .iter() + .find(|stub| stub_key(stub) == key) + .map_or(ext.to_ascii_lowercase(), |s| s.to_string()) + }); + let configured = s.languages.php.stubs.iter().map(|stub| stub.trim().to_owned()); + let mut stubs: Vec = DEFAULT_STUBS.iter().map(|s| s.to_string()).collect(); + let mut known: std::collections::HashSet = stubs.iter().map(|s| stub_key(s)).collect(); + for stub in extensions.chain(configured) { + if !stub.is_empty() && known.insert(stub_key(&stub)) { + stubs.push(stub); + } + } + (stubs.len() > DEFAULT_STUBS.len()).then_some(stubs) +} + +/// The folders in Settings › PHP › Include paths, resolved against the project. +pub fn include_paths(root: &Path, s: &Settings) -> Vec { + s.languages + .php + .include_paths + .iter() + .filter_map(|p| setting_path(root, p)) + .map(|p| p.to_string_lossy().into_owned()) + .collect() +} + +type Version = (u32, u32, u32); + +impl PhpLanguageLevel { + fn new((major, minor, patch): Version, source: PhpLevelSource) -> Self { + Self { version: format!("{major}.{minor}.{patch}"), source } + } +} + +/// `7`, `7.4`, `v7.4.3`, `8.1.*`, `7.4.0-dev`: missing parts are 0. +fn version(s: &str) -> Option { + let s = s.trim().trim_start_matches(['v', 'V']); + let mut parts = s.split('.').map(|p| { + let digits: &str = &p[..p.find(|c: char| !c.is_ascii_digit()).unwrap_or(p.len())]; + digits.parse::().ok() + }); + let major = parts.next()??; + let minor = parts.next().flatten().unwrap_or(0); + let patch = parts.next().flatten().unwrap_or(0); + Some((major, minor, patch)) +} + +/// The lowest version a Composer constraint allows: the lowest lower bound of its +/// `||` alternatives. `None` when an alternative has no lower bound (`*`, `<8.0`). +fn lowest_allowed(constraint: &str) -> Option { + let bounds: Option> = + constraint.split('|').filter(|alt| !alt.trim().is_empty()).map(lower_bound).collect(); + bounds?.into_iter().min() +} + +/// The lower bound of constraints that must all hold (`>=7.2 <8.4`, `^7.4, !=7.4.1`, +/// `7.2 - 8.0`): the highest of their lower bounds. +fn lower_bound(alternative: &str) -> Option { + // A hyphen range: its first version. + let alternative = alternative.split(" - ").next().unwrap_or(alternative); + let mut tokens: Vec = Vec::new(); + for token in alternative.split([' ', ',']).filter(|t| !t.is_empty()) { + // `>= 7.2`: the operator and its version. + match tokens.last_mut() { + Some(op) if op.chars().all(|c| "<>=!~^".contains(c)) => op.push_str(token), + _ => tokens.push(token.to_owned()), + } + } + tokens + .iter() + .filter_map(|t| { + let t = t.split('@').next().unwrap_or(t); + if t.starts_with('<') || t.starts_with("!=") { + return None; + } + version(t.trim_start_matches(['>', '=', '^', '~'])) + }) + .max() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn level(composer: Option<&str>, explicit: &str) -> Option { + levels(composer, explicit).0.map(|l| l.version) + } + + /// The effective level and the composer one. + fn levels(composer: Option<&str>, explicit: &str) -> (Option, Option) { + let tmp = tempfile::tempdir().unwrap(); + if let Some(composer) = composer { + std::fs::write(tmp.path().join("composer.json"), composer).unwrap(); + } + let mut s = Settings::default(); + s.languages.php.language_level = explicit.into(); + (language_level(tmp.path(), &s), composer_level(tmp.path())) + } + + fn required(constraint: &str) -> Option { + level(Some(&serde_json::json!({ "require": { "php": constraint } }).to_string()), "") + } + + #[test] + fn lowest_version_require_php_allows() { + assert_eq!(required("^7.4 || ^8.0").as_deref(), Some("7.4.0")); + assert_eq!(required("^8.0 || ^7.4").as_deref(), Some("7.4.0")); + assert_eq!(required(">=7.2 <8.4").as_deref(), Some("7.2.0")); + assert_eq!(required(">= 7.2, < 8.4").as_deref(), Some("7.2.0")); + assert_eq!(required("8.1.*").as_deref(), Some("8.1.0")); + assert_eq!(required("~7.4.3").as_deref(), Some("7.4.3")); + assert_eq!(required("7.2 - 8.0").as_deref(), Some("7.2.0")); + assert_eq!(required("^7.3|^8.0").as_deref(), Some("7.3.0")); + assert_eq!(required(">=8.1@dev").as_deref(), Some("8.1.0")); + assert_eq!(required("8").as_deref(), Some("8.0.0")); + } + + #[test] + fn no_lower_bound_gives_nothing() { + for constraint in ["*", "<8.0", "", "^7.4 || *", "dev-main"] { + assert_eq!(required(constraint), None, "{constraint}"); + } + } + + #[test] + fn platform_overrides_require() { + let composer = r#"{"require": {"php": "^8.2"}, "config": {"platform": {"php": "7.2.34"}}}"#; + assert_eq!(level(Some(composer), "").as_deref(), Some("7.2.34")); + } + + #[test] + fn no_requirement_gives_nothing() { + assert_eq!(level(None, ""), None); + assert_eq!(level(Some(r#"{"require": {"ext-json": "*"}}"#), ""), None); + assert_eq!(level(Some("not json"), ""), None); + } + + #[test] + fn each_php_version_gets_its_own_intelephense_storage() { + let data = Path::new("/data"); + let server = |name: &str, options: Value| { + let mut cfg: LanguageServerConfig = + toml::from_str(&format!("name = \"{name}\"\ncommand = \"{name}\"")).unwrap(); + if !options.is_null() { + merge_init_options(&mut cfg, options); + } + cfg + }; + let options = |cfg: &LanguageServerConfig| serde_json::to_value(&cfg.initialization_options).unwrap(); + + let mut cfg = server("intelephense", json!({"environment": {"phpVersion": "7.4.0"}, "clearCache": false})); + configure_intelephense_storage(&mut cfg, data); + let storage = data.join("lsp").join("intelephense").join("7.4.0"); + assert_eq!( + options(&cfg), + json!({"environment": {"phpVersion": "7.4.0"}, "clearCache": false, "storagePath": storage.to_string_lossy()}) + ); + + // The version Intelephense reads wins: Settings > Language Servers `settings` over the options. + let storage_of = |init: Value, settings: Value| { + let mut cfg = server("intelephense", init); + cfg.settings = Some(settings); + configure_intelephense_storage(&mut cfg, data); + options(&cfg)["storagePath"].clone() + }; + let path = |v: &str| json!(data.join("lsp").join("intelephense").join(v).to_string_lossy()); + let init = json!({"environment": {"phpVersion": "7.4.0"}}); + assert_eq!(storage_of(init.clone(), json!({"environment": {"phpVersion": "8.3.0"}})), path("8.3.0")); + assert_eq!(storage_of(init, json!({"other": 1})), path("7.4.0")); + let nested = json!({"settings": {"intelephense": {"environment": {"phpVersion": "8.0.0"}}}}); + assert_eq!(storage_of(Value::Null, nested), path("8.0.0")); + + // No version, the user's own storage, not a version, another server: unchanged. + for unchanged in [ + server("intelephense", Value::Null), + server("intelephense", json!({"environment": {"includePaths": []}})), + server("intelephense", json!({"environment": {"phpVersion": "8.1.0"}, "storagePath": "/mine"})), + server("intelephense", json!({"environment": {"phpVersion": "../8.1"}})), + server("phpactor", json!({"environment": {"phpVersion": "8.1.0"}})), + ] { + let mut cfg = unchanged.clone(); + configure_intelephense_storage(&mut cfg, data); + assert_eq!(cfg, unchanged); + } + } + + fn stubs(composer: Option, configured: &[&str]) -> Option> { + let tmp = tempfile::tempdir().unwrap(); + if let Some(composer) = composer { + std::fs::write(tmp.path().join("composer.json"), composer.to_string()).unwrap(); + } + let mut s = Settings::default(); + s.languages.php.stubs = configured.iter().map(|s| s.to_string()).collect(); + intelephense_stubs(tmp.path(), &s) + } + + fn added(stubs: Option>) -> Vec { + let stubs = stubs.expect("stubs"); + assert_eq!(stubs[..DEFAULT_STUBS.len()], DEFAULT_STUBS.iter().map(|s| s.to_string()).collect::>()); + stubs[DEFAULT_STUBS.len()..].to_vec() + } + + #[test] + fn required_extensions_and_configured_stubs_follow_the_defaults() { + let composer = json!({ + "require": {"php": "^8.1", "ext-redis": "*", "ext-json": "*", "monolog/monolog": "^3"}, + "require-dev": {"ext-Imagick": "*", "ext-ffi": "*", "ext-redis": "*"} + }); + assert_eq!( + added(stubs(Some(composer), &[" wordpress ", "Redis", ""])), + ["redis", "imagick", "FFI", "wordpress"] + ); + assert_eq!(added(stubs(None, &["wordpress"])), ["wordpress"]); + } + + #[test] + fn nothing_beyond_the_defaults_sends_nothing() { + assert_eq!(stubs(None, &[]), None); + assert_eq!(stubs(Some(json!({"require": {"php": "^8.1"}})), &[]), None); + // Default stub sets under their extension names: `ext-pdo`, `ext-zend-opcache`, `ext-simplexml`. + let defaults = + json!({"require": {"ext-pdo": "*", "ext-zend-opcache": "*", "ext-simplexml": "*", "ext-json": "*"}}); + assert_eq!(stubs(Some(defaults), &["SPL", "core"]), None); + assert_eq!(stubs(Some(json!("not an object")), &[]), None); + } + + #[test] + fn include_paths_resolve_against_the_project() { + let root = Path::new("/work/module"); + let mut s = Settings::default(); + s.languages.php.include_paths = vec!["../platform".into(), "/opt/prestashop".into(), " ".into()]; + assert_eq!( + include_paths(root, &s), + [root.join("../platform").to_string_lossy().into_owned(), "/opt/prestashop".to_string()] + ); + } + + #[test] + fn each_level_names_its_source() { + let found = |version: &str, source| Some(PhpLanguageLevel { version: version.into(), source }); + let require = r#"{"require": {"php": "^7.4"}}"#; + let platform = r#"{"require": {"php": "^7.4"}, "config": {"platform": {"php": "8.2.1"}}}"#; + + let require_level = found("7.4.0", PhpLevelSource::Require); + assert_eq!(levels(Some(require), ""), (require_level.clone(), require_level.clone())); + let platform_level = found("8.2.1", PhpLevelSource::Platform); + assert_eq!(levels(Some(platform), ""), (platform_level.clone(), platform_level)); + // An explicit level: the composer one is still what Auto would pick. + assert_eq!(levels(Some(require), "8.1"), (found("8.1.0", PhpLevelSource::Settings), require_level)); + assert_eq!(levels(None, "8.1"), (found("8.1.0", PhpLevelSource::Settings), None)); + assert_eq!(levels(None, ""), (None, None)); + assert_eq!(levels(Some(r#"{"require": {"php": "*"}}"#), ""), (None, None)); + } + + #[test] + fn explicit_level_wins() { + let composer = r#"{"require": {"php": "^7.4"}, "config": {"platform": {"php": "7.4.33"}}}"#; + assert_eq!(level(Some(composer), "8.1").as_deref(), Some("8.1.0")); + assert_eq!(level(None, " 5.6 ").as_deref(), Some("5.6.0")); + } +} diff --git a/crates/ide-sdk/src/tests.rs b/crates/ide-sdk/src/tests.rs index 4a61ad1d..7a8bd103 100644 --- a/crates/ide-sdk/src/tests.rs +++ b/crates/ide-sdk/src/tests.rs @@ -38,6 +38,7 @@ fn server(name: &str) -> LanguageServerConfig { max_java: None, launch_error: None, tsserver_project: false, + resync_on_configuration: false, pinned: None, } } @@ -1293,6 +1294,61 @@ fn php_sources_and_selected_interpreter() { assert!(configs[3].env.get("PATH").is_none_or(|p| !p.contains("php82"))); } +#[test] +fn intelephense_gets_the_php_language_level() { + use serde_json::json; + let tmp = tempfile::tempdir().unwrap(); + let configured = |name: &str, s: &Settings| { + let mut cfg = server(name); + configure_server(&mut cfg, tmp.path(), tmp.path(), s); + cfg + }; + let options = |cfg: LanguageServerConfig| serde_json::to_value(cfg.initialization_options).unwrap(); + + // Nothing to resolve: the options stay as they were (the rest depends on the machine's tools). + assert_eq!(configured("intelephense", &Settings::default()).initialization_options, None); + + std::fs::write(tmp.path().join("composer.json"), r#"{"require": {"php": "^7.4 || ^8.0"}}"#).unwrap(); + assert_eq!( + options(configured("intelephense", &Settings::default())), + json!({"environment": {"phpVersion": "7.4.0"}}) + ); + for other in ["phpactor", "basedpyright", "vtsls"] { + assert_eq!(options(configured(other, &Settings::default())).get("environment"), None, "{other}"); + } + + let mut s = Settings::default(); + s.languages.php.language_level = "8.1".into(); + let mut cfg = server("intelephense"); + cfg.initialization_options = Some(toml::Value::try_from(json!({"clearCache": true})).unwrap()); + configure_server(&mut cfg, tmp.path(), tmp.path(), &s); + assert_eq!(options(cfg), json!({"clearCache": true, "environment": {"phpVersion": "8.1.0"}})); +} + +#[test] +fn intelephense_gets_the_required_stubs_and_include_paths() { + use serde_json::json; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("composer.json"), r#"{"require": {"php": "^8.1", "ext-redis": "*"}}"#).unwrap(); + let mut s = Settings::default(); + s.languages.php.stubs = vec!["wordpress".into()]; + s.languages.php.include_paths = vec!["../platform".into()]; + let mut cfg = server("intelephense"); + configure_server(&mut cfg, tmp.path(), tmp.path(), &s); + let options = serde_json::to_value(cfg.initialization_options).unwrap(); + let stubs: Vec<&str> = options["stubs"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect(); + assert_eq!(stubs.len(), 70); + assert_eq!((stubs[0], &stubs[68..]), ("apache", &["redis", "wordpress"][..])); + assert_eq!( + options["environment"], + json!({"phpVersion": "8.1.0", "includePaths": [tmp.path().join("../platform").to_string_lossy()]}) + ); + + let mut other = server("phpactor"); + configure_server(&mut other, tmp.path(), tmp.path(), &s); + assert_eq!(other.initialization_options, None); +} + #[test] fn go_constraints_change_only_protocol_settings() { use serde_json::json; diff --git a/crates/ide-sdk/tests/version_managers.rs b/crates/ide-sdk/tests/version_managers.rs index 4fbd3bd5..d60a790d 100644 --- a/crates/ide-sdk/tests/version_managers.rs +++ b/crates/ide-sdk/tests/version_managers.rs @@ -35,13 +35,15 @@ fn mise_and_asdf_installs_are_detected_and_pinned_versions_used() { let tmp = dunce::canonicalize(tmp.path()).unwrap(); let mise = tmp.join("mise"); let asdf = tmp.join("asdf"); - // SAFETY: the only test in this binary; nothing else reads the environment yet. + // SAFETY: the only test in this binary that writes the environment; the daemon + // tests below read it, so they run after this one's fixtures are in place. unsafe { std::env::set_var("HOME", tmp.join("home")); std::env::set_var("MISE_DATA_DIR", &mise); std::env::set_var("ASDF_DATA_DIR", &asdf); std::env::set_var("MISE_GLOBAL_CONFIG_FILE", tmp.join("no-global.toml")); std::env::remove_var("JAVA_HOME"); + std::env::remove_var("MVND_HOME"); } let installs = mise.join("installs"); @@ -131,11 +133,99 @@ fn mise_and_asdf_installs_are_detected_and_pinned_versions_used() { ide_sdk::apply::apply_to_run_configs(&mut plain, &project, &s); assert_eq!(plain[0].command, installs.join("gradle/8.7/bin/gradle").to_string_lossy()); + // The Maven daemon replaces the pinned Maven when it is on and installed; the + // project JDK (21, from mise.toml) can host its JVM. + let mut daemon = s.clone(); + daemon.languages.java.maven.daemon = true; + daemon.languages.java.maven.mvnd_home = tmp.join("missing/mvnd").to_string_lossy().into_owned(); + assert_eq!( + ide_sdk::apply::maven_command(&project, &daemon), + Some(maven.join("bin/mvn")), + "a dead-end mvnd home falls back to the pinned Maven" + ); + let mvnd = tmp.join("mvnd"); + script(&mvnd.join("bin/mvnd"), "mvnd 1.0-m7"); + daemon.languages.java.maven.mvnd_home = mvnd.to_string_lossy().into_owned(); + assert_eq!( + ide_sdk::apply::maven_command(&project, &daemon), + Some(mvnd.join("bin/mvnd")), + "a bare home resolves to /bin/mvnd" + ); + // The executable itself, like `maven.home`, also works. + daemon.languages.java.maven.mvnd_home = mvnd.join("bin/mvnd").to_string_lossy().into_owned(); + assert_eq!(ide_sdk::apply::maven_command(&project, &daemon), Some(mvnd.join("bin/mvnd"))); + // The project's Maven wrapper keeps precedence over a pinned Maven. script(&project.join("mvnw"), ""); assert_eq!(ide_sdk::apply::maven_command(&project, &s), None); + // ...but not over the daemon. + assert_eq!(ide_sdk::apply::maven_command(&project, &daemon), Some(mvnd.join("bin/mvnd"))); // A version the project pins but that isn't installed: nothing is made up. write(&project.join("mise.toml"), "[tools]\njava = '25'\n"); assert_eq!(ide_sdk::apply::project_jdk(&project, &s), None); } + +#[test] +fn maven_daemon_run_configs_follow_the_setting_and_the_jdk() { + let tmp = tempfile::tempdir().unwrap(); + let tmp = dunce::canonicalize(tmp.path()).unwrap(); + let project = tmp.join("project"); + write(&project.join("pom.xml"), ""); + let mvnd = tmp.join("mvnd"); + script(&mvnd.join("bin/mvnd"), "mvnd 1.0-m7"); + let mvn = tmp.join("mvn"); + script(&mvn.join("bin/mvn"), "Apache Maven 3.9.6"); + let mvnd_path = mvnd.join("bin/mvnd").to_string_lossy().into_owned(); + let mvn_path = mvn.join("bin/mvn").to_string_lossy().into_owned(); + + let mut s = Settings::default(); + s.languages.java.maven.daemon = true; + s.languages.java.maven.mvnd_home = mvnd_path.clone(); + s.languages.java.maven.home = mvn_path.clone(); + + let run = |command: &str, detected: bool| RunConfig { + id: "maven:x".into(), + name: "x".into(), + kind: "maven".into(), + command: command.into(), + args: vec!["package".into()], + cwd: None, + env: BTreeMap::new(), + detected, + is_test: false, + debug_adapter: None, + allow_parallel: false, + extras: Default::default(), + }; + + // No project JDK: the daemon may pick its own, so it stays on mvnd. + let mut configs = [run("mvn", true), run("mvn", false)]; + ide_sdk::apply::apply_to_run_configs(&mut configs, &project, &s); + assert_eq!(configs[0].command, mvnd_path, "detected config runs the daemon"); + assert_eq!(configs[1].command, mvnd_path, "saved `mvn` config is pointed at the daemon"); + + // A launcher the user wrote into a configuration is theirs: an absolute path + // or a bare `mvnd` stays as configured. + let mut configs = [run(&mvn_path, false), run("mvnd", false)]; + ide_sdk::apply::apply_to_run_configs(&mut configs, &project, &s); + assert_eq!(configs[0].command, mvn_path, "a custom absolute launcher is kept"); + assert_eq!(configs[1].command, "mvnd", "a user-written mvnd is kept"); + + // A project JDK older than the daemon's Java 17 requirement keeps mvn. + let jdk8 = tmp.join("jdk8"); + jdk(&jdk8, "1.8.0_392"); + s.languages.java.jdk = jdk8.to_string_lossy().into_owned(); + let mut configs = [run("mvn", true), run("mvn", false)]; + ide_sdk::apply::apply_to_run_configs(&mut configs, &project, &s); + assert_eq!(configs[0].command, mvn_path, "JDK 8 cannot host the daemon JVM"); + assert_eq!(configs[1].command, mvn_path); + + // With the daemon off, the same configurations run the configured Maven. + s.languages.java.maven.daemon = false; + s.languages.java.jdk = String::new(); + let mut configs = [run("mvn", true), run("mvn", false)]; + ide_sdk::apply::apply_to_run_configs(&mut configs, &project, &s); + assert_eq!(configs[0].command, mvn_path); + assert_eq!(configs[1].command, mvn_path); +} diff --git a/crates/ide-search/src/engine.rs b/crates/ide-search/src/engine.rs index 802595b0..638c1016 100644 --- a/crates/ide-search/src/engine.rs +++ b/crates/ide-search/src/engine.rs @@ -142,42 +142,16 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult { } } - if let Some((first, rest)) = on_disk.split_first() - && !ctx.ctl.should_stop() - { - let mut walk = WalkBuilder::new(first); - for path in rest { - walk.add(path); - } - walk.hidden(true) - .ignore(true) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .parents(true) - .require_git(false) - .follow_links(false) - .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) - .filter_entry(|e| e.file_name() != ".git"); - walk.build_parallel().run(|| { + if !ctx.ctl.should_stop() { + walk_files(&on_disk, || { let mut searcher = searcher_builder.build(); let matcher = matcher.clone(); let ctx = &ctx; - Box::new(move |entry| { + Box::new(move |path| { if ctx.ctl.should_stop() { return WalkState::Quit; } - let entry = match entry { - Ok(entry) => entry, - Err(err) => { - tracing::debug!("search walk: {err}"); - return WalkState::Continue; - } - }; - // Archives (jar, zip) are browsed, not searched (IDEA skips them too). - if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) { - ctx.search_file(entry.path(), &matcher, &mut searcher); - } + ctx.search_file(path, &matcher, &mut searcher); if ctx.ctl.stopped() { WalkState::Quit } else { WalkState::Continue } }) }); @@ -193,6 +167,67 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult { }) } +/// A per-thread file visitor of [`walk_files`]. +pub(crate) type VisitFile<'s> = Box WalkState + Send + 's>; + +/// Visits the files below `roots` (and the roots that are files) in parallel, +/// the way IDEA's project scope sees them: directories ignored by +/// `.gitignore` / `.ignore` / `info/exclude` are skipped with everything in +/// them, while ignored files elsewhere (`.env`) and hidden files and folders +/// (`.github`, `.eslintrc`) are visited. `.git`, symlinks and archives (jar, +/// zip: browsed, not searched) are never visited. +pub(crate) fn walk_files<'s>(roots: &[PathBuf], mut visitor: impl FnMut() -> VisitFile<'s> + Send) { + let Some((first, rest)) = roots.split_first() else { return }; + let mut walk = WalkBuilder::new(first); + for path in rest { + walk.add(path); + } + walk.hidden(false) + .ignore(true) + .git_ignore(true) + .git_global(true) + .git_exclude(true) + .parents(true) + .require_git(ide_core::ignored::inside_git_repo(first)) + .follow_links(false) + .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) + .filter_entry(|e| e.file_name() != ".git"); + walk.build_parallel().run(|| { + let mut visit = visitor(); + Box::new(move |entry| { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + tracing::debug!("search walk: {err}"); + return WalkState::Continue; + } + }; + let Some(file_type) = entry.file_type() else { return WalkState::Continue }; + if file_type.is_dir() { + // The walker drops ignored files unseen, so the files of every walked + // directory come from listing it; the walker only picks the directories. + let Ok(children) = std::fs::read_dir(entry.path()) else { return WalkState::Continue }; + for child in children.flatten() { + let path = child.path(); + if child.file_name() != ".git" + && child.file_type().is_ok_and(|t| t.is_file()) + && !ide_vfs::is_archive_file(&path) + && matches!(visit(&path), WalkState::Quit) + { + return WalkState::Quit; + } + } + WalkState::Continue + } else if file_type.is_file() && entry.depth() == 0 && !ide_vfs::is_archive_file(entry.path()) { + // A root that is a file (an explicit file list); files below roots come from listings. + visit(entry.path()) + } else { + WalkState::Continue + } + }) + }); +} + /// Walk roots for a scope; relative paths are resolved against `root`. pub(crate) fn roots(root: &Path, scope: &SearchScope) -> IdeResult> { let dir = |path: &Path| -> IdeResult { diff --git a/crates/ide-search/src/files.rs b/crates/ide-search/src/files.rs index ba85acfc..97b3805d 100644 --- a/crates/ide-search/src/files.rs +++ b/crates/ide-search/src/files.rs @@ -1,10 +1,10 @@ //! The files of a search scope, for analyses that read whole files //! (Inspect Code, Locate Duplicates, Structural Search). -use crate::engine::{rel_path, roots}; +use crate::engine::{rel_path, roots, walk_files}; use crate::{FileMask, PathFilter, SearchScope}; use ide_core::{CancellationToken, IdeResult}; -use ignore::{WalkBuilder, WalkState}; +use ignore::WalkState; use std::path::PathBuf; use std::sync::Mutex; @@ -16,7 +16,7 @@ pub struct ScopeFile { } /// Lists the files of `scope` under `root` the way Find in Files walks them -/// (`.gitignore` honoured, hidden files and `.git` skipped), filtered by +/// (ignored folders and `.git` skipped; ignored and hidden files kept), filtered by /// `mask` and `filter`, sorted by relative path. Explicitly listed files are /// kept even when they do not exist on disk (unsaved buffers). Returns what was /// found so far when cancelled. @@ -43,42 +43,23 @@ pub fn scope_files( on_disk.push(path); } } - if let Some((first, rest)) = on_disk.split_first() { - let mut walk = WalkBuilder::new(first); - for path in rest { - walk.add(path); - } - walk.hidden(true) - .ignore(true) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .parents(true) - .require_git(false) - .follow_links(false) - .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) - .filter_entry(|e| e.file_name() != ".git"); - walk.build_parallel().run(|| { - let found = &found; - let keep = &keep; - Box::new(move |entry| { - if cancel.is_cancelled() { - return WalkState::Quit; - } - let Ok(entry) = entry else { return WalkState::Continue }; - if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) { - let rel = rel_path(root, entry.path()); - if keep(&rel) { - found - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(ScopeFile { path: entry.into_path(), rel_path: rel }); - } - } - WalkState::Continue - }) - }); - } + walk_files(&on_disk, || { + let found = &found; + let keep = &keep; + Box::new(move |path| { + if cancel.is_cancelled() { + return WalkState::Quit; + } + let rel = rel_path(root, path); + if keep(&rel) { + found + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(ScopeFile { path: path.to_path_buf(), rel_path: rel }); + } + WalkState::Continue + }) + }); let mut files = found.into_inner().unwrap_or_else(|e| e.into_inner()); files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path)); files.dedup_by(|a, b| a.path == b.path); @@ -107,12 +88,12 @@ mod tests { } #[test] - fn project_scope_honours_gitignore_and_masks() { + fn project_scope_skips_ignored_folders_and_honours_masks() { let dir = tree(); let none = FileMask::new(&[], &[]).unwrap(); let cancel = CancellationToken::new(); let all = scope_files(dir.path(), &SearchScope::Project, &none, None, &cancel).unwrap(); - assert_eq!(rels(&all), vec!["src/a.rs", "src/c.txt", "src/sub/b.rs"]); + assert_eq!(rels(&all), vec![".gitignore", "src/a.rs", "src/c.txt", "src/sub/b.rs"]); let rs = FileMask::new(&["*.rs".into()], &[]).unwrap(); let only_rs = scope_files(dir.path(), &SearchScope::Project, &rs, None, &cancel).unwrap(); @@ -120,7 +101,7 @@ mod tests { let not_sub: PathFilter<'_> = &|rel: &str| !rel.starts_with("src/sub/"); let filtered = scope_files(dir.path(), &SearchScope::Project, &none, Some(not_sub), &cancel).unwrap(); - assert_eq!(rels(&filtered), vec!["src/a.rs", "src/c.txt"]); + assert_eq!(rels(&filtered), vec![".gitignore", "src/a.rs", "src/c.txt"]); } #[test] diff --git a/crates/ide-search/src/lib.rs b/crates/ide-search/src/lib.rs index 97515510..eb220fbb 100644 --- a/crates/ide-search/src/lib.rs +++ b/crates/ide-search/src/lib.rs @@ -1,7 +1,8 @@ //! Find / Replace in Files (⌘⇧F / ⌘⇧R). //! -//! Built on ripgrep's engine crates: a parallel `ignore` walk (honouring -//! `.gitignore` / `.ignore`, skipping hidden files and `.git`) feeding +//! Built on ripgrep's engine crates: a parallel `ignore` walk (skipping the +//! folders `.gitignore` / `.ignore` exclude and `.git`, but like IDEA keeping +//! ignored files elsewhere and hidden files) feeding //! `grep-searcher` with a `grep-regex` matcher. Binary files are skipped, //! unsaved editor buffers (the overlay) take precedence over disk content, //! and all columns are UTF-16. diff --git a/crates/ide-search/src/words.rs b/crates/ide-search/src/words.rs index ceb94a25..722ebe63 100644 --- a/crates/ide-search/src/words.rs +++ b/crates/ide-search/src/words.rs @@ -25,8 +25,8 @@ pub struct WordFiles { } /// Files under `query.dir` containing `query.word` as a whole word, in walk -/// order (unsaved buffers through `overlay`, `.gitignore`d and hidden files -/// skipped like Find in Files). Stops at `query.limit` files or when cancelled. +/// order (unsaved buffers through `overlay`; ignored folders skipped, ignored +/// and hidden files searched, like Find in Files). Stops at `query.limit` files or when cancelled. pub fn files_with_word( root: &std::path::Path, query: &WordQuery, @@ -74,7 +74,9 @@ mod tests { let root = dir.path(); fs::create_dir_all(root.join("pkg/ignored")).unwrap(); fs::create_dir_all(root.join("build")).unwrap(); - fs::write(root.join(".gitignore"), "ignored/\n").unwrap(); + fs::write(root.join(".gitignore"), "ignored/\nlocal.py\n").unwrap(); + // An ignored file outside ignored folders is still a candidate, as in IDEA. + fs::write(root.join("pkg/local.py"), "get_value\n").unwrap(); fs::write(root.join("pkg/use.py"), "x.get_value(1)\n").unwrap(); fs::write(root.join("pkg/longer.py"), "get_values()\n").unwrap(); fs::write(root.join("pkg/case.py"), "Get_Value()\n").unwrap(); @@ -90,7 +92,10 @@ mod tests { found.files.sort(); assert_eq!( found, - WordFiles { files: vec![root.join("pkg/open.py"), root.join("pkg/use.py")], limit_hit: false } + WordFiles { + files: vec![root.join("pkg/local.py"), root.join("pkg/open.py"), root.join("pkg/use.py")], + limit_hit: false + } ); } diff --git a/crates/ide-search/tests/search.rs b/crates/ide-search/tests/search.rs index 412fe9b1..f4207d05 100644 --- a/crates/ide-search/tests/search.rs +++ b/crates/ide-search/tests/search.rs @@ -187,24 +187,77 @@ fn file_masks() { assert_eq!(with(&[], &["*.kt,*.ts"]), ["notes.md", "src/Util.java"]); } +/// The files of `scope` under `root` (relative), sorted. +fn scope_rels(root: &Path, scope: &SearchScope) -> Vec { + let none = ide_search::FileMask::new(&[], &[]).unwrap(); + let files = ide_search::scope_files(root, scope, &none, None, &CancellationToken::new()).unwrap(); + let mut rels: Vec<_> = files.into_iter().map(|f| f.rel_path).collect(); + rels.sort(); + rels +} + #[test] -fn honours_ignore_files_hidden_and_git() { - let p = project(&[ - (".gitignore", b"target/\n*.log\n"), - (".ignore", b"generated.rs\n"), - ("src/a.rs", b"needle\n"), - ("src/generated.rs", b"needle\n"), - ("target/debug/out.rs", b"needle\n"), - ("build.log", b"needle\n"), - (".hidden/secret.rs", b"needle\n"), - (".env", b"needle\n"), - (".git/config", b"needle\n"), - ("sub/.gitignore", b"local.txt\n"), - ("sub/local.txt", b"needle\n"), - ("sub/kept.txt", b"needle\n"), - ]); +fn searches_ignored_and_hidden_files_but_not_ignored_folders() { + // #1317: like IDEA, only ignored folders are left out; ignored files and dotfiles are searched. + // (path, content, walked): every file mentions `needle`. + let files: &[(&str, &[u8], bool)] = &[ + (".gitignore", b"# needle\ntarget/\n*.log\n.env\nsrc/local/\n", true), + (".ignore", b"# needle\ngenerated.rs\nout/\n", true), + ("src/a.rs", b"needle\n", true), + // Ignored files outside ignored folders. + ("src/generated.rs", b"needle\n", true), + ("build.log", b"needle\n", true), + (".env", b"needle\n", true), + ("sub/.gitignore", b"# needle\nlocal.txt\ncache/\n", true), + ("sub/local.txt", b"needle\n", true), + ("sub/kept.txt", b"needle\n", true), + // Hidden files and folders. + (".eslintrc", b"needle\n", true), + (".github/workflows/ci.yml", b"needle\n", true), + (".hidden/secret.rs", b"needle\n", true), + // Ignored folders, whatever they hold. + ("target/debug/out.rs", b"needle\n", false), + ("target/.env", b"needle\n", false), + ("src/local/notes.txt", b"needle\n", false), + ("out/gen.rs", b"needle\n", false), + ("sub/cache/blob.txt", b"needle\n", false), + // Never: Git's own folder. + (".git/config", b"needle\n", false), + ]; + let p = project(&files.iter().map(|(path, text, _)| (*path, *text)).collect::>()); + let mut expected: Vec<&str> = files.iter().filter(|(_, _, walked)| *walked).map(|(path, _, _)| *path).collect(); + expected.sort(); + let (views, _) = run(p.path(), &query("needle")); - assert_eq!(paths(&views), ["src/a.rs", "sub/kept.txt"]); + assert_eq!(paths(&views), expected, "Find in Files"); + + assert_eq!(scope_rels(p.path(), &SearchScope::Project), expected, "scope files"); + + // A directory scope and an explicit ignored file are searched the same way. + let dir = SearchScope::Directory { path: ide_core::IpcPath(p.path().join("sub")) }; + let (views, _) = run(p.path(), &SearchQuery { scope: dir, ..query("needle") }); + assert_eq!(paths(&views), ["sub/.gitignore", "sub/kept.txt", "sub/local.txt"]); + let one = SearchScope::Files { paths: vec![ide_core::IpcPath(p.path().join("build.log"))] }; + let (views, _) = run(p.path(), &SearchQuery { scope: one, ..query("needle") }); + assert_eq!(paths(&views), ["build.log"]); +} + +#[test] +fn nested_repository_ignores_outer_repository_rules() { + // #1254: the opened project is a Git repository inside a folder its parent repository ignores. + let p = project(&[ + (".git/HEAD", b"ref: refs/heads/main\n"), + ("projects/.gitignore", b"*\n!.gitignore\n"), + ("projects/nested/.git/HEAD", b"ref: refs/heads/main\n"), + ("projects/nested/.gitignore", b"logs/\n"), + ("projects/nested/search-test.txt", b"somesss\n"), + ("projects/nested/src/main.txt", b"somesss\n"), + ("projects/nested/logs/debug.txt", b"somesss\n"), + ]); + let nested = p.path().join("projects/nested"); + let (views, _) = run(&nested, &query("somesss")); + assert_eq!(paths(&views), ["search-test.txt", "src/main.txt"]); + assert_eq!(scope_rels(&nested, &SearchScope::Project), [".gitignore", "search-test.txt", "src/main.txt"]); } #[test] @@ -643,3 +696,31 @@ fn archives_are_skipped() { let (views, _) = run(p.path(), &query("needle")); assert_eq!(paths(&views), ["src/a.txt"]); } + +#[test] +fn a_nested_repository_is_not_hidden_by_the_outer_repositorys_gitignore() { + // parent/.git, parent/projects/.gitignore ignores everything, parent/projects/nested is its own repository. + let p = project(&[ + ("projects/.gitignore", b"*\n!.gitignore\n"), + ("projects/nested/search-test.txt", b"somesss\n"), + ("projects/nested/src/main.txt", b"somesss\n"), + ("projects/nested/.gitignore", b"logs/\n"), + ("projects/nested/logs/debug.txt", b"somesss\n"), + ]); + std::fs::create_dir(p.path().join(".git")).unwrap(); + std::fs::create_dir(p.path().join("projects/nested/.git")).unwrap(); + let (views, _) = run(&p.path().join("projects/nested"), &query("somesss")); + assert_eq!( + paths(&views), + ["search-test.txt", "src/main.txt"], + "the nested repository's own .gitignore still applies" + ); +} + +#[test] +fn a_project_inside_a_repository_keeps_the_repositorys_gitignore() { + let p = project(&[(".gitignore", b"build/\n"), ("app/build/out.txt", b"hit\n"), ("app/src/a.txt", b"hit\n")]); + std::fs::create_dir(p.path().join(".git")).unwrap(); + let (views, _) = run(&p.path().join("app"), &query("hit")); + assert_eq!(paths(&views), ["src/a.txt"]); +} diff --git a/crates/ide-ssh/Cargo.toml b/crates/ide-ssh/Cargo.toml index feb93e12..3d99cdf0 100644 --- a/crates/ide-ssh/Cargo.toml +++ b/crates/ide-ssh/Cargo.toml @@ -14,15 +14,20 @@ tokio.workspace = true tracing.workspace = true parking_lot.workspace = true dirs.workspace = true +uuid.workspace = true +keyring.workspace = true +keyring-core.workspace = true russh = { version = "0.64", default-features = false, features = ["ring", "rsa", "flate2"] } russh-config = "0.58.0" russh-sftp = "2.4" flate2 = "1" [features] -# `testing`: an in-process SSH / SFTP server for tests of crates built on this one. -testing = [] +# `testing`: an in-process SSH / SFTP server for tests of crates built on this one, +# and an in-memory credential store in place of the OS one. +testing = ["keyring-core/sample"] [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tempfile.workspace = true +keyring-core = { workspace = true, features = ["sample"] } diff --git a/crates/ide-ssh/src/git_creds.rs b/crates/ide-ssh/src/git_creds.rs index bdc818e1..4e839aa4 100644 --- a/crates/ide-ssh/src/git_creds.rs +++ b/crates/ide-ssh/src/git_creds.rs @@ -216,6 +216,7 @@ mod tests { use super::*; fn store() -> (tempfile::TempDir, GitCredentials) { + crate::secrets::testing::use_memory_store(); let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("git-credentials.json"); (tmp, GitCredentials { saved: Arc::new(SecretFile::new(path)), sessions: Arc::default() }) diff --git a/crates/ide-ssh/src/secrets.rs b/crates/ide-ssh/src/secrets.rs index 4cd6a9cb..7359f157 100644 --- a/crates/ide-ssh/src/secrets.rs +++ b/crates/ide-ssh/src/secrets.rs @@ -1,61 +1,289 @@ -//! Saved passwords (SSH passwords / key passphrases by configuration id, the HTTP -//! proxy's), in the app's data directory rather than the OS credential store (no -//! system prompts). One JSON object, key → secret; owner-only on Unix. +//! Saved passwords (SSH passwords / key passphrases by configuration id, git HTTPS +//! credentials, the HTTP proxy's) in the OS credential store: Windows Credential +//! Manager, the macOS Keychain, the Secret Service on Linux. The file at `path` only +//! indexes them (an id and the keys, never a secret); owner-only on Unix. +//! +//! Older versions kept the secrets themselves in that file: they are moved to the +//! credential store on first use. Without a credential store (a headless Linux +//! host) such a file is still read and forgetting a secret rewrites it without +//! that one, but no new secret is written to disk: saving fails and callers keep +//! the secret for the session only. use ide_core::{IdeError, IdeResult}; +pub use keyring_core::{Error as StoreError, Result as StoreResult}; use parking_lot::Mutex; -use std::collections::BTreeMap; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; +use std::sync::Arc; + +const SERVICE: &str = "ai.l0ng.openidea"; + +/// Serialises read-modify-write of the index files (several stores may share one). +static LOCK: Mutex<()> = Mutex::new(()); + +/// Reads the secrets of an older version's file, by key (`None`: not that format). +pub type Legacy = fn(serde_json::Value) -> Option>; + +/// Where the secrets are kept: the default credential store, or a stand-in in tests. +pub trait Vault: Send + Sync { + fn available(&self) -> bool; + fn get(&self, service: &str, key: &str) -> StoreResult; + fn set(&self, service: &str, key: &str, secret: &str) -> StoreResult<()>; + fn delete(&self, service: &str, key: &str) -> StoreResult<()>; +} + +/// `keyring_core`'s default store: the OS's, unless a test installed another one. +struct DefaultStore; + +impl DefaultStore { + fn entry(&self, service: &str, key: &str) -> StoreResult { + if !self.available() { + return Err(StoreError::NoDefaultStore); + } + keyring_core::Entry::new(service, key) + } +} + +impl Vault for DefaultStore { + fn available(&self) -> bool { + if keyring_core::get_default_store().is_some() { + return true; + } + static WARNED: std::sync::Once = std::sync::Once::new(); + match keyring::Entry::store_status() { + Ok(()) => true, + Err(e) => { + WARNED.call_once(|| tracing::warn!("saved passwords: no credential store: {e}")); + false + } + } + } + + fn get(&self, service: &str, key: &str) -> StoreResult { + self.entry(service, key)?.get_password() + } + + fn set(&self, service: &str, key: &str, secret: &str) -> StoreResult<()> { + self.entry(service, key)?.set_password(secret) + } + + fn delete(&self, service: &str, key: &str) -> StoreResult<()> { + self.entry(service, key)?.delete_credential() + } +} pub struct SecretFile { path: PathBuf, - lock: Mutex<()>, + legacy: Legacy, + vault: Arc, +} + +/// The file's contents. `id` keeps the credentials of different data directories +/// (and of tests) apart in the one OS store. +#[derive(Serialize, Deserialize)] +struct Index { + id: String, + keys: BTreeSet, +} + +enum Contents { + Index(Index), + /// An older version's file that could not be moved, and why. + Plaintext(BTreeMap, String), +} + +fn service(id: &str) -> String { + format!("{SERVICE}/{id}") +} + +fn new_id() -> String { + uuid::Uuid::new_v4().to_string() } impl SecretFile { pub fn new(path: PathBuf) -> Self { - Self { path, lock: Mutex::new(()) } + Self::with_legacy(path, |v| serde_json::from_value(v).ok()) + } + + pub fn with_legacy(path: PathBuf, legacy: Legacy) -> Self { + Self::with_vault(path, legacy, Arc::new(DefaultStore)) + } + + pub fn with_vault(path: PathBuf, legacy: Legacy, vault: Arc) -> Self { + Self { path, legacy, vault } } pub fn get(&self, key: &str) -> Option { - let _g = self.lock.lock(); - self.read().remove(key) + let _g = LOCK.lock(); + match self.load() { + Contents::Index(mut index) if index.keys.contains(key) => match self.vault.get(&service(&index.id), key) { + Ok(secret) => Some(secret), + Err(StoreError::NoEntry) => { + // Removed outside the app: stop listing it. + index.keys.remove(key); + self.write_logged(&index); + None + } + Err(e) => { + tracing::warn!("saved passwords: {key}: {e}"); + None + } + }, + Contents::Index(_) => None, + Contents::Plaintext(mut old, _) => old.remove(key), + } } pub fn set(&self, key: &str, secret: &str) -> IdeResult<()> { - let _g = self.lock.lock(); - let mut all = self.read(); - all.insert(key.to_owned(), secret.to_owned()); - self.write(&all) + let _g = LOCK.lock(); + let mut index = match self.load() { + Contents::Index(index) => index, + Contents::Plaintext(_, why) => return Err(unavailable(&why)), + }; + let service = service(&index.id); + self.vault.set(&service, key, secret).map_err(|e| unavailable(&e.to_string()))?; + if index.keys.insert(key.to_owned()) + && let Err(e) = self.write(&index) + { + let _ = self.vault.delete(&service, key); + return Err(e); + } + Ok(()) } pub fn delete(&self, key: &str) { - let _g = self.lock.lock(); - let mut all = self.read(); - if all.remove(key).is_some() - && let Err(e) = self.write(&all) - { - tracing::warn!("{}", e.message); + let _g = LOCK.lock(); + match self.load() { + Contents::Index(mut index) => { + if !index.keys.contains(key) { + return; + } + match self.vault.delete(&service(&index.id), key) { + Ok(()) | Err(StoreError::NoEntry) => {} + Err(e) => { + // Kept in the index so that the next delete tries again. + tracing::warn!("saved passwords: could not forget {key}: {e}"); + return; + } + } + index.keys.remove(key); + self.write_logged(&index); + } + Contents::Plaintext(mut old, _) => { + if old.remove(key).is_none() { + return; + } + // A move halfway may have copied it already. + if let Some(id) = self.pending_id() + && let Err(e) = self.vault.delete(&service(&id), key) + && !matches!(e, StoreError::NoEntry) + { + tracing::warn!("saved passwords: could not forget {key}: {e}"); + return; + } + self.write_logged(&old); + } } } /// Every stored key, sorted (the secrets themselves stay private). pub fn keys(&self) -> Vec { - let _g = self.lock.lock(); - self.read().into_keys().collect() - } - - fn read(&self) -> BTreeMap { - match std::fs::read(&self.path) { - Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| { - tracing::warn!("saved passwords: {}: {e}", self.path.display()); - BTreeMap::new() - }), - Err(_) => BTreeMap::new(), + let _g = LOCK.lock(); + match self.load() { + Contents::Index(index) => index.keys.into_iter().collect(), + Contents::Plaintext(old, _) => old.into_keys().collect(), } } - fn write(&self, all: &BTreeMap) -> IdeResult<()> { + fn load(&self) -> Contents { + let value = match std::fs::read(&self.path) { + Err(_) => return Contents::Index(Index { id: new_id(), keys: BTreeSet::new() }), + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(v) => v, + Err(e) => return self.set_aside(&e.to_string()), + }, + }; + if let Ok(index) = serde_json::from_value::(value.clone()) { + return Contents::Index(index); + } + match (self.legacy)(value) { + Some(old) => self.migrate(old), + None => self.set_aside("unknown format"), + } + } + + /// An unreadable file is renamed rather than overwritten (it may index + /// credentials, or hold secrets); the store starts over. + fn set_aside(&self, why: &str) -> Contents { + let aside = self.path.with_extension(format!("json.unreadable-{}", new_id())); + tracing::warn!("saved passwords: {}: {why}; moved to {}", self.path.display(), aside.display()); + if let Err(e) = std::fs::rename(&self.path, &aside) { + // Not overwritten either: nothing is saved until it can be moved. + return Contents::Plaintext(BTreeMap::new(), format!("{}: {e}", self.path.display())); + } + Contents::Index(Index { id: new_id(), keys: BTreeSet::new() }) + } + + fn pending_path(&self) -> PathBuf { + self.path.with_extension("json.migrating") + } + + /// The id of a move that has not finished (see [`SecretFile::migrate`]). + fn pending_id(&self) -> Option { + let id = std::fs::read_to_string(self.pending_path()).ok()?; + Some(id.trim().to_owned()).filter(|id| !id.is_empty()) + } + + /// Moves an older version's secrets to the credential store, key by key; a secret + /// the store refuses for good (too long, say) is forgotten, and the user is asked + /// for it again. The old file is replaced only when everything else was moved. + /// Until then the id waits in `.migrating`, so a retry after a failure or a + /// crash overwrites the same credentials instead of copying them again. + fn migrate(&self, old: BTreeMap) -> Contents { + if !self.vault.available() { + return Contents::Plaintext(old, "no credential store".into()); + } + let pending = self.pending_path(); + let id = match self.pending_id() { + Some(id) => id, + None => { + let id = new_id(); + if let Err(e) = std::fs::write(&pending, &id) { + return Contents::Plaintext(old, format!("{}: {e}", pending.display())); + } + id + } + }; + let mut index = Index { id, keys: BTreeSet::new() }; + for (key, secret) in &old { + match self.vault.set(&service(&index.id), key, secret) { + Ok(()) => { + index.keys.insert(key.clone()); + } + Err(e @ (StoreError::TooLong(..) | StoreError::Invalid(..))) => { + tracing::warn!("saved passwords: {key} does not fit the credential store and is forgotten: {e}"); + } + Err(e) => { + tracing::warn!("saved passwords: could not move {key} to the credential store: {e}"); + return Contents::Plaintext(old, e.to_string()); + } + } + } + if let Err(e) = self.write(&index) { + return Contents::Plaintext(old, e.message); + } + let _ = std::fs::remove_file(&pending); + Contents::Index(index) + } + + fn write_logged(&self, contents: &impl Serialize) { + if let Err(e) = self.write(contents) { + tracing::warn!("{}", e.message); + } + } + + fn write(&self, contents: &impl Serialize) -> IdeResult<()> { let err = |e: std::io::Error| IdeError::unavailable(format!("saved passwords: {}: {e}", self.path.display())); if let Some(dir) = self.path.parent() { std::fs::create_dir_all(dir).map_err(err)?; @@ -65,18 +293,258 @@ impl SecretFile { options.write(true).create(true).truncate(true); #[cfg(unix)] std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); - let bytes = serde_json::to_vec_pretty(all).map_err(|e| IdeError::internal(e.to_string()))?; + let bytes = serde_json::to_vec_pretty(contents).map_err(|e| IdeError::internal(e.to_string()))?; std::io::Write::write_all(&mut options.open(&tmp).map_err(err)?, &bytes).map_err(err)?; std::fs::rename(&tmp, &self.path).map_err(err) } } +fn unavailable(why: &str) -> IdeError { + IdeError::unavailable(format!("saved passwords: {why}")) +} + +/// Helpers for tests of crates that save secrets (feature `testing`). +#[cfg(any(test, feature = "testing"))] +pub mod testing { + use std::path::{Path, PathBuf}; + + /// Makes an in-memory credential store the default, so that the test never + /// touches the user's keychain. Call it before the first secret is saved. + pub fn use_memory_store() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| keyring_core::set_default_store(keyring_core::sample::Store::new().expect("memory store"))); + } + + /// Every file under `dir` whose bytes contain `needle`. + pub fn files_containing(dir: &Path, needle: &str) -> Vec { + let mut found = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(d) = pending.pop() { + for entry in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let path = entry.path(); + if path.is_dir() { + pending.push(path); + } else if std::fs::read(&path).is_ok_and(|b| b.windows(needle.len()).any(|w| w == needle.as_bytes())) { + found.push(path); + } + } + } + found + } +} + #[cfg(test)] mod tests { + use super::testing::{files_containing, use_memory_store}; use super::*; + use std::collections::HashMap; + + /// A credential store that can be absent, or fail on chosen keys. + #[derive(Default)] + struct FakeVault { + absent: bool, + entries: Mutex>, + failing: Mutex StoreError>>, + } + + impl FakeVault { + fn fail(&self, key: &str, error: fn() -> StoreError) { + self.failing.lock().insert(key.to_owned(), error); + } + + fn heal(&self) { + self.failing.lock().clear(); + } + + fn check(&self, key: &str) -> StoreResult<()> { + if self.absent { + return Err(StoreError::NoDefaultStore); + } + self.failing.lock().get(key).map_or(Ok(()), |e| Err(e())) + } + + fn secrets(&self) -> Vec { + let mut all: Vec<_> = self.entries.lock().values().cloned().collect(); + all.sort(); + all + } + } + + impl Vault for FakeVault { + fn available(&self) -> bool { + !self.absent + } + fn get(&self, service: &str, key: &str) -> StoreResult { + self.check(key)?; + self.entries.lock().get(&(service.to_owned(), key.to_owned())).cloned().ok_or(StoreError::NoEntry) + } + fn set(&self, service: &str, key: &str, secret: &str) -> StoreResult<()> { + self.check(key)?; + self.entries.lock().insert((service.to_owned(), key.to_owned()), secret.to_owned()); + Ok(()) + } + fn delete(&self, service: &str, key: &str) -> StoreResult<()> { + self.check(key)?; + self.entries.lock().remove(&(service.to_owned(), key.to_owned())).map(|_| ()).ok_or(StoreError::NoEntry) + } + } + + fn platform() -> StoreError { + StoreError::PlatformFailure("locked".into()) + } + + fn with_fake(path: &std::path::Path, vault: &Arc) -> SecretFile { + SecretFile::with_vault(path.to_path_buf(), |v| serde_json::from_value(v).ok(), vault.clone()) + } + + const OLD: &str = r#"{"https://a.example": "me\nold-a", "https://b.example": "me\nold-b"}"#; + + #[test] + fn saved_secrets_never_reach_disk() { + use_memory_store(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("ssh-passwords.json"); + SecretFile::new(path.clone()).set("a", "hunter2-ssh").unwrap(); + assert_eq!(files_containing(tmp.path(), "hunter2-ssh"), Vec::::new()); + let reopened = SecretFile::new(path); + assert_eq!(reopened.get("a").as_deref(), Some("hunter2-ssh")); + assert_eq!(reopened.keys(), ["a"]); + } + + #[test] + fn moves_plaintext_files_of_older_versions_off_disk() { + use_memory_store(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, r#"{"https://example.com": "me\nold-pw"}"#).unwrap(); + let file = SecretFile::new(path.clone()); + assert_eq!(file.keys(), ["https://example.com"]); + assert_eq!(files_containing(tmp.path(), "old-pw"), Vec::::new()); + assert_eq!(SecretFile::new(path).get("https://example.com").as_deref(), Some("me\nold-pw")); + } + + #[test] + fn a_failed_move_keeps_the_old_file_and_the_retry_converges() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, OLD).unwrap(); + let vault = Arc::new(FakeVault::default()); + vault.fail("https://b.example", platform); + + let file = with_fake(&path, &vault); + assert_eq!(file.get("https://a.example").as_deref(), Some("me\nold-a")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), OLD, "the old file is untouched"); + assert!(file.set("https://c.example", "me\nnew").unwrap_err().message.contains("locked")); + + vault.heal(); + assert_eq!(file.keys(), ["https://a.example", "https://b.example"]); + assert_eq!(files_containing(tmp.path(), "old-a"), Vec::::new()); + assert_eq!(vault.secrets(), ["me\nold-a", "me\nold-b"], "the retry overwrote, not copied"); + assert!(!path.with_extension("json.migrating").exists()); + assert_eq!(with_fake(&path, &vault).get("https://b.example").as_deref(), Some("me\nold-b")); + } + + #[test] + fn forgetting_during_a_pending_move_forgets_the_copy_too() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, OLD).unwrap(); + let vault = Arc::new(FakeVault::default()); + vault.fail("https://b.example", platform); + + let file = with_fake(&path, &vault); + file.delete("https://a.example"); + vault.heal(); + assert_eq!(file.keys(), ["https://b.example"]); + assert_eq!(vault.secrets(), ["me\nold-b"]); + } + + #[test] + fn a_secret_the_store_refuses_for_good_is_forgotten_and_does_not_block_saving() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, OLD).unwrap(); + let vault = Arc::new(FakeVault::default()); + vault.fail("https://b.example", || StoreError::TooLong("password".into(), 2560)); + + let file = with_fake(&path, &vault); + assert_eq!(file.keys(), ["https://a.example"]); + assert_eq!(files_containing(tmp.path(), "old-b"), Vec::::new()); + file.set("https://c.example", "me\nnew").unwrap(); + assert_eq!(file.get("https://c.example").as_deref(), Some("me\nnew")); + } + + #[test] + fn without_a_credential_store_the_old_file_is_read_and_nothing_new_is_saved() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, OLD).unwrap(); + let vault = Arc::new(FakeVault { absent: true, ..Default::default() }); + + let file = with_fake(&path, &vault); + assert_eq!(file.get("https://a.example").as_deref(), Some("me\nold-a")); + assert!(file.set("https://c.example", "me\nnew").unwrap_err().message.contains("no credential store")); + assert_eq!(files_containing(tmp.path(), "me\nnew"), Vec::::new()); + assert!(!path.with_extension("json.migrating").exists()); + + file.delete("https://a.example"); + assert_eq!(file.keys(), ["https://b.example"]); + assert_eq!(files_containing(tmp.path(), "old-a"), Vec::::new()); + } + + #[test] + fn forgets_only_what_the_store_forgot() { + let tmp = tempfile::tempdir().unwrap(); + let vault = Arc::new(FakeVault::default()); + let file = with_fake(&tmp.path().join("ssh-passwords.json"), &vault); + file.set("a", "pw").unwrap(); + vault.fail("a", platform); + file.delete("a"); + assert_eq!(file.keys(), ["a"], "still listed, so that forgetting can be retried"); + vault.heal(); + file.delete("a"); + assert_eq!(file.keys(), Vec::::new()); + assert_eq!(vault.secrets(), Vec::::new()); + } + + #[test] + fn a_secret_removed_outside_the_app_is_no_longer_listed() { + let tmp = tempfile::tempdir().unwrap(); + let vault = Arc::new(FakeVault::default()); + let file = with_fake(&tmp.path().join("ssh-passwords.json"), &vault); + file.set("a", "pw").unwrap(); + vault.entries.lock().clear(); + assert_eq!(file.get("a"), None); + assert_eq!(file.keys(), Vec::::new()); + } + + #[test] + fn a_failed_index_write_takes_the_secret_back_out_of_the_store() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("ssh-passwords.json"); + std::fs::create_dir(&path).unwrap(); + let vault = Arc::new(FakeVault::default()); + assert!(with_fake(&path, &vault).set("a", "pw").is_err()); + assert_eq!(vault.secrets(), Vec::::new()); + } + + #[test] + fn an_unreadable_file_is_set_aside_not_overwritten() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("ssh-passwords.json"); + std::fs::write(&path, "{ not json").unwrap(); + let vault = Arc::new(FakeVault::default()); + let file = with_fake(&path, &vault); + assert_eq!(file.keys(), Vec::::new()); + file.set("a", "pw").unwrap(); + std::fs::write(&path, "{ not json either").unwrap(); + assert_eq!(file.keys(), Vec::::new()); + assert_eq!(files_containing(tmp.path(), "{ not json").len(), 2, "both kept, neither overwritten"); + } #[test] fn stores_and_forgets_secrets() { + use_memory_store(); let tmp = tempfile::tempdir().unwrap(); let file = SecretFile::new(tmp.path().join("ssh-passwords.json")); file.set("a", "pw").unwrap(); diff --git a/crates/ide-ssh/src/session.rs b/crates/ide-ssh/src/session.rs index 47f9cd21..c1e959d4 100644 --- a/crates/ide-ssh/src/session.rs +++ b/crates/ide-ssh/src/session.rs @@ -267,19 +267,29 @@ impl Session { }), ) .await?; + let failed = |code: Option| { + let err = String::from_utf8_lossy(&stderr.lock()).trim().to_owned(); + io(format!("cannot write {remote} on {} ({code:?}): {err}", self.label)) + }; + // A command that exits early (no gzip, a folder it cannot write) stops reading: a + // write would wait for channel window forever, so each one races the exit. + let mut exit = std::pin::pin!(process.wait()); let mut sent = 0; for chunk in packed.chunks(256 * 1024) { - process.write(chunk).await?; + tokio::select! { + written = process.write(chunk) => written?, + code = &mut exit => return Err(failed(code)), + } sent += chunk.len(); progress(sent as f64 / packed.len().max(1) as f64); } - process.eof().await?; - match process.wait().await { + tokio::select! { + closed = process.eof() => closed?, + code = &mut exit => return Err(failed(code)), + } + match exit.await { Some(0) => Ok(()), - code => { - let err = String::from_utf8_lossy(&stderr.lock()).trim().to_owned(); - Err(io(format!("cannot write {remote} on {} ({code:?}): {err}", self.label))) - } + code => Err(failed(code)), } } diff --git a/crates/ide-ssh/tests/os_credential_store.rs b/crates/ide-ssh/tests/os_credential_store.rs new file mode 100644 index 00000000..5382b6d7 --- /dev/null +++ b/crates/ide-ssh/tests/os_credential_store.rs @@ -0,0 +1,44 @@ +//! Saved passwords against the real OS credential store (Windows Credential +//! Manager, macOS Keychain, Secret Service). + +use ide_ssh::secrets::SecretFile; + +/// Forgets every key of the file, also when the test panics. +struct Forget<'a>(&'a SecretFile); + +impl Drop for Forget<'_> { + fn drop(&mut self) { + for key in self.0.keys() { + self.0.delete(&key); + } + } +} + +#[test] +#[ignore = "uses the OS credential store"] +fn saves_moves_and_forgets_in_the_os_store() { + assert!(keyring::Entry::store_status().is_ok(), "no OS credential store"); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("git-credentials.json"); + std::fs::write(&path, r#"{"https://old.example": "me\nold-os-pw"}"#).unwrap(); + let file = SecretFile::new(path.clone()); + let forget = Forget(&file); + file.set("https://new.example", "me\nnew-os-pw").unwrap(); + let on_disk = std::fs::read_to_string(&path).unwrap(); + assert!(!on_disk.contains("old-os-pw") && !on_disk.contains("new-os-pw"), "{on_disk}"); + + let reopened = SecretFile::new(path); + assert_eq!(reopened.get("https://old.example").as_deref(), Some("me\nold-os-pw")); + assert_eq!(reopened.get("https://new.example").as_deref(), Some("me\nnew-os-pw")); + #[cfg(windows)] + { + let index: serde_json::Value = serde_json::from_str(&on_disk).unwrap(); + let service = format!("ai.l0ng.openidea/{}", index["id"].as_str().unwrap()); + let listed = std::process::Command::new("cmdkey").arg("/list").output().unwrap(); + let listed = String::from_utf8_lossy(&listed.stdout); + assert_eq!(listed.matches(service.as_str()).count(), 2, "{listed}"); + } + drop(forget); + assert_eq!(reopened.keys(), Vec::::new()); + assert_eq!(reopened.get("https://old.example"), None); +} diff --git a/crates/ide-ssh/tests/session.rs b/crates/ide-ssh/tests/session.rs index b1a00e28..6c89baa3 100644 --- a/crates/ide-ssh/tests/session.rs +++ b/crates/ide-ssh/tests/session.rs @@ -269,3 +269,26 @@ async fn runs_commands_and_streams_processes() { assert_eq!(process.wait().await, Some(3)); assert_eq!(*seen.lock(), ("ran: python3 main.py".to_owned(), "oops".to_owned())); } + +#[tokio::test] +async fn upload_reports_a_command_that_stops_reading() { + let env = env().await; + let session = env.connect_trusted(&env.target(SshAuth::Password), Some("secret")).await.ok().unwrap(); + // Incompressible and larger than the channel window: the command exits (status 3, `oops`) + // without reading it, as when the host has no gzip or the folder is not writable. + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + let data: Vec = (0..8 << 20) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect(); + let put = session.put("/home/dev/.openidea/server/openidea-server", &data); + let err = tokio::time::timeout(std::time::Duration::from_secs(20), put) + .await + .expect("the upload must end when the command exits") + .unwrap_err(); + assert!(err.message.contains("oops"), "{}", err.message); +} diff --git a/crates/ide-text/Cargo.toml b/crates/ide-text/Cargo.toml index 8867ff2b..fac639ab 100644 --- a/crates/ide-text/Cargo.toml +++ b/crates/ide-text/Cargo.toml @@ -12,6 +12,7 @@ ide-core.workspace = true # Hence no `unicode_lines` / `cr_lines` (a test guards against feature unification). ropey = { version = "1.6", default-features = false, features = ["simd"] } serde.workspace = true +similar = { version = "2.7", default-features = false } specta.workspace = true [dev-dependencies] diff --git a/crates/ide-text/benches/text.rs b/crates/ide-text/benches/text.rs index 34d89857..d6461953 100644 --- a/crates/ide-text/benches/text.rs +++ b/crates/ide-text/benches/text.rs @@ -73,6 +73,10 @@ fn bench(c: &mut Criterion) { BatchSize::SmallInput, ) }); + let two_edits = text.replacen("value_100 ", "renamed_100 ", 1).replacen("value_9000 ", "renamed_9000 ", 1); + c.bench_function("text/diff_two_edits_1mb", |b| b.iter(|| base.diff(black_box(&two_edits)))); + let every_line = text.replace("compute", "calculate"); + c.bench_function("text/diff_every_line_1mb", |b| b.iter(|| base.diff(black_box(&every_line)))); c.bench_function("text/offset_to_pos_round_trip", |b| { let mut rng = Lcg(3); let len = base.len_utf16(); diff --git a/crates/ide-text/src/lib.rs b/crates/ide-text/src/lib.rs index 500cc20a..3a1c53f2 100644 --- a/crates/ide-text/src/lib.rs +++ b/crates/ide-text/src/lib.rs @@ -14,6 +14,7 @@ use ide_core::{ChangeSet, IdeError, IdeResult, Pos, Range, TextChange, TextEdit} use serde::{Deserialize, Serialize}; use specta::Type; use std::borrow::Cow; +use std::time::{Duration, Instant}; pub use ropey; @@ -66,6 +67,36 @@ impl LineEnding { } } +/// How long [`Document::diff`] may look for separate regions, so a reload never stalls on a huge file. +const DIFF_BUDGET: Duration = Duration::from_millis(50); + +type LineRange = std::ops::Range; + +/// Byte lengths of the common prefix and suffix; they never overlap. +fn common_ends(a: &str, b: &str) -> (usize, usize) { + let shared = |a: &mut dyn Iterator, b: &mut dyn Iterator| -> usize { + a.zip(b).take_while(|(x, y)| x == y).map(|(x, _)| x.len_utf8()).sum() + }; + let prefix = shared(&mut a.chars(), &mut b.chars()); + let suffix = shared(&mut a[prefix..].chars().rev(), &mut b[prefix..].chars().rev()); + (prefix, suffix) +} + +/// Byte offset of each line, then the total length. +fn line_starts(lines: &[&str]) -> Vec { + let mut at = 0; + let mut starts = vec![0]; + starts.extend(lines.iter().map(|line| { + at += line.len(); + at + })); + starts +} + +fn utf16_len(text: &str) -> u32 { + text.chars().map(|c| c.len_utf16() as u32).sum() +} + /// Converts `\r\n` and lone `\r` to `\n`. Borrows when there is nothing to do. pub fn normalize_line_endings(text: &str) -> Cow<'_, str> { if !text.contains('\r') { @@ -334,42 +365,63 @@ impl Document { cs } - /// The minimal single-change (common prefix/suffix) change set turning the - /// current text into `text` (after line-ending normalization). Empty when equal. + /// The change set turning the current text into `text` (after line-ending + /// normalization), one change per changed region. Empty when equal. A single change + /// spanning two distant edits would drag the editor's caret and view to the first one. pub fn diff(&self, text: &str) -> ChangeSet { + self.diff_until(text, Instant::now() + DIFF_BUDGET) + } + + /// Past `deadline` the regions get coarser, never wrong. + fn diff_until(&self, text: &str, deadline: Instant) -> ChangeSet { let new = normalize_line_endings(text); - let old_chars = self.rope.len_chars(); + let old = self.rope.to_string(); + let (prefix, suffix) = common_ends(&old, &new); + let old_rest = &old[prefix..old.len() - suffix]; + let new_rest = &new[prefix..new.len() - suffix]; - let mut prefix_chars = 0usize; - let mut prefix_bytes = 0usize; - for (a, b) in self.rope.chars().zip(new.chars()) { - if a != b { - break; + let old_lines: Vec<&str> = old_rest.split_inclusive('\n').collect(); + let new_lines: Vec<&str> = new_rest.split_inclusive('\n').collect(); + let old_starts = line_starts(&old_lines); + let new_starts = line_starts(&new_lines); + + let mut changes = Vec::new(); + let mut utf16_at = utf16_len(&old[..prefix]); + let mut old_at = 0; + let mut region: Option<(LineRange, LineRange)> = None; + let ops = + similar::capture_diff_slices_deadline(similar::Algorithm::Myers, &old_lines, &new_lines, Some(deadline)); + let mut flush = |region: Option<(LineRange, LineRange)>| { + let Some((old_range, new_range)) = region else { return }; + let region_old = &old_rest[old_starts[old_range.start]..old_starts[old_range.end]]; + let region_new = &new_rest[new_starts[new_range.start]..new_starts[new_range.end]]; + let (head, tail) = common_ends(region_old, region_new); + let start = old_starts[old_range.start] + head; + let end = old_starts[old_range.end] - tail; + let from = utf16_at + utf16_len(&old_rest[old_at..start]); + let to = from + utf16_len(&old_rest[start..end]); + changes.push(TextChange { from, to, insert: region_new[head..region_new.len() - tail].to_owned() }); + utf16_at = to; + old_at = end; + }; + // Positions come from the lengths: `similar` can report a stale start for the empty + // side of an op (an insert after an equal run still names the run's first old line). + let (mut old_line, mut new_line) = (0, 0); + for op in ops { + let old_end = old_line + op.old_range().len(); + let new_end = new_line + op.new_range().len(); + if op.tag() == similar::DiffTag::Equal { + flush(region.take()); + } else { + region = Some(match region.take() { + Some((o, n)) => (o.start..old_end, n.start..new_end), + None => (old_line..old_end, new_line..new_end), + }); } - prefix_chars += 1; - prefix_bytes += b.len_utf8(); + (old_line, new_line) = (old_end, new_end); } - let new_rest = &new[prefix_bytes..]; - let new_rest_chars = new_rest.chars().count(); - if prefix_chars == old_chars && new_rest_chars == 0 { - return ChangeSet::default(); - } - - let max_suffix = (old_chars - prefix_chars).min(new_rest_chars); - let mut suffix_chars = 0usize; - let mut suffix_bytes = 0usize; - for (a, b) in self.rope.chars_at(old_chars).reversed().zip(new_rest.chars().rev()).take(max_suffix) { - if a != b { - break; - } - suffix_chars += 1; - suffix_bytes += b.len_utf8(); - } - - let from = self.rope.char_to_utf16_cu(prefix_chars) as u32; - let to = self.rope.char_to_utf16_cu(old_chars - suffix_chars) as u32; - let insert = new_rest[..new_rest.len() - suffix_bytes].to_owned(); - ChangeSet { changes: vec![TextChange { from, to, insert }] } + flush(region); + ChangeSet { changes } } // ---- coordinate conversion (UTF-16 offset ⇄ Pos ⇄ byte offset) ---- @@ -606,4 +658,45 @@ mod tests { assert_eq!(set.changes, vec![tc(2, 2, "x")]); assert_eq!(doc.text(), "😀x😀"); } + + fn numbered_lines(n: usize) -> String { + (0..n).map(|i| format!("line {i}\n")).collect() + } + + #[test] + fn diff_keeps_distant_edits_apart() { + let old = numbered_lines(400); + let new = old.replace("line 20\n", "line 20x\n").replace("line 299\n", "line 299x\n"); + let at = |line: &str| (old.find(line).unwrap() + line.len() - 1) as u32; + let set = Document::new(&old).diff(&new); + assert_eq!( + set.changes, + vec![tc(at("line 20\n"), at("line 20\n"), "x"), tc(at("line 299\n"), at("line 299\n"), "x")] + ); + } + + #[test] + fn diff_handles_added_and_removed_lines() { + let old = "a\nb\nc\nd\ne\nf\n"; + let set = Document::new(old).diff("a\nnew\nb\nc\ne\nf\n"); + assert_eq!(set.changes, vec![tc(2, 2, "new\n"), tc(5, 7, "")]); + } + + #[test] + fn diff_counts_utf16_past_astral_text() { + let old = "😀 one\nsame\n😀 two\n"; + let set = Document::new(old).diff("😀 ONE\nsame\n😀 TWO\n"); + assert_eq!(set.changes, vec![tc(3, 6, "ONE"), tc(15, 18, "TWO")]); + } + + #[test] + fn diff_past_its_deadline_is_coarser_but_still_correct() { + let old = numbered_lines(5000); + let new = old.replace("line 20\n", "line 20x\n").replace("line 4000\n", "line 4000x\n"); + let doc = Document::new(&old); + let set = doc.diff_until(&new, std::time::Instant::now()); + let mut replayed = Document::new(&old); + replayed.apply(0, &set).unwrap(); + assert_eq!(replayed.text(), new); + } } diff --git a/crates/ide-text/tests/props.proptest-regressions b/crates/ide-text/tests/props.proptest-regressions index c1187945..ab2c31ca 100644 --- a/crates/ide-text/tests/props.proptest-regressions +++ b/crates/ide-text/tests/props.proptest-regressions @@ -5,3 +5,4 @@ # It is recommended to check this file in to source control so that # everyone who runs the test benefits from these saved cases. cc afa0785335731ca8adc050b09312843aeb3ff05dc895963d09d9f728c553717c # shrinks to doc_text = "a", picks = [(0, 0, ""), (0, 0, ""), (658842129343018477, 1922606258417419361, ""), (0, 4297423518677873183, ""), (953525717999286983, 557348908835472853, "a"), (0, 0, "b"), (0, 1653396554766652711, "")], seed = 7072935092038531240 +cc dd6ec32be77e8ad96d0ce3ef14ad9d72d5ba2a06f227ea693bfdee62debf08b7 # shrinks to a = "中é\n\n", b = "\n\na" diff --git a/crates/ide-text/tests/props.rs b/crates/ide-text/tests/props.rs index b7324d10..bbf9d3a7 100644 --- a/crates/ide-text/tests/props.rs +++ b/crates/ide-text/tests/props.rs @@ -176,7 +176,7 @@ proptest! { let set = doc.replace_all(&b); prop_assert_eq!(doc.text(), b.clone()); prop_assert_eq!(naive_apply(&a, &set), b.clone()); - prop_assert!(set.changes.len() <= 1); + prop_assert!(set.changes.windows(2).all(|w| w[0].to < w[1].from)); prop_assert_eq!(set.changes.is_empty(), a == b); } diff --git a/crates/ide-tools/src/lib.rs b/crates/ide-tools/src/lib.rs index 749b18f3..881393d6 100644 --- a/crates/ide-tools/src/lib.rs +++ b/crates/ide-tools/src/lib.rs @@ -8,6 +8,12 @@ //! //! Language servers find managed tools once [`extend_process_path`] has put //! [`tools_bin_dirs`] on the process `PATH` at startup. +//! +//! GitHub API downloads prefer `GITHUB_TOKEN`, then `GH_TOKEN`, then the active +//! `github.com` account from `gh auth token --hostname github.com`. The CLI is +//! optional; absent credentials retain anonymous downloads. Tokens are passed +//! to curl over stdin, never stored in tool manifests or command arguments, and +//! are not sent to other download origins. CLI lookup is cancellable and bounded. pub mod archive; mod bundled; @@ -389,6 +395,11 @@ pub fn parse_version(text: &str) -> Option { None } +/// A Windows App Execution Alias (`%LOCALAPPDATA%\Microsoft\WindowsApps\python.exe`). +fn is_app_execution_alias(path: &Path) -> bool { + path.parent().and_then(Path::file_name).is_some_and(|d| d.eq_ignore_ascii_case("WindowsApps")) +} + /// A tool whose "command" is a plugin jar (e.g. a jdtls bundle) rather than an executable. fn is_bundle(command: &str) -> bool { command.ends_with(".jar") @@ -485,8 +496,32 @@ impl Tools { cmd } + /// The interpreter that installs and probes Python tools. Windows installs `python.exe` + /// and the `py` launcher, not `python3.exe`; its `WindowsApps` entries are App Execution + /// Aliases that only open the Store unless the Store's Python is installed, so they count + /// only when they run. fn python(&self) -> Option { - self.which_any(&["python3", "python"]) + if self.platform.os != Os::Windows { + return self.which_any(&["python3", "python"]); + } + ["python", "py", "python3"] + .into_iter() + .flat_map(|command| { + std::env::split_paths(&self.system_path).filter_map(move |dir| { + if dir.as_os_str().is_empty() { + return None; + } + let search = std::env::join_paths([dir]).ok()?; + ide_core::tool_path::find_command(command, &search, &self.root) + }) + }) + .find(|p| { + !is_app_execution_alias(p) || { + let mut c = self.python_command(p); + c.arg("--version"); + run_quiet(c, Duration::from_secs(5)).is_some_and(|(ok, _)| ok) + } + }) } /// Executable `cmd` inside `dir`, honouring Windows suffixes. diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs index 889cbe49..ac8a48b2 100644 --- a/crates/ide-tools/src/process.rs +++ b/crates/ide-tools/src/process.rs @@ -6,9 +6,9 @@ use ide_core::error::ErrorKind; use ide_core::output_encoding::decode_tool_output; use ide_core::process::{kill_tree, own_group}; use ide_core::{CancellationToken, IdeError, IdeResult}; -use std::io::{BufRead, BufReader, Read}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Stdio}; use std::sync::mpsc; use std::time::{Duration, Instant}; @@ -144,6 +144,99 @@ fn strip_ansi(s: &str) -> String { out } +const MAX_GITHUB_TOKEN: usize = 4096; + +fn valid_github_token(value: String) -> Option { + let value = value.trim(); + (!value.is_empty() && value.len() <= MAX_GITHUB_TOKEN && value.bytes().all(|b| b.is_ascii_graphic())) + .then(|| value.to_owned()) +} + +fn environment_github_token(read: impl Fn(&str) -> Option) -> Option { + ["GITHUB_TOKEN", "GH_TOKEN"].into_iter().find_map(|key| read(key).and_then(valid_github_token)) +} + +/// Ask gh for the active github.com account on each request, so switching or +/// logging out takes effect without persisting a second copy of the credential. +fn read_gh_token(gh: &Path, cancel: &CancellationToken, timeout: Duration) -> IdeResult> { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let mut cmd = Command::new(gh); + own_group(ide_core::process::prepare(&mut cmd)); + cmd.args(["auth", "token", "--hostname", "github.com"]) + .env("GH_PROMPT_DISABLED", "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + let Ok(mut child) = cmd.spawn() else { return Ok(None) }; + let stdout = child.stdout.take().expect("piped"); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = stdout.take((MAX_GITHUB_TOKEN + 2) as u64).read_to_end(&mut bytes); + let _ = tx.send(bytes); + }); + let started = Instant::now(); + let mut status = None; + let mut output = None; + let bytes = loop { + if cancel.is_cancelled() || started.elapsed() >= timeout { + kill_tree(&mut child); + return if cancel.is_cancelled() { Err(IdeError::cancelled()) } else { Ok(None) }; + } + if status.is_none() { + match child.try_wait() { + Ok(value) => status = value, + Err(_) => { + kill_tree(&mut child); + return Ok(None); + } + } + } + if let Some(status) = status { + if !status.success() { + kill_tree(&mut child); + return Ok(None); + } + if let Some(bytes) = output.take() { + break bytes; + } + } + match rx.recv_timeout(Duration::from_millis(15)) { + Ok(bytes) => output = Some(bytes), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => std::thread::sleep(Duration::from_millis(15)), + } + }; + if bytes.len() > MAX_GITHUB_TOKEN + 1 { + return Ok(None); + } + Ok(String::from_utf8(bytes).ok().and_then(valid_github_token)) +} + +fn send_auth(child: &mut Child, token: Option<&str>) -> IdeResult<()> { + if let Some(token) = token { + let result = child + .stdin + .take() + .expect("authenticated request has piped stdin") + .write_all(format!("Authorization: Bearer {token}\n").as_bytes()); + if let Err(error) = result { + kill_tree(child); + return Err(IdeError::unavailable(format!("cannot pass download credentials to curl: {error}"))); + } + } + Ok(()) +} + +fn redact_auth(message: &str, token: Option<&str>) -> String { + match token { + Some(token) => message.replace(token, "[redacted]"), + None => message.to_owned(), + } +} + /// Whether `curl` checks certificate revocation with Schannel and takes /// `--ssl-revoke-best-effort` (7.70+). Schannel fails the whole download with /// CRYPT_E_REVOCATION_OFFLINE when the CRL/OCSP servers are unreachable (corporate @@ -174,6 +267,8 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool { /// mainstream Linux), so the IDE carries no TLS stack of its own. pub struct Http { curl: std::path::PathBuf, + gh: Option, + explicit_github_token: Option, https_only: bool, revoke_best_effort: bool, } @@ -182,8 +277,10 @@ impl Http { pub fn new(search_path: &std::ffi::OsStr) -> IdeResult { let curl = which::which_in("curl", Some(search_path), ".") .map_err(|_| IdeError::unavailable("`curl` is required to download tools"))?; + let gh = which::which_in("gh", Some(search_path), ".").ok(); + let explicit_github_token = environment_github_token(|key| std::env::var(key).ok()); let revoke_best_effort = cfg!(windows) && revoke_best_effort(&curl); - Ok(Self { curl, https_only: false, revoke_best_effort }) + Ok(Self { curl, gh, explicit_github_token, https_only: false, revoke_best_effort }) } /// Refuses every protocol but HTTPS, redirects included. @@ -192,8 +289,28 @@ impl Http { self } - fn base(&self, url: &str) -> Command { + fn base(&self, url: &str, cancel: &CancellationToken) -> IdeResult<(Command, Option)> { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let github = url.starts_with("https://api.github.com/"); + let token = if github { + match &self.explicit_github_token { + Some(token) => Some(token.clone()), + None => match &self.gh { + Some(gh) => read_gh_token(gh, cancel, Duration::from_secs(5))?, + None => None, + }, + } + } else { + None + }; let mut cmd = Command::new(&self.curl); + // A user curlrc must not enable verbose token logging or forwarding + // Authorization across origins on authenticated API requests. + if token.is_some() { + cmd.arg("-q"); + } own_group(ide_core::process::prepare(&mut cmd)); cmd.args(["-fsSL", "--retry", "2", "--connect-timeout", "20", "-A", "openidea"]); // System Settings › HTTP Proxy (a manual proxy comes in through `prepare`'s environment). @@ -204,13 +321,14 @@ impl Http { if self.revoke_best_effort { cmd.arg("--ssl-revoke-best-effort"); } - if url.starts_with("https://api.github.com/") - && let Ok(token) = std::env::var("GITHUB_TOKEN").or_else(|_| std::env::var("GH_TOKEN")) - && !token.is_empty() - { - cmd.arg("-H").arg(format!("Authorization: Bearer {token}")); + cmd.stdin(Stdio::null()); + if token.is_some() { + // Keep credentials out of process arguments, inherited environment, + // and files. curl strips Authorization on cross-origin redirects. + cmd.args(["--header", "@-", "--proto-redir", "=https"]); + cmd.env_remove("GH_TOKEN").env_remove("GITHUB_TOKEN").stdin(Stdio::piped()); } - cmd + Ok((cmd, token)) } /// GETs `url` as text. @@ -235,15 +353,16 @@ impl Http { empty_on_404: bool, cancel: &CancellationToken, ) -> IdeResult { - let mut cmd = self.base(url); + let (mut cmd, token) = self.base(url, cancel)?; if empty_on_404 { cmd.args(["--write-out", "\n%{http_code}"]); } if let Some(max) = max_bytes { cmd.arg("--max-filesize").arg(max.to_string()); } - cmd.arg(url).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.arg(url).stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?; + send_auth(&mut child, token.as_deref())?; let stdout = child.stdout.take().expect("piped"); let mut stderr = child.stderr.take().expect("piped"); // Past the limit the pipe closes, which stops curl. @@ -280,7 +399,10 @@ impl Http { let mut err = Vec::new(); let _ = stderr.read_to_end(&mut err); let err = decode_tool_output(&err); - return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim()))); + return Err(IdeError::new( + ErrorKind::Io, + format!("GET {url} failed: {}", redact_auth(err.trim(), token.as_deref())), + )); } String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8"))) } @@ -294,9 +416,10 @@ impl Http { cancel: &CancellationToken, emit: Emit, ) -> IdeResult<()> { - let mut cmd = self.base(url); - cmd.arg("-o").arg(dest).arg(url).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::piped()); + let (mut cmd, token) = self.base(url, cancel)?; + cmd.arg("-o").arg(dest).arg(url).stdout(Stdio::null()).stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?; + send_auth(&mut child, token.as_deref())?; let mut stderr = child.stderr.take().expect("piped"); let err_t = std::thread::spawn(move || { let mut s = Vec::new(); @@ -327,7 +450,10 @@ impl Http { if !status.success() { return Err(IdeError::new( ErrorKind::Io, - format!("download of {url} failed: {}", err_t.join().unwrap_or_default().trim()), + format!( + "download of {url} failed: {}", + redact_auth(err_t.join().unwrap_or_default().trim(), token.as_deref()) + ), )); } let bytes = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0); @@ -490,7 +616,13 @@ mod tests { use std::os::unix::fs::PermissionsExt; let tmp = tempfile::tempdir().unwrap(); let curl = tmp.path().join("curl"); - let http = Http { curl: curl.clone(), https_only: false, revoke_best_effort: false }; + let http = Http { + curl: curl.clone(), + gh: None, + explicit_github_token: None, + https_only: false, + revoke_best_effort: false, + }; for (code, exit, expected) in [(404, 22, Some("[]")), (200, 0, Some("payload")), (403, 22, None), (0, 7, None)] { std::fs::write(&curl, format!("#!/bin/sh\nprintf 'payload\\n{code:03}'\nexit {exit}\n")).unwrap(); @@ -595,3 +727,200 @@ mod input_tests { ); } } + +#[cfg(test)] +mod github_auth_tests { + use super::*; + + #[test] + fn explicit_tokens_keep_precedence_and_reject_header_injection() { + let read = |key: &str| Some(if key == "GITHUB_TOKEN" { "first" } else { "second" }.to_owned()); + assert_eq!(environment_github_token(read).as_deref(), Some("first")); + assert_eq!( + environment_github_token(|key| Some(if key == "GITHUB_TOKEN" { "" } else { "second" }.into())).as_deref(), + Some("second") + ); + for bad in ["", "two words", "token\r\nX-Injected: yes", "token\0"] { + assert!(valid_github_token(bad.into()).is_none()); + } + assert!(valid_github_token("a".repeat(MAX_GITHUB_TOKEN + 1)).is_none()); + assert_eq!(valid_github_token("gho_example\n".into()).as_deref(), Some("gho_example")); + } + + #[cfg(unix)] + mod unix { + use super::*; + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + + fn quote(path: &Path) -> String { + format!("'{}'", path.display().to_string().replace('\'', "'\\''")) + } + struct Fixture { + dir: tempfile::TempDir, + http: Http, + } + impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let curl = dir.path().join("curl"); + let gh = dir.path().join("gh"); + let fixture = Self { + dir, + http: Http { + curl, + gh: Some(gh), + explicit_github_token: None, + https_only: false, + revoke_best_effort: false, + }, + }; + fixture.script( + "gh", + &format!( + "printf '%s\\n' \"$@\" >> {}\n/bin/cat {}", + quote(&fixture.path("gh-args")), + quote(&fixture.path("token")) + ), + ); + fs::write(fixture.path("token"), "gho_fake_first\n").unwrap(); + fixture.curl("printf '{\"ok\":true}'"); + fixture + } + fn path(&self, name: &str) -> PathBuf { + self.dir.path().join(name) + } + fn script(&self, name: &str, body: &str) { + let path = self.path(name); + fs::write(&path, format!("#!/bin/sh\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + fn curl(&self, body: &str) { + self.script( + "curl", + &format!( + "printf '%s\\n' \"$@\" > {}\n/bin/cat > {}\n{body}", + quote(&self.path("curl-args")), + quote(&self.path("headers")) + ), + ); + } + fn get(&self, url: &str) -> IdeResult { + self.http.get_text(url, &CancellationToken::new()) + } + } + const API: &str = "https://api.github.com/repos/example/project/releases/latest"; + + #[test] + fn active_gh_account_is_read_per_request_and_token_is_not_an_argument() { + let f = Fixture::new(); + assert_eq!(f.get(API).unwrap(), "{\"ok\":true}"); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), "Authorization: Bearer gho_fake_first\n"); + let args = fs::read_to_string(f.path("curl-args")).unwrap(); + assert!(!args.contains("gho_fake")); + assert!(args.starts_with("-q\n")); + assert!(args.contains("--header\n@-\n--proto-redir\n=https\n")); + assert!(!args.contains("--location-trusted")); + assert_eq!(fs::read_to_string(f.path("gh-args")).unwrap(), "auth\ntoken\n--hostname\ngithub.com\n"); + fs::write(f.path("token"), "gho_fake_second\n").unwrap(); + f.get(API).unwrap(); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), "Authorization: Bearer gho_fake_second\n"); + } + + #[test] + fn explicit_token_avoids_gh_and_non_github_urls_receive_no_credentials() { + let mut f = Fixture::new(); + f.http.explicit_github_token = Some("explicit_fake".into()); + f.get(API).unwrap(); + assert!(!f.path("gh-args").exists()); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), "Authorization: Bearer explicit_fake\n"); + f.http.explicit_github_token = None; + for url in [ + "https://example.com/file", + "http://api.github.com/repos/x", + "https://api.github.com.evil.example/file", + "https://api.github.com@evil.example/file", + "https://github.com/example/project/releases/download/v1/file", + ] { + f.get(url).unwrap(); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), ""); + assert!(!f.path("gh-args").exists()); + } + } + + #[test] + fn missing_logged_out_or_malformed_gh_falls_back_to_anonymous() { + let mut f = Fixture::new(); + for body in ["exit 1", "printf 'fake\\nX-Injected: yes\\n'", "printf ''", "printf 'fake'; exit 1"] { + f.script("gh", body); + f.get(API).unwrap(); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), ""); + } + f.http.gh = None; + f.get(API).unwrap(); + assert_eq!(fs::read_to_string(f.path("headers")).unwrap(), ""); + } + + #[test] + fn download_auth_and_errors_do_not_expose_the_token() { + let f = Fixture::new(); + let dest = f.path("download"); + f.curl(&format!("printf downloaded > {}", quote(&dest))); + f.http.download(API, &dest, None, &CancellationToken::new(), &|_| {}).unwrap(); + assert_eq!(fs::read_to_string(&dest).unwrap(), "downloaded"); + assert!(fs::read_to_string(f.path("headers")).unwrap().contains("gho_fake_first")); + f.curl(&format!("/bin/cat {} >&2; exit 22", quote(&f.path("headers")))); + let error = f.get(API).unwrap_err().to_string(); + assert!(error.contains("[redacted]")); + assert!(!error.contains("gho_fake_first")); + let error = f.http.download(API, &dest, None, &CancellationToken::new(), &|_| {}).unwrap_err().to_string(); + assert!(!error.contains("gho_fake_first")); + } + + #[test] + fn credential_lookup_is_bounded_and_cancellable_even_with_inherited_stdout() { + let f = Fixture::new(); + // The parent exits but a child keeps stdout open: joining a reader + // without checking the deadline would hang here. + f.script("gh", "/bin/sleep 5 &\nexit 0"); + let started = Instant::now(); + assert!( + read_gh_token(f.http.gh.as_ref().unwrap(), &CancellationToken::new(), Duration::from_millis(60)) + .unwrap() + .is_none() + ); + assert!(started.elapsed() < Duration::from_secs(3)); + f.script("gh", "/bin/sleep 5"); + let cancel = CancellationToken::new(); + let trigger = cancel.clone(); + let thread = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + trigger.cancel(); + }); + assert_eq!(f.http.get_text(API, &cancel).unwrap_err().kind, ErrorKind::Cancelled); + thread.join().unwrap(); + assert!(!f.path("curl-args").exists()); + } + + #[test] + #[ignore = "requires an active gh github.com login and network access"] + fn live_github_release_uses_authenticated_quota() { + let http = Http::new(&std::env::var_os("PATH").unwrap()).unwrap(); + assert!(http.gh.is_some()); + let cancel = CancellationToken::new(); + let body = http.get_text_limited("https://api.github.com/rate_limit", Some(64 * 1024), &cancel).unwrap(); + let response: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert!(response["resources"]["core"]["limit"].as_u64().unwrap() > 60); + let body = http + .get_text_limited( + "https://api.github.com/repos/redhat-developer/vscode-xml/releases/latest", + Some(1024 * 1024), + &cancel, + ) + .unwrap(); + let response: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert!(response["tag_name"].is_string()); + } + } +} diff --git a/crates/ide-tools/src/registry.rs b/crates/ide-tools/src/registry.rs index 116dbfc8..494482f7 100644 --- a/crates/ide-tools/src/registry.rs +++ b/crates/ide-tools/src/registry.rs @@ -641,11 +641,11 @@ pub static TOOLS: &[ToolSpec] = &[ // ECJ 3.46.200 keeps the constructor call in a constructor's statements, which // @SneakyThrows / @NonNull must not wrap or precede (lombok#4089). // Return to Maven Central once an official release includes both. - // Source commit, build instructions and checksum: plugins/lombok/README.md. + // Source commit, build instructions and checksum: java/lombok/README.md. method: Method::Bundled { filename: "lombok-1.18.48-openidea.2.jar", version: "1.18.48-openidea.2", - bytes: include_bytes!("../../../plugins/lombok/lombok-1.18.48-openidea.2.jar"), + bytes: include_bytes!("../../../java/lombok/lombok-1.18.48-openidea.2.jar"), }, fallback: None, version: V, @@ -680,7 +680,9 @@ pub static TOOLS: &[ToolSpec] = &[ rule(Linux, Arm64, "kotlin-lsp-*-linux-aarch64.zip"), rule(Windows, X64, "kotlin-lsp-*-win-x64.zip"), ], - locate: &["kotlin-lsp", "kotlin-lsp.sh", "kotlin-lsp.cmd"], + // `bin/intellij-server` is the launcher; current releases still ship the + // `kotlin-lsp.sh` / `kotlin-lsp.cmd` wrappers around it, deprecated, and old ones only those. + locate: &["intellij-server", "kotlin-lsp", "kotlin-lsp.sh", "kotlin-lsp.cmd"], }, fallback: None, version: V, @@ -1044,6 +1046,25 @@ mod tests { /// The v263 kotlin-server-* release notes: the CLI distribution archives plus /// the VS Code extension and checksums; each platform must pick its native /// distribution and the aarch64 archive must never be picked on x64. + #[test] + fn kotlin_lsp_runs_its_launcher_rather_than_the_deprecated_wrapper() { + let Method::Release { locate, .. } = &find("kotlin-lsp").unwrap().method else { unreachable!() }; + let names: Vec = locate.iter().map(|n| n.to_string()).collect(); + let tmp = tempfile::tempdir().unwrap(); + // Current releases: `kotlin-lsp.sh` warns that it will go and execs `bin/intellij-server`. + let current = tmp.path().join("current"); + let dist = current.join("kotlin-server-263.7084.0"); + std::fs::create_dir_all(dist.join("bin")).unwrap(); + std::fs::write(dist.join("kotlin-lsp.sh"), "").unwrap(); + std::fs::write(dist.join("bin/intellij-server"), "").unwrap(); + assert_eq!(crate::archive::locate(¤t, &names), Some(dist.join("bin/intellij-server"))); + // Older releases have only the script. + let old = tmp.path().join("old"); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("kotlin-lsp.sh"), "").unwrap(); + assert_eq!(crate::archive::locate(&old, &names), Some(old.join("kotlin-lsp.sh"))); + } + #[test] fn kotlin_lsp_picks_the_native_archive_for_each_platform() { use crate::platform::{Platform, select_asset}; diff --git a/crates/ide-tools/src/tests.rs b/crates/ide-tools/src/tests.rs index be6878be..de4ff670 100644 --- a/crates/ide-tools/src/tests.rs +++ b/crates/ide-tools/src/tests.rs @@ -37,6 +37,12 @@ impl Env { fs::write(&p, format!("#!/bin/sh\n{body}\n")).unwrap(); archive::make_executable(&p).unwrap(); } + + #[cfg(windows)] + fn script(&self, dir: &Path, name: &str, body: &str) { + fs::create_dir_all(dir).unwrap(); + fs::write(dir.join(format!("{name}.cmd")), format!("@echo off\r\n{body}\r\n")).unwrap(); + } } fn find<'a>(list: &'a [ToolInfo], name: &str) -> &'a ToolInfo { @@ -71,7 +77,7 @@ fn bundled_lombok_is_available_offline_and_repairs_missing_or_corrupt_files() { let env = Env::new(); let data = env.root.parent().unwrap(); let agent = managed_bundle(data, "lombok").unwrap(); - let checksum = include_str!("../../../plugins/lombok/SHA256SUMS").split_whitespace().next().unwrap(); + let checksum = include_str!("../../../java/lombok/SHA256SUMS").split_whitespace().next().unwrap(); assert_eq!(sources::sha256_file(&agent).unwrap(), checksum); for contents in [None, Some(b"incomplete jar".as_slice())] { if let Some(contents) = contents { @@ -93,7 +99,7 @@ fn concurrent_windows_get_the_same_complete_bundled_lombok() { let windows: Vec<_> = (0..8).map(|_| scope.spawn(|| managed_bundle(data, "lombok").unwrap())).collect(); let paths: Vec<_> = windows.into_iter().map(|window| window.join().unwrap()).collect(); assert!(paths.iter().all(|path| path == &paths[0])); - let checksum = include_str!("../../../plugins/lombok/SHA256SUMS").split_whitespace().next().unwrap(); + let checksum = include_str!("../../../java/lombok/SHA256SUMS").split_whitespace().next().unwrap(); assert_eq!(sources::sha256_file(&paths[0]).unwrap(), checksum); }); } @@ -700,6 +706,34 @@ fn rust_analyzer_managed_copy_survives_rustup_becoming_available() { assert!(!env.root.join("rust-analyzer").exists()); } +/// Windows has `python.exe` and `py`, not `python3.exe`; the `WindowsApps` aliases only count +/// when the Store's Python behind them runs. +#[test] +fn windows_python_skips_store_aliases_that_do_not_run() { + let env = Env::new(); + let aliases = env.sys.parent().unwrap().join("Microsoft").join("WindowsApps"); + env.script(&aliases, "python", "exit 9009"); + env.script(&aliases, "python3", "exit 9009"); + env.script(&env.sys, "py", "echo Python 3.12.0"); + let windows = Platform { os: Os::Windows, arch: platform::Arch::X64 }; + let found = || { + let dirs = std::env::join_paths([aliases.clone(), env.sys.clone()]).unwrap(); + Tools::with_paths(env.root.clone(), dirs, windows).python().map(|python| { + let dir = python.parent().unwrap().file_name().unwrap().to_string_lossy().into_owned(); + format!("{dir}/{}", python.file_stem().unwrap().to_string_lossy()) + }) + }; + assert_eq!(found().as_deref(), Some("sysbin/py")); + fs::remove_file(env.sys.join(if cfg!(windows) { "py.cmd" } else { "py" })).unwrap(); + assert_eq!(found(), None, "only broken aliases are available"); + env.script(&env.sys, "python3", "echo Python 3.12.0"); + assert_eq!(found().as_deref(), Some("sysbin/python3"), "python3 is the final fallback"); + env.script(&env.sys, "python", "echo Python 3.12.0"); + assert_eq!(found().as_deref(), Some("sysbin/python"), "a broken alias must not hide a later interpreter"); + env.script(&aliases, "python", "echo Python 3.13.0"); + assert_eq!(found().as_deref(), Some("WindowsApps/python"), "the Store's Python runs"); +} + #[cfg(unix)] #[test] fn python_probes_run_in_an_empty_private_directory() { diff --git a/crates/ide-vcs/Cargo.toml b/crates/ide-vcs/Cargo.toml index 4d79193c..512b1bde 100644 --- a/crates/ide-vcs/Cargo.toml +++ b/crates/ide-vcs/Cargo.toml @@ -15,6 +15,8 @@ serde_json.workspace = true specta.workspace = true gix = { workspace = true, features = ["sha1", "sha256", "status", "blob-diff", "revision", "index", "dirwalk", "parallel"] } tracing.workspace = true +redb.workspace = true +blake3.workspace = true tempfile.workspace = true which = "8" regex-lite = "0.1" diff --git a/crates/ide-vcs/src/bek.rs b/crates/ide-vcs/src/bek.rs new file mode 100644 index 00000000..ad94665b --- /dev/null +++ b/crates/ide-vcs/src/bek.rs @@ -0,0 +1,192 @@ +//! Bek branch partitioning and bottom-up block merge. +//! Adapted from JetBrains intellij-community BekBranchCreator/BekBranch/Merger, +//! Copyright 2000-2026 JetBrains s.r.o. and contributors, Apache-2.0. +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashSet}; + +pub(crate) fn sort(parents: &[Vec], times: &[i64], heads: &[usize]) -> Vec { + let n = parents.len(); + let mut children = vec![Vec::new(); n]; + for (i, ps) in parents.iter().enumerate() { + for &p in ps { + children[p].push(i); + } + } + let mut layout = vec![0usize; n]; + let mut important = Vec::new(); + let mut next_layout = 1; + for &head in heads { + if layout[head] != 0 { + continue; + } + important.push(head); + let mut stack = vec![head]; + while let Some(&node) = stack.last() { + let first = layout[node] == 0; + if first { + layout[node] = next_layout; + } + if let Some(&p) = parents[node].iter().find(|&&p| layout[p] == 0) { + stack.push(p); + } else { + if first { + next_layout += 1; + } + stack.pop(); + } + } + } + let mut done = vec![false; n]; + let mut restrictions = vec![HashSet::new(); n]; + let mut releases = vec![Vec::new(); n]; + let mut branches = Vec::new(); + for head in important { + if done[head] { + continue; + } + let start = layout[head]; + done[head] = true; + let mut branch = vec![head]; + let mut stack = vec![head]; + while let Some(&node) = stack.last() { + let mut next = None; + for &p in parents[node].iter().rev() { + if done[p] { + if layout[p] < start && restrictions[node].insert(p) { + releases[p].push(node); + } + } else if layout[node] <= layout[p] && !children[p].iter().any(|&c| !done[c] && layout[c] <= layout[p]) + { + next = Some(p); + break; + } + } + if let Some(p) = next { + done[p] = true; + branch.push(p); + stack.push(p); + } else { + stack.pop(); + } + } + branches.push(branch); + } + let mut owner = vec![0usize; n]; + for (b, nodes) in branches.iter().enumerate() { + for &node in nodes { + owner[node] = b; + } + } + let mut remaining: Vec<_> = branches.iter().map(Vec::len).collect(); + let mut prepared = vec![None; branches.len()]; + let mut queue = BinaryHeap::new(); + let prepare = |b: usize, + remaining: &[usize], + restrictions: &[HashSet], + prepared: &mut [Option], + queue: &mut BinaryHeap>| { + let end = remaining[b]; + if end == 0 || prepared[b].is_some() || !restrictions[branches[b][end - 1]].is_empty() { + return; + } + let mut start = end - 1; + while start > 0 { + let up = branches[b][start - 1]; + let down = branches[b][start]; + if !restrictions[up].is_empty() { + break; + } + let connects = parents[up].contains(&down); + if !(parents[up].len() > 1 && connects) { + if !connects { + break; + } + let delta = times[up].abs_diff(times[down]); + if delta > 3 * 24 * 3600 || (start + 20 < end && delta > 4 * 3600) { + break; + } + } + start -= 1; + } + prepared[b] = Some(start); + queue.push(Reverse((times[branches[b][start]], b))); + }; + for b in 0..branches.len() { + prepare(b, &remaining, &restrictions, &mut prepared, &mut queue); + } + let mut result = vec![0; n]; + let mut offset = n; + while let Some(Reverse((_, b))) = queue.pop() { + let start = prepared[b].take().unwrap(); + let part = &branches[b][start..remaining[b]]; + offset -= part.len(); + result[offset..offset + part.len()].copy_from_slice(part); + remaining[b] = start; + let mut released = HashSet::from([b]); + for &node in part { + for &up in &releases[node] { + restrictions[up].remove(&node); + let branch = owner[up]; + if restrictions[up].is_empty() && remaining[branch] > 0 && branches[branch][remaining[branch] - 1] == up + { + released.insert(branch); + } + } + } + for branch in released { + prepare(branch, &remaining, &restrictions, &mut prepared, &mut queue); + } + } + debug_assert_eq!(offset, 0, "Bek must place every node"); + result +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn branch_blocks_preserve_every_edge_under_clock_skew_and_interleaved_refs() { + let mut seed = 17u64; + let mut random = || { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + seed as usize + }; + for _ in 0..200 { + let n = 70; + let parents: Vec> = (0..n) + .map(|i| { + let mut ps = Vec::new(); + if i + 1 < n { + for _ in 0..(random() % 3 + 1) { + let p = i + 1 + random() % (n - i - 1); + if !ps.contains(&p) { + ps.push(p); + } + } + } + ps + }) + .collect(); + let times: Vec<_> = (0..n).map(|_| (random() % 1_000_000) as i64).collect(); + let mut children = vec![0; n]; + for ps in &parents { + for &p in ps { + children[p] += 1; + } + } + let mut heads: Vec<_> = (0..n).filter(|&i| children[i] == 0 || i % 11 == 0).collect(); + heads.sort_by_cached_key(|_| random()); + let result = sort(&parents, ×, &heads); + assert_eq!(result.iter().copied().collect::>().len(), n); + let mut positions = vec![0; n]; + for (i, &node) in result.iter().enumerate() { + positions[node] = i; + } + for (node, ps) in parents.iter().enumerate() { + for &p in ps { + assert!(positions[node] < positions[p]); + } + } + } + } +} diff --git a/crates/ide-vcs/src/block_history.rs b/crates/ide-vcs/src/block_history.rs new file mode 100644 index 00000000..c47d1cb8 --- /dev/null +++ b/crates/ide-vcs/src/block_history.rs @@ -0,0 +1,227 @@ +//! History for Selection: the commits that changed a block of lines +//! (`git log -L`), each with the block as it was before and after. + +use crate::text_encoding::{TextEncoding, decode}; +use crate::{BlockHistory, BlockRevision, LineHunk, Repo, cli}; +use ide_core::{CancellationToken, IdeResult}; +use std::path::Path; + +/// Separates commits in the log output; fields of the header are separated by `\x1f`. +const RECORD: u8 = 0x1e; +const FORMAT: &str = "--format=%x1e%H%x1f%h%x1f%an%x1f%ae%x1f%at%x1f%s"; + +pub(crate) fn block_history( + r: &Repo, + path: &Path, + current_text: Option<&str>, + lines: (u32, u32), + encoding: TextEncoding, + cancel: &CancellationToken, +) -> IdeResult { + let rel = r.rel_path(path)?; + let head = { + let repo = r.gix()?; + crate::repo::blob_at(&repo, &rel, "HEAD")? + }; + let Some(head) = head else { + return Ok(BlockHistory { head_lines: None, revisions: Vec::new() }); + }; + let head = decode(&head, encoding); + let current = match current_text { + Some(text) => text.to_owned(), + None => decode(&std::fs::read(r.abs_path(&rel))?, encoding).into_owned(), + }; + let Some((start, end)) = head_range(&crate::diff::line_hunks(&head, ¤t), lines) else { + return Ok(BlockHistory { head_lines: None, revisions: Vec::new() }); + }; + let range = format!("-L{},{}:{rel}", start + 1, end + 1); + let args = ["log", "--no-color", "--no-ext-diff", "--no-textconv", FORMAT, &range, "HEAD"]; + let out = cli::run_cancellable(r.root(), &args, cancel)?; + Ok(BlockHistory { head_lines: Some((start, end)), revisions: parse_log(&out.stdout, encoding) }) +} + +/// The HEAD lines (0-based, inclusive) of `lines` in the current text, given the +/// hunks between HEAD and it. A selection end inside a changed region takes the +/// whole region; None when every selected line is new. +pub(crate) fn head_range(hunks: &[LineHunk], (start, end): (u32, u32)) -> Option<(u32, u32)> { + let (start, end) = (start.min(end), start.max(end)); + let to_head = |line: u32, at_end: bool| -> i64 { + let mut shift = 0i64; + for h in hunks { + if line < h.start { + break; + } + if line < h.start + h.lines { + return if at_end { + i64::from(h.old_start) + i64::from(h.old_lines) - 1 + } else { + i64::from(h.old_start) + }; + } + shift += i64::from(h.lines) - i64::from(h.old_lines); + } + i64::from(line) - shift + }; + let (from, to) = (to_head(start, false), to_head(end, true)); + (from >= 0 && from <= to).then_some((from as u32, to as u32)) +} + +/// Commits of `git log -L` output, newest first. +pub(crate) fn parse_log(stdout: &[u8], encoding: TextEncoding) -> Vec { + stdout + .split(|b| *b == RECORD) + .filter(|r| !r.is_empty()) + .filter_map(|record| parse_record(record, encoding)) + .collect() +} + +fn parse_record(record: &[u8], encoding: TextEncoding) -> Option { + let newline = record.iter().position(|b| *b == b'\n').unwrap_or(record.len()); + let header = String::from_utf8_lossy(&record[..newline]); + let mut fields = header.split('\x1f'); + let id = fields.next()?.to_owned(); + let short_id = fields.next()?.to_owned(); + let author = fields.next()?.to_owned(); + let email = fields.next()?.to_owned(); + let time_ms = fields.next()?.trim().parse::().ok()? * 1000.0; + let summary = fields.next().unwrap_or_default().to_owned(); + let patch = decode(record.get(newline + 1..).unwrap_or_default(), encoding); + let mut revision = BlockRevision { + id, + short_id, + author, + email, + time_ms, + summary, + path: String::new(), + before: String::new(), + after: String::new(), + before_line: 0, + after_line: 0, + }; + let mut in_hunk = false; + for line in patch.lines() { + if let Some(path) = line.strip_prefix("+++ ") { + revision.path = unquote(path).trim_start_matches("b/").to_owned(); + } else if let Some(ranges) = line.strip_prefix("@@ ") { + in_hunk = true; + let mut parts = ranges.split_whitespace(); + revision.before_line = hunk_start(parts.next().unwrap_or_default()); + revision.after_line = hunk_start(parts.next().unwrap_or_default()); + } else if in_hunk { + match line.as_bytes().first() { + Some(b' ') => { + push_line(&mut revision.before, &line[1..]); + push_line(&mut revision.after, &line[1..]); + } + Some(b'-') => push_line(&mut revision.before, &line[1..]), + Some(b'+') => push_line(&mut revision.after, &line[1..]), + _ => {} + } + } + } + Some(revision) +} + +/// 0-based first line of a hunk side (`-12,3` → 11; `+0,0` for a file that did not exist → 0). +fn hunk_start(side: &str) -> u32 { + let n = side.trim_start_matches(['-', '+']).split(',').next().unwrap_or_default(); + n.parse::().unwrap_or(0).saturating_sub(1) +} + +fn push_line(block: &mut String, line: &str) { + block.push_str(line); + block.push('\n'); +} + +/// A C-quoted path (`"b/a \"b\".txt"`), as git writes unusual names. +fn unquote(path: &str) -> String { + let Some(inner) = path.strip_prefix('"').and_then(|p| p.strip_suffix('"')) else { + return path.to_owned(); + }; + let mut out = String::new(); + let mut chars = inner.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('t') => out.push('\t'), + Some('n') => out.push('\n'), + Some(other) => out.push(other), + None => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HunkKind; + + fn hunk(start: u32, lines: u32, old_start: u32, old_lines: u32) -> LineHunk { + LineHunk { kind: HunkKind::Modified, start, lines, old_start, old_lines, old_text: String::new() } + } + + #[test] + fn maps_the_selection_to_head_lines() { + // Unchanged file. + assert_eq!(head_range(&[], (3, 5)), Some((3, 5))); + // Two lines inserted at 1: current 4..=6 were HEAD 2..=4; a reversed selection too. + let inserted = [hunk(1, 2, 1, 0)]; + assert_eq!(head_range(&inserted, (4, 6)), Some((2, 4))); + assert_eq!(head_range(&inserted, (6, 4)), Some((2, 4))); + // Selecting only the inserted lines: nothing committed. + assert_eq!(head_range(&inserted, (1, 2)), None); + // From an inserted line on: the HEAD lines after the insertion. + assert_eq!(head_range(&inserted, (2, 4)), Some((1, 2))); + // Three HEAD lines (5..=7) replaced by one: a selection ending in it takes all three. + let replaced = [hunk(5, 1, 5, 3)]; + assert_eq!(head_range(&replaced, (4, 5)), Some((4, 7))); + assert_eq!(head_range(&replaced, (6, 6)), Some((8, 8))); + // Two HEAD lines deleted at 2: later lines move down by two. + let deleted = [hunk(2, 0, 2, 2)]; + assert_eq!(head_range(&deleted, (2, 3)), Some((4, 5))); + } + + #[test] + fn parses_git_log_line_ranges() { + let out = [ + "\x1eaaaa1111\x1faaaa\x1fAda\x1fada@example.com\x1f1700000000\x1fRename greet", + "", + "diff --git a/src/a.txt b/src/b.txt", + "--- a/src/a.txt", + "+++ b/src/b.txt", + "@@ -2,2 +2,2 @@", + " keep", + "-hello", + "+hi", + "\x1ebbbb2222\x1fbbbb\x1fBo\x1fbo@example.com\x1f1600000000\x1fAdd", + "", + "diff --git a/src/a.txt b/src/a.txt", + "--- /dev/null", + "+++ b/src/a.txt", + "@@ -0,0 +2,2 @@", + "+keep", + "+hello", + ] + .join("\n"); + let revisions = parse_log(out.as_bytes(), TextEncoding::from(encoding_rs::UTF_8)); + assert_eq!(revisions.len(), 2); + let [newer, older] = &revisions[..] else { unreachable!() }; + assert_eq!((newer.id.as_str(), newer.short_id.as_str(), newer.author.as_str()), ("aaaa1111", "aaaa", "Ada")); + assert_eq!((newer.time_ms, newer.summary.as_str(), newer.path.as_str()), (1.7e12, "Rename greet", "src/b.txt")); + assert_eq!((newer.before.as_str(), newer.after.as_str()), ("keep\nhello\n", "keep\nhi\n")); + assert_eq!((newer.before_line, newer.after_line), (1, 1)); + assert_eq!((older.before.as_str(), older.after.as_str()), ("", "keep\nhello\n")); + assert_eq!((older.before_line, older.after_line), (0, 1)); + } + + #[test] + fn unquotes_c_quoted_paths() { + assert_eq!(unquote(r#""b/a \"q\"\tb.txt""#), "b/a \"q\"\tb.txt"); + assert_eq!(unquote("b/plain.txt"), "b/plain.txt"); + } +} diff --git a/crates/ide-vcs/src/history.rs b/crates/ide-vcs/src/history.rs index ebdc7b27..be0da59b 100644 --- a/crates/ide-vcs/src/history.rs +++ b/crates/ide-vcs/src/history.rs @@ -4,9 +4,8 @@ use crate::repo::{gx, resolve, to_string}; use crate::{ChangeKind, ChangedFile, Commit, LogOrder, LogQuery, Repo, RevisionChange}; use gix::ObjectId; use gix::bstr::ByteSlice; -use ide_core::{IdeError, IdeResult}; +use ide_core::{CancellationToken, IdeError, IdeResult}; use std::collections::{HashMap, HashSet}; -use std::path::Path; /// Log "Text or hash" filter: the message (substring or regex), or a hash prefix. enum TextFilter { @@ -50,52 +49,283 @@ impl TextFilter { } } -pub(crate) fn log(r: &Repo, q: &LogQuery) -> IdeResult> { +pub(crate) fn log(r: &Repo, q: &LogQuery, cancel: &CancellationToken) -> IdeResult> { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let mut index = r.log_index.lock().unwrap_or_else(|p| p.into_inner()); + let result = if q.summary_only { + let mut all = q.clone(); + all.skip = 0; + all.limit = 0; + all.seek = None; + all.retain_until = None; + all.request = None; + all.stop_at = None; + let refs = r.log_refs()?; + let shallow = r + .gix()? + .shallow_commits() + .map_err(gx)? + .map(|ids| ids.iter().copied().collect::>()) + .unwrap_or_default(); + let key = format!("{}:{refs:?}:{shallow:?}", serde_json::to_string(&all).map_err(gx)?); + let cached = index.views.get(&key).cloned(); + let view = if let Some(cached) = cached { + cached + } else { + // Immutable metadata is reused across roots, filters, merges and ref changes. + all.request = q.request; + let commits = + std::sync::Arc::new(crate::log_index::IndexedView::new(log_indexed(r, &all, cancel, &mut index)?)); + if index.views.len() >= 4 { + index.views.clear(); + } + index.views.insert(key, commits.clone()); + commits + }; + let start = q + .seek + .as_ref() + .and_then(|id| view.positions.get(id).copied()) + .map(|i| i.saturating_sub(if q.limit == 0 { 50 } else { 50.min(q.limit as usize / 2) })) + .unwrap_or(q.skip as usize); + let retain = q.retain_until.as_ref().and_then(|id| view.positions.get(id).copied()).map(|i| i + 1).unwrap_or(0); + let limit = if q.limit == 0 { usize::MAX } else { (q.limit as usize).max(retain.saturating_sub(start)) }; + let mut result = Vec::new(); + for (i, c) in view.commits.iter().enumerate().skip(start).take(limit) { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let mut c = c.clone(); + c.log_position = Some(i as u32); + c.log_total = Some(view.commits.len() as u32); + let stop = q.stop_at.as_ref() == Some(&c.id); + result.push(c); + if stop { + break; + } + } + Ok(result) + } else { + log_indexed(r, q, cancel, &mut index) + }; + index.flush(); + result +} + +fn log_indexed( + r: &Repo, + q: &LogQuery, + cancel: &CancellationToken, + index: &mut crate::log_index::LogIndex, +) -> IdeResult> { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } let repo = r.gix()?; + let shallow: HashSet = + repo.shallow_commits().map_err(gx)?.map(|ids| ids.iter().copied().collect()).unwrap_or_default(); let range = q.range.as_deref().map(str::trim).filter(|s| !s.is_empty()); let revs: Vec<&str> = q.branch.iter().chain(&q.branches).map(|b| b.trim()).filter(|b| !b.is_empty()).collect(); - let tips = if range.is_some() { Vec::new() } else { tips(&repo, &revs)? }; - if tips.is_empty() && range.is_none() { + let ranges: Vec<_> = q.ranges.iter().map(|s| s.trim()).filter(|s| !s.is_empty()).collect(); + let tips = + if range.is_some() || (!ranges.is_empty() && revs.is_empty()) { Vec::new() } else { tips(&repo, &revs)? }; + if tips.is_empty() && range.is_none() && ranges.is_empty() { return Ok(Vec::new()); } let decorations = decorations(&repo)?; - let path_filter = match q.path.as_deref() { - Some(p) => { - let rel = r.rel_path(p)?; - if rel.is_empty() { None } else { Some(rel) } + let paths: Vec = q + .path + .iter() + .chain(&q.paths) + .map(|p| r.rel_path(p)) + .collect::>>()? + .into_iter() + .filter(|p| !p.is_empty()) + .collect(); + let range_ids = if let Some(range) = range { + Some(range_ids(r, range, q.order, q.first_parent, cancel)?) + } else if !ranges.is_empty() { + let mut ids = Vec::new(); + let mut seen = HashSet::new(); + for range in &ranges { + for id in range_ids(r, range, q.order, q.first_parent, cancel)? { + if seen.insert(id) { + ids.push(id); + } + } } - None => None, + if !tips.is_empty() { + for id in range_ids( + r, + &tips.iter().map(ToString::to_string).collect::>().join(" "), + q.order, + q.first_parent, + cancel, + )? { + if seen.insert(id) { + ids.push(id); + } + } + } + Some(sort_subset(&repo, ids, q.order, q.first_parent, &shallow, cancel, &index.nodes)?) + } else { + None }; - let authors: Vec = - q.author.iter().chain(&q.authors).map(|a| a.trim().to_lowercase()).filter(|a| !a.is_empty()).collect(); - let text = q.text.as_deref().map(|t| TextFilter::new(t, q.text_regex, q.text_match_case)).transpose()?.flatten(); - let limit = if q.limit == 0 { usize::MAX } else { q.limit as usize }; - let mut to_skip = q.skip as usize; - - let path_walk = path_filter.is_some(); - type Walk<'a> = Box)>> + 'a>; - let walk: Walk<'_> = if let Some(range) = range { - let ids = range_ids(r, range, path_filter.as_deref())?; - let repo = &repo; - Box::new(ids.into_iter().map(move |id| { - let commit = repo.find_commit(id).map_err(gx)?; - Ok((id, commit.parent_ids().map(|p| p.detach()).collect())) - })) - } else if let Some(rel) = path_filter { - Box::new(PathWalk::new(&repo, tips, rel, q.first_parent)?) - } else if repo.is_shallow().unwrap_or(false) { - // Topological order needs complete history; shallow clones fall back to date order. - let mut walk = repo.rev_walk(tips).sorting(gix::revision::walk::Sorting::ByCommitTime(Default::default())); - if q.first_parent { - walk = walk.first_parent_only(); + let mut history_paths = HashMap::new(); + let path_ids = if paths.is_empty() { + None + } else { + let mut ids = HashSet::new(); + if q.follow && paths.len() == 1 { + let rel = &paths[0]; + let mut args = vec![ + "log".to_string(), + "--format=%x00%H".into(), + "--name-status".into(), + "-z".into(), + "--follow".into(), + "--topo-order".into(), + ]; + if q.first_parent { + args.push("--first-parent".into()); + } + args.push("--end-of-options".into()); + let scopes = if let Some(range) = range { + vec![range.to_string()] + } else { + let mut scopes = ranges.iter().map(|s| s.to_string()).collect::>(); + if !tips.is_empty() { + scopes.push(tips.iter().map(ToString::to_string).collect::>().join(" ")); + } + scopes + }; + for scope in scopes { + let mut args = args.clone(); + args.extend(scope.split_whitespace().map(str::to_string)); + args.extend(["--".into(), rel.clone()]); + let output = crate::cli::run_cancellable( + r.root(), + &args.iter().map(String::as_str).collect::>(), + cancel, + )?; + let text = output.stdout_str(); + let mut tokens = text.split('\0').peekable(); + let mut current_path = rel.clone(); + while let Some(token) = tokens.next() { + let token = token.trim_start_matches('\n'); + if let Ok(id) = ObjectId::from_hex(token.as_bytes()) { + ids.insert(id); + history_paths.insert(id, current_path.clone()); + } else if token.starts_with('R') && token[1..].bytes().all(|b| b.is_ascii_digit()) { + if let (Some(old), Some(new)) = (tokens.next(), tokens.next()) + && new == current_path + { + current_path = old.to_string(); + } + } else if ["A", "D", "M", "T", "U"].contains(&token) { + tokens.next(); + } + } + } + } else { + let starts = range_ids.as_ref().cloned().unwrap_or_else(|| tips.clone()); + ids.extend( + PathWalk::new(&repo, starts, paths.clone(), q.first_parent, cancel, index)? + .map(|item| item.map(|(id, _)| id)) + .collect::>>()?, + ); } - let walk = walk.all().map_err(gx)?; - Box::new(walk.map(|info| info.map(|i| (i.id, i.parent_ids.to_vec())).map_err(gx))) + Some(ids) + }; + let authors: Vec<_> = + q.author.iter().chain(&q.authors).map(|a| a.trim().to_lowercase()).filter(|a| !a.is_empty()).collect(); + let user_candidates = if q.authors_exact && !authors.is_empty() { index.user_candidates(&authors) } else { None }; + let path_candidates = if !q.follow && !paths.is_empty() { index.path_candidates(&paths) } else { None }; + let text_candidates = q.text.as_deref().filter(|_| !q.text_regex).and_then(|s| index.message_candidates(s)); + let mut visibility = Visibility { + speed_refs: &decorations, + text_candidates, + user_candidates, + path_candidates, + query: q, + path_ids, + range_ids: range_ids.as_ref().map(|ids| ids.iter().copied().collect()), + authors, + text: q.text.as_deref().map(|t| TextFilter::new(t, q.text_regex, q.text_match_case)).transpose()?.flatten(), + matches: HashMap::new(), + projected: HashMap::new(), + cancel, + index, + shallow: shallow.clone(), + }; + type Walk<'a> = Box> + 'a>; + let walk: Walk<'_> = if let Some(ids) = range_ids { + Box::new(ids.into_iter().map(Ok)) + } else if q.summary_only { + let ids = visibility.index.graph(&repo, &tips, &shallow, q.first_parent, cancel)?; + Box::new( + sort_subset(&repo, ids, q.order, q.first_parent, &shallow, cancel, &visibility.index.nodes)? + .into_iter() + .map(Ok), + ) + } else if repo.is_shallow().unwrap_or(false) || q.request.is_some() { + // The process can be interrupted even during indegree preparation. Git also + // respects shallow boundaries while retaining a topological order. + let mut args = vec![ + "rev-list".to_string(), + match q.order { + LogOrder::Topo | LogOrder::Standard => "--topo-order", + LogOrder::Date => "--date-order", + } + .into(), + ]; + if q.first_parent { + args.push("--first-parent".into()); + } + if q.limit > 0 + && q.order != LogOrder::Standard + && paths.is_empty() + && q.authors.is_empty() + && q.author.is_none() + && q.text.as_deref().is_none_or(|text| text.trim().is_empty()) + && !q.no_merges + && q.since_ms.is_none() + && q.until_ms.is_none() + { + args.push(format!("--max-count={}", u64::from(q.skip) + u64::from(q.limit))); + } + args.push("--end-of-options".into()); + args.extend(tips.iter().map(ToString::to_string)); + let result = + crate::cli::run_cancellable(r.root(), &args.iter().map(String::as_str).collect::>(), cancel)?; + let ids = result + .stdout_str() + .lines() + .map(|line| ObjectId::from_hex(line.as_bytes()).map_err(gx)) + .collect::>>()?; + if repo.is_shallow().unwrap_or(false) { + // A separately bounded walk provides the domain; pagination must never + // turn a shallow boundary into a connection through a locally present object. + let mut boundary_args = vec!["rev-list", "--end-of-options"]; + let ids_as_text: Vec<_> = tips.iter().map(ToString::to_string).collect(); + boundary_args.extend(ids_as_text.iter().map(String::as_str)); + let boundaries = crate::cli::run_cancellable(r.root(), &boundary_args, cancel)?; + visibility.range_ids = Some( + boundaries + .stdout_str() + .lines() + .map(|id| ObjectId::from_hex(id.as_bytes()).map_err(gx)) + .collect::>>()?, + ); + } + Box::new(ids.into_iter().map(Ok)) } else { use gix::traverse::commit::{Parents, topo}; let graph = repo.commit_graph_if_enabled().ok().flatten(); let sorting = match q.order { - LogOrder::Topo => topo::Sorting::TopoOrder, + LogOrder::Topo | LogOrder::Standard => topo::Sorting::TopoOrder, LogOrder::Date => topo::Sorting::DateOrder, }; let topo = topo::Builder::from_iters(&repo.objects, tips, None::>) @@ -104,226 +334,385 @@ pub(crate) fn log(r: &Repo, q: &LogQuery) -> IdeResult> { .with_commit_graph(graph) .build() .map_err(gx)?; - Box::new(topo.map(|info| info.map(|i| (i.id, i.parent_ids.to_vec())).map_err(gx))) + Box::new(topo.map(|info| info.map(|i| i.id).map_err(gx))) }; - - let shape = Shape { - no_merges: q.no_merges, - first_parent: q.first_parent, - since: q.since_ms.map(|ms| (ms / 1000.0).floor() as i64), - until: q.until_ms.map(|ms| (ms / 1000.0).ceil() as i64), + let walk: Walk<'_> = if q.order == LogOrder::Standard { + let ids = walk.collect::>>()?; + Box::new( + sort_subset(&repo, ids, q.order, q.first_parent, &shallow, cancel, &visibility.index.nodes)? + .into_iter() + .map(Ok), + ) + } else { + walk }; - // Walks in (roughly) commit time order can stop once they are well past `since`. - let time_ordered = range.is_none() && (q.order == LogOrder::Date || path_walk); - let mut older_run = 0usize; - // A user or text filter hides commits within the history: the shown ones are linked to their - // nearest shown ancestors instead (dashed past hidden ones), as IDEA's filtered graph does. - // A path walk already links commits that way. - let mut links = (!path_walk && (!authors.is_empty() || text.is_some())).then(FilteredLinks::default); - let mut extra = 0usize; - - let mut out = Vec::new(); - for item in walk { - let (id, mut parents) = item?; - if q.first_parent { - parents.truncate(1); - } - if out.len() >= limit { - // Full: walk on only to link the last commits to their shown ancestors. - let Some(l) = links.as_ref() else { break }; - extra += 1; - if l.is_empty() || extra > LINK_LIMIT { - break; - } - } - let commit = repo.find_commit(id).map_err(gx)?; - let commit_time = commit.time().map(|t| t.seconds).unwrap_or(0); - let mut shown = true; - if shape.is_active() { - let merge = commit.parent_ids().nth(1).is_some(); - if shape.since.is_some_and(|s| commit_time < s) { - older_run += 1; - if time_ordered && older_run >= OLDER_RUN_LIMIT { + let head = repo.head_id().ok().map(|id| id.detach()); + let shallow_stamp = + repo.shallow_file().ok().and_then(|path| std::fs::metadata(path).ok()).and_then(|m| m.modified().ok()); + let stamp = head.map(|id| (id, shallow_stamp)); + if visibility.index.head != stamp { + let mut additions = Vec::new(); + let mut cursor = head; + let old_head = visibility.index.head; + if let (Some((old, old_shallow)), Some(_)) = (old_head, head) + && old_shallow == shallow_stamp + { + while let Some(id) = cursor { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + if id == old { break; } - shown = false; - } else { - older_run = 0; - if !shape.shows(merge, commit_time) { - shown = false; - } else if links.is_none() { - parents = shape.rewrite_parents(&repo, &parents)?; + let record = visibility.index.get(&repo, id)?; + if record.parents.len() != 1 { + cursor = None; + break; + } + additions.push(id); + cursor = Some(ObjectId::from_hex(record.parents[0].as_bytes()).map_err(gx)?); + } + } else { + cursor = None; + } + if cursor.is_some() && cursor == old_head.map(|(id, _)| id) { + visibility.index.reachable.extend(additions); + visibility.index.head = stamp; + } else { + let mut reachable = HashSet::new(); + if let Some(head) = head { + for item in repo.rev_walk([head]).all().map_err(gx)? { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + reachable.insert(item.map_err(gx)?.id); } } - } - let sig = commit.author().map_err(gx)?; - let name = to_string(sig.name.trim().as_bstr()); - let email = to_string(sig.email.trim().as_bstr()); - if shown && !authors.is_empty() { - let (name, email) = (name.to_lowercase(), email.to_lowercase()); - shown = authors.iter().any(|a| name.contains(a.as_str()) || email.contains(a.as_str())); - } - let message = to_string(commit.message_raw_sloppy()); - if shown && let Some(filter) = text.as_ref() { - shown = filter.matches(&message, &id); - } - let listed = shown && to_skip == 0 && out.len() < limit; - if let Some(l) = links.as_mut() { - l.visit(id, &parents, shown, listed.then_some(out.len())); - } - if !shown { - continue; - } - if to_skip > 0 { - to_skip -= 1; - continue; - } - if !listed { - continue; - } - let time_ms = sig.time().map(|t| t.seconds as f64 * 1000.0).unwrap_or(0.0); - let summary = message.lines().find(|l| !l.trim().is_empty()).unwrap_or("").trim().to_string(); - let hex = id.to_string(); - out.push(Commit { - short_id: hex[..hex.len().min(8)].to_string(), - id: hex, - parents: parents.iter().map(ToString::to_string).collect(), - author: name, - email, - time_ms, - summary, - message: message.trim_end().to_string(), - refs: decorations.get(&id).cloned().unwrap_or_default(), - commit_time_ms: commit_time as f64 * 1000.0, - dashed: Vec::new(), - }); - if out.len() >= limit && links.is_none() { - break; + visibility.index.reachable = reachable; + visibility.index.head = stamp; } } - if let Some(l) = links { - l.apply(&mut out); + let mut out = Vec::new(); + let mut skip = q.skip as usize; + let limit = if q.limit == 0 { usize::MAX } else { q.limit as usize }; + for item in walk { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let id = item?; + if !visibility.shows(&repo, id)? { + continue; + } + if skip > 0 { + skip -= 1; + continue; + } + let record = visibility.index.get(&repo, id)?; + let parents = record + .parents + .iter() + .map(|p| ObjectId::from_hex(p.as_bytes()).map_err(gx)) + .collect::>>()?; + let shown_parents = if shallow.contains(&id) { Vec::new() } else { visibility.parents(&repo, &parents)? }; + let mut commit = (*record).clone(); + commit.graph_dashed = shown_parents.iter().map(|p| !parents.contains(p)).collect(); + commit.dashed = if commit.graph_dashed.iter().any(|d| *d) { commit.graph_dashed.clone() } else { Vec::new() }; + commit.graph_parents = Some(shown_parents.iter().map(ToString::to_string).collect()); + if shallow.contains(&id) { + commit.graph_boundary_parents = commit.parents.clone(); + } + commit.on_current_branch = Some(visibility.index.reachable.contains(&id)); + commit.refs = decorations.get(&id).cloned().unwrap_or_default(); + commit.history_path = history_paths.get(&id).cloned(); + if !q.summary_only { + commit.message = visibility.index.message(&repo, id)?; + commit.message_truncated = false; + } + out.push(commit); + if out.len() >= limit || q.stop_at.as_deref() == Some(id.to_string().as_str()) { + break; + } } Ok(out) } -/// Commits walked past a full page to link its last commits to their shown ancestors. -const LINK_LIMIT: usize = 20_000; - -/// The links of a filtered log: each listed commit's nearest shown ancestors. The walk visits -/// children before parents, so a listed commit waits on its parents; a hidden commit passes -/// the waits on to its own parents, and a shown one takes them. -#[derive(Default)] -struct FilteredLinks { - /// Commit → listed commits (by index) waiting on it, and whether the way passed a hidden one. - waiting: HashMap>, - /// Per listed commit: its shown ancestors in the order found, and whether each link is dashed. - found: HashMap>, -} - -impl FilteredLinks { - fn is_empty(&self) -> bool { - self.waiting.is_empty() - } - - fn visit(&mut self, id: ObjectId, parents: &[ObjectId], shown: bool, listed: Option) { - let waits = self.waiting.remove(&id).unwrap_or_default(); - if shown { - for (child, dashed) in waits { - let links = self.found.entry(child).or_default(); - match links.iter_mut().find(|(p, _)| *p == id) { - Some(link) => link.1 &= dashed, - None => links.push((id, dashed)), - } - } - if let Some(index) = listed { - for p in parents { - Self::wait(&mut self.waiting, *p, index, false); - } - } - } else { - for p in parents { - for (child, _) in &waits { - Self::wait(&mut self.waiting, *p, *child, true); - } - } - } - } - - fn wait(waiting: &mut HashMap>, on: ObjectId, child: usize, dashed: bool) { - let list = waiting.entry(on).or_default(); - match list.iter_mut().find(|(c, _)| *c == child) { - Some(w) => w.1 &= dashed, - None => list.push((child, dashed)), - } - } - - /// Replaces the listed commits' parents by their shown ancestors (none found: none). - fn apply(mut self, out: &mut [Commit]) { - for (index, commit) in out.iter_mut().enumerate() { - let links = self.found.remove(&index).unwrap_or_default(); - commit.parents = links.iter().map(|(id, _)| id.to_string()).collect(); - commit.dashed = - if links.iter().any(|(_, d)| *d) { links.iter().map(|(_, d)| *d).collect() } else { Vec::new() }; - } - } -} - -/// Consecutive commits older than `since` after which a time-ordered walk stops. -const OLDER_RUN_LIMIT: usize = 64; - -/// Which commits the Log graph shows independently of the detail filters (author, text): -/// `--no-merges` and the date range. Hidden commits are skipped and the parents of shown -/// ones rewritten past them, like git's parent rewriting. -struct Shape { - no_merges: bool, +/// A ready-node stack gives incoming merge parents priority, while child counts +/// prevent an ancestor shared with another branch from moving ahead of its children. +fn sort_subset( + repo: &gix::Repository, + ids: Vec, + order: LogOrder, first_parent: bool, - /// Commit time bounds in seconds; `since` inclusive, `until` exclusive. - since: Option, - until: Option, + shallow: &HashSet, + cancel: &CancellationToken, + nodes: &HashMap, i64)>, +) -> IdeResult> { + let positions: HashMap<_, _> = ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + let mut parents = Vec::with_capacity(ids.len()); + let mut children = vec![0usize; ids.len()]; + let mut times = Vec::with_capacity(ids.len()); + for id in &ids { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let (all_parents, time) = if let Some(record) = nodes.get(id) { + record.clone() + } else { + let commit = repo.find_commit(*id).map_err(gx)?; + (commit.parent_ids().map(|p| p.detach()).collect(), commit.time().map(|t| t.seconds).unwrap_or(0)) + }; + times.push(time); + let mut ordered = Vec::new(); + let mut pending: Vec<_> = all_parents + .into_iter() + .take(if shallow.contains(id) { + 0 + } else if first_parent { + 1 + } else { + usize::MAX + }) + .collect(); + let mut visited = HashSet::new(); + while let Some(parent) = pending.pop() { + if !visited.insert(parent) { + continue; + } + if let Some(position) = positions.get(&parent) { + if !ordered.contains(position) { + ordered.push(*position); + } + } else if !shallow.contains(&parent) && repo.has_object(parent) { + pending.extend( + repo.find_commit(parent) + .map_err(gx)? + .parent_ids() + .take(if first_parent { 1 } else { usize::MAX }) + .map(|p| p.detach()), + ); + } + } + ordered.reverse(); + for p in &ordered { + children[*p] += 1; + } + parents.push(ordered); + } + if order == LogOrder::Standard { + let refs = decorations(repo)?; + let mut remote_names = HashSet::new(); + for reference in repo.references().map_err(gx)?.remote_branches().map_err(gx)?.flatten() { + if let Some(name) = reference.name().as_bstr().strip_prefix(b"refs/remotes/") { + remote_names.insert(to_string(name.as_bstr())); + } + } + let priority = |i: usize| { + let labels = refs.get(&ids[i]); + let best = labels + .into_iter() + .flatten() + .map(|name| { + ( + if name == "HEAD" { + 0 + } else if name == "main" || name == "master" { + 1 + } else if name.starts_with("tag: ") { + 4 + } else if remote_names.contains(name) { + 3 + } else { + 2 + }, + name.clone(), + ) + }) + .min() + .unwrap_or((5, ids[i].to_string())); + (best, i) + }; + let mut heads: Vec<_> = (0..ids.len()).filter(|i| children[*i] == 0 || refs.contains_key(&ids[*i])).collect(); + heads.sort_by_key(|i| priority(*i)); + let sorted = crate::bek::sort(&parents, ×, &heads); + return Ok(sorted.into_iter().map(|i| ids[i]).collect()); + } + let mut ready: std::collections::BTreeSet<_> = + children.iter().enumerate().filter_map(|(i, count)| (*count == 0).then_some(i)).collect(); + let mut stack = Vec::new(); + let mut dates = std::collections::BinaryHeap::new(); + if order == LogOrder::Date { + for &i in &ready { + dates.push((times[i], std::cmp::Reverse(i))); + } + } + let mut emitted = HashSet::new(); + let mut out = Vec::with_capacity(ids.len()); + while let Some(next) = + if order == LogOrder::Date { dates.pop().map(|(_, i)| i.0) } else { stack.pop().or_else(|| ready.pop_first()) } + { + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + if !emitted.insert(next) { + continue; + } + ready.remove(&next); + out.push(ids[next]); + let ordered: Vec<_> = + if order == LogOrder::Topo { parents[next].iter().rev().copied().collect() } else { parents[next].clone() }; + for parent in ordered { + children[parent] -= 1; + if children[parent] == 0 { + ready.insert(parent); + stack.push(parent); + if order == LogOrder::Date { + dates.push((times[parent], std::cmp::Reverse(parent))); + } + } + } + } + Ok(out) } -/// Upper bound of commits visited while rewriting one commit's parents. -const REWRITE_LIMIT: usize = 4096; +/// Visibility and graph projection share exactly the same predicates. Revision parents +/// remain immutable; only graph parents skip hidden commits. Memoisation avoids walking +/// the same hidden ancestry for each visible child, without a truncation limit. +struct Visibility<'a> { + speed_refs: &'a HashMap>, + query: &'a LogQuery, + path_ids: Option>, + range_ids: Option>, + authors: Vec, + text: Option, + text_candidates: Option>, + user_candidates: Option>, + path_candidates: Option>, + matches: HashMap, + projected: HashMap>, + cancel: &'a CancellationToken, + index: &'a mut crate::log_index::LogIndex, + shallow: HashSet, +} -impl Shape { - fn is_active(&self) -> bool { - self.no_merges || self.since.is_some() || self.until.is_some() +impl Visibility<'_> { + fn shows(&mut self, repo: &gix::Repository, id: ObjectId) -> IdeResult { + if self.cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + if let Some(shown) = self.matches.get(&id) { + return Ok(*shown); + } + let indexed = self.index.indexed(&id.to_string()); + let shown = if (indexed && self.user_candidates.as_ref().is_some_and(|ids| !ids.contains(&id.to_string()))) + || (indexed && self.path_candidates.as_ref().is_some_and(|ids| !ids.contains(&id.to_string()))) + || !repo.has_object(id) + || self.path_ids.as_ref().is_some_and(|ids| !ids.contains(&id)) + || self.range_ids.as_ref().is_some_and(|ids| !ids.contains(&id)) + { + false + } else { + let commit = self.index.get(repo, id)?; + let time = commit.commit_time_ms; + let name = commit.author.to_lowercase(); + let email = commit.email.to_lowercase(); + !(self.query.no_merges && commit.parents.len() > 1) + && self.query.since_ms.is_none_or(|s| time >= s) + && self.query.until_ms.is_none_or(|u| time < u) + && self.query.speed_search.as_ref().is_none_or(|query| { + let mut columns = + vec![commit.summary.clone(), self.speed_refs.get(&id).cloned().unwrap_or_default().join(" ")]; + if self.query.search_fields.iter().any(|f| f == "author") { + columns.push(commit.author.clone()); + } + if self.query.search_fields.iter().any(|f| f == "hash") { + columns.push(commit.short_id.clone()); + } + query.to_lowercase().split_whitespace().all(|word| { + columns.iter().any(|column| { + let mut chars = word.chars().peekable(); + for c in column.to_lowercase().chars() { + if chars.peek() == Some(&c) { + chars.next(); + } + } + chars.peek().is_none() + }) + }) + }) + && (self.authors.is_empty() + || self.authors.iter().any(|a| { + if self.query.authors_exact { + crate::log_index::matches_user(a, &name, &email) + } else { + name.contains(a.as_str()) || email.contains(a.as_str()) + } + })) + && match &self.text { + Some(text) => { + self.text_candidates.as_ref().is_none_or(|ids| { + ids.contains(&id.to_string()) + || !self.index.indexed(&id.to_string()) + || id.to_string().starts_with( + self.query.text.as_deref().unwrap_or("").trim().to_ascii_lowercase().as_str(), + ) + }) && text.matches(&self.index.message(repo, id)?, &id) + } + None => true, + } + }; + self.matches.insert(id, shown); + Ok(shown) } - fn shows(&self, merge: bool, time: i64) -> bool { - !(self.no_merges && merge) && self.since.is_none_or(|s| time >= s) && self.until.is_none_or(|u| time < u) - } - - /// `parents` with each hidden one replaced by its nearest shown ancestors; parents - /// older than `since` are dropped (their history is not shown). - fn rewrite_parents(&self, repo: &gix::Repository, parents: &[ObjectId]) -> IdeResult> { - let mut out: Vec = Vec::new(); - let mut seen: HashSet = HashSet::new(); - let mut stack: Vec = parents.iter().rev().copied().collect(); - let mut visited = 0usize; - while let Some(p) = stack.pop() { - if !seen.insert(p) || !repo.has_object(p) { - continue; + fn parents(&mut self, repo: &gix::Repository, parents: &[ObjectId]) -> IdeResult> { + let parents = if self.query.first_parent { &parents[..parents.len().min(1)] } else { parents }; + let mut out = Vec::new(); + for parent in parents { + // Iterative postorder: a long hidden chain must not consume the call stack. + let mut stack = vec![(*parent, false)]; + while let Some((id, expanded)) = stack.pop() { + if self.projected.contains_key(&id) { + continue; + } + if self.shows(repo, id)? { + self.projected.insert(id, vec![id]); + } else if !repo.has_object(id) { + self.projected.insert(id, Vec::new()); + } else { + let commit = self.index.get(repo, id)?; + let ancestors = commit + .parents + .iter() + .take(if self.shallow.contains(&id) { + 0 + } else if self.query.first_parent { + 1 + } else { + usize::MAX + }) + .map(|p| ObjectId::from_hex(p.as_bytes()).map_err(gx)) + .collect::>>()?; + if !expanded { + stack.push((id, true)); + stack.extend(ancestors.iter().rev().map(|p| (*p, false))); + } else { + let mut nearest = Vec::new(); + for p in ancestors { + for ancestor in self.projected.get(&p).into_iter().flatten() { + if !nearest.contains(ancestor) { + nearest.push(*ancestor); + } + } + } + self.projected.insert(id, nearest); + } + } } - visited += 1; - if visited > REWRITE_LIMIT { - break; - } - let commit = repo.find_commit(p).map_err(gx)?; - let time = commit.time().map(|t| t.seconds).unwrap_or(0); - if self.since.is_some_and(|s| time < s) { - continue; - } - let ids: Vec = commit.parent_ids().map(|i| i.detach()).collect(); - if self.shows(ids.len() > 1, time) { - if !out.contains(&p) { - out.push(p); + for ancestor in &self.projected[parent] { + if !out.contains(ancestor) { + out.push(*ancestor); } - continue; } - let next = if self.first_parent { &ids[..ids.len().min(1)] } else { &ids[..] }; - stack.extend(next.iter().rev()); } Ok(out) } @@ -331,18 +720,25 @@ impl Shape { /// Commit ids of a revision range (`A..B`, `A...B`, `^A B`…), topologically ordered, via /// `git rev-list` so every rev syntax git understands (`@{u}`, `HEAD~3`) works. -fn range_ids(r: &Repo, range: &str, path: Option<&str>) -> IdeResult> { - let mut args: Vec<&str> = vec!["rev-list", "--topo-order", "--end-of-options"]; +fn range_ids( + r: &Repo, + range: &str, + order: LogOrder, + first_parent: bool, + cancel: &CancellationToken, +) -> IdeResult> { + let mut args: Vec<&str> = vec!["rev-list", if order == LogOrder::Date { "--date-order" } else { "--topo-order" }]; + if first_parent { + args.push("--first-parent"); + } + args.push("--end-of-options"); for part in range.split_whitespace() { if part.starts_with('-') { return Err(ide_core::IdeError::invalid(format!("invalid revision range '{range}'"))); } args.push(part); } - if let Some(rel) = path { - args.extend(["--", rel]); - } - let out = crate::cli::run(r.root(), &args, None)?; + let out = crate::cli::run_cancellable(r.root(), &args, cancel)?; out.stdout_str() .lines() .filter(|l| !l.trim().is_empty()) @@ -354,11 +750,69 @@ fn range_ids(r: &Repo, range: &str, path: Option<&str>) -> IdeResult IdeResult> { let mut tips = Vec::new(); if !revs.is_empty() { - for rev in revs { - let id = resolve(repo, rev)?; - let commit = repo.find_object(id).map_err(gx)?.peel_to_commit().map_err(gx)?; - if !tips.contains(&commit.id) { - tips.push(commit.id); + let refs = decorations(repo)?; + let branches: Vec<_> = refs + .iter() + .flat_map(|(id, names)| { + names + .iter() + .filter(|name| *name != "HEAD" && !name.starts_with("tag: ")) + .map(move |name| (name.as_str(), *id)) + }) + .collect(); + let mut excludes = Vec::new(); + let includes: Vec<_> = revs.iter().filter(|rev| !rev.starts_with('-')).collect(); + for rev in revs.iter().filter_map(|rev| rev.strip_prefix('-')) { + let re = if branches.iter().any(|(name, _)| *name == rev) { + None + } else { + Some( + regex_lite::Regex::new(&format!("^(?:{rev})$")) + .map_err(|e| IdeError::invalid(format!("invalid branch pattern: {e}")))?, + ) + }; + excludes.push((rev, re)); + } + if includes.is_empty() { + for (name, id) in &branches { + if !excludes.iter().any(|(exact, re)| name == exact || re.as_ref().is_some_and(|re| re.is_match(name))) + && !tips.contains(id) + { + tips.push(*id); + } + } + } + for rev in includes { + if excludes.iter().any(|(exact, re)| *rev == *exact || re.as_ref().is_some_and(|re| re.is_match(rev))) { + continue; + } + match resolve(repo, rev) { + Ok(id) => { + let commit = repo.find_object(id).map_err(gx)?.peel_to_commit().map_err(gx)?; + if !tips.contains(&commit.id) { + tips.push(commit.id); + } + } + Err(error) => { + let re = regex_lite::Regex::new(&format!("^(?:{rev})$")).map_err(|_| error.clone())?; + let matches: Vec<_> = branches + .iter() + .filter(|(name, _)| { + re.is_match(name) + && !excludes + .iter() + .any(|(exact, re)| name == exact || re.as_ref().is_some_and(|re| re.is_match(name))) + }) + .collect(); + if matches.is_empty() && !rev.chars().any(|c| ".*+?[]()|^$".contains(c)) { + return Err(error); + } + for (_, id) in matches { + if !tips.contains(id) { + tips.push(*id); + } + } + } } } return Ok(tips); @@ -367,7 +821,9 @@ fn tips(repo: &gix::Repository, revs: &[&str]) -> IdeResult> { tips.push(id.detach()); } let platform = repo.references().map_err(gx)?; - for iter in [platform.local_branches().map_err(gx)?, platform.remote_branches().map_err(gx)?] { + for iter in + [platform.local_branches().map_err(gx)?, platform.remote_branches().map_err(gx)?, platform.tags().map_err(gx)?] + { for reference in iter { let Ok(mut reference) = reference else { continue }; if let Ok(id) = reference.peel_to_id() { @@ -382,7 +838,7 @@ fn tips(repo: &gix::Repository, revs: &[&str]) -> IdeResult> { } /// Commit id → decoration labels. -fn decorations(repo: &gix::Repository) -> IdeResult>> { +pub(crate) fn decorations(repo: &gix::Repository) -> IdeResult>> { let mut map: HashMap> = HashMap::new(); if let Ok(id) = repo.head_id() { map.entry(id.detach()).or_default().push("HEAD".to_string()); @@ -414,24 +870,35 @@ fn decorations(repo: &gix::Repository) -> IdeResult { repo: &'r gix::Repository, - rel: String, + paths: Vec, queue: std::collections::BinaryHeap<(i64, u64, ObjectId)>, seen: HashSet, - memo: HashMap>, + index: &'r mut crate::log_index::LogIndex, seq: u64, first_parent: bool, + cancel: &'r CancellationToken, + shallow: HashSet, } impl<'r> PathWalk<'r> { - fn new(repo: &'r gix::Repository, tips: Vec, rel: String, first_parent: bool) -> IdeResult { + fn new( + repo: &'r gix::Repository, + tips: Vec, + paths: Vec, + first_parent: bool, + cancel: &'r CancellationToken, + index: &'r mut crate::log_index::LogIndex, + ) -> IdeResult { let mut walk = PathWalk { repo, - rel, + paths, queue: Default::default(), seen: Default::default(), - memo: Default::default(), + index, seq: 0, first_parent, + cancel, + shallow: repo.shallow_commits().map_err(gx)?.map(|ids| ids.iter().copied().collect()).unwrap_or_default(), }; for tip in tips { walk.push(tip)?; @@ -440,6 +907,9 @@ impl<'r> PathWalk<'r> { } fn push(&mut self, id: ObjectId) -> IdeResult<()> { + if self.cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } if !self.seen.insert(id) || !self.repo.has_object(id) { return Ok(()); } @@ -458,13 +928,23 @@ impl<'r> PathWalk<'r> { let present: Vec = parents .iter() .copied() + .take(if self.shallow.contains(&id) { 0 } else { usize::MAX }) .filter(|p| self.repo.has_object(p)) .take(if self.first_parent { 1 } else { usize::MAX }) .collect(); - let mine = path_entry(self.repo, id, &self.rel, &mut self.memo)?; + let mine = self + .paths + .iter() + .map(|path| self.index.path_entry(self.repo, id, path)) + .collect::>>()?; let mut same_parent = None; for p in &present { - if path_entry(self.repo, *p, &self.rel, &mut self.memo)? == mine { + let theirs = self + .paths + .iter() + .map(|path| self.index.path_entry(self.repo, *p, path)) + .collect::>>()?; + if theirs == mine { same_parent = Some(*p); break; } @@ -475,7 +955,7 @@ impl<'r> PathWalk<'r> { for p in &present { self.push(*p)?; } - if !present.is_empty() || mine.is_some() { + if !present.is_empty() || mine.iter().any(Option::is_some) { return Ok(Some((id, parents))); } } @@ -492,23 +972,6 @@ impl Iterator for PathWalk<'_> { } } -/// Id of the tree entry at `rel` in `commit` (memoised). -fn path_entry( - repo: &gix::Repository, - commit: ObjectId, - rel: &str, - memo: &mut HashMap>, -) -> IdeResult> { - if let Some(v) = memo.get(&commit) { - return Ok(*v); - } - let commit_obj = repo.find_commit(commit).map_err(gx)?; - let tree = commit_obj.tree().map_err(gx)?; - let id = tree.lookup_entry_by_path(Path::new(rel)).map_err(gx)?.map(|e| e.object_id()); - memo.insert(commit, id); - Ok(id) -} - pub(crate) fn commit_files(r: &Repo, rev: &str) -> IdeResult> { let repo = r.gix()?; Ok(first_parent_changes(&repo, rev)?.files) diff --git a/crates/ide-vcs/src/lib.rs b/crates/ide-vcs/src/lib.rs index a14c2344..7adf881e 100644 --- a/crates/ide-vcs/src/lib.rs +++ b/crates/ide-vcs/src/lib.rs @@ -77,28 +77,99 @@ pub struct BlameLine { pub summary: String, } +/// History for Selection: the commits that changed a block of lines, newest first. +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BlockHistory { + /// The selected lines in HEAD (0-based, inclusive); None when none of them is committed. + pub head_lines: Option<(u32, u32)>, + pub revisions: Vec, +} + +/// A commit that changed the block, with the block in its first parent and in it. +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BlockRevision { + pub id: String, + pub short_id: String, + pub author: String, + pub email: String, + pub time_ms: f64, + pub summary: String, + /// Repository-relative path of the file in this commit (before a later rename). + pub path: String, + /// The block before the commit; empty when the commit added it. + pub before: String, + pub after: String, + /// 0-based first line of the block before and after the commit. + pub before_line: u32, + pub after_line: u32, +} + #[derive(Clone, Debug, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] pub struct Commit { pub id: String, pub short_id: String, pub parents: Vec, + /// Connections in the filtered graph; never use these as revision/diff parents. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub graph_parents: Option>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub graph_dashed: Vec, + /// Repository-relative filename at this revision in rename-following history. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub history_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub log_position: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub log_total: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub graph_boundary_parents: Vec, + /// Reachability from HEAD, computed independently of the visible filters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub on_current_branch: Option, pub author: String, pub email: String, + #[serde(default)] + pub committer: String, + #[serde(default)] + pub committer_email: String, pub time_ms: f64, pub summary: String, pub message: String, + #[serde(default)] + pub message_truncated: bool, pub refs: Vec, /// Committer time (Log › View Options › Commit Timestamp). #[serde(default)] pub commit_time_ms: f64, - /// Per parent, under a user or text filter: the link passes commits the filter hides - /// (`parents` then name the nearest shown ancestors). Empty when no parent is rewritten. + /// Legacy alias of `graph_dashed`, indexed by displayed parents. #[serde(default)] #[specta(optional)] pub dashed: Vec, } +#[derive(Clone, Debug, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct LogDetails { + pub message: String, + pub containing_branches: Vec, + /// Git's verification code: G/U good, B bad, E error, N unsigned, X/Y expired, R revoked. + pub signature: String, + pub signer: String, + pub key: String, + pub fingerprint: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct LogIndexProgress { + pub indexed: u32, + pub total: u32, + pub running: bool, +} + /// Order of the Git Log (Graph Options › Sort). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] @@ -108,13 +179,29 @@ pub enum LogOrder { Topo, /// `--date-order`: commit time, but never a parent before its children. Date, + // Topological grouping with incoming merge parents visited before the main line. + Standard, } #[derive(Clone, Debug, Default, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] pub struct LogQuery { + #[serde(default)] + pub speed_search: Option, + #[serde(default)] + pub search_fields: Vec, + /// Jump directly to an indexed window containing this commit (50 rows of context). + #[serde(default)] + pub seek: Option, + #[serde(default)] + pub retain_until: Option, pub branch: Option, pub path: Option, + #[serde(default)] + pub paths: Vec, + /// File history follows renames; structure filters retain fixed paths. + #[serde(default)] + pub follow: bool, pub author: Option, pub text: Option, pub skip: u32, @@ -123,6 +210,9 @@ pub struct LogQuery { /// not the left). Takes precedence over `branch`. #[serde(default)] pub range: Option, + /// Additional ranges unioned with each other and with selected branches. + #[serde(default)] + pub ranges: Vec, /// `text` is a regular expression (Log filter `.*`). #[serde(default)] pub text_regex: bool, @@ -136,6 +226,8 @@ pub struct LogQuery { #[serde(default)] pub authors: Vec, #[serde(default)] + pub authors_exact: bool, + #[serde(default)] pub order: LogOrder, /// Follow only the first parent of merges (`--first-parent`). #[serde(default)] @@ -150,6 +242,13 @@ pub struct LogQuery { pub since_ms: Option, #[serde(default)] pub until_ms: Option, + #[serde(default)] + pub request: Option, + #[serde(default)] + pub summary_only: bool, + /// Inclusive refresh boundary. Clients join only verified linear prefixes. + #[serde(default)] + pub stop_at: Option, } #[derive(Clone, Debug, Serialize, Deserialize, Type)] @@ -458,7 +557,9 @@ pub struct ChangelistState { pub files: std::collections::BTreeMap, } +mod bek; mod blame; +mod block_history; pub mod changelists; mod cli; mod compare; @@ -466,6 +567,7 @@ mod diff; mod history; mod ignore; mod index_writes; +mod log_index; mod manage; mod ops; mod patch; @@ -569,6 +671,9 @@ pub struct Repo { git_dir: PathBuf, inner: std::sync::Mutex, ignored: std::sync::RwLock, + log_index: std::sync::Mutex, + index_progress: std::sync::Mutex, + log_signatures: std::sync::Mutex>, } impl Repo { @@ -685,6 +790,20 @@ impl Repo { let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); write::blame(self, path, current_text, revision, encoding) } + /// History for Selection: commits that changed `lines` (0-based, inclusive, of the + /// current text: `current_text`, or the file on disk), mapped to HEAD's lines first. + pub fn block_history( + &self, + path: &Path, + current_text: Option<&str>, + lines: (u32, u32), + encoding: impl Into, + cancel: &CancellationToken, + ) -> IdeResult { + let encoding = encoding.into(); + let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); + block_history::block_history(self, path, current_text, lines, encoding, cancel) + } /// Commits newest-first in topological order (children before parents). /// /// `branch` is any revision; when absent, HEAD plus all local and remote branches are @@ -694,7 +813,183 @@ impl Repo { /// `refs` decorations are `HEAD`, branch names (`main`, `origin/main`) and `tag: `. pub fn log(&self, query: &LogQuery) -> IdeResult> { let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); - history::log(self, query) + history::log(self, query, &CancellationToken::new()) + } + pub fn log_cancellable(&self, query: &LogQuery, cancel: &CancellationToken) -> IdeResult> { + let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); + history::log(self, query, cancel) + } + /// Enables the disposable, content-addressed log cache outside the worktree. + pub fn enable_log_index(&self, cache_dir: &Path) { + let key = blake3::hash(self.git_dir.to_string_lossy().as_bytes()).to_hex().to_string(); + self.log_index.lock().unwrap_or_else(|p| p.into_inner()).open(&cache_dir.join(key)); + } + /// Reference names and peeled commit ids, including tags and detached HEAD. + pub fn log_refs(&self) -> IdeResult> { + let repo = self.gix()?; + Ok(history::decorations(&repo)? + .into_iter() + .flat_map(|(id, names)| names.into_iter().map(move |name| (name, id.to_string()))) + .collect()) + } + pub fn log_users(&self) -> IdeResult> { + let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); + let out = cli::run(self.root(), &["log", "--all", "--format=%an <%ae>"], None)?; + Ok(out + .stdout_str() + .lines() + .filter(|line| !line.is_empty()) + .map(str::to_string) + .collect::>() + .into_iter() + .collect()) + } + pub fn log_index_progress(&self) -> LogIndexProgress { + self.index_progress.lock().unwrap_or_else(|p| p.into_inner()).clone() + } + + /// Incremental persistent inverted index; completed objects survive cancellation/restart. + pub fn index_log(&self, cancel: &CancellationToken) -> IdeResult { + { + let mut progress = self.index_progress.lock().unwrap_or_else(|p| p.into_inner()); + if progress.running { + return Ok(progress.clone()); + } + progress.running = true; + } + let result = (|| { + let repo = self.gix()?; + let out = cli::run_cancellable(self.root(), &["rev-list", "--all", "--topo-order"], cancel)?; + let ids: Vec<_> = out + .stdout_str() + .lines() + .map(|id| gix::ObjectId::from_hex(id.as_bytes()).map_err(repo::gx)) + .collect::>()?; + { + let mut p = self.index_progress.lock().unwrap_or_else(|p| p.into_inner()); + p.total = ids.len() as u32; + p.indexed = 0; + } + for (i, id) in ids.iter().enumerate() { + if cancel.is_cancelled() { + return Err(ide_core::IdeError::cancelled()); + } + let indexed = self.log_index.lock().unwrap_or_else(|p| p.into_inner()).indexed(&id.to_string()); + if !indexed { + let files = self.commit_files(&id.to_string())?; + let paths = + files.into_iter().flat_map(|f| std::iter::once(f.path).chain(f.old_path)).collect::>(); + let mut index = self.log_index.lock().unwrap_or_else(|p| p.into_inner()); + let c = index.get(&repo, *id)?; + let message = index.message(&repo, *id)?; + index.index_commit(&c.id, &message, &c.author, &c.email, &paths); + if i % 100 == 0 { + index.flush(); + } + } + self.index_progress.lock().unwrap_or_else(|p| p.into_inner()).indexed = i as u32 + 1; + } + Ok(()) + })(); + self.log_index.lock().unwrap_or_else(|p| p.into_inner()).flush(); + self.index_progress.lock().unwrap_or_else(|p| p.into_inner()).running = false; + result?; + Ok(self.log_index_progress()) + } + + pub fn log_details(&self, revision: &str) -> IdeResult { + self.log_details_cancellable(revision, &CancellationToken::new()) + } + pub fn log_details_cancellable(&self, revision: &str, cancel: &CancellationToken) -> IdeResult { + let allowed = self.execution_allowed.load(std::sync::atomic::Ordering::Acquire); + let _scope = crate::ExecutionScope::enter(allowed); + let repo = self.gix()?; + let object = + repo.find_object(repo::resolve(&repo, revision)?).map_err(repo::gx)?.peel_to_commit().map_err(repo::gx)?; + let commit_id = object.id; + let id = commit_id.to_string(); + let contains = format!("--contains={id}"); + let branches = + cli::run_cancellable(self.root(), &["branch", "-a", &contains, "--format=%(refname:short)"], cancel)?; + let cache_key = format!("{allowed}:{id}"); + let cached = self.log_signatures.lock().unwrap_or_else(|p| p.into_inner()).get(&cache_key).cloned(); + let mut details = if let Some(details) = cached { + details + } else { + let end = object.data.windows(2).position(|s| s == b"\n\n").unwrap_or(object.data.len()); + let headers = &object.data[..end]; + let signed = headers.windows(8).any(|s| s == b"\ngpgsig ") || headers.windows(8).any(|s| s == b"\ngpgsig-"); + let output = if allowed && signed { + Some(cli::run_cancellable( + self.root(), + &["log", "-1", "--format=%G?%x00%GS%x00%GK%x00%GF", "--end-of-options", &id], + cancel, + )?) + } else { + None + }; + let text = output.as_ref().map(|out| out.stdout_str()).unwrap_or_default(); + let fields: Vec<_> = text.trim_end().split('\0').collect(); + let message = { + let mut index = self.log_index.lock().unwrap_or_else(|p| p.into_inner()); + let message = index.message(&repo, commit_id)?; + index.flush(); + message + }; + let details = LogDetails { + message, + containing_branches: Vec::new(), + signature: if !signed { + "N".into() + } else { + fields.first().filter(|s| !s.is_empty()).unwrap_or(&"E").to_string() + }, + signer: fields.get(1).unwrap_or(&"").to_string(), + key: fields.get(2).unwrap_or(&"").to_string(), + fingerprint: fields.get(3).unwrap_or(&"").to_string(), + }; + self.log_signatures.lock().unwrap_or_else(|p| p.into_inner()).insert(cache_key, details.clone()); + details + }; + details.containing_branches = + branches.stdout_str().lines().filter(|s| !s.ends_with("/HEAD")).map(str::to_string).collect(); + Ok(details) + } + + /// Lists one immutable revision directory. Directory names end in `/`. + pub fn revision_children(&self, revision: &str, directory: &str) -> IdeResult> { + let repo = self.gix()?; + let id = repo::resolve(&repo, revision)?.to_string(); + let directory = self.rel_path(std::path::Path::new(directory))?; + let tree = if directory.is_empty() { id } else { format!("{id}:{directory}") }; + let out = cli::run(self.root(), &["ls-tree", "-z", "--", &tree], None)?; + Ok(out + .stdout + .split(|b| *b == 0) + .filter_map(|line| { + let split = line.iter().position(|b| *b == b'\t')?; + let name = String::from_utf8_lossy(&line[split + 1..]); + let dir = line.starts_with(b"040000 "); + Some(format!( + "{}{name}{}", + if directory.is_empty() { String::new() } else { format!("{directory}/") }, + if dir { "/" } else { "" } + )) + }) + .collect()) + } + + pub fn revision_files(&self, revision: &str) -> IdeResult> { + let _scope = crate::ExecutionScope::enter(self.execution_allowed.load(std::sync::atomic::Ordering::Acquire)); + let repo = self.gix()?; + let id = repo::resolve(&repo, revision)?.to_string(); + let out = cli::run(self.root(), &["ls-tree", "-r", "-z", "--name-only", "--", &id], None)?; + Ok(out + .stdout + .split(|b| *b == 0) + .filter(|p| !p.is_empty()) + .map(|p| String::from_utf8_lossy(p).into_owned()) + .collect()) } /// Files changed by `commit` relative to its first parent (renames detected). pub fn commit_files(&self, commit: &str) -> IdeResult> { diff --git a/crates/ide-vcs/src/log_index.rs b/crates/ide-vcs/src/log_index.rs new file mode 100644 index 00000000..c1d013aa --- /dev/null +++ b/crates/ide-vcs/src/log_index.rs @@ -0,0 +1,385 @@ +//! Immutable commit metadata, persisted by object id. Ref decorations and graph +//! projections are deliberately never cached as commit metadata. + +use crate::Commit; +use crate::repo::{gx, to_string}; +use gix::ObjectId; +use gix::bstr::ByteSlice; +use ide_core::IdeResult; +use redb::{Database, MultimapTableDefinition, ReadableDatabase, TableDefinition}; +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::SystemTime; + +const COMMITS: TableDefinition<&str, &[u8]> = TableDefinition::new("summaries-v2"); +const MESSAGES: TableDefinition<&str, &str> = TableDefinition::new("messages-v1"); +const PATHS: TableDefinition<&str, &[u8]> = TableDefinition::new("paths-v2"); +const POSTINGS: MultimapTableDefinition<&str, &str> = MultimapTableDefinition::new("search-v1"); +const INDEXED: TableDefinition<&str, bool> = TableDefinition::new("indexed-v1"); + +pub(crate) struct IndexedView { + pub commits: Vec, + pub positions: HashMap, +} +impl IndexedView { + pub fn new(commits: Vec) -> Self { + let positions = commits.iter().enumerate().map(|(i, c)| (c.id.clone(), i)).collect(); + Self { commits, positions } + } +} + +#[derive(Default)] +pub(crate) struct LogIndex { + opened: bool, + db: Option>, + memory: HashMap>, + dirty: Vec>, + dirty_messages: Vec<(String, String)>, + dirty_search: Vec<(String, Vec)>, + path_memory: HashMap<(ObjectId, String), Option<(ObjectId, u16)>>, + dirty_paths: Vec<(String, Option<(String, u16)>)>, + pub head: Option<(ObjectId, Option)>, + pub reachable: HashSet, + pub views: HashMap>, + pub nodes: HashMap, i64)>, + pub shallow: HashSet, +} + +impl LogIndex { + /// A permanent DAG: refresh discovers only new objects, then joins arbitrary + /// changed tips against known ancestry. Reachability is recomputed for resets/deleted refs. + pub fn graph( + &mut self, + repo: &gix::Repository, + tips: &[ObjectId], + shallow: &HashSet, + first_parent: bool, + cancel: &ide_core::CancellationToken, + ) -> IdeResult> { + if &self.shallow != shallow { + self.nodes.clear(); + self.shallow = shallow.clone(); + } + let mut pending = tips.to_vec(); + while let Some(id) = pending.pop() { + if cancel.is_cancelled() { + return Err(ide_core::IdeError::cancelled()); + } + if self.nodes.contains_key(&id) { + continue; + } + let record = self.get(repo, id)?; + let parents: Vec<_> = if shallow.contains(&id) { + Vec::new() + } else { + record.parents.iter().map(|s| ObjectId::from_hex(s.as_bytes()).map_err(gx)).collect::>()? + }; + pending.extend(parents.iter().copied()); + self.nodes.insert(id, (parents, (record.commit_time_ms / 1000.0) as i64)); + } + let mut seen = HashSet::new(); + let mut ids = Vec::new(); + let mut pending = tips.to_vec(); + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + ids.push(id); + if let Some((parents, _)) = self.nodes.get(&id) { + pending.extend(parents.iter().take(if first_parent { 1 } else { usize::MAX }).rev().copied()); + } + } + Ok(ids) + } + pub fn indexed(&self, id: &str) -> bool { + self.db + .as_ref() + .and_then(|db| { + let txn = db.begin_read().ok()?; + let table = txn.open_table(INDEXED).ok()?; + Some(table.get(id).ok()??.value()) + }) + .unwrap_or(false) + } + + pub fn index_commit(&mut self, id: &str, message: &str, author: &str, email: &str, paths: &[String]) { + let mut keys: HashSet = Self::grams(message).into_iter().map(|s| format!("m:{s}")).collect(); + for user in [author, email, email.split('@').next().unwrap_or(email)] { + keys.insert(format!("u:{}", normalize_user(user))); + } + for path in paths { + keys.insert(format!("p:{path}")); + } + self.dirty_search.push((id.into(), keys.into_iter().collect())); + } + + fn grams(text: &str) -> Vec { + let chars: Vec<_> = text.to_lowercase().chars().collect(); + chars.windows(3).map(|s| s.iter().collect()).collect::>().into_iter().collect() + } + + /// Missing/unindexed commits are always checked directly by the caller. + pub fn message_candidates(&self, text: &str) -> Option> { + let grams = Self::grams(text); + if grams.is_empty() { + return None; + } + let db = self.db.as_ref()?; + let txn = db.begin_read().ok()?; + let table = txn.open_multimap_table(POSTINGS).ok()?; + let mut candidates: Option> = None; + for gram in grams { + let values = table.get(format!("m:{gram}").as_str()).ok()?; + let ids: HashSet<_> = values.filter_map(Result::ok).map(|s| s.value().to_string()).collect(); + candidates = Some(match candidates { + Some(old) => old.intersection(&ids).cloned().collect(), + None => ids, + }); + if candidates.as_ref().is_some_and(HashSet::is_empty) { + break; + } + } + Some(candidates.unwrap_or_default()) + } + + pub fn user_candidates(&self, authors: &[String]) -> Option> { + let txn = self.db.as_ref()?.begin_read().ok()?; + let table = txn.open_multimap_table(POSTINGS).ok()?; + let mut ids = HashSet::new(); + for author in authors { + let name = author.split('<').next().unwrap_or(author).trim(); + let email = author.split('<').nth(1).map(|s| s.trim_end_matches('>')).unwrap_or(author); + for alias in [name, email, email.split('@').next().unwrap_or(email)] { + for value in table.get(format!("u:{}", normalize_user(alias)).as_str()).ok()?.filter_map(Result::ok) { + ids.insert(value.value().to_string()); + } + } + } + Some(ids) + } + + pub fn path_candidates(&self, paths: &[String]) -> Option> { + let txn = self.db.as_ref()?.begin_read().ok()?; + let table = txn.open_multimap_table(POSTINGS).ok()?; + let mut ids = HashSet::new(); + for path in paths { + for value in table.get(format!("p:{path}").as_str()).ok()?.filter_map(Result::ok) { + ids.insert(value.value().to_string()); + } + let prefix = format!("p:{path}/"); + let end = format!("{prefix}{}", char::MAX); + for row in table.range(prefix.as_str()..end.as_str()).ok()?.filter_map(Result::ok) { + for value in row.1.filter_map(Result::ok) { + ids.insert(value.value().to_string()); + } + } + } + Some(ids) + } + pub fn open(&mut self, dir: &Path) { + if self.opened { + return; + } + self.opened = true; + // Several windows can read the same worktree; redb permits one owning + // handle per file, shared here rather than silently disabling its index. + static DATABASES: OnceLock>>> = OnceLock::new(); + let path = dir.join("commits.redb"); + let mut databases = DATABASES.get_or_init(Mutex::default).lock().unwrap_or_else(|p| p.into_inner()); + self.db = databases.get(&path).and_then(Weak::upgrade).or_else(|| { + let db = Arc::new(std::fs::create_dir_all(dir).ok().and_then(|_| Database::create(&path).ok())?); + databases.insert(path, Arc::downgrade(&db)); + Some(db) + }); + } + + pub fn get(&mut self, repo: &gix::Repository, id: ObjectId) -> IdeResult> { + if let Some(commit) = self.memory.get(&id) { + return Ok(commit.clone()); + } + let key = id.to_string(); + let cached = self.db.as_ref().and_then(|db| { + let txn = db.begin_read().ok()?; + let table = txn.open_table(COMMITS).ok()?; + let value = table.get(key.as_str()).ok()??; + serde_json::from_slice::(value.value()).ok().filter(|c| c.id == key) + }); + let commit = if let Some(commit) = cached { + Arc::new(commit) + } else { + let commit = repo.find_commit(id).map_err(gx)?; + let author = commit.author().map_err(gx)?; + let committer = commit.committer().map_err(gx)?; + let raw = commit.message_raw_sloppy(); + let summary = raw.split(|b| *b == b'\n').find(|l| !l.trim().is_empty()).unwrap_or_default().trim(); + let summary_text = to_string(summary.as_bstr()); + let record = Arc::new(Commit { + id: key.clone(), + short_id: key[..key.len().min(8)].to_string(), + parents: commit.parent_ids().map(|p| p.to_string()).collect(), + graph_parents: None, + graph_dashed: Vec::new(), + dashed: Vec::new(), + on_current_branch: None, + author: to_string(author.name.trim().as_bstr()), + email: to_string(author.email.trim().as_bstr()), + committer: to_string(committer.name.trim().as_bstr()), + committer_email: to_string(committer.email.trim().as_bstr()), + time_ms: author.time().map(|t| t.seconds as f64 * 1000.0).unwrap_or(0.0), + commit_time_ms: committer.time().map(|t| t.seconds as f64 * 1000.0).unwrap_or(0.0), + summary: summary_text.clone(), + message: summary_text, + message_truncated: raw.trim() != summary, + history_path: None, + log_position: None, + log_total: None, + graph_boundary_parents: Vec::new(), + refs: Vec::new(), + }); + if self.db.is_some() { + self.dirty.push(record.clone()); + } + record + }; + // Keep hot pages in memory; older metadata stays in the persistent index. + if self.memory.len() >= 4096 { + self.flush(); + self.memory.clear(); + } + self.memory.insert(id, commit.clone()); + Ok(commit) + } + + pub fn message(&mut self, repo: &gix::Repository, id: ObjectId) -> IdeResult { + if let Some((_, message)) = self.dirty_messages.iter().find(|(key, _)| key == &id.to_string()) { + return Ok(message.clone()); + } + let key = id.to_string(); + let cached = self.db.as_ref().and_then(|db| { + let txn = db.begin_read().ok()?; + let table = txn.open_table(MESSAGES).ok()?; + Some(table.get(key.as_str()).ok()??.value().to_string()) + }); + if let Some(message) = cached { + return Ok(message); + } + let message = to_string(repo.find_commit(id).map_err(gx)?.message_raw_sloppy()).trim_end().to_string(); + if self.db.is_some() { + self.dirty_messages.push((key, message.clone())); + } + Ok(message) + } + + pub fn flush(&mut self) { + if self.dirty.is_empty() + && self.dirty_paths.is_empty() + && self.dirty_messages.is_empty() + && self.dirty_search.is_empty() + { + return; + } + let result = (|| -> Result<(), Box> { + let Some(db) = &self.db else { + return Ok(()); + }; + let txn = db.begin_write()?; + { + let mut table = txn.open_table(COMMITS)?; + for commit in &self.dirty { + let bytes = serde_json::to_vec(commit.as_ref())?; + table.insert(commit.id.as_str(), bytes.as_slice())?; + } + } + { + let mut table = txn.open_table(PATHS)?; + for (key, value) in &self.dirty_paths { + let bytes = serde_json::to_vec(value)?; + table.insert(key.as_str(), bytes.as_slice())?; + } + } + { + let mut table = txn.open_table(MESSAGES)?; + for (id, message) in &self.dirty_messages { + table.insert(id.as_str(), message.as_str())?; + } + } + { + let mut postings = txn.open_multimap_table(POSTINGS)?; + let mut indexed = txn.open_table(INDEXED)?; + for (id, keys) in &self.dirty_search { + for key in keys { + postings.insert(key.as_str(), id.as_str())?; + } + indexed.insert(id.as_str(), true)?; + } + } + txn.commit()?; + Ok(()) + })(); + if let Err(error) = result { + tracing::warn!("Git log cache write failed: {error}"); + } + self.dirty_messages.clear(); + self.dirty_search.clear(); + self.dirty.clear(); + self.dirty_paths.clear(); + } + + pub fn path_entry( + &mut self, + repo: &gix::Repository, + id: ObjectId, + path: &str, + ) -> IdeResult> { + let memory_key = (id, path.to_string()); + if let Some(value) = self.path_memory.get(&memory_key) { + return Ok(*value); + } + let key = format!("{id}\0{path}"); + let cached = self.db.as_ref().and_then(|db| { + let txn = db.begin_read().ok()?; + let table = txn.open_table(PATHS).ok()?; + let value = table.get(key.as_str()).ok()??; + serde_json::from_slice::>(value.value()).ok() + }); + let cached = match cached { + Some(None) => Some(None), + Some(Some((hex, mode))) => ObjectId::from_hex(hex.as_bytes()).ok().map(|id| Some((id, mode))), + None => None, + }; + let value = if let Some(value) = cached { + value + } else { + let tree = repo.find_commit(id).map_err(gx)?.tree().map_err(gx)?; + let value = tree + .lookup_entry_by_path(Path::new(path)) + .map_err(gx)? + .map(|entry| (entry.object_id(), entry.mode().value())); + if self.db.is_some() { + self.dirty_paths.push((key, value.map(|(id, mode)| (id.to_string(), mode)))); + } + value + }; + if self.path_memory.len() >= 4096 { + self.flush(); + self.path_memory.clear(); + } + self.path_memory.insert(memory_key, value); + Ok(value) + } +} + +pub(crate) fn normalize_user(user: &str) -> String { + user.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect() +} + +pub(crate) fn matches_user(query: &str, name: &str, email: &str) -> bool { + let query_name = query.split('<').next().unwrap_or(query).trim(); + let query_email = query.split('<').nth(1).map(|s| s.trim_end_matches('>')).unwrap_or(query); + let target = + [normalize_user(name), normalize_user(email), normalize_user(email.split('@').next().unwrap_or(email))]; + [query_name, query_email, query_email.split('@').next().unwrap_or(query_email)] + .iter() + .any(|alias| target.contains(&normalize_user(alias))) +} diff --git a/crates/ide-vcs/src/repo.rs b/crates/ide-vcs/src/repo.rs index 47e145b6..6ad81787 100644 --- a/crates/ide-vcs/src/repo.rs +++ b/crates/ide-vcs/src/repo.rs @@ -64,6 +64,9 @@ pub(crate) fn discover(path: &Path) -> IdeResult> { git_dir, inner: std::sync::Mutex::new(cache), ignored: Default::default(), + log_index: Default::default(), + index_progress: Default::default(), + log_signatures: Default::default(), })) } diff --git a/crates/ide-vcs/tests/block_history.rs b/crates/ide-vcs/tests/block_history.rs new file mode 100644 index 00000000..96128273 --- /dev/null +++ b/crates/ide-vcs/tests/block_history.rs @@ -0,0 +1,124 @@ +//! History for Selection (`git log -L`) against throwaway repositories built with the git CLI. + +use ide_core::CancellationToken; +use ide_core::error::ErrorKind; +use ide_vcs::*; +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Fixture { + _tmp: tempfile::TempDir, + dir: PathBuf, +} + +impl Fixture { + fn new() -> Self { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("work"); + std::fs::create_dir(&dir).unwrap(); + let fx = Fixture { _tmp: tmp, dir }; + fx.git(&["init", "-q"]); + for (k, v) in [("user.name", "Test User"), ("user.email", "test@example.com"), ("commit.gpgsign", "false")] { + fx.git(&["config", k, v]); + } + fx + } + fn git(&self, args: &[&str]) -> String { + let out = Command::new("git") + .current_dir(&self.dir) + .args(["-c", "init.defaultBranch=main", "-c", "core.autocrlf=false"]) + .args(args) + .output() + .expect("git runs"); + assert!(out.status.success(), "git {args:?} failed: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + fn write(&self, rel: &str, content: &str) { + std::fs::write(self.dir.join(rel), content).unwrap(); + } + fn commit(&self, msg: &str) { + self.git(&["add", "-A"]); + self.git(&["commit", "-q", "-m", msg]); + } + fn repo(&self) -> Repo { + Repo::discover(&self.dir).unwrap().expect("repository found") + } + fn path(&self, rel: &str) -> PathBuf { + self.dir.join(rel) + } +} + +fn history(repo: &Repo, path: &Path, text: Option<&str>, lines: (u32, u32)) -> BlockHistory { + repo.block_history(path, text, lines, encoding_rs::UTF_8, &CancellationToken::new()).unwrap() +} + +/// a.txt: added, line 2 edited, line 5 edited, renamed to b.txt, line 3 edited. +fn renamed_history() -> Fixture { + let fx = Fixture::new(); + fx.write("a.txt", "one\ntwo\nthree\nfour\nfive\n"); + fx.commit("Add a"); + fx.write("a.txt", "one\nTWO\nthree\nfour\nfive\n"); + fx.commit("Shout two"); + fx.write("a.txt", "one\nTWO\nthree\nfour\nFIVE\n"); + fx.commit("Shout five"); + fx.git(&["mv", "a.txt", "b.txt"]); + fx.commit("Rename a to b"); + fx.write("b.txt", "one\nTWO\nTHREE\nfour\nFIVE\n"); + fx.commit("Shout three"); + fx +} + +#[test] +fn follows_the_block_through_edits_and_a_rename() { + let fx = renamed_history(); + let h = history(&fx.repo(), &fx.path("b.txt"), None, (1, 2)); + assert_eq!(h.head_lines, Some((1, 2))); + let shown: Vec<_> = h.revisions.iter().map(|r| (r.summary.as_str(), r.path.as_str())).collect(); + // Edits outside the block and the pure rename are left out. + assert_eq!(shown, [("Shout three", "b.txt"), ("Shout two", "a.txt"), ("Add a", "a.txt")]); + let blocks: Vec<_> = h.revisions.iter().map(|r| (r.before.as_str(), r.after.as_str())).collect(); + assert_eq!(blocks, [("TWO\nthree\n", "TWO\nTHREE\n"), ("two\nthree\n", "TWO\nthree\n"), ("", "two\nthree\n")]); + let newest = &h.revisions[0]; + assert_eq!((newest.before_line, newest.after_line, newest.author.as_str()), (1, 1, "Test User")); + assert!(newest.id.starts_with(&newest.short_id) && newest.time_ms > 0.0); +} + +#[test] +fn maps_an_edited_selection_to_head_lines() { + let fx = renamed_history(); + let repo = fx.repo(); + // Two unsaved lines inserted at the top: lines 3..=4 are HEAD's 1..=2. + let edited = "new\nnew\none\nTWO\nTHREE\nfour\nFIVE\n"; + let h = history(&repo, &fx.path("b.txt"), Some(edited), (3, 4)); + assert_eq!(h.head_lines, Some((1, 2))); + assert_eq!(h.revisions.len(), 3); + // Only new lines selected, or a file Git does not know: nothing committed to show. + let h = history(&repo, &fx.path("b.txt"), Some(edited), (0, 1)); + assert_eq!((h.head_lines, h.revisions.len()), (None, 0)); + fx.write("untracked.txt", "x\n"); + let h = history(&repo, &fx.path("untracked.txt"), None, (0, 0)); + assert_eq!((h.head_lines, h.revisions.len()), (None, 0)); +} + +#[test] +fn restricted_mode_runs_no_diff_driver_of_the_repository() { + let fx = renamed_history(); + let marker = fx.path("textconv-ran"); + fx.write(".gitattributes", "*.txt diff=probe\n"); + fx.commit("Attributes"); + fx.git(&["config", "diff.probe.textconv", &format!("sh -c 'touch {}; cat \"$0\"'", marker.display())]); + let repo = fx.repo(); + repo.set_execution_allowed(false); + let h = history(&repo, &fx.path("b.txt"), None, (1, 2)); + assert_eq!(h.revisions.len(), 3); + assert!(!marker.exists(), "a textconv driver ran in restricted mode"); +} + +#[test] +fn cancelled_requests_stop() { + let fx = renamed_history(); + let cancel = CancellationToken::new(); + cancel.cancel(); + let err = fx.repo().block_history(&fx.path("b.txt"), None, (1, 2), encoding_rs::UTF_8, &cancel).unwrap_err(); + assert_eq!(err.kind, ErrorKind::Cancelled); +} diff --git a/crates/ide-vcs/tests/log_projection.rs b/crates/ide-vcs/tests/log_projection.rs new file mode 100644 index 00000000..10c9c61a --- /dev/null +++ b/crates/ide-vcs/tests/log_projection.rs @@ -0,0 +1,438 @@ +use ide_vcs::{LogOrder, LogQuery, Repo}; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .current_dir(dir) + .args([ + "-c", + "init.defaultBranch=main", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn commit(dir: &Path, file: &str, message: &str) -> String { + std::fs::write(dir.join(file), message).unwrap(); + git(dir, &["add", "-A"]); + git(dir, &["commit", "-qm", message]); + git(dir, &["rev-parse", "HEAD"]) +} + +#[test] +fn branch_and_independent_ranges_form_a_union() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "a", "root"); + let first = commit(dir, "a", "first"); + let second = commit(dir, "a", "second"); + let third = commit(dir, "a", "third"); + git(dir, &["checkout", "-qb", "side", &root]); + let side = commit(dir, "b", "side"); + let repo = Repo::discover(dir).unwrap().unwrap(); + for order in [LogOrder::Topo, LogOrder::Date, LogOrder::Standard] { + let log = repo + .log(&LogQuery { + branches: vec!["side".into()], + ranges: vec![format!("{root}..{first}"), format!("{second}..{third}")], + order, + ..Default::default() + }) + .unwrap(); + let got: std::collections::HashSet<_> = log.iter().map(|c| c.id.clone()).collect(); + assert_eq!(got, [root.clone(), first.clone(), third.clone(), side.clone()].into()); + let ranges = repo + .log(&LogQuery { + ranges: vec![format!("{root}..{first}"), format!("{second}..{third}")], + order, + ..Default::default() + }) + .unwrap(); + assert_eq!( + ranges.iter().map(|c| c.id.clone()).collect::>(), + [first.clone(), third.clone()].into() + ); + let positions: std::collections::HashMap<_, _> = + log.iter().enumerate().map(|(i, c)| (c.id.as_str(), i)).collect(); + for (i, c) in log.iter().enumerate() { + for p in c.graph_parents.as_ref().unwrap() { + assert!(positions.get(p.as_str()).is_none_or(|p| *p > i)); + } + } + } +} + +#[test] +fn lazy_summaries_seek_and_index_survive_refresh_and_restart() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let mut ids = Vec::new(); + for i in 0..70 { + ids.push(commit(dir, "a", &format!("summary {i}\n\nneedle body {i}"))); + } + let cache = tempfile::tempdir().unwrap(); + let repo = Repo::discover(dir).unwrap().unwrap(); + repo.enable_log_index(cache.path()); + let query = LogQuery { summary_only: true, limit: 5, ..Default::default() }; + let page = repo.log(&query).unwrap(); + assert_eq!(page[0].message, "summary 69"); + assert!(page[0].message_truncated); + assert_eq!(page[0].log_position, Some(0)); + assert_eq!(page[0].log_total, Some(70)); + let window = repo.log(&LogQuery { seek: Some(ids[5].clone()), limit: 20, ..query.clone() }).unwrap(); + assert!(window.iter().any(|c| c.id == ids[5])); + assert!(window[0].log_position.unwrap() > 0); + assert!(repo.log(&LogQuery { request: Some(ide_core::RequestId(1)), ..query.clone() }).is_ok()); + let cancel = ide_core::CancellationToken::new(); + cancel.cancel(); + assert!(repo.log_cancellable(&query, &cancel).is_err()); + let progress = repo.index_log(&ide_core::CancellationToken::new()).unwrap(); + assert_eq!((progress.indexed, progress.total, progress.running), (70, 70, false)); + assert_eq!(repo.log(&LogQuery { text: Some("needle body 5".into()), ..query.clone() }).unwrap().len(), 5); + let new = commit(dir, "b", "new head"); + let refreshed = repo.log(&LogQuery { retain_until: Some(ids[0].clone()), ..query.clone() }).unwrap(); + assert_eq!(refreshed[0].id, new); + assert_eq!(refreshed.len(), 71); + drop(repo); + let repo = Repo::discover(dir).unwrap().unwrap(); + repo.enable_log_index(cache.path()); + let hit = repo.log(&LogQuery { text: Some("needle body 69".into()), ..query }).unwrap(); + assert_eq!(hit[0].id, ids[69]); + assert_eq!(repo.log_details(&ids[69]).unwrap().message, "summary 69\n\nneedle body 69"); + assert_eq!(repo.log_details(&ids[69]).unwrap().signature, "N"); +} + +#[test] +fn followed_history_carries_the_filename_at_each_revision() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let first = commit(dir, "old.txt", "original"); + git(dir, &["mv", "old.txt", "new.txt"]); + git(dir, &["commit", "-qm", "rename"]); + let renamed = git(dir, &["rev-parse", "HEAD"]); + let tip = commit(dir, "new.txt", "latest"); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo.log(&LogQuery { path: Some("new.txt".into()), follow: true, ..Default::default() }).unwrap(); + assert_eq!( + log.iter().map(|c| (c.id.as_str(), c.history_path.as_deref())).collect::>(), + vec![(tip.as_str(), Some("new.txt")), (renamed.as_str(), Some("new.txt")), (first.as_str(), Some("old.txt"))] + ); + assert_eq!(repo.revision_children(&first, "").unwrap(), vec!["old.txt"]); +} + +#[test] +fn detail_filters_connect_visible_ancestors_without_changing_revision_parents() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let a = commit(dir, "a", "keep root"); + commit(dir, "b", "hidden middle"); + git(dir, &["commit", "--amend", "--no-edit", "--author=Other "]); + let b = git(dir, &["rev-parse", "HEAD"]); + let c = commit(dir, "a", "keep tip"); + let repo = Repo::discover(dir).unwrap().unwrap(); + for q in [ + LogQuery { text: Some("keep".into()), ..Default::default() }, + LogQuery { path: Some("a".into()), ..Default::default() }, + LogQuery { authors: vec!["Test".into()], authors_exact: true, ..Default::default() }, + ] { + let page = repo.log(&LogQuery { limit: 1, ..q.clone() }).unwrap(); + assert_eq!(page[0].id, c); + assert_eq!(page[0].parents, vec![b.clone()]); + assert_eq!(page[0].graph_parents, Some(vec![a.clone()])); + assert_eq!(page[0].graph_dashed, vec![true]); + assert_eq!(page[0].on_current_branch, Some(true)); + let next = repo.log(&LogQuery { skip: 1, limit: 1, ..q }).unwrap(); + assert_eq!(next[0].id, a); + assert_eq!(next[0].graph_parents, Some(vec![])); + } +} + +#[test] +fn all_history_includes_commits_reachable_only_from_tags() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "a", "root"); + let tip = commit(dir, "a", "tagged tip"); + git(dir, &["tag", "v1"]); + git(dir, &["reset", "--hard", &root]); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo.log(&LogQuery::default()).unwrap(); + let tagged = log.iter().find(|c| c.id == tip).unwrap(); + assert!(tagged.refs.contains(&"tag: v1".into())); + assert_eq!(tagged.on_current_branch, Some(false)); +} + +#[test] +fn projection_does_not_truncate_long_hidden_chains() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let mut stream = String::new(); + for i in 0..4100 { + let message = if i == 0 || i == 4099 { "keep" } else { "hide" }; + stream.push_str(&format!( + "commit refs/heads/main\ncommitter Test {} +0000\ndata {}\n{}\n\n", + 1000 + i, + message.len(), + message + )); + } + let mut child = Command::new("git") + .current_dir(dir) + .args(["fast-import", "--quiet"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(stream.as_bytes()).unwrap(); + assert!(child.wait().unwrap().success()); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo.log(&LogQuery { text: Some("keep".into()), ..Default::default() }).unwrap(); + assert_eq!(log.len(), 2); + assert_eq!(log[0].graph_parents, Some(vec![log[1].id.clone()])); +} + +#[test] +fn path_and_shallow_history_remain_topological_with_clock_skew() { + let tmp = tempfile::tempdir().unwrap(); + let source = tmp.path().join("source"); + std::fs::create_dir(&source).unwrap(); + git(&source, &["init", "-q"]); + let make = |message: &str, parent: Option<&str>, time: &str| { + std::fs::write(source.join("a.txt"), message).unwrap(); + git(&source, &["add", "a.txt"]); + let tree = git(&source, &["write-tree"]); + let mut cmd = Command::new("git"); + cmd.current_dir(&source).args([ + "-c", + "user.name=Test", + "-c", + "user.email=t@example.com", + "commit-tree", + &tree, + "-m", + message, + ]); + if let Some(p) = parent { + cmd.args(["-p", p]); + } + let out = cmd.env("GIT_AUTHOR_DATE", time).env("GIT_COMMITTER_DATE", time).output().unwrap(); + assert!(out.status.success()); + String::from_utf8(out.stdout).unwrap().trim().to_string() + }; + let root = make("root", None, "@5000 +0000"); + let left = make("left", Some(&root), "@1000 +0000"); + let right = make("right", Some(&root), "@2000 +0000"); + git(&source, &["update-ref", "refs/heads/main", &left]); + git(&source, &["update-ref", "refs/heads/side", &right]); + let shallow = tmp.path().join("shallow"); + git( + tmp.path(), + &[ + "clone", + "-q", + "--depth=2", + "--no-single-branch", + &format!("file://{}", source.display()), + shallow.to_str().unwrap(), + ], + ); + for dir in [&source, &shallow] { + let repo = Repo::discover(dir).unwrap().unwrap(); + for order in [LogOrder::Date, LogOrder::Topo, LogOrder::Standard] { + let log = repo.log(&LogQuery { order, path: Some("a.txt".into()), ..Default::default() }).unwrap(); + assert_eq!(log.len(), 3); + let positions: std::collections::HashMap<_, _> = + log.iter().enumerate().map(|(i, c)| (c.id.as_str(), i)).collect(); + for (i, c) in log.iter().enumerate() { + for parent in c.graph_parents.as_ref().unwrap() { + assert!(positions.get(parent.as_str()).is_none_or(|p| *p > i)); + } + } + } + } +} + +#[test] +fn multiple_paths_exact_authors_patterns_and_rename_history() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "old.txt", "root"); + git(dir, &["mv", "old.txt", "new.txt"]); + git(dir, &["commit", "-qm", "rename"]); + let rename = git(dir, &["rev-parse", "HEAD"]); + let tip = commit(dir, "other.txt", "another file"); + git(dir, &["branch", "feature/log"]); + git(dir, &["branch", "feature/skip", &root]); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo.log(&LogQuery { paths: vec!["new.txt".into(), "other.txt".into()], ..Default::default() }).unwrap(); + assert_eq!(log.iter().map(|c| c.id.clone()).collect::>(), vec![tip.clone(), rename.clone()]); + let log = repo.log(&LogQuery { path: Some("new.txt".into()), follow: true, ..Default::default() }).unwrap(); + assert_eq!(log.iter().map(|c| c.id.clone()).collect::>(), vec![rename, root.clone()]); + let log = repo + .log(&LogQuery { branches: vec!["feature/.*".into(), "-feature/skip".into()], ..Default::default() }) + .unwrap(); + assert_eq!(log[0].id, tip); + std::fs::write(dir.join("other.txt"), "different author").unwrap(); + git(dir, &["add", "-A"]); + git(dir, &["-c", "user.name=Test User", "-c", "user.email=other@example.com", "commit", "-qm", "different author"]); + let log = repo.log(&LogQuery { authors: vec!["Test".into()], authors_exact: true, ..Default::default() }).unwrap(); + assert_eq!(log.len(), 3); + assert!(log.iter().all(|c| c.author == "Test")); + assert!(repo.log_users().unwrap().contains(&"Test User ".to_string())); +} + +#[test] +fn metadata_and_path_cache_survive_restart_without_caching_refs_or_filters() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("repo"); + std::fs::create_dir(&dir).unwrap(); + git(&dir, &["init", "-q"]); + let first = commit(&dir, "a", "first\n\nComplete commit body"); + git(&dir, &["tag", "v1"]); + git(&dir, &["tag", "-a", "annotated", "-m", "release"]); + let cache = tmp.path().join("cache"); + { + let repo = Repo::discover(&dir).unwrap().unwrap(); + repo.enable_log_index(&cache); + let log = repo.log(&LogQuery { path: Some("a".into()), summary_only: true, ..Default::default() }).unwrap(); + assert!(log[0].message_truncated); + assert_eq!(log[0].message, "first"); + assert_eq!(repo.log_details(&first).unwrap().message, "first\n\nComplete commit body"); + let details = repo.log_details("annotated").unwrap(); + assert_eq!(details.message, "first\n\nComplete commit body"); + assert_eq!(details.signature, "N"); + assert!(details.containing_branches.contains(&"main".to_string())); + assert_eq!(repo.revision_files(&first).unwrap(), vec!["a"]); + } + let second = commit(&dir, "a", "second"); + git(&dir, &["tag", "-f", "v1"]); + let repo = Repo::discover(&dir).unwrap().unwrap(); + repo.enable_log_index(&cache); + let log = repo.log(&LogQuery { path: Some("a".into()), ..Default::default() }).unwrap(); + assert_eq!(log[0].id, second); + assert!(log[0].refs.contains(&"tag: v1".into())); + assert!(!log[1].refs.contains(&"tag: v1".into())); + assert!(log[1].message.contains("Complete commit body")); + assert!(!log[1].message_truncated); +} + +#[test] +fn cancelled_query_does_not_publish_a_partial_page() { + let tmp = tempfile::tempdir().unwrap(); + git(tmp.path(), &["init", "-q"]); + commit(tmp.path(), "a", "root"); + let repo = Repo::discover(tmp.path()).unwrap().unwrap(); + let cancel = ide_core::CancellationToken::new(); + cancel.cancel(); + assert_eq!( + repo.log_cancellable(&LogQuery::default(), &cancel).unwrap_err().kind, + ide_core::error::ErrorKind::Cancelled + ); +} + +#[test] +fn shallow_boundaries_do_not_reconnect_to_locally_present_ancestors() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "a", "keep root"); + let boundary = commit(dir, "a", "hidden boundary"); + let tip = commit(dir, "a", "keep tip"); + git(dir, &["tag", "root", &root]); + std::fs::write(dir.join(".git/shallow"), format!("{boundary}\n")).unwrap(); + let repo = Repo::discover(dir).unwrap().unwrap(); + for order in [LogOrder::Date, LogOrder::Topo, LogOrder::Standard] { + for text in [None, Some("keep".to_string())] { + let log = repo.log(&LogQuery { order, text, path: Some("a".into()), ..Default::default() }).unwrap(); + let boundary_row = log.iter().find(|c| c.id == boundary); + if let Some(row) = boundary_row { + assert_eq!(row.parents, vec![root.clone()]); + assert_eq!(row.graph_parents, Some(vec![])); + } else { + assert_eq!(log.iter().find(|c| c.id == tip).unwrap().graph_parents, Some(vec![])); + } + } + } +} + +#[test] +fn exact_branch_exclusions_treat_punctuation_literally() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "a", "root"); + git(dir, &["branch", "release/1.0"]); + let tip = commit(dir, "a", "tip"); + git(dir, &["branch", "release/1x0"]); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo + .log(&LogQuery { branches: vec!["release/.*".into(), "-release/1.0".into()], ..Default::default() }) + .unwrap(); + assert_eq!(log.iter().map(|c| c.id.as_str()).collect::>(), vec![tip.as_str(), root.as_str()]); +} + +#[test] +fn mode_only_changes_appear_in_path_history() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + let root = commit(dir, "script", "root"); + git(dir, &["update-index", "--chmod=+x", "script"]); + git(dir, &["commit", "-qm", "make executable"]); + let tip = git(dir, &["rev-parse", "HEAD"]); + let repo = Repo::discover(dir).unwrap().unwrap(); + let log = repo.log(&LogQuery { path: Some("script".into()), ..Default::default() }).unwrap(); + assert_eq!(log.iter().map(|c| c.id.as_str()).collect::>(), vec![tip.as_str(), root.as_str()]); +} + +#[test] +fn multiple_paths_are_simplified_together_at_merges() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + git(dir, &["init", "-q"]); + commit(dir, "a", "a0"); + commit(dir, "b", "b0"); + git(dir, &["branch", "side"]); + commit(dir, "a", "left"); + git(dir, &["checkout", "side"]); + commit(dir, "b", "right"); + git(dir, &["checkout", "main"]); + git(dir, &["merge", "--no-ff", "side", "-m", "merge both paths"]); + let repo = Repo::discover(dir).unwrap().unwrap(); + let expected = git(dir, &["log", "--topo-order", "--format=%H", "--", "a", "b"]); + for order in [LogOrder::Topo, LogOrder::Date, LogOrder::Standard] { + let log = repo + .log(&LogQuery { + branch: Some("HEAD".into()), + paths: vec!["a".into(), "b".into()], + order, + ..Default::default() + }) + .unwrap(); + assert_eq!(log[0].summary, "merge both paths"); + assert_eq!( + log.iter().map(|c| c.id.as_str()).collect::>(), + expected.lines().collect() + ); + assert_eq!(log[0].parents.len(), 2); + assert_eq!(log[0].graph_parents.as_ref().unwrap().len(), 2); + } +} diff --git a/crates/ide-vcs/tests/log_shape.rs b/crates/ide-vcs/tests/log_shape.rs index b26236ec..4414d6d5 100644 --- a/crates/ide-vcs/tests/log_shape.rs +++ b/crates/ide-vcs/tests/log_shape.rs @@ -64,7 +64,7 @@ fn ids(commits: &[Commit]) -> Vec { } fn parents_of(commits: &[Commit], id: &str) -> Vec { - commits.iter().find(|c| c.id == id).unwrap().parents.clone() + commits.iter().find(|c| c.id == id).unwrap().graph_parents.clone().unwrap() } #[test] @@ -75,6 +75,14 @@ fn date_order_follows_commit_time_and_reports_it() { assert_eq!(out[0].commit_time_ms, 5_000_000.0); } +#[test] +fn incoming_order_groups_the_merged_branch_before_the_main_parent() { + let h = history(); + let out = log(&h, LogQuery { order: LogOrder::Standard, ..Default::default() }); + assert_eq!(ids(&out), vec![h.c3.clone(), h.m.clone(), h.s1.clone(), h.c2.clone(), h.c1.clone()]); + assert_eq!(out[1].parents, vec![h.c2.clone(), h.s1.clone()]); +} + #[test] fn no_merges_links_children_to_the_merge_parents() { let h = history(); @@ -90,6 +98,7 @@ fn first_parent_skips_merged_branches() { let out = log(&h, LogQuery { first_parent: true, ..Default::default() }); assert_eq!(ids(&out), vec![h.c3.clone(), h.m.clone(), h.c2.clone(), h.c1.clone()]); assert_eq!(parents_of(&out, &h.m), vec![h.c2.clone()]); + assert_eq!(out.iter().find(|c| c.id == h.m).unwrap().parents, vec![h.c2.clone(), h.s1.clone()]); } #[test] @@ -201,19 +210,26 @@ fn a_user_filter_links_shown_commits_past_hidden_ones() { let log = repo.log(&LogQuery { author: Some("me@".into()), ..Default::default() }).unwrap(); let links: Vec<(&str, Vec<&str>, Vec)> = log .iter() - .map(|c| (c.summary.as_str(), c.parents.iter().map(String::as_str).collect(), c.dashed.clone())) + .map(|c| { + ( + c.summary.as_str(), + c.graph_parents.as_ref().unwrap().iter().map(String::as_str).collect(), + if c.graph_dashed.iter().any(|d| *d) { c.graph_dashed.clone() } else { Vec::new() }, + ) + }) .collect(); assert_eq!( links, [("m3", vec![m2.as_str()], vec![]), ("m2", vec![m1.as_str()], vec![true]), ("m1", vec![], vec![]),], "m2's parent o1 is hidden: m2 links to m1, dashed" ); - assert!(!log.iter().any(|c| c.parents.contains(&o1))); + assert!(!log.iter().any(|c| c.graph_parents.as_ref().unwrap().contains(&o1))); + assert_eq!(log[1].parents, vec![o1]); // A page ends at m2: walking on still links it to m1. let page = repo.log(&LogQuery { author: Some("me@".into()), limit: 2, ..Default::default() }).unwrap(); assert_eq!(page.len(), 2); - assert_eq!(page[1].parents, std::slice::from_ref(&m1)); + assert_eq!(page[1].graph_parents.as_deref().unwrap(), std::slice::from_ref(&m1)); let next = repo.log(&LogQuery { author: Some("me@".into()), skip: 2, limit: 2, ..Default::default() }).unwrap(); assert_eq!(next.iter().map(|c| c.id.clone()).collect::>(), std::slice::from_ref(&m1)); let _ = m3; diff --git a/crates/ide-vfs/Cargo.toml b/crates/ide-vfs/Cargo.toml index 42d430ec..dd4bfdf3 100644 --- a/crates/ide-vfs/Cargo.toml +++ b/crates/ide-vfs/Cargo.toml @@ -13,7 +13,7 @@ serde.workspace = true specta.workspace = true ignore.workspace = true globset.workspace = true -notify-debouncer-full.workspace = true +notify.workspace = true encoding_rs.workspace = true chardetng.workspace = true trash.workspace = true diff --git a/crates/ide-vfs/src/lib.rs b/crates/ide-vfs/src/lib.rs index 6ac24e21..3941e8ba 100644 --- a/crates/ide-vfs/src/lib.rs +++ b/crates/ide-vfs/src/lib.rs @@ -256,6 +256,11 @@ pub fn set_writable(path: &Path, writable: bool) -> IdeResult<()> { pub fn rename(from: &Path, to: &Path) -> IdeResult<()> { ops::rename(from, to) } +/// Checks source existence and destination conflicts without mutation, using +/// the same filesystem identity rules as [`rename`]. +pub fn validate_rename(from: &Path, to: &Path) -> IdeResult<()> { + ops::validate_rename(from, to) +} /// Moves to the OS trash (recoverable), like IDEA's safe delete for files. /// Deletes permanently when the trash is unavailable. pub fn delete(path: &Path) -> IdeResult<()> { @@ -274,13 +279,14 @@ pub fn copy(from: &Path, to: &Path) -> IdeResult<()> { /// honouring .gitignore, .ignore and `excludes`. Shared by index and search. /// /// Hidden files are included; `.git` is always skipped. `.gitignore` applies -/// even outside a git repository. Symlinks to files are visited, symlinked +/// even outside a git repository; inside one, a nested repository's root is a +/// boundary for the outer repository's rules. Symlinks to files are visited, symlinked /// directories are not followed. pub fn walk_files(root: &Path, excludes: &[String], visit: impl Fn(&Path) + Send + Sync) { let excludes = Arc::new(ExcludeMatcher::new(excludes)); let base = root.to_path_buf(); let mut builder = ignore::WalkBuilder::new(root); - builder.hidden(false).require_git(false).follow_links(false); + builder.hidden(false).require_git(ide_core::ignored::inside_git_repo(root)).follow_links(false); builder.filter_entry(move |entry| { if entry.depth() > 0 && entry.file_name() == ".git" { return false; @@ -324,6 +330,11 @@ pub struct FsBatch { /// Folders holding changes filtered out as excluded or ignored. Only a view already /// showing such a folder cares, so they don't count as worktree changes. pub excluded_dirs: Vec, + /// Files changed, created or removed that an ignore rule matches while their folder + /// is not ignored (`.env`). Kept apart from the worktree changes above (no Local + /// History, deployment or `fsChanged`), but the index lists them and open editors + /// follow them, like IDEA, where only ignored folders leave the project. + pub ignored: Vec, } impl FsBatch { @@ -333,6 +344,7 @@ impl FsBatch { && self.removed.is_empty() && self.vcs.is_empty() && self.excluded_dirs.is_empty() + && self.ignored.is_empty() } } diff --git a/crates/ide-vfs/src/ops.rs b/crates/ide-vfs/src/ops.rs index d256aa34..7ebe171f 100644 --- a/crates/ide-vfs/src/ops.rs +++ b/crates/ide-vfs/src/ops.rs @@ -88,7 +88,7 @@ fn same_file(a: &Path, b: &Path) -> bool { } } -pub(crate) fn rename(from: &Path, to: &Path) -> IdeResult<()> { +pub(crate) fn validate_rename(from: &Path, to: &Path) -> IdeResult<()> { if !exists(from) { return Err(not_found(from)); } @@ -99,6 +99,14 @@ pub(crate) fn rename(from: &Path, to: &Path) -> IdeResult<()> { if exists(to) && !same_file(from, to) { return Err(already_exists(to)); } + Ok(()) +} + +pub(crate) fn rename(from: &Path, to: &Path) -> IdeResult<()> { + validate_rename(from, to)?; + if from == to { + return Ok(()); + } ensure_parent(to)?; match fs::rename(from, to) { Ok(()) => Ok(()), diff --git a/crates/ide-vfs/src/watch.rs b/crates/ide-vfs/src/watch.rs index d9309d90..253d7a82 100644 --- a/crates/ide-vfs/src/watch.rs +++ b/crates/ide-vfs/src/watch.rs @@ -12,9 +12,8 @@ use crate::FsBatch; use crate::rules::{ExcludeMatcher, GitignoreStack}; use ide_core::{IdeError, IdeEvent, IdeResult, SharedSink}; -use notify_debouncer_full::notify::event::{ModifyKind, RenameMode}; -use notify_debouncer_full::notify::{self, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, WatcherKind}; -use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, Debouncer, NoCache, new_debouncer_opt}; +use notify::event::{ModifyKind, RenameMode}; +use notify::{Event, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, WatcherKind}; use parking_lot::{Condvar, Mutex, RwLock}; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::path::{Component, Path, PathBuf}; @@ -62,10 +61,9 @@ pub struct Watcher { type SharedExcludes = Arc>; -enum Inner { - Recursive(#[allow(dead_code)] Debouncer), - Polling(#[allow(dead_code)] Debouncer), - PerDirectory { state: Arc, thread: Option> }, +struct Inner { + state: Arc, + thread: Option>, } impl Watcher { @@ -131,80 +129,32 @@ impl Watcher { /// Number of OS-level watches (1 in recursive and flat mode). pub fn watch_count(&self) -> usize { - match &self.inner { - Inner::Recursive(_) | Inner::Polling(_) => 1, - Inner::PerDirectory { state, .. } => state.watched.lock().len(), - } + self.inner.state.watched.lock().len() } fn start_native(root: &Path, filter: EventFilter, emit: Emit, mode: WatchMode) -> IdeResult { - let config = notify::Config::default(); - match mode { - WatchMode::Recursive => { - Self::start_debounced(root, filter, emit, RecursiveMode::Recursive, config).map(Inner::Recursive) - } - WatchMode::Flat => { - Self::start_debounced(root, filter, emit, RecursiveMode::NonRecursive, config).map(Inner::Recursive) - } - WatchMode::PerDirectory => Self::start_per_directory::(root, filter, emit, config), - } + Self::start_queued::(root, filter, emit, mode, notify::Config::default()) } /// Polls per directory unless flat: a recursive poll would rescan /// `node_modules` and `target` every round. fn start_polling(root: &Path, filter: EventFilter, emit: Emit, mode: WatchMode) -> IdeResult { let config = notify::Config::default().with_poll_interval(POLL_INTERVAL); - match mode { - WatchMode::Flat => { - Self::start_debounced(root, filter, emit, RecursiveMode::NonRecursive, config).map(Inner::Polling) - } - WatchMode::Recursive | WatchMode::PerDirectory => { - Self::start_per_directory::(root, filter, emit, config) - } - } + let mode = if mode == WatchMode::Flat { mode } else { WatchMode::PerDirectory }; + Self::start_queued::(root, filter, emit, mode, config) } - fn start_debounced( - root: &Path, - filter: EventFilter, - emit: Emit, - recursive: RecursiveMode, - config: notify::Config, - ) -> IdeResult> { - let polled = W::kind() == WatcherKind::PollWatcher; - // No file-id cache: building it walks and stats the whole tree before `watch` returns, - // which made opening large projects block for seconds. Renames arrive as remove + create. - let handler = move |result: DebounceEventResult| match result { - Ok(mut events) => { - if polled { - events.retain(|e| !is_dir_touch(&e.event)); - } - let (batch, _) = filter.batch_detailed(&events); - if !batch.is_empty() { - emit(&batch); - } - } - Err(errors) => { - for e in errors { - tracing::warn!("file watcher: {e}"); - } - } - }; - let mut debouncer = new_debouncer_opt::<_, W, _>(DEBOUNCE, None, handler, NoCache, config) - .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?; - debouncer.watch(root, recursive).map_err(|e| io_error(format!("cannot watch {}: {e}", root.display())))?; - Ok(debouncer) - } - - fn start_per_directory( + fn start_queued( root: &Path, filter: EventFilter, emit: Emit, + mode: WatchMode, config: notify::Config, ) -> IdeResult { let polled = W::kind() == WatcherKind::PollWatcher; let state = Arc::new(DirState { filter, + mode, watcher: Mutex::new(None), watched: Mutex::default(), queue: Mutex::default(), @@ -228,8 +178,11 @@ impl Watcher { ) .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?; *state.watcher.lock() = Some(Box::new(watcher)); - let mut created = Vec::new(); - state.watch_tree(root, false, &mut created); + if mode == WatchMode::PerDirectory { + state.watch_tree(root, false, &mut Vec::new()); + } else { + state.add_watch(root); + } if state.watched.lock().is_empty() { return Err(io_error(format!("cannot watch {}", root.display()))); } @@ -240,20 +193,22 @@ impl Watcher { .spawn(move || state.run(&*emit)) .map_err(|e| io_error(format!("cannot start file watcher: {e}")))? }; - Ok(Inner::PerDirectory { state, thread: Some(thread) }) + Ok(Inner { state, thread: Some(thread) }) } } -impl Drop for Watcher { +impl Drop for Inner { fn drop(&mut self) { - if let Inner::PerDirectory { state, thread } = &mut self.inner { + let state = &self.state; + { + let _queue = state.queue.lock(); state.stop.store(true, Ordering::SeqCst); state.wake.notify_all(); - let watcher = state.watcher.lock().take(); - drop(watcher); - if let Some(t) = thread.take() { - let _ = t.join(); - } + } + let watcher = state.watcher.lock().take(); + drop(watcher); + if let Some(t) = self.thread.take() { + let _ = t.join(); } } } @@ -268,10 +223,10 @@ fn is_dir_touch(event: ¬ify::Event) -> bool { matches!(event.kind, EventKind::Modify(_)) && event.paths.iter().all(|p| p.is_dir()) } -/// State of the per-directory strategy, shared with the notify callback -/// (weakly) and the debounce thread. +/// Shared with the notify callback (weakly) and the debounce thread. struct DirState { filter: EventFilter, + mode: WatchMode, watcher: Mutex>>, watched: Mutex>, queue: Mutex>, @@ -310,9 +265,13 @@ impl DirState { } std::mem::take(&mut *q) }; - let events: Vec = events.into_iter().map(|(e, t)| DebouncedEvent::new(e, t)).collect(); + // A scan may publish a path before its create event is delivered: + // keep create + remove pairs so the consumer still sees the removal. + let events: Vec = events.into_iter().map(|(e, _)| e).collect(); let (mut batch, rule_dirs) = self.filter.batch_detailed(&events); - self.update_watches(&mut batch, &rule_dirs); + if self.mode == WatchMode::PerDirectory { + self.update_watches(&mut batch, &rule_dirs); + } if !batch.is_empty() && !self.stopped() { emit(&batch); } @@ -363,7 +322,9 @@ impl DirState { } let mut guard = self.watcher.lock(); let Some(w) = guard.as_mut() else { return false }; - match w.watch(dir, RecursiveMode::NonRecursive) { + let recursive = + if self.mode == WatchMode::Recursive { RecursiveMode::Recursive } else { RecursiveMode::NonRecursive }; + match w.watch(dir, recursive) { Ok(()) => { self.watched.lock().insert(dir.to_path_buf()); true @@ -443,6 +404,9 @@ fn merge(prev: Option, next: Change) -> Change { match (prev, next) { (None, n) => n, (Some(Vcs), _) | (_, Vcs) => Vcs, + // FSEvents reports a deleted file's history as create, remove, then modify: a + // modify can't bring back a path that went away, only a create can. + (Some(Transient), Modified) => Transient, (Some(Created | Transient), Modified | Created) => Created, (Some(Created | Transient), Removed) => Transient, (Some(Removed), Created | Modified) => Modified, @@ -531,6 +495,13 @@ impl EventFilter { && !self.gitignore.is_ignored(path, is_dir) } + /// Whether a filtered-out file is ignored by a rule on the file itself, in a folder + /// that is neither excluded nor ignored. + fn is_ignored_file(&self, path: &Path, rel: &Path) -> bool { + !self.excludes.read().is_excluded(rel) + && path.parent().is_some_and(|dir| dir == self.root || !self.gitignore.is_ignored(dir, true)) + } + fn classify(&self, path: &Path, change: Change, out: &mut BTreeMap) { let rel = match path.strip_prefix(&self.root) { Ok(rel) if rel.as_os_str().is_empty() => return, @@ -548,17 +519,17 @@ impl EventFilter { } #[cfg(test)] - pub fn batch(&self, events: &[DebouncedEvent]) -> FsBatch { + pub fn batch(&self, events: &[Event]) -> FsBatch { self.batch_detailed(events).0 } /// The coalesced, filtered batch plus the directories whose ignore rules /// changed (their cached matchers have been rebuilt already). - pub fn batch_detailed(&self, events: &[DebouncedEvent]) -> (FsBatch, Vec) { + pub fn batch_detailed(&self, events: &[Event]) -> (FsBatch, Vec) { let mut changes: BTreeMap = BTreeMap::new(); for ev in events { - let paths = &ev.event.paths; - match ev.event.kind { + let paths = &ev.paths; + match ev.kind { EventKind::Access(_) => {} EventKind::Create(_) => paths.iter().for_each(|p| self.classify(p, Change::Created, &mut changes)), EventKind::Remove(_) => paths.iter().for_each(|p| self.classify(p, Change::Removed, &mut changes)), @@ -594,7 +565,7 @@ impl EventFilter { rule_dirs.insert(dir.to_path_buf()); } } - if events.iter().flat_map(|e| &e.event.paths).any(|p| p.ends_with(".git/info/exclude")) { + if events.iter().flat_map(|e| &e.paths).any(|p| p.ends_with(".git/info/exclude")) { self.gitignore.reload_repo_level(); rule_dirs.insert(self.root.clone()); } @@ -642,6 +613,9 @@ impl EventFilter { if change != Change::Modified { excluded_dirs.extend(path.parent().map(Path::to_path_buf)); } + if !is_dir && self.is_ignored_file(&path, rel) { + batch.ignored.push(path); + } continue; } } @@ -659,11 +633,125 @@ impl EventFilter { #[cfg(test)] mod tests { use super::*; - use notify_debouncer_full::notify::Event; - use notify_debouncer_full::notify::event::{CreateKind, DataChange, RemoveKind}; + use notify::event::{CreateKind, DataChange, MetadataKind, RemoveKind}; - fn ev(kind: EventKind, paths: Vec) -> DebouncedEvent { - DebouncedEvent::new(Event { kind, paths, attrs: Default::default() }, Instant::now()) + fn ev(kind: EventKind, paths: Vec) -> Event { + Event { kind, paths, attrs: Default::default() } + } + + const TRANSIENT_PATHS: &[&str] = &["fresh.txt", "src/fresh.rs", "src/tasks/fresh.py"]; + + /// Delivers a whole debounce window without depending on OS scheduling. + struct TransientWatcher(Box); + + impl notify::Watcher for TransientWatcher { + fn new(handler: F, _: notify::Config) -> notify::Result { + Ok(Self(Box::new(handler))) + } + + fn watch(&mut self, root: &Path, _: RecursiveMode) -> notify::Result<()> { + for rel in TRANSIENT_PATHS { + for kind in [ + EventKind::Create(CreateKind::File), + EventKind::Modify(ModifyKind::Data(DataChange::Content)), + EventKind::Remove(RemoveKind::File), + ] { + self.0.handle_event(Ok(Event::new(kind).add_path(root.join(rel)))); + } + } + for kind in [EventKind::Create(CreateKind::Folder), EventKind::Remove(RemoveKind::Folder)] { + self.0.handle_event(Ok(Event::new(kind).add_path(root.join("new")))); + } + for (rel, kind) in [ + ("from.py", EventKind::Create(CreateKind::File)), + ("from.py", EventKind::Modify(ModifyKind::Name(RenameMode::From))), + ("to.py", EventKind::Modify(ModifyKind::Name(RenameMode::To))), + ("to.py", EventKind::Remove(RemoveKind::File)), + ] { + self.0.handle_event(Ok(Event::new(kind).add_path(root.join(rel)))); + } + for kind in [EventKind::Remove(RemoveKind::File), EventKind::Create(CreateKind::File)] { + self.0.handle_event(Ok(Event::new(kind).add_path(root.join("kept.py")))); + } + Ok(()) + } + + fn unwatch(&mut self, _: &Path) -> notify::Result<()> { + Ok(()) + } + + fn kind() -> WatcherKind { + WatcherKind::NullWatcher + } + } + + #[test] + fn debounce_preserves_transient_removals_and_atomic_replacements() { + for mode in [WatchMode::Recursive, WatchMode::Flat, WatchMode::PerDirectory] { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + std::fs::write(root.join("kept.py"), "class Kept: pass\n").unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + let emit: Emit = Arc::new(move |b| tx.send(b.clone()).unwrap()); + let _watcher = Watcher::start_queued::( + &root, + EventFilter::new(root.clone(), &[]), + emit, + mode, + notify::Config::default(), + ) + .unwrap(); + let batch = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let removed: BTreeSet<_> = batch.removed.into_iter().collect(); + let expected = + TRANSIENT_PATHS.iter().copied().chain(["new", "from.py", "to.py"]).map(|rel| root.join(rel)).collect(); + assert_eq!(removed, expected, "{mode:?}: scans may have already published transient paths"); + assert_eq!(batch.modified, vec![root.join("kept.py")]); + assert!(batch.created.is_empty()); + } + } + + #[cfg(windows)] + #[test] + fn native_watch_reports_quickly_removed_files() { + for mode in [WatchMode::Recursive, WatchMode::Flat, WatchMode::PerDirectory] { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + std::fs::create_dir_all(root.join("src/tasks")).unwrap(); + let batches: Arc>> = Arc::default(); + let watcher = { + let batches = batches.clone(); + Watcher::start_with( + &root, + Vec::new(), + Arc::new(ide_core::NullSink), + Box::new(move |b| batches.lock().push(b.clone())), + mode, + ) + .unwrap() + }; + assert!(watcher.polling().is_none()); + assert_eq!(watcher.watch_count(), if mode == WatchMode::PerDirectory { 3 } else { 1 }); + let paths: Vec<_> = TRANSIENT_PATHS + .iter() + .filter(|rel| mode != WatchMode::Flat || !rel.contains('/')) + .map(|rel| root.join(rel)) + .collect(); + for path in &paths { + std::fs::write(path, "class FreshWidget:\n pass\n").unwrap(); + std::fs::remove_file(path).unwrap(); + } + assert!( + wait_for(&batches, |b| paths.iter().all(|p| b.iter().any(|batch| batch.removed.contains(p)))), + "{mode:?}: {:?}", + batches.lock() + ); + drop(watcher); + let count = batches.lock().len(); + std::fs::write(root.join("after-drop.txt"), "x").unwrap(); + std::thread::sleep(DEBOUNCE * 2); + assert_eq!(batches.lock().len(), count, "no callback after drop in {mode:?}"); + } } #[test] @@ -700,6 +788,25 @@ mod tests { assert_eq!(batch.vcs, vec![root.join(".git/HEAD"), root.join(".git/refs/heads/feature")]); assert_eq!(batch.modified, vec![root.join("kept.txt")]); assert_eq!(batch.removed, vec![root.join("from.txt"), root.join("gone.txt"), root.join("old.txt")]); + assert_eq!(batch.ignored, vec![root.join("debug.log")], "ignored files are reported apart"); + } + + /// What FSEvents delivers when a file is deleted: its coalesced history flags as + /// create, remove, then modify events, all for a path that no longer exists. + #[test] + fn fsevents_deletion_with_history_flags_is_a_removal() { + let tmp = tempfile::tempdir().unwrap(); + let root = dunce::canonicalize(tmp.path()).unwrap(); + let filter = EventFilter::new(root.clone(), &[]); + let gone = root.join("src/Fresh.java"); + let batch = filter.batch(&[ + ev(EventKind::Create(CreateKind::File), vec![gone.clone()]), + ev(EventKind::Remove(RemoveKind::File), vec![gone.clone()]), + ev(EventKind::Modify(ModifyKind::Metadata(MetadataKind::Extended)), vec![gone.clone()]), + ev(EventKind::Modify(ModifyKind::Data(DataChange::Content)), vec![gone.clone()]), + ]); + assert_eq!(batch.removed, vec![gone]); + assert!(batch.created.is_empty() && batch.modified.is_empty()); } #[test] @@ -751,6 +858,20 @@ mod tests { ]); assert!(batch.created.is_empty() && batch.modified.is_empty() && batch.removed.is_empty(), "{batch:?}"); assert_eq!(batch.excluded_dirs, vec![root.clone(), root.join("dist"), root.join("target/debug")]); + assert_eq!(batch.ignored, vec![root.join("debug.log")], "only the ignored file outside ignored folders"); + + // An ignored file's edit and removal reach the index too (nothing else changes). + let modify = EventKind::Modify(ModifyKind::Data(DataChange::Content)); + let batch = filter.batch(&[ev(modify, vec![root.join("debug.log")])]); + assert_eq!(batch.ignored, vec![root.join("debug.log")]); + assert!(!batch.is_empty() && batch.excluded_dirs.is_empty()); + std::fs::remove_file(root.join("debug.log")).unwrap(); + let batch = filter.batch(&[ + ev(EventKind::Remove(RemoveKind::File), vec![root.join("debug.log")]), + ev(EventKind::Remove(RemoveKind::File), vec![root.join("dist/app.js")]), + ]); + // A removed path can't tell file from folder: it goes out as a removal, as before. + assert_eq!(batch.removed, vec![root.join("debug.log")]); // Sibling files in unignored directories still come through. let batch = filter.batch(&[ev(create_file, vec![root.join("src/main.rs")])]); @@ -797,10 +918,10 @@ mod tests { } let filter = EventFilter::new(root.clone(), &[]); let modify = EventKind::Modify(ModifyKind::Data(DataChange::Content)); - let touch = - |paths: &[&str]| -> Vec { paths.iter().map(|p| ev(modify, vec![root.join(p)])).collect() }; + let touch = |paths: &[&str]| -> Vec { paths.iter().map(|p| ev(modify, vec![root.join(p)])).collect() }; let (batch, rules) = filter.batch_detailed(&touch(&["pkg/a.tmp", "pkg/gen/x.rs", "pkg/keep.rs", "b.tmp"])); assert_eq!(batch.modified, vec![root.join("b.tmp"), root.join("pkg/keep.rs")]); + assert_eq!(batch.ignored, vec![root.join("pkg/a.tmp")], "nested rules, but not inside ignored folders"); assert!(rules.is_empty()); assert!(!filter.should_watch_dir(&root.join("pkg/gen"))); @@ -818,7 +939,8 @@ mod tests { std::fs::create_dir_all(root.join(".git/info")).unwrap(); std::fs::write(root.join(".git/info/exclude"), "b.tmp\n").unwrap(); let (batch, rules) = filter.batch_detailed(&touch(&[".git/info/exclude", "b.tmp"])); - assert!(batch.is_empty(), "{batch:?}"); + assert!(batch.modified.is_empty() && batch.vcs.is_empty(), "{batch:?}"); + assert_eq!(batch.ignored, vec![root.join("b.tmp")]); assert_eq!(rules, vec![root.clone()]); } @@ -846,10 +968,7 @@ mod tests { } fn watcher_state(w: &Watcher) -> &DirState { - match &w.inner { - Inner::PerDirectory { state, .. } => state, - Inner::Recursive(_) | Inner::Polling(_) => panic!("not per-directory"), - } + &w.inner.state } fn wait_for(events: &Mutex>, pred: impl Fn(&[FsBatch]) -> bool) -> bool { diff --git a/crates/ide-vfs/tests/vfs.rs b/crates/ide-vfs/tests/vfs.rs index 76d2cdc7..0a08f732 100644 --- a/crates/ide-vfs/tests/vfs.rs +++ b/crates/ide-vfs/tests/vfs.rs @@ -259,6 +259,47 @@ fn walk_honours_ignores() { assert_eq!(found, expected); } +#[test] +fn walk_scopes_gitignore_to_each_repository() { + let (_t, r) = root(); + // An outer repository ignores everything under `projects/`, nested repository included (#1254). + write(&r.join("outer/.git/HEAD"), "ref: refs/heads/main\n"); + write(&r.join("outer/top.txt"), ""); + write(&r.join("outer/projects/.gitignore"), "*\n!.gitignore\n"); + write(&r.join("outer/projects/nested/.git/HEAD"), "ref: refs/heads/main\n"); + write(&r.join("outer/projects/nested/.git/info/exclude"), "local.txt\n"); + write(&r.join("outer/projects/nested/.gitignore"), "*.log\n"); + write(&r.join("outer/projects/nested/search-test.txt"), ""); + write(&r.join("outer/projects/nested/src/a.rs"), ""); + write(&r.join("outer/projects/nested/debug.log"), ""); + write(&r.join("outer/projects/nested/local.txt"), ""); + write(&r.join("outer/projects/plain/file.txt"), ""); + write(&r.join("loose/.gitignore"), "*.log\n"); + write(&r.join("loose/a.txt"), ""); + write(&r.join("loose/b.log"), ""); + + let walk = |dir: &str| { + let base = r.join(dir); + let found = Mutex::new(BTreeSet::new()); + walk_files(&base, &[], |p| { + found.lock().insert(p.strip_prefix(&base).unwrap().to_string_lossy().replace('\\', "/")); + }); + found.into_inner().into_iter().collect::>() + }; + let cases: [(&str, &[&str]); 4] = [ + // The nested repository's own rules apply; the outer repository's do not. + ("outer/projects/nested", &[".gitignore", "search-test.txt", "src/a.rs"]), + // A plain folder inside the outer repository is still governed by it, as in Git. + ("outer/projects/plain", &[]), + ("outer", &["projects/.gitignore", "top.txt"]), + // Outside any repository `.gitignore` still applies. + ("loose", &[".gitignore", "a.txt"]), + ]; + for (dir, expected) in cases { + assert_eq!(walk(dir), expected, "{dir}"); + } +} + fn wait_for(events: &Mutex>, pred: impl Fn(&[FsBatch]) -> bool) -> bool { let deadline = Instant::now() + Duration::from_secs(10); while Instant::now() < deadline { diff --git a/crates/ide-workspace/Cargo.toml b/crates/ide-workspace/Cargo.toml index bf51d793..5ebf071f 100644 --- a/crates/ide-workspace/Cargo.toml +++ b/crates/ide-workspace/Cargo.toml @@ -21,6 +21,7 @@ blake3.workspace = true tracing.workspace = true tempfile.workspace = true globset.workspace = true +uuid.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/ide-workspace/src/documents/workspace_edit.rs b/crates/ide-workspace/src/documents/workspace_edit.rs index 46698708..6e5e67f9 100644 --- a/crates/ide-workspace/src/documents/workspace_edit.rs +++ b/crates/ide-workspace/src/documents/workspace_edit.rs @@ -4,7 +4,7 @@ use super::{DocState, Documents, RegEntry, disk_len, keys_for}; use ide_core::error::ErrorKind; use ide_core::{DocId, FileEdit, IdeError, IdeEvent, IdeResult, NullSink, WorkspaceEdit, WorkspaceEditPrecondition}; use parking_lot::Mutex; -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; @@ -259,7 +259,9 @@ fn hash_bytes(hash: &mut blake3::Hasher, bytes: &[u8]) { hash.update(bytes); } -fn hash_disk(hash: &mut blake3::Hasher, path: &Path) -> IdeResult<()> { +/// Hashes `path` (recursively for a directory). A file in `unsaved` contributes the bytes its +/// buffer would save, not its disk bytes, so saving a buffer leaves the hash unchanged. +fn hash_disk(hash: &mut blake3::Hasher, path: &Path, unsaved: &HashMap>) -> IdeResult<()> { let meta = match fs::symlink_metadata(path) { Ok(meta) => meta, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { @@ -277,6 +279,10 @@ fn hash_disk(hash: &mut blake3::Hasher, path: &Path) -> IdeResult<()> { if meta.file_type().is_symlink() { hash.update(b"link"); hash_bytes(hash, fs::read_link(path)?.as_os_str().as_encoded_bytes()); + } else if let Some(bytes) = unsaved.get(path).filter(|_| meta.is_file()) { + hash.update(b"file"); + hash.update(&(bytes.len() as u64).to_le_bytes()); + hash.update(bytes); } else if meta.is_file() { hash.update(b"file"); hash.update(&meta.len().to_le_bytes()); @@ -295,7 +301,7 @@ fn hash_disk(hash: &mut blake3::Hasher, path: &Path) -> IdeResult<()> { entries.sort_by_key(|e| e.file_name()); for entry in entries { hash_bytes(hash, entry.file_name().as_encoded_bytes()); - hash_disk(hash, &entry.path())?; + hash_disk(hash, &entry.path(), unsaved)?; } } else { return Err(IdeError::invalid(format!("{} is not a regular file or directory", path.display()))); @@ -303,14 +309,19 @@ fn hash_disk(hash: &mut blake3::Hasher, path: &Path) -> IdeResult<()> { Ok(()) } -/// A reverted edit may leave the dirty flag set even when all saved bytes match. -fn differs_from_disk(state: &DocState) -> IdeResult { +/// The bytes saving `state` would write. +fn saved_bytes(state: &DocState) -> IdeResult> { let text = state.doc.text_for_save(); - let bytes = if state.native_to_ascii { + Ok(if state.native_to_ascii { ide_vfs::encode_properties_ascii(&text).into_bytes() } else { ide_vfs::encode(&text, &state.encoding, state.has_bom)? - }; + }) +} + +/// A reverted edit may leave the dirty flag set even when all saved bytes match. +fn differs_from_disk(state: &DocState) -> IdeResult { + let bytes = saved_bytes(state)?; match fs::read(&state.path) { Ok(disk) => Ok(disk != bytes), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), @@ -329,35 +340,31 @@ impl Documents { Ok(()) } + /// What each path holds: disk contents, with unsaved buffers in place of their files. Saving + /// a buffer doesn't change it; edits, external writes and file operations do. fn fingerprints(&self, paths: &[PathBuf]) -> IdeResult> { + let mut unsaved = HashMap::new(); + for id in self.open_documents() { + self.with_doc(id, |state| { + let file = text_path(&state.path)?; + if state.dirty && paths.iter().any(|path| file.starts_with(path)) && differs_from_disk(state)? { + unsaved.insert(file, saved_bytes(state)?); + } + Ok(()) + })?; + } paths .iter() .map(|path| { let mut hash = blake3::Hasher::new(); - hash_disk(&mut hash, path)?; - let mut dirty = Vec::new(); - for id in self.open_documents() { - self.with_doc(id, |state| { - let file = text_path(&state.path)?; - if state.dirty && file.starts_with(path) && differs_from_disk(state)? { - dirty.push(( - file, - state.doc.text(), - state.encoding.clone(), - state.doc.line_ending(), - state.has_bom, - )); - } - Ok(()) - })?; - } - dirty.sort_by(|a, b| a.0.cmp(&b.0)); - for (file, text, encoding, ending, bom) in dirty { + hash_disk(&mut hash, path, &unsaved)?; + // Unsaved files not on disk (yet) or outside what the walk above reached. + let mut extra: Vec<_> = + unsaved.iter().filter(|(file, _)| file.starts_with(path) && !file.exists()).collect(); + extra.sort_by(|a, b| a.0.cmp(b.0)); + for (file, bytes) in extra { hash_bytes(&mut hash, file.as_os_str().as_encoded_bytes()); - hash_bytes(&mut hash, text.as_bytes()); - hash_bytes(&mut hash, encoding.as_bytes()); - hash_bytes(&mut hash, ending.as_str().as_bytes()); - hash.update(&[u8::from(bom)]); + hash_bytes(&mut hash, bytes); } Ok(WorkspaceEditPrecondition { path: path.clone().into(), hash: hash.finalize().to_hex().to_string() }) }) @@ -394,14 +401,22 @@ impl Documents { if edit.files.is_empty() { return Ok(()); } + // Validate project rules before path resolution can erase linked components. + crate::file_ops::validate_workspace_edit_paths(edit)?; let mut edit = normalize_edit(edit)?; + let captured = !edit.preconditions.is_empty(); + crate::file_ops::expand_workspace_edit(&mut edit)?; self.prepare_workspace_edit(&mut edit)?; self.validate_preconditions(&edit.preconditions)?; for file in &edit.files { if let FileEdit::Edit { path, version: Some(version), .. } = file { - let doc = self.by_path(path).ok_or_else(|| stale(path))?; - if i64::from(self.version(doc)?) != i64::from(*version) { - return Err(stale(path)); + match self.by_path(path) { + Some(doc) if i64::from(self.version(doc)?) != i64::from(*version) => return Err(stale(path)), + Some(_) => {} + // Servers version files the editor never opened (gopls renames). The + // preconditions captured with the edit already cover its disk contents. + None if captured => {} + None => return Err(stale(path)), } } } diff --git a/crates/ide-workspace/src/file_moves.rs b/crates/ide-workspace/src/file_moves.rs index ad6444b8..7a1cae91 100644 --- a/crates/ide-workspace/src/file_moves.rs +++ b/crates/ide-workspace/src/file_moves.rs @@ -120,6 +120,8 @@ impl FileMoves { if moves.is_empty() { return Err(IdeError::invalid("nothing to move")); } + let expanded = crate::file_ops::expand_moves(moves)?; + let moves = expanded.as_slice(); for m in moves { if !m.from.exists() { return Err(IdeError::not_found(format!("{} does not exist", m.from.display()))); @@ -145,16 +147,9 @@ impl FileMoves { restore(documents, &paths, &before); return Err(e); } - let mut done: Vec<&FileMove> = Vec::new(); - for m in moves { - if let Err(e) = ide_vfs::rename(&m.from, &m.to) { - for d in done.iter().rev() { - let _ = ide_vfs::rename(&d.to, &d.from); - } - restore(documents, &paths, &before); - return Err(e); - } - done.push(m); + if let Err(e) = crate::file_ops::move_files(moves) { + restore(documents, &paths, &before); + return Err(e); } let mut edited = Vec::with_capacity(paths.len()); for (p, before) in paths.iter().zip(before) { @@ -196,27 +191,21 @@ impl FileMoves { .ok_or_else(|| IdeError::not_found("the refactoring can no longer be undone"))?; records.remove(i) }; - for m in &record.moves { - if !m.to.exists() { - return Err(IdeError::not_found(format!("{} no longer exists", m.to.display()))); - } - if m.from.exists() { - return Err(IdeError::new( - ide_core::error::ErrorKind::AlreadyExists, - format!("{} exists again", m.from.display()), - )); - } + let back: Vec<_> = + record.moves.iter().rev().map(|m| FileMove { from: m.to.clone(), to: m.from.clone() }).collect(); + if let Err(e) = crate::file_ops::move_files(&back) { + self.records.lock().push(record); + return Err(e); } - let mut outcome = UndoOutcome::default(); - for m in record.moves.iter().rev() { - ide_vfs::rename(&m.to, &m.from)?; - outcome.moves.push(FileMove { from: m.to.clone(), to: m.from.clone() }); - } - let back: Vec = outcome.moves.clone(); + let mut outcome = UndoOutcome { moves: back.clone(), conflicts: Vec::new() }; for f in &record.edited { let path = moved_path(&f.path, &back); match read_text(documents, &path) { - Ok(now) if now == f.after => write_text(documents, &path, &f.before)?, + Ok(now) if now == f.after => { + if write_text(documents, &path, &f.before).is_err() { + outcome.conflicts.push(path.into()); + } + } Ok(now) if now == f.before => {} _ => outcome.conflicts.push(path.into()), } diff --git a/crates/ide-workspace/src/file_ops.rs b/crates/ide-workspace/src/file_ops.rs new file mode 100644 index 00000000..6f3324d1 --- /dev/null +++ b/crates/ide-workspace/src/file_ops.rs @@ -0,0 +1,94 @@ +//! Project-aware file operations. Plain files retain the VFS behavior; project +//! rules may expand a resource operation before the common transaction runs. + +use crate::FileMove; +use ide_core::{IdeError, IdeResult, WorkspaceEdit}; +use std::path::{Path, PathBuf}; + +mod project; + +pub fn copy(from: &Path, to: &Path) -> IdeResult<()> { + project::copy(from, to).unwrap_or_else(|| ide_vfs::copy(from, to)) +} + +pub fn delete(path: &Path) -> IdeResult<()> { + project::delete(path).unwrap_or_else(|| ide_vfs::delete(path)) +} + +/// Files Local History must capture before deletion. +pub fn deletion_paths(path: &Path) -> IdeResult> { + project::deletion_paths(path).unwrap_or_else(|| Ok(vec![path.to_path_buf()])) +} + +pub fn rename(from: &Path, to: &Path) -> IdeResult<()> { + if !project::has_move_rules(from, to) { + return ide_vfs::rename(from, to); + } + let moves = expand_moves(&[FileMove { from: from.into(), to: to.into() }])?; + move_files(&moves) +} + +pub(crate) fn expand_moves(moves: &[FileMove]) -> IdeResult> { + let expanded = project::expand_moves(moves)?; + validate_moves(&expanded)?; + Ok(expanded) +} + +fn path_key(path: &Path) -> PathBuf { + #[cfg(windows)] + return path.to_string_lossy().replace('\\', "/").to_lowercase().into(); + #[cfg(not(windows))] + path.to_path_buf() +} + +fn validate_moves(moves: &[FileMove]) -> IdeResult<()> { + let mut sources = std::collections::HashSet::new(); + let mut targets = std::collections::HashSet::new(); + for m in moves { + ide_vfs::validate_rename(&m.from, &m.to)?; + let from = path_key(&m.from); + let to = path_key(&m.to); + if to != from && to.starts_with(&from) { + return Err(IdeError::invalid("Cannot move a directory into itself")); + } + if !sources.insert(from) || !targets.insert(to) { + return Err(IdeError::invalid("Duplicate source or destination in file operation")); + } + } + Ok(()) +} + +pub(crate) fn move_files(moves: &[FileMove]) -> IdeResult<()> { + execute_moves(moves, &|from, to| project::rename_entry(from, to).unwrap_or_else(|| ide_vfs::rename(from, to))) +} + +/// Executes an expanded batch. Rollback never overwrites a newly occupied path; +/// failures report where the caller can recover the remaining files. +pub(crate) fn execute_moves(moves: &[FileMove], rename: &dyn Fn(&Path, &Path) -> IdeResult<()>) -> IdeResult<()> { + validate_moves(moves)?; + for (i, m) in moves.iter().enumerate() { + if let Err(mut error) = rename(&m.from, &m.to) { + for done in moves[..i].iter().rev() { + if let Err(rollback) = + ide_vfs::validate_rename(&done.to, &done.from).and_then(|()| rename(&done.to, &done.from)) + { + error.message.push_str(&format!( + "; rollback {} -> {} failed: {rollback}", + done.to.display(), + done.from.display() + )); + } + } + return Err(error); + } + } + Ok(()) +} + +pub(crate) fn validate_workspace_edit_paths(edit: &WorkspaceEdit) -> IdeResult<()> { + project::validate_workspace_edit_paths(edit) +} + +pub(crate) fn expand_workspace_edit(edit: &mut WorkspaceEdit) -> IdeResult<()> { + project::expand_workspace_edit(edit) +} diff --git a/crates/ide-workspace/src/file_ops/project.rs b/crates/ide-workspace/src/file_ops/project.rs new file mode 100644 index 00000000..d38b6ccd --- /dev/null +++ b/crates/ide-workspace/src/file_ops/project.rs @@ -0,0 +1,52 @@ +//! The integration boundary for project-specific resource rules. These rules +//! never replace the plain VFS path unless an operation touches project assets. + +use crate::{FileMove, unity::assets}; +use ide_core::{FileEdit, IdeResult, WorkspaceEdit}; +use std::path::{Path, PathBuf}; + +pub(super) fn has_move_rules(from: &Path, to: &Path) -> bool { + assets::project_root(from).is_some() || assets::project_root(to).is_some() +} + +pub(super) fn copy(from: &Path, to: &Path) -> Option> { + has_move_rules(from, to).then(|| assets::copy(from, to)) +} + +pub(super) fn delete(path: &Path) -> Option> { + assets::project_root(path).map(|_| assets::delete(path)) +} + +pub(super) fn deletion_paths(path: &Path) -> Option>> { + assets::project_root(path).map(|_| assets::deletion_paths(path)) +} + +pub(super) fn rename_entry(from: &Path, to: &Path) -> Option> { + has_move_rules(from, to).then(|| assets::move_one(from, to)) +} + +pub(super) fn expand_moves(moves: &[FileMove]) -> IdeResult> { + if moves.iter().any(|m| has_move_rules(&m.from, &m.to)) { assets::expand_moves(moves) } else { Ok(moves.to_vec()) } +} + +fn has_edit_rules(edit: &WorkspaceEdit) -> bool { + edit.files.iter().any(|file| match file { + FileEdit::Rename { from, to } => has_move_rules(from, to), + FileEdit::Delete { path, .. } => assets::project_root(path).is_some(), + _ => false, + }) +} + +pub(super) fn validate_workspace_edit_paths(edit: &WorkspaceEdit) -> IdeResult<()> { + if has_edit_rules(edit) { + assets::validate_workspace_edit_paths(edit)?; + } + Ok(()) +} + +pub(super) fn expand_workspace_edit(edit: &mut WorkspaceEdit) -> IdeResult<()> { + if has_edit_rules(edit) { + assets::expand_workspace_edit(edit)?; + } + Ok(()) +} diff --git a/crates/ide-workspace/src/lib.rs b/crates/ide-workspace/src/lib.rs index 38ba30ca..18c29fc1 100644 --- a/crates/ide-workspace/src/lib.rs +++ b/crates/ide-workspace/src/lib.rs @@ -7,6 +7,7 @@ pub mod documents; pub mod editorconfig; pub mod file_defaults; pub mod file_moves; +pub mod file_ops; pub mod file_refs; pub mod history; pub mod memory; @@ -15,6 +16,7 @@ pub mod recent; pub mod roots; pub mod settings; pub mod text_refs; +pub mod unity; pub mod window_groups; pub mod workspace; diff --git a/crates/ide-workspace/src/roots/detect.rs b/crates/ide-workspace/src/roots/detect.rs index 3212bcff..1e243bc2 100644 --- a/crates/ide-workspace/src/roots/detect.rs +++ b/crates/ide-workspace/src/roots/detect.rs @@ -62,7 +62,10 @@ const ROOT_NAMES: &[&str] = &[ pub fn affects_roots(rel: &Path) -> bool { let mut names = rel.iter().rev().filter_map(|c| c.to_str()); let Some(name) = names.next() else { return false }; - BUILD_FILES.contains(&name) || ROOT_NAMES.contains(&name) || names.next() == Some("src") + BUILD_FILES.contains(&name) + || ROOT_NAMES.contains(&name) + || matches!(name, "Assets" | "Packages" | "ProjectSettings" | "ProjectVersion.txt") + || names.next() == Some("src") } /// Detects roots below `root`; paths in the result are unique and sorted. @@ -77,6 +80,29 @@ pub fn detect(root: &Path) -> Vec { break; } let dir = if rel.is_empty() { root.to_path_buf() } else { root.join(&rel) }; + if crate::unity::is_project(&dir) { + for (name, kind) in + [("Assets", RootKind::Source), ("Packages", RootKind::Source), ("ProjectSettings", RootKind::Resources)] + { + if dir.join(name).is_dir() { + out.push(SourceFolder { + path: join(&rel, name), + kind, + detected_from: Some(join(&rel, crate::unity::VERSION_FILE)), + }); + } + } + // Exclude before the index/watchers start, even if Unity creates its caches later. + for name in crate::unity::GENERATED_DIRS { + out.push(SourceFolder { + path: join(&rel, name), + kind: RootKind::Excluded, + detected_from: Some(join(&rel, crate::unity::VERSION_FILE)), + }); + } + // Assets and PackageCache may contain arbitrary build files; they aren't nested modules. + continue; + } let Ok(entries) = std::fs::read_dir(&dir) else { continue }; let mut files: Vec = Vec::new(); let mut dirs: Vec = Vec::new(); @@ -244,6 +270,43 @@ mod tests { roots.iter().map(|r| format!("{}={:?}@{}", r.path, r.kind, r.detected_from.as_deref().unwrap_or(""))).collect() } + #[test] + fn unity_roots_exclude_future_caches_and_do_not_scan_assets_as_modules() { + let dir = tree(&[ + "Assets/Scripts/Player.cs", + "Assets/Plugins/Tool/package.json", + "ProjectSettings/ProjectVersion.txt", + "Packages/manifest.json", + "Library/PackageCache/tool/Cargo.toml", + ]); + let roots = detect(dir.path()); + assert!(roots.iter().any(|r| r.path == "Assets" && r.kind == RootKind::Source)); + assert!(roots.iter().any(|r| r.path == "Packages" && r.kind == RootKind::Source)); + assert!(!roots.iter().any(|r| r.path.starts_with("Assets/") || r.path.starts_with("Library/"))); + let effective = super::super::ProjectRoots::compute(roots, &Default::default()); + for name in crate::unity::GENERATED_DIRS { + assert!(effective.excluded_globs().contains(&format!("/{name}"))); + } + for path in ["Assets", "Packages", "ProjectSettings", crate::unity::VERSION_FILE] { + assert!(affects_roots(Path::new(path))); + } + } + + #[test] + fn nested_unity_roots_are_anchored_without_excluding_other_projects() { + let dir = tree(&[ + "game/Assets/Scripts/Player.cs", + "game/ProjectSettings/ProjectVersion.txt", + "server/Cargo.toml", + "server/src/main.rs", + "Library/Source.cs", + ]); + let roots = super::super::ProjectRoots::compute(detect(dir.path()), &Default::default()); + assert!(roots.excluded_globs().contains(&"/game/Library".to_owned())); + assert!(!roots.excluded_globs().contains(&"/Library".to_owned())); + assert!(roots.roots.iter().any(|r| r.path == "server/src" && r.kind == RootKind::Source)); + } + #[test] fn maven_conventional_roots_and_target() { let dir = tree(&[ diff --git a/crates/ide-workspace/src/unity.rs b/crates/ide-workspace/src/unity.rs new file mode 100644 index 00000000..1d348b9d --- /dev/null +++ b/crates/ide-workspace/src/unity.rs @@ -0,0 +1,94 @@ +//! Unity project identity. Generated C# solutions are optional, not root markers. + +pub(crate) mod assets; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::io::Read; +use std::path::Path; + +pub const VERSION_FILE: &str = "ProjectSettings/ProjectVersion.txt"; +/// Root-relative caches and build outputs, including folders not created yet. +pub const GENERATED_DIRS: &[&str] = &["Library", "Temp", "Obj", "obj", "Logs", "Build", "Builds", "UserSettings"]; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct UnityProjectInfo { + pub editor_version: Option, + /// File names relative to the Unity root, in stable order. + pub solutions: Vec, + pub projects: Vec, +} + +pub fn is_project(root: &Path) -> bool { + root.join("Assets").is_dir() && root.join(VERSION_FILE).is_file() +} + +pub fn detect(root: &Path) -> Option { + if !is_project(root) { + return None; + } + let mut text = String::new(); + let _ = std::fs::File::open(root.join(VERSION_FILE)).and_then(|f| f.take(64 * 1024).read_to_string(&mut text)); + let editor_version = text.lines().find_map(|line| { + let (key, value) = line.trim_start_matches('\u{feff}').split_once(':')?; + let value = value.trim(); + (key.trim() == "m_EditorVersion" && !value.is_empty() && value.len() <= 128).then(|| value.to_owned()) + }); + let mut info = UnityProjectInfo { editor_version, solutions: Vec::new(), projects: Vec::new() }; + if let Ok(entries) = std::fs::read_dir(root) { + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_file()) { + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + match entry.path().extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref() { + Some("sln" | "slnx") => info.solutions.push(name), + Some("csproj") => info.projects.push(name), + _ => {} + } + } + } + info.solutions.sort(); + info.projects.sort(); + Some(info) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_fresh_and_legacy_projects_without_packages_or_solutions() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + std::fs::create_dir(root.join("Assets")).unwrap(); + assert!(detect(root).is_none()); + std::fs::create_dir(root.join("ProjectSettings")).unwrap(); + std::fs::write( + root.join(VERSION_FILE), + "\u{feff}m_EditorVersion: 2022.3.62f1\r\nm_EditorVersionWithRevision: ignored\r\n", + ) + .unwrap(); + let info = detect(root).unwrap(); + assert_eq!(info.editor_version.as_deref(), Some("2022.3.62f1")); + assert!(info.solutions.is_empty() && info.projects.is_empty()); + std::fs::write(root.join("Game.sln"), "").unwrap(); + std::fs::write(root.join("Game.slnx"), "").unwrap(); + std::fs::write(root.join("Assembly-CSharp.csproj"), "").unwrap(); + std::fs::create_dir(root.join("NotAProject.csproj")).unwrap(); + let info = detect(root).unwrap(); + assert_eq!(info.solutions, ["Game.sln", "Game.slnx"]); + assert_eq!(info.projects, ["Assembly-CSharp.csproj"]); + std::fs::write(root.join(VERSION_FILE), "malformed").unwrap(); + assert_eq!(detect(root).unwrap().editor_version, None); + } + + #[test] + fn ordinary_csharp_projects_are_not_unity_projects() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("App.csproj"), "").unwrap(); + std::fs::create_dir(temp.path().join("Assets")).unwrap(); + assert!(detect(temp.path()).is_none()); + } +} diff --git a/crates/ide-workspace/src/unity/assets.rs b/crates/ide-workspace/src/unity/assets.rs new file mode 100644 index 00000000..2aa69bdb --- /dev/null +++ b/crates/ide-workspace/src/unity/assets.rs @@ -0,0 +1,254 @@ +//! Filesystem operations for assets under a Unity project's Assets directory. +//! Metadata moves with the asset; duplication gets new GUIDs before publication. + +use crate::{file_moves::FileMove, file_ops::execute_moves}; +use ide_core::{IdeError, IdeResult, error::ErrorKind}; +use std::{ + collections::HashMap, + fs, + path::{Component, Path, PathBuf}, +}; + +mod duplicate; +mod workspace_edit; +pub(crate) use workspace_edit::{expand_workspace_edit, validate_workspace_edit_paths}; +#[cfg(test)] +mod tests; + +pub fn meta_path(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".meta"); + name.into() +} + +/// Includes Assets itself so callers can protect the asset root. +pub fn project_root(path: &Path) -> Option { + path.ancestors().find_map(|p| { + (p.file_name().is_some_and(|n| if cfg!(windows) { n.eq_ignore_ascii_case("Assets") } else { n == "Assets" }) + && p.parent().is_some_and(super::is_project)) + .then(|| p.parent().unwrap().to_path_buf()) + }) +} + +fn exists(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok() +} + +fn is_meta(path: &Path) -> bool { + path.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("meta")) +} + +fn validate_asset(path: &Path) -> IdeResult<()> { + if let Some(root) = project_root(path) { + if key(path) == key(&root.join("Assets")) { + return Err(IdeError::invalid("The Unity Assets root cannot be moved, copied or deleted")); + } + if is_meta(path) && exists(&path.with_extension("")) { + return Err(IdeError::invalid( + "Select the Unity asset instead of its .meta file; metadata is handled together", + )); + } + } + no_links(path) +} + +fn no_links(path: &Path) -> IdeResult<()> { + if path.components().any(|c| matches!(c, Component::ParentDir)) { + return Err(IdeError::invalid("Unity asset operations require paths without '..'")); + } + // Trust the opened project's location, including a linked root or parent + // (macOS /var and /tmp). Check inside it, including Library staging paths. + // The outermost root prevents a nested Unity project from hiding a link. + let root = path.ancestors().filter(|p| super::is_project(p)).last(); + for p in path.ancestors().take_while(|p| Some(*p) != root) { + match fs::symlink_metadata(p) { + Ok(m) if is_link(&m) => { + return Err(IdeError::invalid(format!("Unity asset operations do not follow links: {}", p.display()))); + } + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + } + Ok(()) +} + +fn is_link(metadata: &fs::Metadata) -> bool { + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + // Also reject directory junctions and other reparse points. + metadata.file_attributes() & 0x400 != 0 + } + #[cfg(not(windows))] + { + metadata.file_type().is_symlink() + } +} + +fn key(path: &Path) -> String { + let s = path.to_string_lossy().replace('\\', "/"); + if cfg!(windows) { s.to_lowercase() } else { s } +} + +fn vacant(from: &Path, to: &Path) -> IdeResult<()> { + if exists(to) { + if !exists(from) { + return Err(IdeError::new(ErrorKind::AlreadyExists, format!("{} already exists", to.display()))); + } + ide_vfs::validate_rename(from, to)?; + } + Ok(()) +} + +/// Expand before applying reference edits, so orphan metadata conflicts cannot +/// leave a partially refactored asset. Metadata goes first on publication. +pub fn expand_moves(moves: &[FileMove]) -> IdeResult> { + let mut out = Vec::new(); + for m in moves { + let unity = project_root(&m.from).is_some() || project_root(&m.to).is_some(); + if unity { + // A caller may explicitly select both an asset and its metadata. + if is_meta(&m.from) + && moves + .iter() + .any(|a| meta_path(&a.from) == m.from.to_path_buf() && meta_path(&a.to) == m.to.to_path_buf()) + { + continue; + } + validate_asset(&m.from)?; + validate_asset(&m.to)?; + let from = meta_path(&m.from); + let to = meta_path(&m.to); + no_links(&from)?; + no_links(&to)?; + vacant(&from, &to)?; + if exists(&from) { + out.push(FileMove { from: from.into(), to: to.into() }); + } + } + out.push(m.clone()); + } + Ok(out) +} + +pub(crate) fn move_one(from: &Path, to: &Path) -> IdeResult<()> { + if project_root(from).is_some() || project_root(to).is_some() { + no_links(from)?; + no_links(to)?; + if let Some(parent) = to.parent() { + fs::create_dir_all(parent)?; + } + // A failed cross-volume copy/delete cannot be rolled back reliably. + // Keep paired Unity moves atomic per entry and let rename report it. + fs::rename(from, to)?; + Ok(()) + } else { + ide_vfs::rename(from, to) + } +} + +/// The paths Local History must snapshot before an asset deletion. +pub fn deletion_paths(path: &Path) -> IdeResult> { + validate_asset(path)?; + let meta = meta_path(path); + no_links(&meta)?; + let mut paths = vec![path.to_path_buf()]; + if exists(&meta) { + paths.push(meta); + } + Ok(paths) +} + +fn staging(root: &Path) -> IdeResult { + let parent = root.join("Library/OpenIDEA/AssetOperations"); + no_links(&parent)?; + fs::create_dir_all(&parent)?; + let dir = parent.join(uuid::Uuid::new_v4().simple().to_string()); + fs::create_dir(&dir)?; + Ok(dir) +} + +pub fn delete(path: &Path) -> IdeResult<()> { + delete_with(path, &ide_vfs::delete) +} + +fn delete_with(path: &Path, remove: &dyn Fn(&Path) -> IdeResult<()>) -> IdeResult<()> { + let paths = deletion_paths(path)?; + let root = project_root(path).ok_or_else(|| IdeError::invalid("Expected a Unity asset path"))?; + let stage = staging(&root)?; + let payload = stage.join("files"); + fs::create_dir(&payload)?; + let moves: Vec<_> = paths + .iter() + .map(|p| FileMove { from: p.clone().into(), to: payload.join(p.file_name().unwrap()).into() }) + .collect(); + // Recycle-bin restore returns the group to staging. Keep its destinations + // with it so originals can also be recovered without the IDE's history. + let manifest = stage.join("restore-paths.json"); + fs::write(&manifest, serde_json::to_vec_pretty(&paths).map_err(|e| IdeError::internal(e.to_string()))?)?; + // Never automatically delete this staging directory on failure: it may + // contain originals when rollback fails. + if let Err(e) = execute_moves(&moves, &move_one) { + if fs::remove_dir(&payload).is_ok() { + let _ = fs::remove_file(&manifest); + let _ = fs::remove_dir(&stage); + } + return Err(e); + } + if let Err(mut e) = remove(&stage) { + let back: Vec<_> = moves.iter().rev().map(|m| FileMove { from: m.to.clone(), to: m.from.clone() }).collect(); + if let Err(rollback) = execute_moves(&back, &move_one) { + e.message.push_str(&format!("; recovery files at {}: {rollback}", stage.display())); + } + if fs::remove_dir(&payload).is_ok() { + let _ = fs::remove_file(&manifest); + let _ = fs::remove_dir(&stage); + } + return Err(e); + } + Ok(()) +} + +pub fn copy(from: &Path, to: &Path) -> IdeResult<()> { + let root = project_root(to) + .or_else(|| project_root(from)) + .ok_or_else(|| IdeError::invalid("Expected a Unity asset path"))?; + validate_asset(from)?; + validate_asset(to)?; + if key(from) == key(to) || key(to).starts_with(&(key(from) + "/")) { + return Err(IdeError::invalid("Cannot copy an asset onto or into itself")); + } + if exists(to) { + return Err(IdeError::new(ErrorKind::AlreadyExists, format!("{} already exists", to.display()))); + } + let src_meta = meta_path(from); + let dst_meta = meta_path(to); + no_links(&src_meta)?; + no_links(&dst_meta)?; + // Copies never overwrite metadata, including orphan destination metadata. + if exists(&dst_meta) { + return Err(IdeError::new(ErrorKind::AlreadyExists, format!("{} already exists", dst_meta.display()))); + } + let stage = staging(&root)?; + let result = (|| { + let asset = stage.join(from.file_name().ok_or_else(|| IdeError::invalid("Missing asset name"))?); + duplicate::copy_tree(from, &asset)?; + let meta = meta_path(&asset); + if exists(&src_meta) { + duplicate::copy_tree(&src_meta, &meta)?; + } + if project_root(to).is_some() { + duplicate::regenerate(&stage)?; + } + let mut moves = Vec::new(); + if exists(&meta) { + moves.push(FileMove { from: meta.into(), to: dst_meta.into() }); + } + moves.push(FileMove { from: asset.into(), to: to.into() }); + execute_moves(&moves, &move_one) + })(); + // Only disposable copies live here, never the source assets. + let _ = fs::remove_dir_all(&stage); + result +} diff --git a/crates/ide-workspace/src/unity/assets/duplicate.rs b/crates/ide-workspace/src/unity/assets/duplicate.rs new file mode 100644 index 00000000..f35642ae --- /dev/null +++ b/crates/ide-workspace/src/unity/assets/duplicate.rs @@ -0,0 +1,236 @@ +use super::*; +use std::io::Read; + +const TEXT_LIMIT: u64 = 32 * 1024 * 1024; + +pub(super) fn copy_tree(from: &Path, to: &Path) -> IdeResult<()> { + no_links(from)?; + let metadata = fs::symlink_metadata(from)?; + if metadata.is_dir() { + fs::create_dir(to)?; + for entry in fs::read_dir(from)? { + let entry = entry?; + copy_tree(&entry.path(), &to.join(entry.file_name()))?; + } + } else if metadata.is_file() { + fs::copy(from, to)?; + } else { + return Err(IdeError::invalid(format!("Not a regular Unity asset: {}", from.display()))); + } + Ok(()) +} + +fn files(dir: &Path, out: &mut Vec) -> IdeResult<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + files(&entry.path(), out)?; + } else { + out.push(entry.path()); + } + } + Ok(()) +} + +fn read_text(path: &Path) -> IdeResult { + let mut bytes = Vec::new(); + fs::File::open(path)?.take(TEXT_LIMIT + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > TEXT_LIMIT { + return Err(IdeError::invalid(format!( + "Unity serialized asset exceeds the 32 MiB copy limit: {}", + path.display() + ))); + } + String::from_utf8(bytes) + .map_err(|_| IdeError::invalid(format!("Cannot rewrite binary Unity metadata: {}", path.display()))) +} + +fn guid(value: &str) -> bool { + value.len() == 32 && value.bytes().all(|b| b.is_ascii_hexdigit()) && value.bytes().any(|b| b != b'0') +} + +pub(super) fn regenerate(stage: &Path) -> IdeResult<()> { + let mut paths = Vec::new(); + files(stage, &mut paths)?; + let mut ids = HashMap::new(); + for path in paths.iter().filter(|p| is_meta(p)) { + let text = read_text(path)?; + let values: Vec<_> = text + .trim_start_matches('\u{feff}') + .lines() + .filter_map(|l| l.strip_prefix("guid:").map(str::trim)) + .collect(); + if values.len() != 1 || !guid(values[0]) { + return Err(IdeError::invalid(format!("Missing or invalid GUID in {}", path.display()))); + } + if ids.insert(values[0].to_ascii_lowercase(), uuid::Uuid::new_v4().simple().to_string()).is_some() { + return Err(IdeError::invalid("Duplicate source GUIDs; resolve them in Unity before copying")); + } + } + for path in paths { + let ext = path.extension().unwrap_or_default().to_string_lossy().to_ascii_lowercase(); + if ext == "asmdef" || ext == "asmref" { + let text = read_text(&path)?; + let mut json: serde_json::Value = + serde_json::from_str(&text).map_err(|e| IdeError::invalid(e.to_string()))?; + if rewrite_json(&mut json, &ids) { + fs::write(&path, serde_json::to_string_pretty(&json).map_err(|e| IdeError::internal(e.to_string()))?)?; + } + continue; + } + let mut header = [0; 32]; + let n = fs::File::open(&path)?.read(&mut header)?; + let header = String::from_utf8_lossy(&header[..n]); + let header = header.trim_start_matches('\u{feff}'); + let yaml = header.starts_with("%YAML") || header.starts_with("--- !u!"); + if !yaml && !is_meta(&path) { + // These formats may be binary under Mixed/Force Binary. We cannot + // safely preserve internal references without Unity deserialization. + if matches!( + ext.as_str(), + "asset" + | "prefab" + | "unity" + | "mat" + | "anim" + | "controller" + | "overridecontroller" + | "playable" + | "mask" + | "rendertexture" + | "physicmaterial" + | "physicsmaterial2d" + | "lighting" + | "terrainlayer" + | "guiskin" + | "flare" + | "cubemap" + | "spriteatlas" + | "spriteatlasv2" + | "preset" + | "shadervariants" + | "giparams" + | "lightingdata" + | "fontsettings" + | "texture2d" + | "mesh" + ) { + return Err(IdeError::invalid(format!( + "Cannot copy binary Unity serialized asset {}; use Force Text serialization in Unity", + path.display() + ))); + } + continue; + } + let before = read_text(&path)?; + let after = rewrite_yaml(&before, &ids); + if before != after { + fs::write(&path, after)?; + } + } + Ok(()) +} + +fn rewrite_json(value: &mut serde_json::Value, ids: &HashMap) -> bool { + match value { + serde_json::Value::String(s) => { + if let Some(id) = s.strip_prefix("GUID:").and_then(|id| ids.get(&id.to_ascii_lowercase())) { + *s = format!("GUID:{id}"); + true + } else { + false + } + } + serde_json::Value::Array(a) => { + let mut changed = false; + for v in a { + changed |= rewrite_json(v, ids); + } + changed + } + serde_json::Value::Object(o) => { + let mut changed = false; + for v in o.values_mut() { + changed |= rewrite_json(v, ids); + } + changed + } + _ => false, + } +} + +/// Patch GUID scalar tokens without reserializing Unity's YAML dialect. Quoted +/// strings, comments and indented block scalars are not asset references. +fn rewrite_yaml(text: &str, ids: &HashMap) -> String { + let mut bytes = text.as_bytes().to_vec(); + let mut offset = 0; + let mut block_indent = None; + let mut quote = None; + for line in text.split_inclusive('\n') { + let raw = line.as_bytes(); + let indent = raw.iter().take_while(|b| **b == b' ').count(); + if block_indent.is_some_and(|n| indent > n || line.trim().is_empty()) { + offset += raw.len(); + continue; + } + block_indent = None; + let mut i = 0; + while i < raw.len() { + let b = raw[i]; + if let Some(q) = quote { + if q == b'"' && b == b'\\' { + i += 2; + continue; + } + if b == q { + if q == b'\'' && raw.get(i + 1) == Some(&q) { + i += 2; + continue; + } + quote = None; + } + i += 1; + continue; + } + if b == b'#' && (i == 0 || raw[i - 1].is_ascii_whitespace()) { + break; + } + if b == b'"' || b == b'\'' { + let prefix = line[..i].trim(); + // Apostrophes inside plain scalars (e.g. Hero's Sword) do not + // open a quoted scalar or suppress references on later lines. + if prefix.is_empty() || prefix.ends_with([':', '-', '[', '{', ',']) { + quote = Some(b); + i += 1; + continue; + } + } + if (b == b'|' || b == b'>') && line[..i].trim_end().ends_with([':', '-']) { + block_indent = Some(indent); + break; + } + if raw[i..].starts_with(b"guid:") { + let prefix = line[..i].trim_start_matches('\u{feff}').trim_end(); + if prefix.is_empty() || prefix.ends_with(['{', ',']) { + let start = i + 5 + raw[i + 5..].iter().take_while(|b| **b == b' ' || **b == b'\t').count(); + let end = start + 32; + if end <= raw.len() + && raw[start..end].iter().all(u8::is_ascii_hexdigit) + && (end == raw.len() + || raw[end].is_ascii_whitespace() + || matches!(raw[end], b',' | b'}' | b'#')) + && let Some(new) = ids.get(&line[start..end].to_ascii_lowercase()) + { + bytes[offset + start..offset + end].copy_from_slice(new.as_bytes()); + i = end; + continue; + } + } + } + // Non-ASCII bytes cannot begin any token handled above. + i += 1; + } + offset += raw.len(); + } + String::from_utf8(bytes).expect("GUID substitutions preserve UTF-8") +} diff --git a/crates/ide-workspace/src/unity/assets/tests.rs b/crates/ide-workspace/src/unity/assets/tests.rs new file mode 100644 index 00000000..3dd670e1 --- /dev/null +++ b/crates/ide-workspace/src/unity/assets/tests.rs @@ -0,0 +1,217 @@ +use super::*; +use crate::file_ops::{move_files, rename}; +use crate::{documents::Documents, file_moves::FileMoves}; +use ide_core::{LanguageId, WorkspaceEdit}; +use std::sync::Arc; + +mod links; + +const A: &str = "11111111111111111111111111111111"; +const B: &str = "22222222222222222222222222222222"; +const C: &str = "33333333333333333333333333333333"; + +fn project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + fs::create_dir(tmp.path().join("Assets")).unwrap(); + fs::create_dir(tmp.path().join("ProjectSettings")).unwrap(); + fs::write(tmp.path().join(super::super::VERSION_FILE), "m_EditorVersion: 6000.0.1f1").unwrap(); + tmp +} + +fn asset(root: &Path, name: &str, content: &str, id: &str) -> PathBuf { + let path = root.join("Assets").join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, content).unwrap(); + fs::write(meta_path(&path), format!("fileFormatVersion: 2\r\nguid: {id}\r\nImporter:\r\n setting: keep\r\n")) + .unwrap(); + path +} + +fn id(path: &Path) -> String { + fs::read_to_string(meta_path(path)) + .unwrap() + .lines() + .find_map(|l| l.strip_prefix("guid: ").map(str::to_owned)) + .unwrap() +} + +#[test] +fn rename_preserves_metadata_and_undo_retries_after_conflict() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "class Player {}", A); + let dst = src.with_file_name("Hero.cs"); + let docs = Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + let stack = FileMoves::new(); + let moves = [FileMove { from: src.clone().into(), to: dst.clone().into() }]; + let token = stack.perform(&docs, None, &moves, &WorkspaceEdit::default(), "Rename", &|_| Ok(())).unwrap(); + assert_eq!(id(&dst), A); + assert!(!meta_path(&src).exists()); + fs::write(meta_path(&src), "conflict").unwrap(); + assert!(stack.undo(&docs, token).is_err()); + assert!(stack.can_undo(token)); + assert!(dst.exists()); + fs::remove_file(meta_path(&src)).unwrap(); + let undone = stack.undo(&docs, token).unwrap(); + assert_eq!(undone.moves.len(), 2); + assert_eq!(id(&src), A); + assert!(!dst.exists()); +} + +#[test] +fn orphan_metadata_conflicts_and_standalone_metadata_are_protected() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "code", A); + let dst = src.with_file_name("Hero.cs"); + fs::write(meta_path(&dst), "orphan").unwrap(); + assert!(rename(&src, &dst).is_err()); + assert!(copy(&src, &dst).is_err()); + assert!(src.exists() && !dst.exists()); + assert_eq!(id(&src), A); + assert!(rename(&meta_path(&src), &src.with_file_name("Other.meta")).is_err()); + assert!(delete_with(&meta_path(&src), &|_| panic!("must reject before deletion")).is_err()); + assert!(rename(&tmp.path().join("Assets"), &tmp.path().join("Renamed")).is_err()); +} + +#[test] +fn folder_copy_regenerates_guids_and_only_internal_references() { + let tmp = project(); + let root = tmp.path(); + let text = format!( + "%YAML 1.1\n--- !u!1 &1\nObject:\n internal: {{fileID: 1, guid: {B}, type: 3}}\n external: {{fileID: 2, guid: {C}, type: 3}}\n quoted: \"guid: {B}\"\n block: |\n guid: {B}\n" + ); + let src = asset(root, "Group/Scene.prefab", &text, A); + asset(root, "Group/Script.cs", "// untouched", B); + fs::write(root.join("Assets/Group.meta"), format!("guid: {C}\nfolderAsset: yes\n")).unwrap(); + // An actual external id not copied into Group. + let external = "44444444444444444444444444444444"; + fs::write( + &src, + text.replace(&format!("external: {{fileID: 2, guid: {C}"), &format!("external: {{fileID: 2, guid: {external}")), + ) + .unwrap(); + fs::write(root.join("Assets/Group/Assembly.asmdef"), format!(r#"{{"references":["GUID:{B}","GUID:{external}"]}}"#)) + .unwrap(); + let to = root.join("Assets/Group Copy"); + copy(&root.join("Assets/Group"), &to).unwrap(); + let script = id(&to.join("Script.cs")); + assert_ne!(script, B); + assert_ne!(id(&to.join("Scene.prefab")), A); + assert_ne!(id(&to), C); + assert_eq!(id(&src), A); + let copied = fs::read_to_string(to.join("Scene.prefab")).unwrap(); + assert!(copied.contains(&format!("internal: {{fileID: 1, guid: {script}"))); + assert!(copied.contains(&format!("external: {{fileID: 2, guid: {external}"))); + assert!(copied.contains(&format!("quoted: \"guid: {B}\""))); + assert!(copied.contains(&format!("block: |\n guid: {B}"))); + assert!(fs::read_to_string(meta_path(&to.join("Script.cs"))).unwrap().contains("setting: keep\r\n")); + let assembly = fs::read_to_string(to.join("Assembly.asmdef")).unwrap(); + assert!(assembly.contains(&format!("GUID:{script}")) && assembly.contains(external)); +} + +#[test] +fn invalid_or_binary_copy_never_publishes_partial_assets() { + let tmp = project(); + let src = asset(tmp.path(), "Group/Data.asset", "binary data", A); + let from = src.parent().unwrap(); + let to = from.with_file_name("Copy"); + assert!(copy(from, &to).is_err()); + assert!(!to.exists() && !meta_path(&to).exists()); + assert_eq!(id(&src), A); + fs::write(&src, "%YAML 1.1\n--- !u!1 &1\n").unwrap(); + fs::write(meta_path(&src), "guid: invalid\n").unwrap(); + assert!(copy(from, &to).is_err()); + assert!(!to.exists()); +} + +#[test] +fn deletion_groups_sidecar_and_restores_both_on_trash_failure() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "code", A); + assert_eq!(deletion_paths(&src).unwrap(), vec![src.clone(), meta_path(&src)]); + let error = delete_with(&src, &|stage| { + assert!(stage.join("files/Player.cs").exists()); + assert!(stage.join("files/Player.cs.meta").exists()); + let destinations: Vec = + serde_json::from_slice(&fs::read(stage.join("restore-paths.json")).unwrap()).unwrap(); + assert_eq!(destinations, vec![src.clone(), meta_path(&src)]); + Err(IdeError::internal("trash unavailable")) + }) + .unwrap_err(); + assert!(error.message.contains("trash unavailable")); + assert!(src.exists()); + assert_eq!(id(&src), A); + delete_with(&src, &ide_vfs::delete_permanently).unwrap(); + assert!(!src.exists() && !meta_path(&src).exists()); +} + +#[test] +fn selecting_both_asset_and_sidecar_does_not_move_twice() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "code", A); + let dst = src.with_file_name("Hero.cs"); + let expanded = expand_moves(&[ + FileMove { from: src.clone().into(), to: dst.clone().into() }, + FileMove { from: meta_path(&src).into(), to: meta_path(&dst).into() }, + ]) + .unwrap(); + assert_eq!(expanded.len(), 2); + move_files(&expanded).unwrap(); + assert_eq!(id(&dst), A); +} + +#[test] +fn non_unity_operations_do_not_touch_adjacent_metadata() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("file.txt"); + let dst = tmp.path().join("copy.txt"); + fs::write(&src, "text").unwrap(); + fs::write(meta_path(&src), "unrelated").unwrap(); + crate::file_ops::copy(&src, &dst).unwrap(); + assert!(!meta_path(&dst).exists()); + rename(&src, &tmp.path().join("moved.txt")).unwrap(); + assert!(meta_path(&src).exists()); +} + +#[test] +fn failed_second_move_rolls_back_metadata() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "code", A); + let meta_dst = src.with_file_name("Moved.cs.meta"); + let blocker = tmp.path().join("Assets/blocked"); + fs::write(&blocker, "not a directory").unwrap(); + let moves = [ + FileMove { from: meta_path(&src).into(), to: meta_dst.clone().into() }, + FileMove { from: src.clone().into(), to: blocker.join("Moved.cs").into() }, + ]; + assert!(move_files(&moves).is_err()); + assert_eq!(id(&src), A); + assert!(src.exists() && !meta_dst.exists()); +} + +#[test] +fn yaml_scalar_text_does_not_change_reference_detection() { + let tmp = project(); + let text = format!( + "%YAML 1.1\n--- !u!1 &1\nObject:\n name: Hero's Sword\n self: {{fileID: 1, guid: {A}, type: 3}}\n text: \"escaped \\\" guid: {A}\"\n multiline: 'value\n guid: {A}'\n selfAgain: {{fileID: 1, guid: {A}, type: 3}}\n" + ); + let src = asset(tmp.path(), "Player.prefab", &text, A); + let dst = src.with_file_name("Copied.prefab"); + copy(&src, &dst).unwrap(); + let new_id = id(&dst); + let copied = fs::read_to_string(&dst).unwrap(); + assert!(copied.contains(&format!("self: {{fileID: 1, guid: {new_id}"))); + assert!(copied.contains(&format!("selfAgain: {{fileID: 1, guid: {new_id}"))); + assert!(copied.contains(&format!("multiline: 'value\n guid: {A}'"))); + assert!(copied.contains(&format!("text: \"escaped \\\" guid: {A}\""))); +} + +#[cfg(windows)] +#[test] +fn case_only_rename_keeps_metadata() { + let tmp = project(); + let src = asset(tmp.path(), "Player.cs", "code", A); + let dst = src.with_file_name("player.cs"); + rename(&src, &dst).unwrap(); + assert_eq!(id(&dst), A); + assert!(fs::read_dir(dst.parent().unwrap()).unwrap().any(|e| e.unwrap().file_name() == "player.cs.meta")); +} diff --git a/crates/ide-workspace/src/unity/assets/tests/links.rs b/crates/ide-workspace/src/unity/assets/tests/links.rs new file mode 100644 index 00000000..cabbb176 --- /dev/null +++ b/crates/ide-workspace/src/unity/assets/tests/links.rs @@ -0,0 +1,233 @@ +use super::*; +use ide_core::FileEdit; + +fn dir_link(target: &Path, link: &Path) { + #[cfg(unix)] + std::os::unix::fs::symlink(target, link).unwrap(); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + let output = std::process::Command::new("cmd") + .args(["/c", "mklink", "/J"]) + .arg(link.to_string_lossy().replace('/', "\\")) + .arg(target.to_string_lossy().replace('/', "\\")) + .creation_flags(0x08000000) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + } +} + +#[test] +fn linked_parent_and_project_root_support_asset_operations_and_undo() { + for parent_alias in [false, true] { + let tmp = project(); + let aliases = tempfile::tempdir().unwrap(); + let link = aliases.path().join("alias"); + let root = if parent_alias { + dir_link(tmp.path().parent().unwrap(), &link); + link.join(tmp.path().file_name().unwrap()) + } else { + dir_link(tmp.path(), &link); + link.clone() + }; + let from = asset(&root, "Player.cs", "class Player {}", A); + let to = root.join("Assets/New/Nested/Hero.cs"); + let docs = Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + let moves = FileMoves::new(); + let token = moves + .perform( + &docs, + None, + &[FileMove { from: from.clone().into(), to: to.clone().into() }], + &WorkspaceEdit::default(), + "Move", + &|_| Ok(()), + ) + .unwrap(); + assert_eq!(id(&to), A); + moves.undo(&docs, token).unwrap(); + assert_eq!(id(&from), A); + copy(&from, &to).unwrap(); + assert_ne!(id(&to), A); + assert_eq!(id(&from), A); + delete_with(&to, &ide_vfs::delete_permanently).unwrap(); + assert!(!to.exists() && !meta_path(&to).exists()); + let intermediate = root.join("Assets/Intermediate.cs"); + docs.apply_workspace_edit(&WorkspaceEdit { + files: vec![ + FileEdit::Rename { from: from.clone().into(), to: intermediate.clone().into() }, + FileEdit::Rename { from: intermediate.into(), to: to.clone().into() }, + ], + ..Default::default() + }) + .unwrap(); + assert_eq!(id(&to), A); + assert!(!from.exists() && !meta_path(&from).exists()); + } +} + +#[test] +fn internal_directory_links_reject_asset_operations_before_mutation() { + for relative in ["Assets/Linked", "Library", "Library/OpenIDEA", "Library/OpenIDEA/AssetOperations"] { + let tmp = project(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("sentinel"), "keep").unwrap(); + let from = asset(tmp.path(), "Player.cs", "code", A); + let link = tmp.path().join(relative); + fs::create_dir_all(link.parent().unwrap()).unwrap(); + dir_link(outside.path(), &link); + let to = if relative.starts_with("Assets") { link.join("Copy.cs") } else { from.with_file_name("Copy.cs") }; + assert!(copy(&from, &to).unwrap_err().message.contains("do not follow links")); + if relative.starts_with("Assets") { + assert!(rename(&from, &to).is_err()); + let docs = + Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + assert!( + docs.apply_workspace_edit(&WorkspaceEdit { + files: vec![FileEdit::Rename { from: from.clone().into(), to: to.clone().into() },], + ..Default::default() + }) + .is_err() + ); + let sentinel = link.join("sentinel"); + assert!( + docs.apply_workspace_edit(&WorkspaceEdit { + files: vec![FileEdit::Delete { path: sentinel.into(), recursive: false },], + ..Default::default() + }) + .is_err() + ); + } else { + assert!(delete_with(&from, &|_| panic!("must reject before removal")).is_err()); + } + assert_eq!(id(&from), A); + assert!(from.exists() && !to.exists() && !meta_path(&to).exists()); + assert_eq!(fs::read_to_string(outside.path().join("sentinel")).unwrap(), "keep"); + assert_eq!(fs::read_dir(outside.path()).unwrap().count(), 1); + } +} + +#[test] +fn linked_assets_root_and_recursive_copy_links_are_rejected() { + let tmp = project(); + let outside = tempfile::tempdir().unwrap(); + fs::remove_dir(tmp.path().join("Assets")).unwrap(); + dir_link(outside.path(), &tmp.path().join("Assets")); + let from = asset(tmp.path(), "Player.cs", "code", A); + assert!(rename(&from, &from.with_file_name("Hero.cs")).is_err()); + assert!(deletion_paths(&from).is_err()); + assert_eq!(id(&from), A); + + let tmp = project(); + let from = asset(tmp.path(), "Group/Player.cs", "code", A); + dir_link(outside.path(), &tmp.path().join("Assets/Group/Linked")); + let to = tmp.path().join("Assets/Copy"); + assert!(copy(from.parent().unwrap(), &to).is_err()); + assert!(!to.exists() && !meta_path(&to).exists()); + assert_eq!(id(&from), A); +} + +#[cfg(unix)] +#[test] +fn linked_asset_and_sidecar_leaves_are_rejected() { + for sidecar in [false, true] { + let tmp = project(); + let outside = tempfile::NamedTempFile::new().unwrap(); + let from = asset(tmp.path(), "Player.cs", "code", A); + let path = if sidecar { meta_path(&from) } else { from.clone() }; + fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink(outside.path(), &path).unwrap(); + assert!(rename(&from, &from.with_file_name("Hero.cs")).is_err()); + assert!(copy(&from, &from.with_file_name("Copy.cs")).is_err()); + assert!(deletion_paths(&from).is_err()); + assert!(fs::symlink_metadata(&path).unwrap().file_type().is_symlink()); + } +} + +#[test] +fn cross_project_operations_validate_each_root_and_keep_metadata() { + let source = project(); + let destination = project(); + let aliases = tempfile::tempdir().unwrap(); + let a = aliases.path().join("source"); + let b = aliases.path().join("destination"); + dir_link(source.path(), &a); + dir_link(destination.path(), &b); + let from = asset(&a, "Player.cs", "code", A); + let copied = b.join("Assets/Copied.cs"); + copy(&from, &copied).unwrap(); + assert_ne!(id(&copied), A); + assert_eq!(id(&from), A); + let moved = b.join("Assets/Moved.cs"); + rename(&from, &moved).unwrap(); + assert_eq!(id(&moved), A); + assert!(!from.exists() && !meta_path(&from).exists()); +} + +#[test] +fn external_links_do_not_become_trusted_by_importing_into_unity() { + let tmp = project(); + let external = tempfile::tempdir().unwrap(); + let aliases = tempfile::tempdir().unwrap(); + fs::write(external.path().join("File.cs"), "code").unwrap(); + let link = aliases.path().join("external"); + dir_link(external.path(), &link); + let from = link.join("File.cs"); + let to = tmp.path().join("Assets/Imported.cs"); + assert!(copy(&from, &to).is_err()); + assert!(rename(&from, &to).is_err()); + assert!(from.exists() && !to.exists()); +} + +#[test] +fn folder_rename_cannot_hide_linked_children_in_later_transaction_operations() { + let tmp = project(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("sentinel"), "keep").unwrap(); + let from = tmp.path().join("Assets/Group"); + fs::create_dir(&from).unwrap(); + dir_link(outside.path(), &from.join("Linked")); + let to = tmp.path().join("Assets/Moved"); + let docs = Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + let error = docs + .apply_workspace_edit(&WorkspaceEdit { + files: vec![ + FileEdit::Rename { from: from.clone().into(), to: to.clone().into() }, + FileEdit::Delete { + path: (if cfg!(windows) { tmp.path().join("Assets/moved") } else { to.clone() }) + .join("Linked/sentinel") + .into(), + recursive: false, + }, + ], + ..Default::default() + }) + .unwrap_err(); + assert!(error.message.contains("do not follow links")); + assert!(from.exists() && !to.exists()); + assert_eq!(fs::read_to_string(outside.path().join("sentinel")).unwrap(), "keep"); +} + +#[test] +fn directory_transactions_reject_nested_links_before_snapshotting() { + for delete in [false, true] { + let tmp = project(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("sentinel"), "keep").unwrap(); + let from = tmp.path().join("Assets/Group"); + fs::create_dir(&from).unwrap(); + dir_link(outside.path(), &from.join("Linked")); + let to = tmp.path().join("Assets/Moved"); + let file = if delete { + FileEdit::Delete { path: from.clone().into(), recursive: true } + } else { + FileEdit::Rename { from: from.clone().into(), to: to.clone().into() } + }; + let docs = Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + let error = docs.apply_workspace_edit(&WorkspaceEdit { files: vec![file], ..Default::default() }).unwrap_err(); + assert!(error.message.contains("do not follow links")); + assert!(from.exists() && !to.exists()); + assert_eq!(fs::read_to_string(outside.path().join("sentinel")).unwrap(), "keep"); + } +} diff --git a/crates/ide-workspace/src/unity/assets/workspace_edit.rs b/crates/ide-workspace/src/unity/assets/workspace_edit.rs new file mode 100644 index 00000000..c3a5baf1 --- /dev/null +++ b/crates/ide-workspace/src/unity/assets/workspace_edit.rs @@ -0,0 +1,99 @@ +//! Include sidecars in the document transaction's preflight, recovery and undo. +use super::{exists, fs, is_link, key, meta_path, no_links, project_root, validate_asset}; +use ide_core::{FileEdit, IdeError, IdeResult, WorkspaceEdit}; +use std::path::Path; + +pub(crate) fn validate_workspace_edit_paths(edit: &WorkspaceEdit) -> IdeResult<()> { + for file in &edit.files { + let validate = |path: &Path| -> IdeResult<()> { + validate_asset(path)?; + no_links(&meta_path(path))?; + no_nested_links(path) + }; + match file { + FileEdit::Rename { from, to } if project_root(from).is_some() || project_root(to).is_some() => { + validate(from)?; + validate(to)?; + } + FileEdit::Delete { path, .. } if project_root(path).is_some() => validate(path)?, + _ => {} + } + } + Ok(()) +} + +// Transactions snapshot directories recursively. Reject linked descendants +// before backup, including junctions and children renamed by a later operation. +fn no_nested_links(path: &Path) -> IdeResult<()> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + if is_link(&metadata) { + return Err(IdeError::invalid(format!("Unity asset operations do not follow links: {}", path.display()))); + } + if metadata.is_dir() { + for entry in fs::read_dir(path)? { + no_nested_links(&entry?.path())?; + } + } + Ok(()) +} + +// Resolve against the state produced by earlier operations without touching disk. +// This also handles renaming a folder before renaming/deleting a child asset. +fn present(path: &Path, preceding: &[FileEdit]) -> bool { + let mut path = path.to_path_buf(); + for op in preceding.iter().rev() { + match op { + FileEdit::Rename { from, to } => { + if let Ok(relative) = path.strip_prefix(to) { + // Joining an empty suffix adds a separator to a file path; + // that makes an existing sidecar look like a missing directory. + path = if relative.as_os_str().is_empty() { from.to_path_buf() } else { from.join(relative) }; + } else if path.starts_with(from) { + return false; + } + } + FileEdit::Delete { path: deleted, .. } if path.starts_with(deleted) => return false, + FileEdit::Create { path: created, .. } if path == created.to_path_buf() => return true, + _ => {} + } + } + exists(&path) +} + +pub(crate) fn expand_workspace_edit(edit: &mut WorkspaceEdit) -> IdeResult<()> { + let mut files = Vec::new(); + for file in &edit.files { + match file { + FileEdit::Rename { from, to } if project_root(from).is_some() || project_root(to).is_some() => { + validate_asset(from)?; + validate_asset(to)?; + let source = meta_path(from); + let destination = meta_path(to); + no_links(&source)?; + no_links(&destination)?; + if present(&destination, &files) && key(&source) != key(&destination) { + return Err(IdeError::invalid(format!("{} already exists", destination.display()))); + } + if present(&source, &files) { + files.push(FileEdit::Rename { from: source.into(), to: destination.into() }); + } + } + FileEdit::Delete { path, .. } if project_root(path).is_some() => { + validate_asset(path)?; + let meta = meta_path(path); + no_links(&meta)?; + if present(&meta, &files) { + files.push(FileEdit::Delete { path: meta.into(), recursive: false }); + } + } + _ => {} + } + files.push(file.clone()); + } + edit.files = files; + Ok(()) +} diff --git a/crates/ide-workspace/tests/documents.rs b/crates/ide-workspace/tests/documents.rs index 16670ce1..52c20e95 100644 --- a/crates/ide-workspace/tests/documents.rs +++ b/crates/ide-workspace/tests/documents.rs @@ -344,6 +344,28 @@ fn backend_edits_emit_patches() { assert_eq!(fs::read_to_string(&path).unwrap(), "ONE\r\n2\r\nTHREE\r\n"); } +#[test] +fn external_change_in_two_places_patches_each() { + let f = fixture(); + let path = f.root.join("two.txt"); + let old: String = (0..400).map(|i| format!("line {i}\n")).collect(); + fs::write(&path, &old).unwrap(); + let id = f.docs.open(&path).unwrap().id; + + std::thread::sleep(std::time::Duration::from_millis(10)); + let new = old.replace("line 20\n", "line 20x\n").replace("line 299\n", "line 299x\n"); + fs::write(&path, &new).unwrap(); + f.docs.on_disk_changed(std::slice::from_ref(&path)); + + assert_eq!(f.docs.snapshot(id).unwrap().text, new); + let events = f.events.lock(); + let changes = events.iter().find_map(|e| match e { + IdeEvent::DocPatched { changes, .. } => Some(changes), + _ => None, + }); + assert_eq!(changes.map(Vec::len), Some(2)); +} + #[test] fn external_changes() { let f = fixture(); diff --git a/crates/ide-workspace/tests/file_operations.rs b/crates/ide-workspace/tests/file_operations.rs new file mode 100644 index 00000000..6999b038 --- /dev/null +++ b/crates/ide-workspace/tests/file_operations.rs @@ -0,0 +1,147 @@ +use ide_workspace::file_ops; +use std::{fs, path::Path}; + +fn metadata_path(path: &Path) -> std::path::PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".meta"); + name.into() +} + +fn unity_project(root: &Path) { + fs::create_dir_all(root.join("Assets")).unwrap(); + fs::create_dir_all(root.join("ProjectSettings")).unwrap(); + fs::write(root.join("ProjectSettings/ProjectVersion.txt"), "m_EditorVersion: 2022.3.40f1").unwrap(); +} + +fn directory_alias(target: &Path, alias: &Path) { + #[cfg(unix)] + std::os::unix::fs::symlink(target, alias).unwrap(); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + let output = std::process::Command::new("cmd") + .args(["/c", "mklink", "/J"]) + .arg(alias.to_string_lossy().replace('/', "\\")) + .arg(target.to_string_lossy().replace('/', "\\")) + .creation_flags(0x08000000) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + } +} + +#[test] +fn ordinary_rename_through_an_alias_keeps_vfs_same_file_behavior() { + let dir = tempfile::tempdir().unwrap(); + let aliases = tempfile::tempdir().unwrap(); + let alias = aliases.path().join("project"); + directory_alias(dir.path(), &alias); + let from = dir.path().join("File.txt"); + let to = alias.join("File.txt"); + fs::write(&from, "keep").unwrap(); + ide_vfs::rename(&from, &to).unwrap(); + file_ops::rename(&from, &to).unwrap(); + let copied = alias.join("Copy.txt"); + file_ops::copy(&to, &copied).unwrap(); + assert_eq!(fs::read_to_string(copied).unwrap(), "keep"); + assert_eq!(fs::read_to_string(from).unwrap(), "keep"); +} + +#[test] +fn plain_files_and_non_assets_in_unity_keep_adjacent_metadata_untouched() { + for unity in [false, true] { + let dir = tempfile::tempdir().unwrap(); + if unity { + unity_project(dir.path()); + } + let from = dir.path().join("File.txt"); + let copied = dir.path().join("Copy.txt"); + let renamed = dir.path().join("file.txt"); + fs::write(&from, "data").unwrap(); + fs::write(metadata_path(&from), "unrelated").unwrap(); + file_ops::copy(&from, &copied).unwrap(); + assert!(!metadata_path(&copied).exists()); + file_ops::rename(&from, &renamed).unwrap(); + assert_eq!(fs::read_to_string(&renamed).unwrap(), "data"); + assert_eq!(file_ops::deletion_paths(&renamed).unwrap(), [renamed]); + assert_eq!(fs::read_to_string(metadata_path(&from)).unwrap(), "unrelated"); + } +} + +#[test] +fn mixed_move_and_undo_apply_rules_only_to_unity_assets() { + use ide_core::{LanguageId, WorkspaceEdit}; + use ide_workspace::{Documents, FileMove, FileMoves}; + use std::sync::Arc; + + let unity = tempfile::tempdir().unwrap(); + unity_project(unity.path()); + let plain = tempfile::tempdir().unwrap(); + let aliases = tempfile::tempdir().unwrap(); + let alias = aliases.path().join("plain"); + directory_alias(plain.path(), &alias); + let source = unity.path().join("Assets/Player.cs"); + let destination = unity.path().join("Assets/Hero.cs"); + let ordinary = alias.join("plain.txt"); + let ordinary_to = alias.join("moved.txt"); + fs::write(&source, "class Player {}").unwrap(); + fs::write(metadata_path(&source), "guid: 11111111111111111111111111111111\n").unwrap(); + fs::write(&ordinary, "plain").unwrap(); + fs::write(metadata_path(&ordinary), "unrelated").unwrap(); + let docs = Documents::new(Arc::new(|_| {}), Box::new(|_: &Path, _: Option<&str>| LanguageId::new("plaintext"))); + let moves = FileMoves::new(); + let token = moves + .perform( + &docs, + None, + &[ + FileMove { from: ordinary.clone().into(), to: ordinary_to.clone().into() }, + FileMove { from: source.clone().into(), to: destination.clone().into() }, + ], + &WorkspaceEdit::default(), + "Move", + &|_| Ok(()), + ) + .unwrap(); + assert!(destination.exists() && metadata_path(&destination).exists()); + assert!(ordinary_to.exists() && !metadata_path(&ordinary_to).exists()); + assert!(metadata_path(&ordinary).exists()); + moves.undo(&docs, token).unwrap(); + assert!(source.exists() && metadata_path(&source).exists()); + assert!(ordinary.exists() && metadata_path(&ordinary).exists()); + assert!(!destination.exists() && !metadata_path(&destination).exists()); + + let blocker = alias.join("blocked"); + fs::write(&blocker, "not a directory").unwrap(); + assert!( + moves + .perform( + &docs, + None, + &[ + FileMove { from: source.clone().into(), to: destination.clone().into() }, + FileMove { from: ordinary.clone().into(), to: blocker.join("file.txt").into() }, + ], + &WorkspaceEdit::default(), + "Move", + &|_| Ok(()), + ) + .is_err() + ); + assert!(source.exists() && metadata_path(&source).exists()); + assert!(ordinary.exists() && metadata_path(&ordinary).exists()); + assert!(!destination.exists() && !metadata_path(&destination).exists()); +} + +#[test] +fn vfs_operations_remain_unaware_of_unity_metadata() { + let dir = tempfile::tempdir().unwrap(); + unity_project(dir.path()); + let from = dir.path().join("Assets/Player.cs"); + let to = dir.path().join("Assets/Hero.cs"); + fs::write(&from, "code").unwrap(); + fs::write(metadata_path(&from), "metadata").unwrap(); + ide_vfs::rename(&from, &to).unwrap(); + assert!(to.exists() && !metadata_path(&to).exists()); + assert!(metadata_path(&from).exists()); +} diff --git a/crates/ide-workspace/tests/workspace_edits.rs b/crates/ide-workspace/tests/workspace_edits.rs index 78bd905b..e58d7f52 100644 --- a/crates/ide-workspace/tests/workspace_edits.rs +++ b/crates/ide-workspace/tests/workspace_edits.rs @@ -9,6 +9,95 @@ fn documents() -> Documents { Documents::new(Arc::new(NullSink), Box::new(|_, _| LanguageId::plain_text())) } +fn meta(path: &std::path::Path) -> std::path::PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".meta"); + name.into() +} + +fn unity_project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("Assets")).unwrap(); + std::fs::create_dir_all(tmp.path().join("ProjectSettings")).unwrap(); + std::fs::write(tmp.path().join("ProjectSettings/ProjectVersion.txt"), "m_EditorVersion: 2022.3.40f1\n").unwrap(); + tmp +} + +#[test] +fn unity_rename_chain_and_delete_share_metadata_undo() { + let tmp = unity_project(); + let original = tmp.path().join("Assets/Player.cs"); + let renamed = tmp.path().join("Assets/Hero.cs"); + std::fs::write(&original, "old").unwrap(); + std::fs::write(meta(&original), "guid: unchanged").unwrap(); + let (docs, events) = recorded_documents(); + docs.apply_workspace_edit(&WorkspaceEdit { + files: vec![ + FileEdit::Rename { from: original.clone().into(), to: renamed.clone().into() }, + FileEdit::Delete { path: renamed.clone().into(), recursive: false }, + ], + ..Default::default() + }) + .unwrap(); + for path in [&original, &renamed, &meta(&original), &meta(&renamed)] { + assert!(!path.exists(), "{} should be removed", path.display()); + } + docs.undo_workspace_edit(undo_id(&events)).unwrap(); + assert_eq!(std::fs::read_to_string(&original).unwrap(), "old"); + assert_eq!(std::fs::read_to_string(meta(&original)).unwrap(), "guid: unchanged"); + assert!(!renamed.exists() && !meta(&renamed).exists()); +} + +#[test] +fn unity_metadata_conflict_keeps_earlier_edits_unchanged() { + let tmp = unity_project(); + let original = tmp.path().join("Assets/Player.cs"); + let renamed = tmp.path().join("Assets/Hero.cs"); + std::fs::write(&original, "old").unwrap(); + std::fs::write(meta(&original), "guid: original").unwrap(); + std::fs::write(meta(&renamed), "guid: orphan").unwrap(); + assert!( + documents() + .apply_workspace_edit(&WorkspaceEdit { + files: vec![ + edit(&original, "new"), + FileEdit::Rename { from: original.clone().into(), to: renamed.clone().into() } + ], + ..Default::default() + }) + .is_err() + ); + assert_eq!(std::fs::read_to_string(&original).unwrap(), "old"); + assert_eq!(std::fs::read_to_string(meta(&original)).unwrap(), "guid: original"); + assert_eq!(std::fs::read_to_string(meta(&renamed)).unwrap(), "guid: orphan"); +} + +#[test] +fn unity_folder_rename_then_child_rename_preserves_metadata() { + let tmp = unity_project(); + let folder = tmp.path().join("Assets/Before"); + let moved = tmp.path().join("Assets/After"); + std::fs::create_dir(&folder).unwrap(); + std::fs::write(meta(&folder), "folder guid").unwrap(); + std::fs::write(folder.join("Player.cs"), "old").unwrap(); + std::fs::write(meta(&folder.join("Player.cs")), "script guid").unwrap(); + let (docs, events) = recorded_documents(); + docs.apply_workspace_edit(&WorkspaceEdit { + files: vec![ + FileEdit::Rename { from: folder.clone().into(), to: moved.clone().into() }, + FileEdit::Rename { from: moved.join("Player.cs").into(), to: moved.join("Hero.cs").into() }, + ], + ..Default::default() + }) + .unwrap(); + assert_eq!(std::fs::read_to_string(meta(&moved)).unwrap(), "folder guid"); + assert_eq!(std::fs::read_to_string(meta(&moved.join("Hero.cs"))).unwrap(), "script guid"); + assert!(!meta(&moved.join("Player.cs")).exists()); + docs.undo_workspace_edit(undo_id(&events)).unwrap(); + assert_eq!(std::fs::read_to_string(meta(&folder.join("Player.cs"))).unwrap(), "script guid"); + assert!(!moved.exists() && !meta(&moved).exists()); +} + fn edit(path: &Path, text: &str) -> FileEdit { FileEdit::Edit { path: path.to_owned().into(), @@ -111,6 +200,37 @@ fn matching_document_version_is_applied() { assert_eq!(std::fs::read_to_string(a).unwrap(), "new\n"); } +#[test] +fn versioned_edit_of_a_closed_file_is_guarded_by_its_preconditions() { + // gopls versions every file of a rename, including ones the editor never opened. + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("closed.txt"); + std::fs::write(&path, "old\n").unwrap(); + let docs = documents(); + let versioned = || { + let mut operation = edit(&path, "new"); + if let FileEdit::Edit { version, .. } = &mut operation { + *version = Some(0); + } + WorkspaceEdit { files: vec![operation], ..Default::default() } + }; + // Without a snapshot of what the server saw, a version for a closed file can't be trusted. + assert_eq!(docs.apply_workspace_edit(&versioned()).unwrap_err().kind, ErrorKind::OutOfSync); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "old\n"); + + let mut changed = versioned(); + docs.prepare_workspace_edit(&mut changed).unwrap(); + std::fs::write(&path, "own\n").unwrap(); + assert_eq!(docs.apply_workspace_edit(&changed).unwrap_err().kind, ErrorKind::OutOfSync); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "own\n"); + + std::fs::write(&path, "old\n").unwrap(); + let mut prepared = versioned(); + docs.prepare_workspace_edit(&mut prepared).unwrap(); + docs.apply_workspace_edit(&prepared).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "new\n"); +} + #[test] fn unversioned_preview_rejects_changed_disk_and_dirty_buffers() { let tmp = tempfile::tempdir().unwrap(); @@ -144,6 +264,38 @@ fn unversioned_preview_rejects_changed_disk_and_dirty_buffers() { } } +#[test] +fn saving_between_preview_and_apply_keeps_the_edit_valid() { + let tmp = tempfile::tempdir().unwrap(); + // A new scratch (empty on disk) and an edited file, saved after the edit was computed. + for (name, disk, typed) in [("scratch.c", "", "int\n"), ("a.txt", "old\n", "own")] { + let path = tmp.path().join(name); + std::fs::write(&path, disk).unwrap(); + let docs = documents(); + let info = docs.open(&path).unwrap(); + let replace = if disk.is_empty() { 0 } else { 3 }; + docs.edit( + info.id, + info.version, + ChangeSet { changes: vec![TextChange { from: 0, to: replace, insert: typed.into() }] }, + ) + .unwrap(); + let mut change = WorkspaceEdit { files: vec![edit(&path, "new")], ..Default::default() }; + docs.prepare_workspace_edit(&mut change).unwrap(); + docs.save(info.id).unwrap(); + docs.apply_workspace_edit(&change).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(docs.snapshot(info.id).unwrap().text.starts_with("new"), "{name}"); + // A real change after the preview still invalidates it. + let mut change = WorkspaceEdit { files: vec![edit(&path, "xyz")], ..Default::default() }; + docs.prepare_workspace_edit(&mut change).unwrap(); + let version = docs.snapshot(info.id).unwrap().version; + docs.edit(info.id, version, ChangeSet { changes: vec![TextChange { from: 0, to: 0, insert: "!".into() }] }) + .unwrap(); + docs.save(info.id).unwrap(); + assert!(docs.apply_workspace_edit(&change).is_err(), "{name}"); + } +} + #[test] fn grouped_undo_restores_open_closed_and_unsaved_files() { let tmp = tempfile::tempdir().unwrap(); diff --git a/docs/plugins/themes.md b/docs/plugins/themes.md index b11082a3..e28db7a8 100644 --- a/docs/plugins/themes.md +++ b/docs/plugins/themes.md @@ -132,9 +132,29 @@ and credit the original authors in `authors` and `description`. ### From IntelliJ IDEA (automatic) +To use a theme you already have in IntelliJ, open **Settings › Plugins**, gear +menu: + +- **Import Theme from IntelliJ IDE…** lists the theme plugins installed in the + JetBrains IDEs (and Android Studio) on this computer, the theme each IDE uses + first. It reads their settings folders: `~/Library/Application Support/JetBrains` + (macOS), `%APPDATA%\JetBrains` (Windows), `~/.config/JetBrains` and + `~/.local/share/JetBrains` (Linux). +- **Import IntelliJ Theme from File…** takes a theme plugin downloaded from the + JetBrains Marketplace (`.jar`, or a `.zip` with the plugin's `lib/*.jar`) or a + `.theme.json` (its `editorScheme` file is read from the same folder). + +The whole plugin becomes one OpenIDEA theme plugin, `intellij.`, with +each of its themes and their editor schemes; it is installed through the usual +install page and the chosen theme (otherwise the first) is applied. Importing +the plugin again replaces it. Both are actions too (`plugins.importIntellijTheme`, +`plugins.importInstalledIntellijTheme`). The conversion is the one described +below. + `scripts/theme/import-intellij-theme.ts` converts an IntelliJ `.theme.json`, -and optionally its `.icls` editor scheme, into a ready-to-install plugin. It -has no dependencies beyond Node 22. +and optionally its `.icls` editor scheme, into a plugin folder you can edit +before installing it — the way to port a theme for publishing. It has no +dependencies beyond Node 22. ```sh pnpm theme:import-intellij path/to/MyTheme.theme.json path/to/MyTheme.icls --out ~/my-theme --author "Original Author" @@ -172,14 +192,16 @@ ship no `.icls` at all keep the built-in syntax colours. | `--danger`, `--warning` | `Component.errorFocusColor`, `Component.warningFocusColor` | | `--bg-tooltip`, `--fg-tooltip`, `--border-tooltip` | `ToolTip.background`, `.foreground`, `.borderColor` | | `--bg-button`, `--border-button` | `Button.startBackground`, `Button.startBorderColor` | -| `--bg-tab-selected`, `--border-tab-selected` | `EditorTabs.underlinedTabBackground`, `EditorTabs.underlineColor` | +| `--bg-tab-selected` | `EditorTabs.underlinedTabBackground` | +| `--border-tab-selected` | `EditorTabs.underlinedBorderColor` (Islands), `EditorTabs.underlineColor` | +| `--fg-completion-match` | `CompletionPopup.matchForeground` | | `--bg-toolbar-hover`, `--bg-toolbar-pressed` | `ActionButton.hoverBackground`, `ActionButton.pressedBackground` | | `--bg-stripe-hover`, `--bg-stripe-selected` | `ToolWindow.Button.hoverBackground`, `.selectedBackground` | | `--scrollbar-thumb`, `--scrollbar-thumb-hover` | `ScrollBar.Mac.thumbColor`, `ScrollBar.Mac.hoverThumbColor` | | `--bg-toggle-*` | `ToggleButton.onBackground`, `.offBackground`, `.buttonColor` | | `--bg-feedback-*`, `--border-feedback-*` | `Banner.infoBackground`, `Banner.errorBorderColor`, … | -The full table is `UI_KEYS` in [`scripts/theme/intellij.ts`](../../scripts/theme/intellij.ts). +The full table is `UI_KEYS` in [`src/features/plugins/intellijTheme.ts`](../../src/features/plugins/intellijTheme.ts). #### `.icls` → editor scheme @@ -189,6 +211,7 @@ The full table is `UI_KEYS` in [`scripts/theme/intellij.ts`](../../scripts/theme | `--caret`, `--bg-current-line` | colour `CARET_COLOR`, `CARET_ROW_COLOR` | | `--bg-editor-selection` | colour `SELECTION_BACKGROUND` | | `--fg-line-number`, `--fg-line-number-current` | colour `LINE_NUMBERS_COLOR`, `LINE_NUMBER_ON_CARET_ROW_COLOR` | +| `--bg-editor-gutter` | colour `EDITOR_GUTTER_BACKGROUND`, else the editor background (as the new UI paints it; `GUTTER_BACKGROUND` is the classic UI's) | | `--border-indent-guide`, `--border-right-margin`, `--fg-whitespace` | colour `INDENT_GUIDE`, `RIGHT_MARGIN_COLOR`, `WHITESPACES` | | `--vcs-*`, `--gutter-*` | colour `FILESTATUS_*`, `MODIFIED_LINES_COLOR`, `ADDED_LINES_COLOR`, `DELETED_LINES_COLOR` | | `--syn-keyword`, `--syn-string`, `--syn-number` | `DEFAULT_KEYWORD`, `DEFAULT_STRING`, `DEFAULT_NUMBER` | @@ -206,7 +229,9 @@ The full table is `UI_KEYS` in [`scripts/theme/intellij.ts`](../../scripts/theme | `--syn-self`, `--syn-keyword-arg`, `--syn-docstring`, `--syn-predefined` | `PY.SELF_PARAMETER`, `PY.KEYWORD_ARGUMENT`, `PY.DOC_COMMENT`, `PY.BUILTIN_NAME` | | `--ansi-*` | `CONSOLE_*_OUTPUT` (`CONSOLE_DARKGRAY_OUTPUT` is bright black, `CONSOLE_GRAY_OUTPUT` white) | -The full tables are `ICLS_COLORS` and `ICLS_ATTRIBUTES` in the same file. +The full tables are `ICLS_COLORS` and `ICLS_ATTRIBUTES` in the same file. An +attribute written as `` has no colours of its own and shows as plain +text (`TEXT`'s foreground), as in IntelliJ, not with the parent scheme's colour. ### From VS Code (by hand) @@ -298,7 +323,7 @@ a token. -223 themeable tokens: 117 in the UI theme file, 106 in the editor colour scheme file. +224 themeable tokens: 117 in the UI theme file, 107 in the editor colour scheme file. ### Surfaces (UI theme) @@ -400,6 +425,7 @@ a token. | `--syn-number` | `#2aacb8` | `#1750eb` | | `--syn-punct` | `#bcbec4` | `#080808` | | `--syn-variable` | `#bcbec4` | `#080808` | +| `--syn-global` | `var(--syn-variable)` | `var(--syn-variable)` | | `--syn-param` | `#bcbec4` | `#080808` | | `--syn-property` | `#c77dbb` | `#871094` | | `--syn-constant` | `#c77dbb` | `#871094` | diff --git a/docs/unity.md b/docs/unity.md new file mode 100644 index 00000000..28c9eba2 --- /dev/null +++ b/docs/unity.md @@ -0,0 +1,23 @@ +# Unity projects + +Open the folder containing `Assets` and `ProjectSettings/ProjectVersion.txt`. +OpenIDEA detects Unity without requiring a generated solution and shows the editor +version in the Unity project view. + +The Unity view hides `.meta` files, generated C# project files and cache/build +folders. **Show Meta Files** and **Show Generated Files** reveal them independently; +the regular Project view is still available. Generated folders are excluded from +indexing and search, including folders Unity creates after the project is opened. + +Operations on assets inside `Assets` include their `.meta` files. Rename and move +preserve GUIDs, deletion removes both files, and copy generates new GUIDs and updates +references within the copied group. References to assets outside the group stay +unchanged. File move undo and transactional workspace edits include metadata too. +Destination metadata conflicts are rejected before making changes. + +Projects may be opened through a linked root or parent directory, including macOS +`/var` and `/tmp`. Links inside the project (including Windows junctions) are not +followed by asset operations. For moves/copies to or from ordinary directories, +the external path must have no linked ancestors. Cross-volume asset moves and +copies that require rewriting binary serialized assets are rejected; use Unity +for those operations. diff --git a/docs/workspaces-and-window-groups.md b/docs/workspaces-and-window-groups.md index db86800e..02136699 100644 --- a/docs/workspaces-and-window-groups.md +++ b/docs/workspaces-and-window-groups.md @@ -22,7 +22,15 @@ Paths in an existing manifest may be relative to the workspace directory. New an Linked folders appear as roots in Project view and participate in indexing, Find in Files, Git repository discovery and language-server root selection. Detected and saved run configurations have project-qualified names and use their member's working directory and launch settings. Build Project combines the linked projects' build plans; Compile File targets its owning project. Workspace editor/settings state belongs to its own directory. Each member retains its repository and project configuration. -The specialized Maven/Gradle tool-window model still describes the workspace directory itself; it does not yet aggregate independent member models. Workspace creation and management currently target local folders. +The specialized Maven/Gradle tool-window model still describes the workspace directory itself; it does not yet aggregate independent member models. Workspace creation and management currently target local folders. To link independent Gradle builds in one Gradle tree, open their parent folder as an ordinary project instead of creating a Workspace manifest, then use **Link Gradle Projects…** in the Gradle toolbar or Find Action. + +## Linked Gradle builds in a project + +In an ordinary project window, **Link Gradle Projects…** accepts one or more `settings.gradle`, `settings.gradle.kts`, `build.gradle`, or `build.gradle.kts` files. Each linked build has its own root in the Gradle tool window. Its tasks, dependencies, wrapper, working directory and sync cache stay separate, even when another build has the same `:app` project or task names. Included builds remain under the build that includes them. + +Right-click a build root for **Reload Gradle Project** or **Unlink Gradle Project**. Unlinking only removes the IDE connection; it does not delete files or saved run configurations. The Link button remains available after the last build is unlinked. Linked builds may live outside the opened folder. + +Links are saved in project settings as `languages.java.gradle.linkedProjects`. Until links are configured, existing automatic discovery is unchanged. An explicit empty list means that no Gradle builds are linked. Opening the project again restores its links. A folder first opened by an older version may need one fresh sync because independent builds now have separate caches. ## Window group @@ -30,7 +38,7 @@ Open two project windows, then use **Window → Merge All Project Windows**. The Use the tab's close button to close that project, or **Move Tab to New Window** to detach the current tab. Right-click a tab for **Close Other Tabs**, **Close Tabs to the Left** and **Close Tabs to the Right**, which close the projects one at a time and keep the clicked tab; cancelling one project's unsaved-changes prompt leaves it and the rest open. The same commands are in the Command Palette and can be assigned shortcuts. Closing uses the existing unsaved-changes guard. Quit visits project windows in sequence so a hidden project's prompt is visible. Groups restore when **Reopen last project** is enabled. -The grouping model lives in `ide-workspace`, independently of project ownership. The desktop adapter keeps each member's WebView and backend services alive, presents the selected window at the group's position/size, and hides its peers. This is an application-drawn tab strip, not the macOS native tab strip. Its persistence is application-local in `window-groups.json`. Language servers, terminals, editor buffers and settings remain independent between group members. +The grouping model lives in `ide-workspace`, independently of project ownership. The desktop adapter keeps each member's WebView and backend services alive, presents the selected window at the group's position/size, and hides its peers. Hidden members are kept at the selected window's frame and display (and follow it after a move or resize), so switching projects only changes which window is visible. This is an application-drawn tab strip, not the macOS native tab strip. Its persistence is application-local in `window-groups.json`. Language servers, terminals, editor buffers and settings remain independent between group members. ## Validation diff --git a/e2e/agentMarkdown.spec.ts b/e2e/agentMarkdown.spec.ts new file mode 100644 index 00000000..da15e0e3 --- /dev/null +++ b/e2e/agentMarkdown.spec.ts @@ -0,0 +1,130 @@ +/** Markdown in agent replies, in the real conversation: browser defaults don't suit a narrow chat. */ +import { expect, test, type Page } from "@playwright/test"; + +const TASK = { + id: "t1", + title: "Repro task", + status: "ready" as const, + isolation: "local" as const, + provider: "codex", + model: null, + cwd: "/dev/acme-api", + branch: null, + base: null, + baseCommit: null, + error: null, + createdMs: 0, + updatedMs: 0, + config: [], + commands: [], + usage: null, + queue: [], +}; + +const PIXEL = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"; +const REPLY = [ + "# Heading one", + "", + "## Heading two", + "", + "##### Heading five", + "", + "> A quote", + "", + "- [x] done", + "- [ ] open", + "", + "---", + "", + `![pixel](${PIXEL})`, + "", + "| Path | Owner | Status | Notes |", + "| --- | --- | --- | --- |", + "| src/features/agents/conversation/very/long/path/file.tsx | someone | modified | a fairly long note |", + "", + "```ts", + "const a = 1;", + "```", +].join("\n"); + +async function openReply(page: Page, text: string) { + await page.goto("/"); + await page.getByRole("tree", { name: "Project" }).waitFor(); + await page.evaluate( + async ({ task, text }) => { + // Paths go through variables: knip can't resolve these browser-absolute URLs. + const storePath = "/src/features/agents/store.ts"; + const store = await import(storePath); + store.useAgents.setState({ + loaded: true, + error: null, + providers: [], + tasks: new Map([[task.id, task]]), + entries: new Map([[task.id, [{ type: "agent", text }]]]), + unread: new Set(), + docked: null, + }); + store.dockTask(task.id); + }, + { task: TASK, text }, + ); + const reply = page.locator("[class*='_markdown_']").first(); + await reply.locator("h1, p").first().waitFor(); + return reply; +} + +const css = (reply: ReturnType, selector: string, property: string) => + reply + .locator(selector) + .first() + .evaluate((el, name) => getComputedStyle(el).getPropertyValue(name), property); +const px = (value: string) => Number.parseFloat(value); + +test("styles tables so a reply reads as a table", async ({ page }) => { + const reply = await openReply(page, REPLY); + expect(await css(reply, "td", "border-top-width")).toBe("1px"); + expect(await css(reply, "td", "padding-left")).toBe("8px"); + expect(await css(reply, "th", "border-top-width")).toBe("1px"); + // A narrow chat must not squeeze a column to a few letters. + const widths = await reply + .locator("td") + .evaluateAll((cells) => cells.map((cell) => cell.getBoundingClientRect().width)); + for (const width of widths) expect(width).toBeGreaterThanOrEqual(70); +}); + +test("scrolls a wide table inside the reply", async ({ page }) => { + const reply = await openReply(page, REPLY); + const sizes = await reply.evaluate((el) => { + const table = el.querySelector("table")!; + return { reply: el.clientWidth, table: table.clientWidth, content: table.scrollWidth }; + }); + expect(sizes.table).toBeLessThanOrEqual(sizes.reply); + expect(sizes.content).toBeGreaterThan(sizes.table); +}); + +test("sizes headings for a chat and keeps small ones readable", async ({ page }) => { + const reply = await openReply(page, REPLY); + const body = px(await css(reply, "td", "font-size")); + const h1 = px(await css(reply, "h1", "font-size")); + expect(h1).toBeGreaterThan(body); + expect(h1).toBeLessThanOrEqual(body * 1.2); + expect(px(await css(reply, "h2", "font-size"))).toBeLessThan(h1); + expect(px(await css(reply, "h5", "font-size"))).toBe(body); +}); + +test("starts and ends a reply without stray margins", async ({ page }) => { + const reply = await openReply(page, REPLY); + expect(await css(reply, "> :first-child", "margin-top")).toBe("0px"); + expect(await css(reply, "> :last-child", "margin-bottom")).toBe("0px"); +}); + +test("styles quotes, rules, images and task lists", async ({ page }) => { + const reply = await openReply(page, REPLY); + expect(await css(reply, "blockquote", "margin-left")).toBe("0px"); + expect(await css(reply, "blockquote", "border-left-width")).toBe("3px"); + expect(await css(reply, "hr", "border-left-width")).toBe("0px"); + expect(await css(reply, "hr", "border-top-width")).toBe("1px"); + expect(await css(reply, "img", "max-width")).toBe("100%"); + // A checkbox stands in for the bullet. + expect(await css(reply, "li:has(> input)", "list-style-type")).toBe("none"); +}); diff --git a/e2e/clickhouse-source.spec.ts b/e2e/clickhouse-source.spec.ts new file mode 100644 index 00000000..8a29e09e --- /dev/null +++ b/e2e/clickhouse-source.spec.ts @@ -0,0 +1,42 @@ +import { expect, test } from "./fixtures"; + +test("configures a ClickHouse HTTP source and keeps transaction controls unavailable", async ({ + workbench: page, +}, info) => { + await page.evaluate(async () => { + const storePath = "/src/features/database/store.ts"; + const bindingsPath = "/src/bindings.ts"; + const { useDatabase } = await import(storePath); + const { commands } = await import(bindingsPath); + commands.dbTest = async (source: unknown) => { + document.documentElement.dataset.testedSource = JSON.stringify(source); + return { status: "ok", data: "ClickHouse" }; + }; + useDatabase.setState({ dialog: { kind: "source", source: null } }); + }); + const dialog = page.getByRole("dialog", { name: "New Data Source", exact: true }); + await dialog.getByRole("button", { name: "Data source type", exact: true }).click(); + await page.getByRole("option", { name: "ClickHouse", exact: true }).click(); + await expect(dialog.locator("#db-port")).toHaveValue("8123"); + await expect(dialog.locator("#db-user")).toHaveValue("default"); + await expect(dialog.locator("#db-database")).toHaveValue("default"); + await expect(dialog.locator("#db-url")).toHaveValue("http://localhost:8123/default"); + await dialog.locator("#db-url").fill("https://reader@db.example:8443/analytics"); + await expect(dialog.locator("#db-host")).toHaveValue("db.example"); + await expect(dialog.locator("#db-port")).toHaveValue("8443"); + await expect(dialog.locator("#db-user")).toHaveValue("reader"); + await expect(dialog.locator("#db-database")).toHaveValue("analytics"); + await dialog.getByRole("button", { name: "Test Connection", exact: true }).click(); + await expect(dialog.getByRole("status")).toHaveText("Succeeded: ClickHouse"); + expect(await page.evaluate(() => JSON.parse(document.documentElement.dataset.testedSource!))).toMatchObject({ + driver: "clickhouse", + host: "db.example", + port: 8443, + user: "reader", + database: "analytics", + ssl: "verifyFull", + }); + await dialog.screenshot({ path: info.outputPath("clickhouse-source.png") }); + await dialog.getByRole("tab", { name: "Options", exact: true }).click(); + await expect(dialog.getByRole("button", { name: "Transaction control", exact: true })).toBeDisabled(); +}); diff --git a/e2e/codeInsight.spec.ts b/e2e/codeInsight.spec.ts new file mode 100644 index 00000000..ed4e4c7f --- /dev/null +++ b/e2e/codeInsight.spec.ts @@ -0,0 +1,219 @@ +/** + * Code intelligence through the editor: completion, navigation, usages, rename, quick fixes and + * diagnostics. The language server is stubbed at the IPC layer; the servers themselves are covered + * by `crates/ide-lsp/tests`. + */ +import type { Locator, Page } from "@playwright/test"; +import { expect, ipcCalls, openFromTree, stubDesktop, test } from "./fixtures"; + +const ROOT = "/dev/acme-api"; +const FILE = `${ROOT}/build.gradle.kts`; + +const at = (line: number, col: number) => ({ line, col }); +const range = (line: number, from: number, to: number) => ({ start: at(line, from), end: at(line, to) }); + +/** Puts the caret just before `word` on the line containing `text`. */ +async function caretAt(editor: Locator, page: Page, text: string, word: string) { + const line = editor.locator(".cm-line", { hasText: text }); + await line.click(); + await page.keyboard.press("Home"); + const content = (await line.textContent()) ?? ""; + const indent = content.length - content.trimStart().length; + for (let i = indent; i < content.indexOf(word); i++) await page.keyboard.press("ArrowRight"); +} + +async function openSample(page: Page) { + const editor = await openFromTree(page, "build.gradle.kts"); + await stubDesktop(page); + return editor; +} + +test("completion lists the server's items and inserts the chosen one", async ({ workbench: page }) => { + const editor = await openSample(page); + await page.evaluate(() => { + const item = (label: string, kind: string, detail: string, insertText: string) => ({ + label, + labelDetail: null, + detail, + kind, + documentation: null, + sortText: null, + filterText: null, + preselect: false, + deprecated: false, + insertText, + isSnippet: false, + replaceRange: null, + replaceRangeTab: null, + additionalEdits: [], + commitCharacters: [], + command: null, + data: null, + server: "kotlin", + }); + window.__ipc.handlers.lsp_completion = () => ({ + isIncomplete: false, + items: [item("toDto", "method", "OrderDto", "toDto()"), item("total", "property", "BigDecimal", "total")], + }); + }); + await editor.locator(".cm-line", { hasText: "return order.toDto()" }).click(); + await page.keyboard.press("End"); + await page.keyboard.press("Enter"); + await page.keyboard.type("order."); + await page.keyboard.press("Control+Space"); + const popup = page.locator(".cm-tooltip-autocomplete"); + await expect(popup.getByText("total", { exact: true })).toBeVisible(); + await expect(popup.getByText("toDto", { exact: true })).toBeVisible(); + await page.keyboard.type("toD"); + await expect(popup.getByText("total", { exact: true })).toBeHidden(); + await page.keyboard.press("Enter"); + await expect(popup).toBeHidden(); + await expect(editor.locator(".cm-activeLine")).toHaveText(/^\s*order\.toDto\(\)$/); + expect((await ipcCalls(page, "lsp_completion")).length).toBeGreaterThan(0); +}); + +test("⌘B goes to the declaration in another file", async ({ workbench: page }) => { + const editor = await openSample(page); + await page.evaluate( + ({ path, target }) => { + window.__ipc.handlers.lsp_goto = () => [{ path, range: target }]; + }, + { path: `${ROOT}/settings.gradle.kts`, target: range(20, 12, 17) }, + ); + await caretAt(editor, page, "val order: Order", "service"); + await page.keyboard.press("Meta+b"); + const tabs = page.getByRole("tablist", { name: "Editor tabs" }); + await expect(tabs.getByRole("tab", { name: "settings.gradle.kts" })).toHaveAttribute("aria-selected", "true"); + const target = page.locator(".cm-editor").filter({ has: page.locator(".cm-activeLine") }); + await expect(target.locator(".cm-activeLine")).toContainText("val order: Order = service.create(req)"); + const [args] = await ipcCalls(page, "lsp_goto"); + expect(args).toMatchObject({ pos: at(20, 27), kind: "definition" }); +}); + +test("Find Usages lists every usage by file and opens one", async ({ workbench: page }) => { + const editor = await openSample(page); + await page.evaluate( + ({ file, other }) => { + const loc = (path: string, line: number, from: number, to: number) => ({ + path, + range: { start: { line, col: from }, end: { line, col: to } }, + }); + window.__ipc.handlers.lsp_references = () => ({ + locations: [loc(file, 16, 8, 15), loc(file, 20, 27, 34), loc(other, 16, 8, 15)], + incomplete: null, + }); + }, + { file: FILE, other: `${ROOT}/settings.gradle.kts` }, + ); + await caretAt(editor, page, "val order: Order", "service"); + await page.keyboard.press("Alt+F7"); + const usages = page.getByRole("tree", { name: /usages/i }); + await expect(usages.getByRole("treeitem", { name: "Found usages 3 results" })).toBeVisible(); + await expect(usages.getByRole("treeitem", { name: "build.gradle.kts 2 usages" })).toBeVisible(); + await expect(usages.getByRole("treeitem", { name: /^17 service\.find\(status/ })).toBeVisible(); + await expect(usages.getByRole("treeitem", { name: /^21 val order: Order = service\.create/ })).toBeVisible(); + const other = usages.getByRole("treeitem", { name: "settings.gradle.kts 1 usage" }); + await expect(other).toBeVisible(); + // Double-clicking a usage in another file opens that file. + await usages.getByRole("treeitem", { name: /^17/ }).last().dblclick(); + await expect( + page.getByRole("tablist", { name: "Editor tabs" }).getByRole("tab", { name: "settings.gradle.kts" }), + ).toHaveAttribute("aria-selected", "true"); +}); + +test("⇧F6 renames in place and sends the server's edit to be applied", async ({ workbench: page }) => { + const editor = await openSample(page); + await page.evaluate( + ({ file, prepared, uses }) => { + window.__ipc.handlers.lsp_prepare_rename = () => prepared; + window.__ipc.handlers.lsp_rename = (args) => ({ + edit: { + files: [ + { + kind: "edit", + path: file, + edits: uses.map((range) => ({ range, new_text: args.newName })), + }, + ], + }, + incomplete: null, + }); + }, + { file: FILE, prepared: range(20, 12, 17), uses: [range(20, 12, 17), range(21, 15, 20)] }, + ); + await caretAt(editor, page, "val order: Order", "order"); + await page.keyboard.press("Shift+F6"); + await expect(page.getByText(/Press ⇧F6 to show dialog/)).toBeVisible(); + // The name starts selected; occurrences follow as it's typed. + await page.keyboard.type("placed"); + await expect(editor.locator(".cm-line", { hasText: "val placed: Order" })).toBeVisible(); + await page.keyboard.press("Enter"); + await expect.poll(() => ipcCalls(page, "apply_workspace_edit")).toHaveLength(1); + const [rename] = await ipcCalls(page, "lsp_rename"); + expect(rename).toMatchObject({ pos: at(20, 12), newName: "placed" }); + const [apply] = await ipcCalls(page, "apply_workspace_edit"); + expect(JSON.stringify(apply)).toContain('"new_text":"placed"'); +}); + +test("diagnostics underline the range and ⌥⏎ applies the preferred quick fix", async ({ workbench: page }) => { + const editor = await openSample(page); + await page.evaluate( + ({ file, bad }) => { + window.__ipc.handlers.lsp_code_actions = () => [ + { + title: "Import 'com.acme.orders.model.Order'", + kind: "quickfix", + isPreferred: true, + disabledReason: null, + edit: { + files: [ + { + kind: "edit", + path: file, + edits: [ + { + range: { start: { line: 4, col: 0 }, end: { line: 4, col: 0 } }, + new_text: "import com.acme.orders.model.Order\n", + }, + ], + }, + ], + }, + command: null, + data: null, + server: "kotlin", + }, + { + title: "Create class 'Order'", + kind: "quickfix", + isPreferred: false, + disabledReason: null, + edit: null, + command: null, + data: null, + server: "kotlin", + }, + ]; + window.__ipc.emit({ + type: "diagnostics", + path: file, + source: "kotlin", + items: [ + { range: bad, severity: "error", message: "Unresolved reference: Order", source: "kotlin", code: null }, + ], + }); + }, + { file: FILE, bad: range(20, 19, 24) }, + ); + const squiggle = editor.locator(".cm-lintRange-error"); + await expect(squiggle).toHaveText("Order"); + await caretAt(editor, page, "val order: Order", "Order "); + await page.keyboard.press("Alt+Enter"); + const fix = page.getByRole("option", { name: /Import 'com\.acme\.orders\.model\.Order'/ }); + await expect(fix).toBeVisible(); + await expect(page.getByRole("option", { name: /Create class 'Order'/ })).toBeVisible(); + await fix.click(); + await expect.poll(() => ipcCalls(page, "lsp_execute_code_action")).toHaveLength(1); + const [applied] = await ipcCalls(page, "lsp_execute_code_action"); + expect(applied).toMatchObject({ action: { title: "Import 'com.acme.orders.model.Order'" } }); +}); diff --git a/e2e/commit.spec.ts b/e2e/commit.spec.ts new file mode 100644 index 00000000..6d0c1852 --- /dev/null +++ b/e2e/commit.spec.ts @@ -0,0 +1,75 @@ +/** + * Committing from the Commit tool window. Git is stubbed at the IPC layer; the repository side is + * covered by `crates/ide-vcs` against real repositories. + */ +import type { Page } from "@playwright/test"; +import { expect, ipcCalls, openFromTree, stubDesktop, test } from "./fixtures"; + +const ROOT = "/dev/acme-api"; + +/** A repository with a modified tracked file and an untracked one; `vcs_commit` clears both. */ +async function stubRepository(page: Page) { + await stubDesktop(page); + await page.evaluate(async (root) => { + let committed = false; + const changes = [ + { + path: `${root}/build.gradle.kts`, + relPath: "build.gradle.kts", + staged: null, + unstaged: "modified", + renamedFrom: null, + }, + { path: `${root}/notes.txt`, relPath: "notes.txt", staged: null, unstaged: "untracked", renamedFrom: null }, + ]; + const h = window.__ipc.handlers; + h.vcs_roots = () => [{ root, operation: null, head: "main", headCommit: "1111111" }]; + h.vcs_branch_info = () => ({ + head: "main", + headCommit: "1111111", + upstream: null, + ahead: 0, + behind: 0, + operation: null, + }); + h.vcs_status = () => (committed ? [] : changes); + h.vcs_stage = () => null; + h.vcs_unstage = () => null; + h.vcs_commit = () => { + committed = true; + return "2222222"; + }; + const vcsModule = "/src/features/vcs/store.ts"; + const { useVcs, refreshNow } = await import(/* @vite-ignore */ vcsModule); + useVcs.setState({ available: true, root, repo: root, branchesLoaded: false, stashesLoaded: false }); + await refreshNow(); + }, ROOT); +} + +test("⌘K commits the checked changes with the typed message", async ({ workbench: page }) => { + await openFromTree(page, "build.gradle.kts"); + await stubRepository(page); + await page.keyboard.press("Meta+k"); + const panel = page.getByRole("region", { name: "Commit", exact: true }); + const commit = panel.getByRole("button", { name: "Commit", exact: true }); + const changes = panel.getByRole("tree", { name: "Changes" }); + // Tracked changes start included; unversioned files start excluded. + await expect(changes.getByRole("checkbox", { name: "Include build.gradle.kts in commit" })).toBeChecked(); + await expect(changes.getByRole("checkbox", { name: "Include Unversioned Files in commit" })).not.toBeChecked(); + const message = panel.getByRole("textbox", { name: "Commit message" }); + await expect(message).toBeFocused(); + + // Committing without a message is refused. + await commit.click(); + await expect(panel.getByRole("alert")).toHaveText("Specify commit message"); + expect(await ipcCalls(page, "vcs_commit")).toHaveLength(0); + + await message.fill("Return order DTOs from create"); + await commit.click(); + await expect(page.getByText(/1 file committed: Return order DTOs from create/)).toBeVisible(); + const [committed] = await ipcCalls(page, "vcs_commit"); + expect(committed).toMatchObject({ repo: ROOT, message: "Return order DTOs from create" }); + const staged = (await ipcCalls(page, "vcs_stage")).flatMap((a) => a.paths as string[]); + expect(staged).toEqual([`${ROOT}/build.gradle.kts`]); + await expect(changes.getByRole("checkbox", { name: "Include build.gradle.kts in commit" })).toBeHidden(); +}); diff --git a/e2e/debug.spec.ts b/e2e/debug.spec.ts new file mode 100644 index 00000000..1af77663 --- /dev/null +++ b/e2e/debug.spec.ts @@ -0,0 +1,115 @@ +/** + * A debugging session: a breakpoint set in the editor reaches the adapter, the session stops there + * with its frames and variables, and stepping and resuming go back to the adapter. The adapter is + * stubbed at the IPC layer; `crates/ide-dap/tests` drives real ones. + */ +import { expect, ipcCalls, openFromTree, stubDesktop, test } from "./fixtures"; + +const ROOT = "/dev/acme-api"; +const FILE = `${ROOT}/build.gradle.kts`; +const SESSION = 3; + +test("stops at a breakpoint, shows frames and variables, steps and resumes", async ({ workbench: page }) => { + const editor = await openFromTree(page, "build.gradle.kts"); + await stubDesktop(page); + await page.evaluate( + async ({ file, root, session }) => { + const config = { + id: "python:app", + name: "app", + kind: "python", + command: "python3", + args: ["app.py"], + cwd: root, + env: {}, + detected: false, + isTest: false, + debugAdapter: "debugpy", + }; + const h = window.__ipc.handlers; + const debug = (event: Record) => window.__ipc.emit({ type: "debug", session, event }); + const stopAt = (line: number) => { + h.debug_stack_trace = () => [ + { id: 11, name: "create", path: file, line, column: 0, presentationHint: null }, + { id: 12, name: "handle", path: file, line: 16, column: 0, presentationHint: null }, + ]; + debug({ kind: "stopped", threadId: 1, reason: "breakpoint", description: null, allThreadsStopped: true }); + }; + h.run_configs = () => [config]; + h.run_apply_settings = (args) => args.configs; + h.debug_set_breakpoints = (args) => + (args.breakpoints as { line: number }[]).map((b, i) => ({ + id: i + 1, + verified: true, + line: b.line, + message: null, + })); + h.debug_start = () => { + setTimeout(() => { + debug({ kind: "started", name: config.name, adapter: "debugpy", parent: null }); + debug({ kind: "initialized" }); + stopAt(20); + }, 50); + return session; + }; + h.debug_threads = () => [{ id: 1, name: "MainThread" }]; + h.debug_scopes = () => [{ name: "Locals", variablesReference: 100, expensive: false }]; + h.debug_variables = () => [ + { + name: "req", + value: "CreateOrderRequest(sku='A-1', qty=2)", + typeName: "CreateOrderRequest", + variablesReference: 0, + evaluateName: "req", + }, + { name: "qty", value: "2", typeName: "int", variablesReference: 0, evaluateName: "qty" }, + ]; + h.debug_step = () => { + setTimeout(() => { + debug({ kind: "continued", threadId: 1, allThreadsContinued: true }); + stopAt(21); + }, 50); + return null; + }; + h.debug_resume = () => { + setTimeout(() => { + debug({ kind: "continued", threadId: 1, allThreadsContinued: true }); + debug({ kind: "exited", code: 0 }); + debug({ kind: "terminated" }); + }, 50); + return null; + }; + const runModule = "/src/features/run/store.ts"; + const { loadConfigs, selectConfig } = await import(/* @vite-ignore */ runModule); + await loadConfigs(); + selectConfig(config.id); + }, + { file: FILE, root: ROOT, session: SESSION }, + ); + + // Breakpoint and execution lines carry CSS-module classes (`_bpLine_`). + const line = (text: string) => editor.locator(".cm-line", { hasText: text }); + await line("val order: Order").click(); + await page.keyboard.press("Meta+F8"); + await expect(line("val order: Order")).toHaveClass(/_bpLine_/); + + await page.keyboard.press("Control+d"); + await expect.poll(() => ipcCalls(page, "debug_start")).toHaveLength(1); + const set = (await ipcCalls(page, "debug_set_breakpoints")).filter((a) => a.path === FILE).at(-1); + expect(set?.breakpoints).toEqual([expect.objectContaining({ line: 20 })]); + + const debug = page.getByRole("region", { name: "Debug", exact: true }); + await expect(debug.getByText("create", { exact: true })).toBeVisible(); + await expect(debug.getByText("handle", { exact: true })).toBeVisible(); + await expect(debug.getByText("CreateOrderRequest(sku='A-1', qty=2)")).toBeVisible(); + await expect(line("val order: Order")).toHaveClass(/_execLine_/); + + await page.keyboard.press("F8"); + await expect.poll(() => ipcCalls(page, "debug_step")).toEqual([{ session: SESSION, threadId: 1, kind: "over" }]); + await expect(line("return order.toDto()")).toHaveClass(/_execLine_/); + await expect(line("val order: Order")).not.toHaveClass(/_execLine_/); + + await page.keyboard.press("F9"); + await expect.poll(() => ipcCalls(page, "debug_resume")).toHaveLength(1); + await expect(debug.getByText("CreateOrderRequest(sku='A-1', qty=2)")).toBeHidden(); +}); diff --git a/e2e/deployment.spec.ts b/e2e/deployment.spec.ts new file mode 100644 index 00000000..9f110d06 --- /dev/null +++ b/e2e/deployment.spec.ts @@ -0,0 +1,287 @@ +/** Deployment diff over an in-memory server: no SSH or filesystem writes. */ +import { test, expect } from "./fixtures"; +import type { Page } from "@playwright/test"; + +test("native deployment entry fills its own window and exposes minimize and maximize controls", async ({ + page, +}, info) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.route("**/deployment-window-test", (route) => + route.fulfill({ + contentType: "text/html", + body: '
', + }), + ); + await page.goto("/deployment-window-test"); + await page.evaluate(async () => { + const refreshModule = "/@react-refresh"; + const { default: refresh } = await import(/* @vite-ignore */ refreshModule); + refresh.injectIntoGlobalHook(window); + Object.assign(window, { + $RefreshReg$: () => {}, + $RefreshSig$: () => (type: unknown) => type, + __vite_plugin_react_preamble_installed__: true, + }); + const mocksModule = "/node_modules/@tauri-apps/api/mocks.js"; + const { mockIPC, mockWindows } = await import(/* @vite-ignore */ mocksModule); + const calls: string[] = []; + (window as unknown as { deploymentCalls: string[] }).deploymentCalls = calls; + const settings = { + appearance: { theme: "dark", syncWithOs: false }, + deployment: { servers: [{ id: "staging", name: "Staging", kind: "local", rootPath: "/srv/app" }] }, + }; + let launch = true; + const local = ") => { + calls.push(command); + if (/deploy_(upload|download|sync|delete)|doc_save|fs_write/.test(command)) throw new Error("Unexpected write"); + if (command === "app_info") + return { platform: "windows", version: "test", configDir: "/config", wayland: false }; + if (command === "settings_get") return settings; + if (command === "project_current") return { root: "/second", name: "Second", settings }; + if (command === "window_take_launch") { + if (!launch) return null; + launch = false; + return { deployment: { server: "staging", paths: ["/second/perfil.php"] } }; + } + if (command === "deploy_coverage") return [{ server: "staging", paths: ["/second/perfil.php"], skipped: [] }]; + if (command === "deploy_compare") + return [ + { + path: "perfil.php", + remotePath: "/srv/app/perfil.php", + local: { size: local.length, mtime: 1700000100 }, + remote: { size: remote.length, mtime: 1700000000 }, + status: "localNewer", + }, + ]; + if (command === "remote_open") return "/snapshot/perfil.php"; + if (command === "fs_probe") return { size: local.length, binary: false, tooLarge: false }; + if (command === "fs_read_text") return String(args.path).startsWith("/snapshot/") ? remote : local; + if (command === "doc_open") throw new Error("Read-only test fixture"); + if (command === "languages") return []; + if (command === "plugin:window|is_maximized" || command === "plugin:window|is_fullscreen") return false; + return null; + }, + { shouldMockEvents: true }, + ); + const entry = "/src/main.tsx"; + await import(/* @vite-ignore */ entry); + }); + const comparison = page.getByRole("region", { name: "Diff Between 'local files' and 'files on Staging'" }); + await expect(comparison.getByRole("grid", { name: "Files to synchronize" })).toBeVisible(); + await expect(comparison.getByRole("region", { name: "perfil.php" }).getByRole("textbox")).toHaveCount(2); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(page.getByRole("tree", { name: "Project" })).toHaveCount(0); + const frame = await comparison.boundingBox(); + expect(frame!.y).toBe(0); + expect(frame!.height).toBe(page.viewportSize()!.height); + await expect(comparison.getByRole("button", { name: "Synchronize All", exact: true })).toBeInViewport(); + await expect(comparison.getByRole("button", { name: "Synchronize Selected", exact: true })).toBeVisible(); + await comparison.getByRole("button", { name: "Minimize", exact: true }).click(); + await comparison.getByRole("button", { name: "Maximize", exact: true }).click(); + // The caption buttons reach the window API asynchronously. + const calls = () => page.evaluate(() => (window as unknown as { deploymentCalls: string[] }).deploymentCalls); + await expect.poll(calls).toContain("plugin:window|minimize"); + await expect.poll(calls).toContain("plugin:window|toggle_maximize"); + expect(errors).toEqual([]); + await page.screenshot({ path: info.outputPath("deployment-native-window.png") }); +}); + +async function showComparison(page: Page) { + await page.evaluate(async () => { + const projectModule = "/src/platform/project.ts"; + const settingsModule = "/src/platform/remoteSettings.ts"; + const stateModule = "/src/features/deployment/store.ts"; + const routeModule = "/src/platform/tauriCore.ts"; + const backendModule = "/src/editor/core/backend.ts"; + const [{ useProject }, { DEFAULT_DEPLOYMENT_SERVER }, { useDeployment }, { setInvokeRoute }, { docBackend }] = + await Promise.all([ + import(/* @vite-ignore */ projectModule), + import(/* @vite-ignore */ settingsModule), + import(/* @vite-ignore */ stateModule), + import(/* @vite-ignore */ routeModule), + import(/* @vite-ignore */ backendModule), + ]); + const local: Record = { + "app.js": "export const version = 'local';\n", + "perfil.php": + "find('list', [\n 'fields' => [$this->alias . '.id', $this->alias . '.descripcion'],\n 'order' => $this->alias . '.descripcion ASC',\n ]);\n }\n}\n", + "hidden.php": "alpha", + "same.js": "same", + }; + const remote: Record = { + "app.js": "export const version = 'deployed';\n", + "perfil.php": + " (path.startsWith("/snapshot/") ? remote : local)[path.split("/").at(-1)!] ?? ""; + setInvokeRoute((command: string, args: Record) => { + const path = String(args?.path ?? ""); + if (command === "remote_open") return Promise.resolve(`/snapshot/${path.split("/").at(-1)}`); + if (command === "fs_probe") return Promise.resolve({ size: text(path).length, binary: false, tooLarge: false }); + if (command === "fs_read_text") return Promise.resolve(text(path)); + if (command === "fs_read_range") { + const content = text(path); + return Promise.resolve({ + total: content.length, + offset: args.offset, + data: btoa(content.slice(Number(args.offset), Number(args.offset) + Number(args.len))), + }); + } + if (command === "doc_open") return Promise.reject(new Error("No live document in this fixture")); + if (command === "deploy_sync" && syncState.enabled) { + const steps = args.steps as { path: string; action: string }[]; + syncState.transfers.push(steps.map((step) => ({ ...step }))); + writes.push(command); + for (const step of steps) { + if (step.action === "upload") remote[step.path] = local[step.path]!; + else if (step.action === "download") local[step.path] = remote[step.path]!; + else if (step.action === "deleteLocal") delete local[step.path]; + else if (step.action === "deleteRemote") delete remote[step.path]; + } + return Promise.resolve({ files: steps.length, skipped: 0, errors: [] }); + } + if (command === "deploy_compare") { + const name = path.slice(root.length + 1); + if (local[name] === undefined && remote[name] === undefined) return Promise.resolve([]); + const identical = local[name] === remote[name]; + return Promise.resolve([ + { + path: name, + remotePath: `/srv/app/${name}`, + local: local[name] === undefined ? null : { size: local[name]!.length, mtime: 1_700_000_100 }, + remote: + remote[name] === undefined + ? null + : { size: remote[name]!.length, mtime: identical ? 1_700_000_100 : 1_700_000_000 }, + status: + local[name] === undefined + ? "remoteOnly" + : remote[name] === undefined + ? "localOnly" + : identical + ? "same" + : "localNewer", + }, + ]); + } + if (/deploy_(upload|download|sync|delete)|doc_save|fs_write/.test(command)) { + writes.push(command); + return Promise.reject(new Error("Unexpected write")); + } + return null; + }); + const entries = Object.keys(local).map((path) => ({ + path, + remotePath: `/srv/app/${path}`, + local: { size: local[path]!.length, mtime: 1_700_000_100 }, + remote: { size: remote[path]!.length, mtime: 1_700_000_000 }, + status: path === "hidden.php" || path === "same.js" ? "same" : "localNewer", + })); + useProject.setState({ + settings: { + ...useProject.getState().settings, + appearance: { ...useProject.getState().settings?.appearance, theme: "dark", syncWithOs: false }, + deployment: { + servers: [ + { ...DEFAULT_DEPLOYMENT_SERVER, id: "staging", kind: "local", name: "Staging", rootPath: "/srv/app" }, + ], + }, + }, + }); + useDeployment.setState({ + sync: { + server: "staging", + serverName: "Staging", + paths: [useProject.getState().project.root], + entries, + skipped: [], + request: null, + error: null, + choices: Object.fromEntries(entries.map((entry) => [entry.path, entry.status === "same" ? "skip" : "upload"])), + hideSame: true, + running: false, + }, + }); + }); +} + +test("deployment shows mirrored lists and a selectable side-by-side content diff", async ({ + workbench: page, +}, info) => { + await showComparison(page); + const dialog = page.getByRole("dialog", { name: "Diff Between 'local files' and 'files on Staging'" }); + await expect(dialog).toBeVisible(); + const grid = dialog.getByRole("grid", { name: "Files to synchronize" }); + await grid.getByRole("combobox", { name: "Action perfil.php" }).locator("..", {}).locator("..").click(); + const preview = dialog.locator('[aria-label="Deployment content comparison"]'); + await expect(preview.locator(".cm-editor")).toHaveCount(2); + await expect(preview.locator(".cm-editor").first()).toContainText("getPerfilesValidos"); + await expect(preview.locator(".cm-editor").last()).not.toContainText("getPerfilesValidos"); + await expect(preview.locator(".cm-editor").last().locator(".cm-content")).toHaveAttribute("contenteditable", "false"); + await expect(dialog.getByRole("toolbar", { name: "Diff" })).toBeVisible(); + await expect(dialog.getByRole("separator", { name: "Resize file list and diff preview" })).toBeVisible(); + await dialog.screenshot({ path: info.outputPath("deployment-comparison.png"), animations: "disabled" }); + await page.getByLabel("Action perfil.php", { exact: true }).selectOption("skip"); + await grid.focus(); + await page.keyboard.press("Enter"); + await expect(dialog).toBeVisible(); + expect(await page.evaluate(() => (window as unknown as { deploymentWrites: string[] }).deploymentWrites)).toEqual([]); +}); + +test("Synchronize Selected and Enter transfer one file at a time while the comparison stays open", async ({ + workbench: page, +}) => { + await showComparison(page); + await page.evaluate(() => { + (window as unknown as { deploymentSyncState: { enabled: boolean } }).deploymentSyncState.enabled = true; + }); + const dialog = page.getByRole("dialog", { name: "Diff Between 'local files' and 'files on Staging'" }); + const grid = dialog.getByRole("grid", { name: "Files to synchronize" }); + await grid.getByRole("combobox", { name: "Action perfil.php" }).locator("..").locator("..").click(); + await dialog.getByRole("button", { name: "Synchronize Selected", exact: true }).click(); + await expect(dialog).toBeVisible(); + await expect(grid.getByRole("combobox", { name: "Action perfil.php" })).toHaveCount(0); + await expect(grid.getByRole("combobox", { name: "Action app.js" })).toHaveValue("upload"); + await expect(dialog.locator('[aria-label="Deployment content comparison"] .cm-editor').last()).toContainText( + "deployed", + ); + await grid.focus(); + await page.keyboard.press("Enter"); + await expect(dialog.getByRole("button", { name: "Synchronize All", exact: true })).toBeDisabled(); + await expect(dialog.getByRole("button", { name: "Synchronize Selected", exact: true })).toBeDisabled(); + await expect(dialog).toBeVisible(); + expect( + await page.evaluate( + () => (window as unknown as { deploymentSyncState: { transfers: unknown[] } }).deploymentSyncState.transfers, + ), + ).toEqual([[{ path: "perfil.php", action: "upload" }], [{ path: "app.js", action: "upload" }]]); +}); + +test("binary content finds changes hidden by metadata and file filters keep the preview in sync", async ({ + workbench: page, +}) => { + await showComparison(page); + await page.getByLabel("Compare by", { exact: true }).selectOption("content"); + await expect(page.getByLabel("Action hidden.php", { exact: true })).toHaveValue("upload"); + await expect(page.getByLabel("Action same.js", { exact: true })).toHaveCount(0); + await page.getByLabel("Filter by name", { exact: true }).fill("hidden.php"); + const preview = page.locator('[aria-label="Deployment content comparison"]'); + await expect(preview.locator(".cm-editor").first()).toContainText("alpha"); + await expect(preview.locator(".cm-editor").last()).toContainText("bravo"); + await page.getByLabel("Action hidden.php", { exact: true }).selectOption("skip"); + expect(await page.evaluate(() => (window as unknown as { deploymentWrites: string[] }).deploymentWrites)).toEqual([]); +}); diff --git a/e2e/editor.spec.ts b/e2e/editor.spec.ts index 4f8163df..6f6a79d5 100644 --- a/e2e/editor.spec.ts +++ b/e2e/editor.spec.ts @@ -1,3 +1,4 @@ +import type { Page } from "@playwright/test"; import { expect, openFromTree, test } from "./fixtures"; test("opens a file from the project tree and types into it", async ({ workbench: page }) => { @@ -25,6 +26,49 @@ test("⌘F opens the find bar and highlights matches; Escape returns to the edit await expect(editor.locator(".cm-content")).toBeFocused(); }); +test("each file keeps its own find bar, query and caret", async ({ workbench: page }) => { + const find = page.getByRole("textbox", { name: "Find" }); + const caret = page.getByRole("button", { name: /^\d+:\d+/ }); + const tab = (name: string) => page.getByRole("tablist", { name: "Editor tabs" }).getByRole("tab", { name }); + + const a = await openFromTree(page, "build.gradle.kts"); + await a.locator(".cm-content").click(); + await page.keyboard.press("Meta+f"); + await find.fill("Order"); + await expect(caret).toHaveText(/\(5 chars\)/); + const caretA = await caret.textContent(); + + // Another file has no bar until ⌘F; opening one starts from the last search without moving the caret. + const b = await openFromTree(page, "settings.gradle.kts"); + await expect(find).toBeHidden(); + await b.locator(".cm-line", { hasText: "@PostMapping" }).click(); + const caretB = await caret.textContent(); + await page.keyboard.press("Meta+f"); + await expect(find).toHaveValue("Order"); + await expect(caret).toHaveText(caretB!); + await find.fill("status"); + await expect(caret).toHaveText(/\(6 chars\)/); + const caretB2 = await caret.textContent(); + + // A third file starts from the latest search. + await openFromTree(page, "README.md"); + await expect(find).toBeHidden(); + await page.keyboard.press("Meta+f"); + await expect(find).toHaveValue("status"); + + // Switching back shows each file's own query, caret untouched. + await tab("build.gradle.kts").click(); + await expect(find).toHaveValue("Order"); + await expect(caret).toHaveText(caretA!); + await tab("settings.gradle.kts").click(); + await expect(find).toHaveValue("status"); + await expect(caret).toHaveText(caretB2!); + await find.press("Escape"); + await expect(find).toBeHidden(); + await tab("build.gradle.kts").click(); + await expect(find).toHaveValue("Order"); +}); + test("opening several files creates tabs and ⌘W closes the active one", async ({ workbench: page }) => { await openFromTree(page, "build.gradle.kts"); await openFromTree(page, "settings.gradle.kts"); @@ -59,3 +103,44 @@ test("breadcrumbs follow the caret into the symbol under it", async ({ workbench await page.keyboard.press("Escape"); await expect(bar).toBeHidden(); }); + +/** What the in-memory backend has saved for `path`. */ +const saved = (page: Page, path: string) => + page.evaluate(async (path) => { + const backendModule = "/src/editor/core/backend.ts"; + const { docBackend } = await import(/* @vite-ignore */ backendModule); + return (docBackend() as { files: Map }).files.get(path) ?? null; + }, path); + +test("an edit marks the tab modified and ⌘S saves it", async ({ workbench: page }) => { + const editor = await openFromTree(page, "build.gradle.kts"); + // The only tab; its modified dot sits next to its close button. + const tabs = page.getByRole("tablist", { name: "Editor tabs" }); + await expect(tabs.getByLabel("Modified")).toHaveCount(0); + await editor.locator(".cm-line", { hasText: "return order.toDto()" }).click(); + await page.keyboard.press("End"); + await page.keyboard.press("Enter"); + await page.keyboard.type("// saved by e2e"); + await expect(tabs.getByLabel("Modified")).toHaveCount(1); + expect(await saved(page, "/dev/acme-api/build.gradle.kts")).toBeNull(); + await page.keyboard.press("Meta+s"); + await expect(tabs.getByLabel("Modified")).toHaveCount(0); + expect(await saved(page, "/dev/acme-api/build.gradle.kts")).toContain("// saved by e2e"); +}); + +test("⌘R replaces every match in the file", async ({ workbench: page }) => { + const editor = await openFromTree(page, "build.gradle.kts"); + await editor.locator(".cm-content").click(); + await page.keyboard.press("Meta+r"); + const bar = page.getByRole("search", { name: "Replace in file" }); + await bar.getByRole("textbox", { name: "Find" }).fill("OrderDto"); + await bar.getByRole("textbox", { name: "Replace" }).fill("OrderView"); + await bar.getByRole("button", { name: "Replace All" }).click(); + const content = editor.locator(".cm-content"); + await expect(content).not.toContainText("OrderDto"); + await expect(editor.locator(".cm-line", { hasText: "OrderView" })).toHaveCount(2); + // One undo step takes the whole replacement back. + await content.click(); + await page.keyboard.press("Meta+z"); + await expect(editor.locator(".cm-line", { hasText: "OrderDto" })).toHaveCount(2); +}); diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts index 05369faf..71cda429 100644 --- a/e2e/fixtures.ts +++ b/e2e/fixtures.ts @@ -23,6 +23,97 @@ export const test = base.extend<{ workbench: Page }>({ export { expect }; +/** What `stubDesktop` installs on `window.__ipc`; handlers are defined inside `page.evaluate`. */ +export interface IpcStub { + /** Command name as the backend registers it (`lsp_goto`) → its answer; throw `{ kind, message }` for an `IdeError`. */ + handlers: Record) => unknown>; + calls: { cmd: string; args: Record }[]; + /** Sends one message down a `Channel` passed to a command. */ + send(channel: unknown, message: unknown): void; + /** Emits an `IdeEvent` the way the backend does. */ + emit(event: { type: string } & Record): void; +} + +declare global { + interface Window { + __ipc: IpcStub; + } +} + +/** + * Switches the page into desktop mode over a stubbed native layer: commands with a handler answer, + * every other command fails as unavailable, the way a backend without that feature does. Documents + * keep their in-memory backend; the project tree keeps its sample only while it isn't reloaded, so + * open files from it before stubbing. + */ +export async function stubDesktop(page: Page) { + await page.evaluate(async () => { + // Documents stay in memory: the editor's backend is chosen once, on first use. + const backendModule = "/src/editor/core/backend.ts"; + const { docBackend } = await import(/* @vite-ignore */ backendModule); + docBackend(); + const callbacks = new Map void>(); + const channelIndex = new Map(); + const listeners: { event: string; handler: number }[] = []; + let nextCallback = 1; + const ipc: IpcStub = { + // This window is the only one. + handlers: { "plugin:window|get_all_windows": () => [] }, + calls: [], + send(channel, message) { + const id = (channel as { id: number }).id; + const index = channelIndex.get(id) ?? 0; + channelIndex.set(id, index + 1); + callbacks.get(id)?.({ index, message }); + }, + emit(event) { + for (const l of listeners) + if (l.event === "ide://event") callbacks.get(l.handler)?.({ event: l.event, id: 0, payload: event }); + }, + }; + window.__ipc = ipc; + Object.defineProperty(window, "__TAURI_EVENT_PLUGIN_INTERNALS__", { + configurable: true, + value: { unregisterListener: () => {} }, + }); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: { + metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } }, + transformCallback(cb: (message: unknown) => void) { + const id = nextCallback++; + callbacks.set(id, cb); + return id; + }, + unregisterCallback(id: number) { + callbacks.delete(id); + }, + convertFileSrc: (path: string) => path, + async invoke(cmd: string, args: Record = {}) { + ipc.calls.push({ cmd, args }); + if (cmd === "plugin:event|listen") { + listeners.push({ event: args.event as string, handler: args.handler as number }); + return listeners.length; + } + const handler = ipc.handlers[cmd]; + // Tauri plugin calls (windows, events) succeed quietly; app commands are what tests stub. + if (!handler && cmd.startsWith("plugin:")) return null; + if (!handler) throw { kind: "unavailable", message: `${cmd} is not stubbed`, detail: null }; + return handler(args); + }, + }, + }); + const ipcModule = "/src/platform/ipc.ts"; + const { ideEventsReady } = await import(/* @vite-ignore */ ipcModule); + await ideEventsReady(); + }); +} + +/** The commands the page has invoked so far under `stubDesktop`, by name. */ +export async function ipcCalls(page: Page, cmd: string): Promise[]> { + return page.evaluate((cmd) => window.__ipc.calls.filter((c) => c.cmd === cmd).map((c) => c.args), cmd); +} + /** Tool windows currently laid out (visible) in the workbench. */ export async function visibleToolWindows(page: Page): Promise { return page diff --git a/e2e/hierarchyRefactor.spec.ts b/e2e/hierarchyRefactor.spec.ts new file mode 100644 index 00000000..1c6ed491 --- /dev/null +++ b/e2e/hierarchyRefactor.spec.ts @@ -0,0 +1,172 @@ +/** + * Pull Members Up / Push Members Down. The language server is stubbed at the IPC layer: the + * Java side is covered by java/java-usages/tests/hierarchy.py against a real JDT LS. + */ +import type { Page } from "@playwright/test"; +import { expect, openFromTree, test } from "./fixtures"; + +const captureDir = process.env.OPENIDEA_ACCEPTANCE_DIR; + +/** Opens the dialog as the action does after the server answered, for the editor's view. */ +async function openDialog(page: Page, kind: "pullUp" | "pushDown") { + await page.evaluate(async (kind) => { + const module = "/src/editor/refactor/hierarchy.ts"; + const { useHierarchyDialog } = await import(/* @vite-ignore */ module); + const view = (document.querySelector(".cm-content") as unknown as { cmTile: { view: { state: { doc: unknown } } } }) + .cmTile.view; + const pullUp = kind === "pullUp"; + useHierarchyDialog.setState({ + request: { + seq: 1, + view, + kind, + doc: 1, + pos: { line: 0, col: 0 }, + text: view.state.doc, + info: { + className: pullUp ? "Dog" : "Animal", + targets: pullUp + ? [ + { id: "animal", name: "Animal", qualifiedName: "zoo.Animal", interface: false }, + { id: "speaker", name: "Speaker", qualifiedName: "zoo.Speaker", interface: true }, + ] + : [ + { id: "dog", name: "Dog", qualifiedName: "zoo.Dog", interface: false }, + { id: "cat", name: "Cat", qualifiedName: "zoo.Cat", interface: false }, + ], + members: pullUp + ? [ + { id: "tail", label: "tail : int", kind: "field", selected: false, abstractable: false }, + { id: "bark", label: "bark() : String", kind: "method", selected: true, abstractable: true }, + { id: "wag", label: "wag() : int", kind: "method", selected: false, abstractable: true }, + ] + : [ + { id: "name", label: "name : String", kind: "field", selected: false, abstractable: false }, + { id: "describe", label: "describe() : String", kind: "method", selected: true, abstractable: true }, + ], + }, + }, + }); + }, kind); +} + +/** Stubs the server's check and the edit application; returns what reached them. */ +async function stubServer(page: Page, problems: { severity: string; message: string }[]) { + await page.evaluate(async (problems) => { + const module = "/src/platform/ipc.ts"; + const { commands } = await import(/* @vite-ignore */ module); + // Only the stubbed commands answer; window lookups by other features keep working. + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: { + transformCallback: () => 1, + invoke: async () => null, + metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } }, + }, + }); + const seen = ((window as unknown as { hierarchySeen: unknown[] }).hierarchySeen = [] as unknown[]); + commands.lspHierarchyRefactor = async (...args: unknown[]) => { + seen.push({ refactor: args.slice(3) }); + return { + status: "ok", + data: { + problems, + edit: { + files: [ + { + kind: "edit", + path: "/dev/acme-api/README.md", + edits: [{ range: { start: { line: 0, col: 0 }, end: { line: 0, col: 0 } }, new_text: "" }], + }, + ], + }, + }, + }; + }; + commands.applyWorkspaceEdit = async (edit: unknown) => { + seen.push({ apply: edit }); + return { status: "ok", data: null }; + }; + }, problems); +} + +const seen = (page: Page) => page.evaluate(() => (window as unknown as { hierarchySeen: unknown[] }).hierarchySeen); + +test("the editor's Refactor menu offers Pull Members Up and Push Members Down", async ({ workbench: page }) => { + const editor = await openFromTree(page, "README.md"); + await editor.locator(".cm-content").click({ button: "right" }); + await page.getByRole("menuitem", { name: "Refactor", exact: true }).hover(); + await expect(page.getByRole("menuitem", { name: /^Pull Members Up…/ })).toBeVisible(); + await expect(page.getByRole("menuitem", { name: /^Push Members Down…/ })).toBeVisible(); + if (captureDir) await page.screenshot({ path: `${captureDir}/01-refactor-menu.png` }); + await page.keyboard.press("Escape"); +}); + +test("Pull Members Up: members, interface destination, conflicts, then the edit", async ({ workbench: page }) => { + await openFromTree(page, "README.md"); + await openDialog(page, "pullUp"); + const dialog = page.getByRole("dialog", { name: "Pull Members Up" }); + await expect(dialog).toBeVisible(); + await expect(dialog.getByText("Pull up members of Dog to:")).toBeVisible(); + const member = (label: string) => dialog.getByRole("row").filter({ hasText: label }); + // The member at the caret starts checked; fields can't be made abstract. + await expect(member("bark() : String").getByRole("checkbox").first()).toBeChecked(); + await expect(member("wag() : int").getByRole("checkbox").first()).not.toBeChecked(); + await member("tail : int").getByRole("checkbox").first().check(); + await expect(member("tail : int").getByRole("checkbox", { name: "Make abstract" })).toBeDisabled(); + if (captureDir) await page.screenshot({ path: `${captureDir}/02-pull-up-dialog.png` }); + + // To an interface, methods are declared abstract there. + await dialog.getByRole("button", { name: "Destination class" }).click(); + await page.getByRole("option", { name: /zoo\.Speaker/ }).click(); + const abstractBark = member("bark() : String").getByRole("checkbox", { name: "Make abstract" }); + await expect(abstractBark).toBeChecked(); + await expect(abstractBark).toBeDisabled(); + await expect(dialog.getByText("Methods pulled up to an interface are declared abstract there.")).toBeVisible(); + if (captureDir) await page.screenshot({ path: `${captureDir}/03-pull-up-to-interface.png` }); + + await stubServer(page, [ + { severity: "error", message: "Field 'zoo.Dog.tail' is not accessible from type 'zoo.Speaker'" }, + ]); + await dialog.getByRole("button", { name: "Refactor", exact: true }).click(); + const problems = page.getByRole("alertdialog").or(page.getByRole("dialog", { name: "Problems Detected" })); + await expect(problems.getByText(/is not accessible from type/)).toBeVisible(); + if (captureDir) await page.screenshot({ path: `${captureDir}/04-problems-detected.png` }); + // Cancel: nothing applied, the dialog stays. + await problems.getByRole("button", { name: "Cancel", exact: true }).click(); + await expect(dialog).toBeVisible(); + expect(await seen(page)).toEqual([ + { + refactor: [ + "pullUp", + "speaker", + [ + { id: "tail", makeAbstract: false }, + { id: "bark", makeAbstract: true }, + ], + ], + }, + ]); + // Continue: the edit is applied and the dialog closes. + await dialog.getByRole("button", { name: "Refactor", exact: true }).click(); + await problems.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(dialog).toBeHidden(); + await expect.poll(async () => (await seen(page)).some((s) => "apply" in (s as object))).toBe(true); +}); + +test("Push Members Down keeps an abstract declaration when asked", async ({ workbench: page }) => { + await openFromTree(page, "README.md"); + await openDialog(page, "pushDown"); + const dialog = page.getByRole("dialog", { name: "Push Members Down" }); + await expect(dialog.getByText("Push members of Animal down to:")).toBeVisible(); + await expect(dialog.getByText("Dog, Cat")).toBeVisible(); + const describe = dialog.getByRole("row").filter({ hasText: "describe() : String" }); + await describe.getByRole("checkbox", { name: "Keep abstract" }).check(); + if (captureDir) await page.screenshot({ path: `${captureDir}/05-push-down-dialog.png` }); + await stubServer(page, []); + await dialog.getByRole("button", { name: "Refactor", exact: true }).click(); + await expect(dialog).toBeHidden(); + const calls = await seen(page); + expect(calls[0]).toEqual({ refactor: ["pushDown", null, [{ id: "describe", makeAbstract: true }]] }); + expect(calls.some((s) => "apply" in (s as object))).toBe(true); +}); diff --git a/e2e/linkedGradle.spec.ts b/e2e/linkedGradle.spec.ts new file mode 100644 index 00000000..0ee3dc8c --- /dev/null +++ b/e2e/linkedGradle.spec.ts @@ -0,0 +1,120 @@ +/** Linked Gradle builds use isolated native stubs; no real builds or files are touched. */ +import type { GradleModel, RunConfig } from "../src/platform/ipc"; +import { expect, ipcCalls, stubDesktop, test } from "./fixtures"; + +const build = (dir: string): GradleModel => ({ + dir, + command: `/dev/acme-api/${dir}/gradlew`, + env: { JAVA_HOME: `/${dir}/jdk` }, + javaVersion: 21, + syncedAt: 1, + stale: false, + syncError: null, + projects: [ + { path: ":", name: dir, dir, buildFile: `${dir}/build.gradle`, tasks: [] }, + { + path: ":app", + name: "app", + dir: `${dir}/app`, + buildFile: `${dir}/app/build.gradle`, + tasks: [{ name: "assemble", group: "build", description: "Assembles this project" }], + }, + ], +}); + +test("links independent Gradle roots, runs identical task paths in their own builds and unlinks without deletion", async ({ + workbench: page, +}, info) => { + await stubDesktop(page); + await page.evaluate( + async ({ first, second }) => { + let linked = [first]; + let runId = 0; + const model = () => ({ maven: null, gradle: linked }); + const h = window.__ipc.handlers; + h.build_model = model; + h.build_sync = model; + h.run_configs = () => []; + h.run_apply_settings = (args) => args.configs; + h.gradle_link_projects = () => { + if (!linked.some((g) => g.dir === second.dir)) linked = [...linked, second]; + return model(); + }; + h.gradle_unlink_projects = (args) => { + linked = linked.filter((g) => !(args.paths as string[]).includes(g.dir)); + return model(); + }; + h["plugin:dialog|open"] = () => ["/dev/acme-api/second/settings.gradle"]; + h.run_start = (args) => { + const config = args.config as RunConfig; + const id = ++runId; + setTimeout(() => window.__ipc.emit({ type: "runExited", run: id, code: 0 }), 50); + return { id, configId: config.id, name: config.name, pid: 4242 + id, running: true, pty: false }; + }; + const storeModule = "/src/features/build-tools/store.ts"; + const { loadModel } = await import(/* @vite-ignore */ storeModule); + await loadModel(); + const contributionsModule = "/src/workbench/contributions.ts"; + const { toolWindowService } = await import(/* @vite-ignore */ contributionsModule); + toolWindowService().show("gradle", true); + }, + { first: build("first"), second: build("second") }, + ); + + const tree = page.getByRole("tree", { name: "Gradle projects" }); + await expect(tree.getByRole("treeitem", { name: "first", exact: true })).toBeVisible(); + await expect(tree.getByRole("treeitem", { name: "second", exact: true })).toHaveCount(0); + await page.screenshot({ path: info.outputPath("gradle-one-root.png"), animations: "disabled" }); + + await page.getByRole("button", { name: "Link Gradle Projects", exact: true }).click(); + await expect + .poll(() => ipcCalls(page, "gradle_link_projects")) + .toEqual([{ paths: ["/dev/acme-api/second/settings.gradle"] }]); + await expect(tree.getByRole("treeitem", { name: "second", exact: true })).toBeVisible(); + // Reloading the model reads the persisted linkage again. + await page.evaluate(async () => { + const storeModule = "/src/features/build-tools/store.ts"; + const { loadModel } = await import(/* @vite-ignore */ storeModule); + await loadModel(); + const actionModule = "/src/platform/actions.ts"; + const { executeAction } = await import(/* @vite-ignore */ actionModule); + await executeAction("gradle.expandAll"); + await executeAction("gradle.toggleOffline"); + }); + const tasks = tree.getByRole("treeitem", { name: /^assemble\b/ }); + await expect(tasks).toHaveCount(2); + await page.screenshot({ path: info.outputPath("gradle-two-roots.png"), animations: "disabled" }); + + await tasks.nth(0).dblclick(); + await expect.poll(async () => (await ipcCalls(page, "run_start")).length).toBe(1); + await tasks.nth(1).dblclick(); + await expect.poll(async () => (await ipcCalls(page, "run_start")).length).toBe(2); + const runs = (await ipcCalls(page, "run_start")).map((args) => args.config as RunConfig); + expect(runs[0]!.id).not.toBe(runs[1]!.id); + for (const [index, dir] of ["first", "second"].entries()) { + expect(runs[index]).toMatchObject({ + command: `/dev/acme-api/${dir}/gradlew`, + cwd: dir, + args: [":app:assemble", "--offline"], + env: { JAVA_HOME: `/${dir}/jdk` }, + }); + } + + for (const dir of ["second", "first"]) { + await tree.getByRole("treeitem", { name: dir, exact: true }).click({ button: "right" }); + await page.getByRole("menuitem", { name: "Unlink Gradle Project", exact: true }).click(); + const dialog = page.getByRole("dialog", { name: "Unlink Gradle project?" }); + await expect(dialog).toContainText("Files will remain"); + await dialog.getByRole("button", { name: "Unlink", exact: true }).click(); + await expect(tree.getByRole("treeitem", { name: dir, exact: true })).toHaveCount(0); + } + await expect + .poll(() => ipcCalls(page, "gradle_unlink_projects")) + .toEqual([{ paths: ["second"] }, { paths: ["first"] }]); + await expect(page.getByText("No Gradle projects", { exact: true })).toBeVisible(); + await expect(page.getByRole("button", { name: "Link Gradle Projects", exact: true })).toBeVisible(); + expect( + await page.evaluate(() => window.__ipc.calls.filter((c) => /^fs_(?:delete|remove|trash)/.test(c.cmd))), + ).toEqual([]); + await page.screenshot({ path: info.outputPath("gradle-unlinked.png"), animations: "disabled" }); +}); diff --git a/e2e/openProject.spec.ts b/e2e/openProject.spec.ts new file mode 100644 index 00000000..12445bd1 --- /dev/null +++ b/e2e/openProject.spec.ts @@ -0,0 +1,71 @@ +/** + * Closing a project returns to the Welcome screen, and a recent project opens from there into the + * workbench. The backend's project handling is stubbed at the IPC layer; `crates/ide-host/tests` + * covers opening and closing real projects. + */ +import { doubleShift, expect, ipcCalls, stubDesktop, test } from "./fixtures"; + +const BILLING = "/dev/billing"; + +test("Close Project shows the Welcome screen; a recent project opens from it once trusted", async ({ + workbench: page, +}) => { + await stubDesktop(page); + await page.evaluate((root) => { + const entry = (name: string, kind: string) => ({ + name, + path: `${root}/${name}`, + kind, + size: 0, + mtimeMs: null, + ignored: false, + excluded: false, + }); + const h = window.__ipc.handlers; + h.close_project = () => null; + h.app_info = () => ({ version: "1.0.0", platform: "macos", configDir: "/Users/dev/.openidea", wayland: false }); + h.recent_projects = () => [ + { path: root, name: "billing", openedAtMs: Date.now() - 3_600_000, branch: "main" }, + { path: "/dev/acme-api", name: "acme-api", openedAtMs: Date.now(), branch: "main" }, + ]; + let trusted = false; + h.project_trust_status = (args) => ({ root: args.path, trusted, decided: trusted }); + h.project_trust_set = (args) => { + trusted = Boolean(args.trusted); + return null; + }; + h.open_project = (args) => ({ + trusted: true, + root: args.path, + name: String(args.path).split("/").pop(), + isGit: false, + settings: {}, + }); + h.fs_read_dir = (args) => + args.path === root ? [entry("src", "dir"), entry("package.json", "file"), entry("README.md", "file")] : []; + }, BILLING); + + await doubleShift(page); + const search = page.getByRole("dialog", { name: "Search Everywhere" }); + await search.getByRole("combobox").fill("Close Project"); + await expect(search.getByRole("option", { name: /^Close Project/ }).first()).toBeVisible(); + await page.keyboard.press("Enter"); + + const projects = page.getByRole("region", { name: "Projects" }); + await expect(page.getByRole("tree", { name: "Project" })).toBeHidden(); + await expect(projects.getByRole("option", { name: /^billing/ })).toBeVisible(); + expect(await ipcCalls(page, "close_project")).toHaveLength(1); + + await projects.getByRole("option", { name: /^billing/ }).click(); + // A project not yet trusted asks first. + const trust = page.getByRole("dialog", { name: "Trust and Open Project?" }); + await expect(trust).toBeVisible(); + expect(await ipcCalls(page, "open_project")).toHaveLength(0); + await trust.getByRole("button", { name: "Trust Project" }).click(); + await expect(trust).toBeHidden(); + expect(await ipcCalls(page, "project_trust_set")).toEqual([{ path: BILLING, trusted: true, folder: false }]); + await expect.poll(() => ipcCalls(page, "open_project")).toEqual([{ path: BILLING }]); + const tree = page.getByRole("tree", { name: "Project" }); + await expect(tree.getByRole("treeitem", { name: /^billing/ })).toBeVisible(); + await expect(tree.getByRole("treeitem", { name: "package.json" })).toBeVisible(); +}); diff --git a/e2e/run.spec.ts b/e2e/run.spec.ts new file mode 100644 index 00000000..75596434 --- /dev/null +++ b/e2e/run.spec.ts @@ -0,0 +1,82 @@ +/** + * Running a configuration: output streams into the Run tool window, the exit shows, and Stop ends + * the process. Processes are stubbed at the IPC layer; `crates/ide-run` covers spawning them. + */ +import type { Page } from "@playwright/test"; +import { expect, ipcCalls, stubDesktop, test } from "./fixtures"; + +const RUN_ID = 7; + +/** One saved `npm run dev` configuration; `run_start` prints `output` and exits with `exit` unless null. */ +async function stubRun(page: Page, output: { kind: string; text: string }[], exit: number | null) { + await stubDesktop(page); + await page.evaluate( + async ({ output, exit, runId }) => { + const config = { + id: "npm:dev", + name: "dev", + kind: "npm", + command: "npm", + args: ["run", "dev"], + cwd: "/dev/acme-api", + env: {}, + detected: false, + isTest: false, + debugAdapter: null, + }; + const h = window.__ipc.handlers; + h.run_configs = () => [config]; + h.run_apply_settings = (args) => args.configs; + h.run_start = (args) => { + setTimeout(() => { + for (const chunk of output) window.__ipc.send(args.onOutput, chunk); + if (exit !== null) setTimeout(() => window.__ipc.emit({ type: "runExited", run: runId, code: exit }), 50); + }, 50); + return { id: runId, configId: config.id, name: config.name, pid: 4242, running: true, pty: false }; + }; + h.run_stop = () => { + setTimeout(() => window.__ipc.emit({ type: "runExited", run: runId, code: 143 }), 50); + return null; + }; + const runModule = "/src/features/run/store.ts"; + const { loadConfigs, selectConfig } = await import(/* @vite-ignore */ runModule); + await loadConfigs(); + selectConfig(config.id); + }, + { output, exit, runId: RUN_ID }, + ); +} + +test("⌃R runs the selected configuration and shows its output and exit", async ({ workbench: page }) => { + await stubRun( + page, + [ + { kind: "stdout", text: "> acme-api@1.0.0 dev\n" }, + { kind: "stdout", text: "Server listening on http://localhost:8080\n" }, + { kind: "stderr", text: "warning: config/dev.json not found, using defaults\n" }, + ], + 0, + ); + await page.keyboard.press("Control+r"); + const run = page.getByRole("region", { name: "Run", exact: true }); + await expect(run).toBeVisible(); + await expect(run.getByText("Server listening on http://localhost:8080")).toBeVisible(); + await expect(run.getByText("warning: config/dev.json not found, using defaults")).toBeVisible(); + // Once the process exits, Stop is off and the session's tab tells the exit code. + await expect(run.getByRole("button", { name: "Stop" })).toBeDisabled(); + await expect(run.getByRole("tab", { name: "dev" })).toHaveAttribute("data-tooltip", /exit code 0$/); + const [started] = await ipcCalls(page, "run_start"); + expect(started).toMatchObject({ config: { id: "npm:dev", command: "npm", args: ["run", "dev"] } }); +}); + +test("⌘F2 stops a running process", async ({ workbench: page }) => { + await stubRun(page, [{ kind: "stdout", text: "Watching for changes…\n" }], null); + await page.keyboard.press("Control+r"); + const run = page.getByRole("region", { name: "Run", exact: true }); + await expect(run.getByText("Watching for changes…")).toBeVisible(); + await expect(run.getByRole("button", { name: "Stop" })).toBeEnabled(); + await page.keyboard.press("Meta+F2"); + await expect.poll(() => ipcCalls(page, "run_stop")).toEqual([{ id: RUN_ID }]); + await expect(run.getByRole("button", { name: "Stop" })).toBeDisabled(); + await expect(run.getByRole("tab", { name: "dev" })).toHaveAttribute("data-tooltip", /exit code 143$/); +}); diff --git a/e2e/search.spec.ts b/e2e/search.spec.ts index cdf4fefc..e27aedc9 100644 --- a/e2e/search.spec.ts +++ b/e2e/search.spec.ts @@ -1,4 +1,4 @@ -import { doubleShift, expect, test } from "./fixtures"; +import { doubleShift, expect, ipcCalls, stubDesktop, test } from "./fixtures"; test("double Shift opens Search Everywhere with its tabs; Escape closes it", async ({ workbench: page }) => { await doubleShift(page); @@ -23,15 +23,92 @@ test("Search Everywhere finds and runs an action", async ({ workbench: page }) = await expect(page.getByRole("list", { name: /language servers/i })).toBeVisible(); }); +const ROOT = "/dev/acme-api"; +const SERVICE = `${ROOT}/src/main/kotlin/com/acme/orders/service/OrderService.kt`; + test("Search Everywhere finds a file and opens it", async ({ workbench: page }) => { - // TODO(search): plain-browser mode has no in-memory `findFiles`, so the Files - // source returns nothing. Un-fixme once search/everywhere gets browser deps - // (e.g. backed by the project-tree sample file system). - test.fixme(true, "Search Everywhere has no in-memory file source in browser mode"); + // The project's file index is stubbed at the IPC layer. + await stubDesktop(page); + await page.evaluate((path) => { + window.__ipc.handlers.find_files = (args) => + String(args.query).startsWith("OrderSer") + ? [{ path, relPath: path.slice("/dev/acme-api/".length), score: 100, indices: [] }] + : []; + }, SERVICE); await doubleShift(page); const dialog = page.getByRole("dialog", { name: "Search Everywhere" }); await dialog.getByRole("combobox").fill("OrderService"); await expect(dialog.getByRole("option", { name: /OrderService\.kt/ })).toBeVisible(); await page.keyboard.press("Enter"); - await expect(page.getByRole("tab", { name: "OrderService.kt" })).toBeVisible(); + await expect(page.getByRole("tab", { name: "OrderService.kt" })).toHaveAttribute("aria-selected", "true"); + await expect(page.getByRole("textbox", { name: `Editor for ${SERVICE}` })).toBeVisible(); +}); + +test("Find in Files streams matches by file and opens the chosen one at its line", async ({ workbench: page }) => { + await stubDesktop(page); + await page.evaluate( + ({ root, service }) => { + const match = (line: number, text: string, word: string) => ({ + line, + text, + ranges: [[text.indexOf(word), text.indexOf(word) + word.length]], + textOffset: 0, + }); + window.__ipc.handlers.search_validate = () => null; + window.__ipc.handlers.search_start = (args) => { + const send = (event: unknown) => window.__ipc.send(args.onEvent, event); + setTimeout(() => { + send({ + type: "file", + file: { + path: `${root}/build.gradle.kts`, + relPath: "build.gradle.kts", + matches: [ + match(7, "class OrderController(private val service: OrderService) {", "OrderService"), + match(3, "import com.acme.orders.service.OrderService", "OrderService"), + ], + }, + }); + send({ + type: "file", + file: { + path: service, + relPath: service.slice(root.length + 1), + matches: [match(0, "// OrderService", "OrderService")], + }, + }); + send({ + type: "done", + summary: { files: 2, matches: 3, limitHit: false, cancelled: false, elapsedMs: 4 }, + }); + }, 20); + return null; + }; + }, + { root: ROOT, service: SERVICE }, + ); + await page.keyboard.press("Meta+Shift+f"); + const dialog = page.getByRole("dialog", { name: "Find in Files" }); + await dialog.getByRole("textbox", { name: "Text to find" }).fill("OrderService"); + const matches = dialog.getByRole("listbox", { name: "Matches" }); + await expect(matches.getByRole("option")).toHaveCount(3); + await expect + .poll(async () => (await ipcCalls(page, "search_start")).at(-1)?.query) + .toMatchObject({ + pattern: "OrderService", + }); + await matches.getByRole("option", { name: /import com\.acme\.orders\.service\.OrderService/ }).dblclick(); + await expect(page.getByRole("tab", { name: "build.gradle.kts" })).toHaveAttribute("aria-selected", "true"); + const caretLine = () => + page.evaluate((label) => { + const content = document.querySelector(`.cm-content[aria-label="${label}"]`) as unknown as { + cmTile: { + view: { state: { doc: { lineAt(pos: number): { text: string } }; selection: { main: { head: number } } } }; + }; + } | null; + if (!content) return null; + const { state } = content.cmTile.view; + return state.doc.lineAt(state.selection.main.head).text; + }, `Editor for ${ROOT}/build.gradle.kts`); + await expect.poll(caretLine).toBe("import com.acme.orders.service.OrderService"); }); diff --git a/e2e/toolwindows.spec.ts b/e2e/toolwindows.spec.ts index e8eb67d9..dcf00a8d 100644 --- a/e2e/toolwindows.spec.ts +++ b/e2e/toolwindows.spec.ts @@ -1,4 +1,4 @@ -import { expect, test, visibleToolWindows } from "./fixtures"; +import { expect, ipcCalls, stubDesktop, test, visibleToolWindows } from "./fixtures"; test("⌘1 focuses, hides and re-shows the Project tool window", async ({ workbench: page }) => { const project = page.getByRole("region", { name: "Project" }); @@ -31,6 +31,34 @@ test("⌥F12 opens the Terminal tool window and toggles it closed", async ({ wor await expect(terminal).toBeHidden(); }); +test("the first ⌥F12 focuses the terminal input once the lazily loaded view mounts", async ({ workbench: page }) => { + await page.keyboard.press("Alt+F12"); + const terminal = page.getByRole("region", { name: "Terminal", exact: true }); + await expect(terminal.getByRole("textbox", { name: "Terminal input" })).toBeFocused(); + // Reopened with the view already mounted, it focuses the input as well. + await page.keyboard.press("Alt+F12"); + await expect(terminal).toBeHidden(); + await page.keyboard.press("Alt+F12"); + await expect(terminal.getByRole("textbox", { name: "Terminal input" })).toBeFocused(); +}); + +test("a terminal view that loads late doesn't take focus moved elsewhere meanwhile", async ({ workbench: page }) => { + let release!: () => void; + const loaded = new Promise((resolve) => (release = resolve)); + await page.route(/\/src\/features\/terminal\/runtime\.ts/, async (route) => { + await loaded; + await route.continue(); + }); + await page.keyboard.press("Alt+F12"); + const terminal = page.getByRole("region", { name: "Terminal", exact: true }); + await expect(terminal).toBeVisible(); + const tree = page.getByRole("tree", { name: "Project" }); + await tree.getByRole("treeitem", { name: "README.md" }).click(); + release(); + await expect(terminal.getByRole("textbox", { name: "Terminal input" })).toBeAttached(); + await expect(tree).toBeFocused(); +}); + test("Split Right gives the Terminal a tab group with its own header; Unsplit merges it back", async ({ workbench: page, }) => { @@ -80,3 +108,47 @@ test("⌘⇧F12 hides all tool windows and restores them", async ({ workbench: p await expect(page.getByRole("region", { name: "Project" })).toBeVisible(); await expect(page.getByRole("region", { name: "Problems" })).toBeVisible(); }); + +test("the Terminal sends typed commands to the shell and shows its output", async ({ workbench: page }) => { + // The shell is stubbed at the IPC layer: it echoes input and answers `echo`. + await stubDesktop(page); + await page.evaluate(() => { + let output: unknown = null; + let line = ""; + const out = (text: string) => window.__ipc.send(output, text); + const h = window.__ipc.handlers; + h.term_default_shell = () => "/bin/zsh"; + h.term_available_shells = () => ["/bin/zsh"]; + h.term_spawn = (args) => { + output = args.onOutput; + setTimeout(() => out("acme-api % "), 20); + return { id: 5, title: "zsh", pid: 99 }; + }; + h.term_resize = () => null; + h.term_is_busy = () => false; + h.term_write = (args) => { + for (const ch of String(args.data)) { + if (ch === "\r") { + const said = /^echo (.*)$/.exec(line); + out(`\r\n${said ? `${said[1]}\r\n` : ""}acme-api % `); + line = ""; + } else { + line += ch; + out(ch); + } + } + return null; + }; + }); + await page.keyboard.press("Alt+F12"); + const terminal = page.getByRole("region", { name: "Terminal", exact: true }); + const rows = terminal.locator(".xterm-rows"); + await expect(rows).toContainText("acme-api %"); + await terminal.locator(".xterm-screen").click(); + await expect(terminal.getByRole("textbox", { name: "Terminal input" })).toBeFocused(); + await page.keyboard.type("echo hello from e2e"); + await page.keyboard.press("Enter"); + await expect(rows.locator("div", { hasText: /^hello from e2e\s*$/ })).toHaveCount(1); + const written = (await ipcCalls(page, "term_write")).map((a) => a.data).join(""); + expect(written).toBe("echo hello from e2e\r"); +}); diff --git a/fixtures/README.md b/fixtures/README.md index c6c97e17..a4353ca3 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -12,11 +12,19 @@ something to show. | `python-app/` | Python 3.10+, pytest | `python -m tasks.cli` (from `src/`) | `pytest` | `OPENIDEA_FAIL=1 pytest` | unused imports in `models.py` | | `go-app/` | Go 1.22+ | `go run ./cmd/server` | `go test ./...` | `OPENIDEA_FAIL=1 go test ./...` | unused `fmt.Sprintf` result in `internal/store/store.go` | | `kotlin-gradle/` | JDK 21, Gradle 8 (no wrapper) | `gradle run` | `gradle test` | `gradle test -PopenideaFail` | unused variable in `App.kt` | +| `java-app/` | JDK 21, Maven 3.9 | `mvn compile exec:java` | `mvn test` | `mvn test -DopenideaFail=true` | unused variable in `App.java` | | `polyglot/` | Go, TS, Python, C, shell, SQL, TOML/YAML/JSON, proto, Terraform | `make` | `make test` | — | unused variable in `native/hash.c` | The fixtures are standalone: `rust-app` declares its own `[workspace]` and `fixtures/eslint.config.js` keeps them out of the IDE's own lint run. +The language smoke test (`crates/ide-host/tests/language_smoke.rs`) runs completion, +navigation, diagnostics, quick fixes, formatting and rename on these projects with real +language servers, `smoke/run.test.ts` runs each project's run and test configurations +(the intentionally failing test included, then Rerun Failed Tests), and `smoke/debug.test.ts` +debugs its entry point from the "Set a breakpoint here" line: when you change a fixture, run +them for that language. + ## Setup Work on a **copy** so git and edits never touch this repository: diff --git a/fixtures/java-app/.gitignore b/fixtures/java-app/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/fixtures/java-app/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/fixtures/java-app/pom.xml b/fixtures/java-app/pom.xml new file mode 100644 index 00000000..916e3138 --- /dev/null +++ b/fixtures/java-app/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + com.example + openidea-fixture-orders + 0.1.0 + OpenIDEA QA fixture: a tiny order book + + + 21 + UTF-8 + + false + + + + + org.junit.jupiter + junit-jupiter + 5.11.3 + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + ${openideaFail} + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + com.example.orders.App + + + + + diff --git a/fixtures/java-app/src/main/java/com/example/orders/App.java b/fixtures/java-app/src/main/java/com/example/orders/App.java new file mode 100644 index 00000000..80bd50d7 --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/App.java @@ -0,0 +1,16 @@ +package com.example.orders; + +public class App { + public static void main(String[] args) { + OrderBook book = new OrderBook(); + book.place("alice", "Mechanical keyboard", 1, 8999); + book.place("bob", "Wireless mouse", 2, 2999); + book.place("alice", "USB-C hub", 1, 3999); + + int unused = 42; // intentional warning: unused variable + // Set a breakpoint here (QA: debugger) and inspect `book`. + long revenue = book.revenueCents(); + System.out.printf("%d orders, revenue %.2f%n", book.size(), revenue / 100.0); + book.largest().ifPresent(o -> System.out.println("Largest: " + o.product())); + } +} diff --git a/fixtures/java-app/src/main/java/com/example/orders/Order.java b/fixtures/java-app/src/main/java/com/example/orders/Order.java new file mode 100644 index 00000000..a082b0c4 --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/Order.java @@ -0,0 +1,14 @@ +package com.example.orders; + +/** A customer order: a product, how many and the unit price in cents. */ +public record Order(String customer, String product, int quantity, long unitPriceCents) { + + public Order { + if (quantity <= 0) throw new IllegalArgumentException("quantity must be positive"); + } + + /** The order's total in cents. */ + public long totalCents() { + return unitPriceCents * quantity; + } +} diff --git a/fixtures/java-app/src/main/java/com/example/orders/OrderBook.java b/fixtures/java-app/src/main/java/com/example/orders/OrderBook.java new file mode 100644 index 00000000..07926151 --- /dev/null +++ b/fixtures/java-app/src/main/java/com/example/orders/OrderBook.java @@ -0,0 +1,35 @@ +package com.example.orders; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +/** Keeps the orders placed so far, in the order they were placed. */ +public class OrderBook { + private final List orders = new ArrayList<>(); + + /** Places an order and returns it. */ + public Order place(String customer, String product, int quantity, long unitPriceCents) { + Order order = new Order(customer, product, quantity, unitPriceCents); + orders.add(order); + return order; + } + + public List byCustomer(String customer) { + return orders.stream().filter(o -> o.customer().equals(customer)).toList(); + } + + public Optional largest() { + return orders.stream().max(Comparator.comparingLong(Order::totalCents)); + } + + // TODO: discounts for repeat customers + public long revenueCents() { + return orders.stream().mapToLong(Order::totalCents).sum(); + } + + public int size() { + return orders.size(); + } +} diff --git a/fixtures/java-app/src/test/java/com/example/orders/OrderBookTest.java b/fixtures/java-app/src/test/java/com/example/orders/OrderBookTest.java new file mode 100644 index 00000000..42b30c2d --- /dev/null +++ b/fixtures/java-app/src/test/java/com/example/orders/OrderBookTest.java @@ -0,0 +1,29 @@ +package com.example.orders; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +class OrderBookTest { + @Test + void sumsRevenueOverAllOrders() { + OrderBook book = new OrderBook(); + book.place("alice", "Keyboard", 2, 1000); + book.place("bob", "Mouse", 1, 500); + assertEquals(2500, book.revenueCents()); + assertEquals(1, book.byCustomer("bob").size()); + } + + @Test + void rejectsEmptyOrders() { + assertThrows(IllegalArgumentException.class, () -> new Order("alice", "Keyboard", 0, 1000)); + } + + @Test + @EnabledIfSystemProperty(named = "openidea.fail", matches = "true") + void intentionallyFails() { + assertEquals(1, new OrderBook().size(), "QA: an intentionally failing test"); + } +} diff --git a/plugins/java-usages/CONFIGURATION.md b/java/java-usages/CONFIGURATION.md similarity index 96% rename from plugins/java-usages/CONFIGURATION.md rename to java/java-usages/CONFIGURATION.md index f0965bcd..11d122a0 100644 --- a/plugins/java-usages/CONFIGURATION.md +++ b/java/java-usages/CONFIGURATION.md @@ -95,10 +95,10 @@ runtime or resolve profile/precedence rules. Use the JDT LS distribution / Java version documented in [README](README.md). ```sh -python3 plugins/java-usages/build.py /path/to/jdtls -python3 plugins/java-usages/tests/configuration.py /path/to/jdtls -OPENIDEA_CONFIGURATION_STRESS=1 python3 plugins/java-usages/tests/configuration.py /path/to/jdtls -OPENIDEA_SPRING_BOOT_JAR=/path/to/spring-boot-autoconfigure.jar python3 plugins/java-usages/tests/configuration.py /path/to/jdtls +python3 java/java-usages/build.py /path/to/jdtls +python3 java/java-usages/tests/configuration.py /path/to/jdtls +OPENIDEA_CONFIGURATION_STRESS=1 python3 java/java-usages/tests/configuration.py /path/to/jdtls +OPENIDEA_SPRING_BOOT_JAR=/path/to/spring-boot-autoconfigure.jar python3 java/java-usages/tests/configuration.py /path/to/jdtls cargo test -p ide-language -p ide-lsp ``` diff --git a/plugins/java-usages/MANIFEST.MF b/java/java-usages/MANIFEST.MF similarity index 72% rename from plugins/java-usages/MANIFEST.MF rename to java/java-usages/MANIFEST.MF index 2d5796c3..252534d5 100644 --- a/plugins/java-usages/MANIFEST.MF +++ b/java/java-usages/MANIFEST.MF @@ -2,9 +2,10 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: OpenIDEA Java usages Bundle-SymbolicName: ai.openidea.usages;singleton:=true -Bundle-Version: 1.2.1 +Bundle-Version: 1.3.0 Bundle-RequiredExecutionEnvironment: JavaSE-21 Require-Bundle: org.eclipse.jdt.ls.core,org.eclipse.jdt.core, org.eclipse.jdt.core.manipulation,org.eclipse.core.runtime, - org.eclipse.lsp4j,org.eclipse.core.resources,com.google.gson + org.eclipse.lsp4j,org.eclipse.core.resources,com.google.gson, + org.eclipse.ltk.core.refactoring diff --git a/plugins/java-usages/README.md b/java/java-usages/README.md similarity index 79% rename from plugins/java-usages/README.md rename to java/java-usages/README.md index 819fd2b3..f5d7b242 100644 --- a/plugins/java-usages/README.md +++ b/java/java-usages/README.md @@ -57,7 +57,7 @@ is not a claim of full IntelliJ feature parity. ## Rebuild and delivery The checked-in jar is embedded by ide-sdk `include_bytes!`, installed as -`java-usages-1.2.1.jar`, and added alongside existing initialization bundles. +`java-usages-1.3.0.jar`, and added alongside existing initialization bundles. Users need no compiler, Maven or Python. Manifest/install version must change when replacing the bundle because OSGi caches bundles. @@ -73,10 +73,12 @@ Version 1.2.0 adds the configuration command; its reproducible SHA-256 is Version 1.2.1 keeps the configuration snapshot across editor open/close, saves of unrelated content and non-Java resource changes; its reproducible SHA-256 is `cbae3e3a0bcf2818b4cc9d98998bd9b81f92e1ae7803dc757134485f9776a4e1`. +Version 1.3.0 adds the class-hierarchy refactoring commands; its reproducible SHA-256 is +`cca4cc385ace286111730d2a6164333b8958d74d69631548eadca8863255c24a`. ```sh export PATH=/path/to/jdk25/bin:$PATH -python3 plugins/java-usages/build.py /path/to/jdtls +python3 java/java-usages/build.py /path/to/jdtls ``` ## Manual upgrades @@ -98,9 +100,10 @@ not rebuild this jar. A maintainer must: ## Real server verification ```sh -python3 plugins/java-usages/tests/verify.py /path/to/jdtls /path/to/lombok.jar -python3 plugins/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write.json -OPENIDEA_USAGES_INDEX_STRESS=1 python3 plugins/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write-indexed.json +python3 java/java-usages/tests/verify.py /path/to/jdtls /path/to/lombok.jar +python3 java/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write.json +OPENIDEA_USAGES_INDEX_STRESS=1 python3 java/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write-indexed.json +python3 java/java-usages/tests/hierarchy.py /path/to/jdtls ``` The harnesses select the JDT LS configuration for the current platform. The first @@ -112,6 +115,24 @@ method references, receiver/array reads and a real cancellation response (`-3280 Use the actual desktop app to check read/write toggles, mutual toggle behavior, Find transfer, persistence and unfiltered non-Java counts with the same fixture. +`hierarchy.py` runs Pull Members Up and Push Members Down: member and destination +listing, moving, declaring abstract, pulling up to an interface, conflicts reported +before any change, and a class without supertypes. Each edit is applied to a copy of +the fixture and compiled with javac. + +## Class-hierarchy refactorings + +`openidea.java.hierarchy.members(uri, line, column, kind)` describes the class at the +position for `kind` `pullUp` or `pushDown`: its movable members (handle id, label, +kind, preselected, can be declared abstract) and the destination supertypes, or the +direct subtypes the members are pushed into. A position on a member preselects it. +`openidea.java.hierarchy.refactor(uri, line, column, kind, target, choices)` runs +JDT's `PullUpRefactoringProcessor` / `PushDownRefactoringProcessor` with the chosen +`{id, abstract}` members and returns `{problems, edit}`. Nothing is applied by the +server; the app shows the problems, then applies the edit as one undoable change. +A method pulled up to an interface is always declared there, as in JDT's wizard. +Either command answers `{refused: reason}` when the position can't be refactored. + ## Review and manual reproduction See [the English reproduction guide](tests/REPRODUCE.md) for a small Java/Lombok @@ -129,6 +150,11 @@ This is an app-owned JDT LS extension, not a standard LSP feature. It uses: objects (`isIncludeAccessors`). - `JDTUtils.resolveTypeRoot`, `findElementAtSelection`. - `org.eclipse.jdt.internal.corext.codemanipulation.GetterSetterUtil`. +- `org.eclipse.jdt.internal.corext.refactoring.structure.PullUpRefactoringProcessor`, + `PushDownRefactoringProcessor`, `RefactoringAvailabilityTesterCore`, `JdtFlags`, + `JavaElementLabelsCore`, `PreferenceManager.getCodeGenerationSettings`, + `JsonRpcHelpers.toOffset`, `JDTUtils.resolveCompilationUnit` and + `ChangeUtil.convertToWorkspaceEdit`. - Public JDT model/DOM APIs for exact target identity and source-expression context. The fingerprint protects native target discovery, not all binary compatibility. diff --git a/plugins/java-usages/build-dependencies.sha256 b/java/java-usages/build-dependencies.sha256 similarity index 100% rename from plugins/java-usages/build-dependencies.sha256 rename to java/java-usages/build-dependencies.sha256 diff --git a/plugins/java-usages/build.py b/java/java-usages/build.py similarity index 100% rename from plugins/java-usages/build.py rename to java/java-usages/build.py diff --git a/plugins/java-usages/openidea-java-usages.jar b/java/java-usages/openidea-java-usages.jar similarity index 80% rename from plugins/java-usages/openidea-java-usages.jar rename to java/java-usages/openidea-java-usages.jar index bedb5a1a..2493c96f 100644 Binary files a/plugins/java-usages/openidea-java-usages.jar and b/java/java-usages/openidea-java-usages.jar differ diff --git a/plugins/java-usages/plugin.xml b/java/java-usages/plugin.xml similarity index 67% rename from plugins/java-usages/plugin.xml rename to java/java-usages/plugin.xml index dad7e8d8..d6c345e0 100644 --- a/plugins/java-usages/plugin.xml +++ b/java/java-usages/plugin.xml @@ -7,5 +7,9 @@ + + + +
diff --git a/plugins/java-usages/src/ai/openidea/configuration/ConfigurationCommand.java b/java/java-usages/src/ai/openidea/configuration/ConfigurationCommand.java similarity index 100% rename from plugins/java-usages/src/ai/openidea/configuration/ConfigurationCommand.java rename to java/java-usages/src/ai/openidea/configuration/ConfigurationCommand.java diff --git a/java/java-usages/src/ai/openidea/hierarchy/HierarchyCommand.java b/java/java-usages/src/ai/openidea/hierarchy/HierarchyCommand.java new file mode 100644 index 00000000..b2504dd7 --- /dev/null +++ b/java/java-usages/src/ai/openidea/hierarchy/HierarchyCommand.java @@ -0,0 +1,261 @@ +package ai.openidea.hierarchy; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.OperationCanceledException; +import org.eclipse.jdt.core.*; +import org.eclipse.jdt.internal.core.manipulation.JavaElementLabelsCore; +import org.eclipse.jdt.internal.corext.refactoring.RefactoringAvailabilityTesterCore; +import org.eclipse.jdt.internal.corext.refactoring.structure.PullUpRefactoringProcessor; +import org.eclipse.jdt.internal.corext.refactoring.structure.PushDownRefactoringProcessor; +import org.eclipse.jdt.internal.corext.refactoring.structure.PushDownRefactoringProcessor.MemberActionInfo; +import org.eclipse.jdt.internal.corext.util.JdtFlags; +import org.eclipse.jdt.ls.core.internal.ChangeUtil; +import org.eclipse.jdt.ls.core.internal.IDelegateCommandHandler; +import org.eclipse.jdt.ls.core.internal.JDTUtils; +import org.eclipse.jdt.ls.core.internal.handlers.JsonRpcHelpers; +import org.eclipse.jdt.ls.core.internal.preferences.PreferenceManager; +import org.eclipse.ltk.core.refactoring.Change; +import org.eclipse.ltk.core.refactoring.RefactoringStatus; +import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; +import org.eclipse.ltk.core.refactoring.participants.ProcessorBasedRefactoring; +import org.eclipse.ltk.core.refactoring.participants.RefactoringProcessor; + +/** + * Pull Members Up / Push Members Down with JDT's own processors. + * + *

{@code openidea.java.hierarchy.members(uri, line, column, kind)} describes the class at the + * position: its movable members and, for pull up, the supertypes in project sources. + * {@code openidea.java.hierarchy.refactor(uri, line, column, kind, target, choices)} checks the + * chosen move and returns its problems and the edit, which is never applied here. {@code kind} + * is {@code pullUp} or {@code pushDown}; {@code choices} are {@code {id, abstract}} objects whose + * ids are the handle identifiers returned by {@code members}. + */ +public final class HierarchyCommand implements IDelegateCommandHandler { + static final String MEMBERS = "openidea.java.hierarchy.members"; + static final String REFACTOR = "openidea.java.hierarchy.refactor"; + + private static final long LABEL = JavaElementLabelsCore.M_PARAMETER_TYPES | JavaElementLabelsCore.M_APP_RETURNTYPE + | JavaElementLabelsCore.F_APP_TYPE_SIGNATURE; + + @Override + public Object executeCommand(String command, List arguments, IProgressMonitor monitor) throws Exception { + String uri = (String) arguments.get(0); + int line = ((Number) arguments.get(1)).intValue(); + int column = ((Number) arguments.get(2)).intValue(); + boolean pullUp = switch ((String) arguments.get(3)) { + case "pullUp" -> true; + case "pushDown" -> false; + default -> throw new IllegalArgumentException("unknown hierarchy refactoring " + arguments.get(3)); + }; + ICompilationUnit unit = JDTUtils.resolveCompilationUnit(uri); + if (unit == null || !unit.exists()) return refused("The file is not part of a Java project"); + IMember start = memberAt(unit, JsonRpcHelpers.toOffset(unit, line, column)); + IMember[] members = start == null ? null : selection(start, pullUp); + if (members == null) { + return refused(pullUp ? "Caret should be inside a class that has members to pull up" + : "Caret should be inside a class that has members to push down"); + } + if (MEMBERS.equals(command)) return pullUp ? pullUpMembers(unit, members, monitor) : pushDownMembers(members, monitor); + if (REFACTOR.equals(command)) { + String target = (String) arguments.get(4); + List choices = new ArrayList<>(); + for (Object o : (List) arguments.get(5)) { + Map map = (Map) o; + IJavaElement element = JavaCore.create((String) map.get("id")); + if (!(element instanceof IMember member) || !member.exists()) return refused("The class changed; open the dialog again"); + choices.add(new Choice(member, Boolean.TRUE.equals(map.get("abstract")))); + } + if (choices.isEmpty()) return refused("No members selected"); + // Started from the class, the processors know it even when no member is moved + // (a method only declared abstract in the destination). + IType declaring = members[0] instanceof IType type ? type : members[0].getDeclaringType(); + IMember[] owner = { declaring }; + return pullUp ? pullUp(unit, owner, target, choices, monitor) : pushDown(owner, choices, monitor); + } + throw new IllegalArgumentException("unknown command " + command); + } + + private record Choice(IMember member, boolean makeAbstract) {} + + /** The innermost member at the offset, or the type itself for an initializer. */ + private static IMember memberAt(ICompilationUnit unit, int offset) throws JavaModelException { + IJavaElement element = unit.getElementAt(offset); + while (element != null && !(element instanceof IMember)) element = element.getParent(); + if (element instanceof IInitializer initializer) return initializer.getDeclaringType(); + return (IMember) element; + } + + /** The members to start from: the one at the caret when it can move, otherwise its class. */ + private static IMember[] selection(IMember start, boolean pullUp) throws JavaModelException { + for (IMember member = start; member != null; member = member.getDeclaringType()) { + if (member.isBinary() || member.isReadOnly()) return null; + if (pullUp ? RefactoringAvailabilityTesterCore.isPullUpAvailable(member) + : RefactoringAvailabilityTesterCore.isPushDownAvailable(member)) return new IMember[] { member }; + if (member instanceof IType type && (pullUp ? RefactoringAvailabilityTesterCore.getPullUpMembers(type) + : RefactoringAvailabilityTesterCore.getPushDownMembers(type)).length > 0) { + return new IMember[] { type }; + } + } + return null; + } + + private static Object pullUpMembers(ICompilationUnit unit, IMember[] members, IProgressMonitor monitor) throws Exception { + var processor = new PullUpRefactoringProcessor(members, PreferenceManager.getCodeGenerationSettings(unit)); + RefactoringStatus status = new ProcessorBasedRefactoring(processor).checkInitialConditions(monitor); + if (status.hasFatalError()) return refused(status.getMessageMatchingSeverity(RefactoringStatus.FATAL)); + IType[] candidates = processor.getCandidateTypes(new RefactoringStatus(), monitor); + cancelled(monitor); + if (candidates.length == 0) { + return refused("'" + processor.getDeclaringType().getElementName() + + "' has no superclass or interface in the project sources to pull members up to"); + } + List> targets = new ArrayList<>(); + for (IType type : candidates) { + Map row = new LinkedHashMap<>(); + row.put("id", type.getHandleIdentifier()); + row.put("name", type.getElementName()); + row.put("qualifiedName", type.getFullyQualifiedName('.')); + row.put("interface", type.isInterface()); + targets.add(row); + } + Set selected = Set.of(processor.getMembersToMove()); + List> rows = new ArrayList<>(); + for (IMember member : processor.getPullableMembersOfDeclaringType()) { + boolean abstractable = member instanceof IMethod method && !JdtFlags.isStatic(method); + rows.add(member(member, selected.contains(member), abstractable)); + } + return info(processor.getDeclaringType(), targets, rows); + } + + private static Object pushDownMembers(IMember[] members, IProgressMonitor monitor) throws Exception { + var processor = new PushDownRefactoringProcessor(members); + RefactoringStatus status = new ProcessorBasedRefactoring(processor).checkInitialConditions(monitor); + if (status.hasFatalError()) return refused(status.getMessageMatchingSeverity(RefactoringStatus.FATAL)); + IType declaring = processor.getDeclaringType(); + List> targets = new ArrayList<>(); + for (IType type : declaring.newTypeHierarchy(declaring.getJavaProject(), monitor).getSubtypes(declaring)) { + if (type.isBinary()) continue; + Map row = new LinkedHashMap<>(); + row.put("id", type.getHandleIdentifier()); + row.put("name", type.getElementName()); + row.put("qualifiedName", type.getFullyQualifiedName('.')); + row.put("interface", type.isInterface()); + targets.add(row); + } + cancelled(monitor); + List> rows = new ArrayList<>(); + for (MemberActionInfo info : processor.getMemberActionInfos()) { + boolean abstractable = false; + for (int action : info.getAvailableActions()) abstractable |= action == MemberActionInfo.PUSH_ABSTRACT_ACTION; + rows.add(member(info.getMember(), info.isToBePushedDown(), abstractable)); + } + return info(declaring, targets, rows); + } + + private static Object pullUp(ICompilationUnit unit, IMember[] members, String target, List choices, + IProgressMonitor monitor) throws Exception { + if (!(JavaCore.create(target) instanceof IType destination) || !destination.exists()) { + return refused("The destination class no longer exists"); + } + var processor = new PullUpRefactoringProcessor(members, PreferenceManager.getCodeGenerationSettings(unit)); + List moved = new ArrayList<>(); + List declaredAbstract = new ArrayList<>(); + List deleted = new ArrayList<>(); + for (Choice choice : choices) { + // A method pulled up to an interface is declared there, as JDT's own wizard does. + if (choice.member instanceof IMethod method && (choice.makeAbstract || destination.isInterface())) { + declaredAbstract.add(method); + } else { + moved.add(choice.member); + if (choice.member instanceof IMethod method) deleted.add(method); + } + } + processor.setMembersToMove(moved.toArray(IMember[]::new)); + processor.setAbstractMethods(declaredAbstract.toArray(IMethod[]::new)); + processor.setDeletedMethods(deleted.toArray(IMethod[]::new)); + processor.setDestinationType(destination); + processor.setCreateMethodStubs(true); + return run(processor, monitor); + } + + private static Object pushDown(IMember[] members, List choices, IProgressMonitor monitor) throws Exception { + var processor = new PushDownRefactoringProcessor(members); + var refactoring = new ProcessorBasedRefactoring(processor); + RefactoringStatus initial = refactoring.checkInitialConditions(monitor); + if (initial.hasFatalError()) return result(initial, null); + Map chosen = new LinkedHashMap<>(); + for (Choice choice : choices) chosen.put(choice.member, choice); + for (MemberActionInfo info : processor.getMemberActionInfos()) { + Choice choice = chosen.get(info.getMember()); + if (choice == null) info.setAction(MemberActionInfo.NO_ACTION); + else info.setAction(choice.makeAbstract && info.getMember() instanceof IMethod + ? MemberActionInfo.PUSH_ABSTRACT_ACTION : MemberActionInfo.PUSH_DOWN_ACTION); + } + RefactoringStatus status = refactoring.checkFinalConditions(monitor); + initial.merge(status); + return finish(refactoring, initial, monitor); + } + + private static Object run(RefactoringProcessor processor, IProgressMonitor monitor) throws Exception { + var refactoring = new ProcessorBasedRefactoring(processor); + RefactoringStatus status = refactoring.checkAllConditions(monitor); + return finish(refactoring, status, monitor); + } + + private static Object finish(ProcessorBasedRefactoring refactoring, RefactoringStatus status, IProgressMonitor monitor) + throws Exception { + cancelled(monitor); + if (status.hasFatalError()) return result(status, null); + Change change = refactoring.createChange(monitor); + cancelled(monitor); + return result(status, ChangeUtil.convertToWorkspaceEdit(change)); + } + + private static Map member(IMember member, boolean selected, boolean abstractable) { + Map row = new LinkedHashMap<>(); + row.put("id", member.getHandleIdentifier()); + row.put("label", JavaElementLabelsCore.getElementLabel(member, LABEL)); + row.put("kind", member instanceof IMethod ? "method" : member instanceof IField ? "field" : "type"); + row.put("selected", selected); + row.put("abstractable", abstractable); + return row; + } + + private static Map info(IType type, List> targets, List> members) { + Map out = new LinkedHashMap<>(); + out.put("className", type.getElementName()); + out.put("targets", targets); + out.put("members", members); + return out; + } + + private static Map result(RefactoringStatus status, Object edit) { + List> problems = new ArrayList<>(); + for (RefactoringStatusEntry entry : status.getEntries()) { + String severity = switch (entry.getSeverity()) { + case RefactoringStatus.FATAL -> "fatal"; + case RefactoringStatus.ERROR -> "error"; + case RefactoringStatus.WARNING -> "warning"; + default -> "info"; + }; + problems.add(Map.of("severity", severity, "message", entry.getMessage())); + } + Map out = new LinkedHashMap<>(); + out.put("problems", problems); + out.put("edit", edit); + return out; + } + + private static Map refused(String reason) { + return Map.of("refused", reason); + } + + private static void cancelled(IProgressMonitor monitor) { + if (monitor.isCanceled()) throw new OperationCanceledException(); + } +} diff --git a/plugins/java-usages/src/ai/openidea/usages/UsageAccess.java b/java/java-usages/src/ai/openidea/usages/UsageAccess.java similarity index 100% rename from plugins/java-usages/src/ai/openidea/usages/UsageAccess.java rename to java/java-usages/src/ai/openidea/usages/UsageAccess.java diff --git a/plugins/java-usages/src/ai/openidea/usages/UsagesCommand.java b/java/java-usages/src/ai/openidea/usages/UsagesCommand.java similarity index 100% rename from plugins/java-usages/src/ai/openidea/usages/UsagesCommand.java rename to java/java-usages/src/ai/openidea/usages/UsagesCommand.java diff --git a/plugins/java-usages/tests/REPRODUCE.md b/java/java-usages/tests/REPRODUCE.md similarity index 98% rename from plugins/java-usages/tests/REPRODUCE.md rename to java/java-usages/tests/REPRODUCE.md index aad80dc6..7dc2d013 100644 --- a/plugins/java-usages/tests/REPRODUCE.md +++ b/java/java-usages/tests/REPRODUCE.md @@ -73,7 +73,7 @@ checks cancellation. No Maven download is needed for this harness. ```sh export PATH=/path/to/jdk25/bin:$PATH -python3 plugins/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write.json +python3 java/java-usages/tests/read-write.py /path/to/jdtls /path/to/lombok.jar > /tmp/read-write.json ``` Then launch the desktop app with `pnpm tauri dev` and open the same Java/Lombok diff --git a/plugins/java-usages/tests/ReadWrite.java b/java/java-usages/tests/ReadWrite.java similarity index 100% rename from plugins/java-usages/tests/ReadWrite.java rename to java/java-usages/tests/ReadWrite.java diff --git a/plugins/java-usages/tests/configuration.py b/java/java-usages/tests/configuration.py similarity index 100% rename from plugins/java-usages/tests/configuration.py rename to java/java-usages/tests/configuration.py diff --git a/java/java-usages/tests/hierarchy.py b/java/java-usages/tests/hierarchy.py new file mode 100644 index 00000000..7e0e4ad5 --- /dev/null +++ b/java/java-usages/tests/hierarchy.py @@ -0,0 +1,272 @@ +"""Pull Members Up / Push Members Down against a real JDT LS. Argument: jdtls home. + +Each case computes the edit from the same unchanged sources, applies it to a copy +of the project and compiles the copy with javac. +""" +import json +import pathlib +import platform +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse + +home = pathlib.Path(sys.argv[1]).resolve() +bundle = pathlib.Path(__file__).resolve().parents[1] / 'openidea-java-usages.jar' +work = pathlib.Path(tempfile.mkdtemp(prefix='openidea-hierarchy-')) +config = work / 'config' +config.mkdir() +system, machine = platform.system(), platform.machine().lower() +config_name = {'Windows': 'config_win', 'Darwin': 'config_mac'}.get(system, 'config_linux') +config_names = ([config_name + '_arm'] if machine in ('arm64', 'aarch64') else []) + [config_name] +config_source = next(home / name / 'config.ini' for name in config_names if (home / name / 'config.ini').is_file()) +(config / 'config.ini').write_bytes(config_source.read_bytes()) +project = work / 'project' +(project / 'src/zoo').mkdir(parents=True) +(project / '.project').write_text('zooorg.eclipse.jdt.core.javabuilderorg.eclipse.jdt.core.javanature') +(project / '.classpath').write_text('') +sources = { + 'Animal.java': '''package zoo; + +public abstract class Animal { + protected String name = "animal"; + + public String describe() { + return "I am " + name; + } +} +''', + 'Speaker.java': '''package zoo; + +public interface Speaker { +} +''', + 'Dog.java': '''package zoo; + +public class Dog extends Animal implements Speaker { + private int tail = 1; + + public String bark() { + return "Woof"; + } + + public int wag() { + return tail * 2; + } +} +''', + 'Cat.java': '''package zoo; + +public class Cat extends Animal { + public String purr() { + return describe() + " purr"; + } +} +''', + 'Lonely.java': '''package zoo; + +public class Lonely { + public void alone() { + } +} +''', + 'Use.java': '''package zoo; + +public class Use { + String run(Dog dog, Animal animal) { + return dog.bark() + dog.wag() + animal.describe(); + } +} +''', +} +for name, text in sources.items(): + (project / 'src/zoo' / name).write_text(text) +uri = lambda name: (project / 'src/zoo' / name).as_uri() + +log = (work / 'stderr.log').open('w') +proc = subprocess.Popen(['java', '-Declipse.application=org.eclipse.jdt.ls.core.id1', '-Dosgi.bundles.defaultStartLevel=4', '-Declipse.product=org.eclipse.jdt.ls.core.product', '-Xmx1G', '--add-modules=ALL-SYSTEM', '--add-opens', 'java.base/java.util=ALL-UNNAMED', '--add-opens', 'java.base/java.lang=ALL-UNNAMED', '-jar', str(next((home / 'plugins').glob('org.eclipse.equinox.launcher_*.jar'))), '-configuration', str(config), '-data', str(work / 'metadata')], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log) +messages = queue.Queue() +lock = threading.Lock() + + +def send(message): + data = json.dumps(message).encode() + with lock: + proc.stdin.write(f'Content-Length: {len(data)}\r\n\r\n'.encode() + data) + proc.stdin.flush() + + +def read(): + while True: + headers = {} + while True: + line = proc.stdout.readline() + if not line: + return + if line == b'\r\n': + break + k, v = line.decode().split(':', 1) + headers[k.lower()] = v.strip() + message = json.loads(proc.stdout.read(int(headers['content-length']))) + if 'method' in message and 'id' in message: + send({'jsonrpc': '2.0', 'id': message['id'], 'result': [] if message['method'] == 'workspace/configuration' else None}) + else: + messages.put(message) + + +threading.Thread(target=read, daemon=True).start() +seq = 0 + + +def request(method, params): + global seq + seq += 1 + send({'jsonrpc': '2.0', 'id': seq, 'method': method, 'params': params}) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + msg = messages.get(timeout=max(1, deadline - time.monotonic())) + if msg.get('id') == seq: + if 'error' in msg: + raise RuntimeError(msg['error']) + return msg.get('result') + raise TimeoutError(method) + + +def position(name, needle): + lines = sources[name].splitlines() + line = next(i for i, t in enumerate(lines) if needle in t) + return line, lines[line].index(needle) + + +def command(name, *args): + return request('workspace/executeCommand', {'command': name, 'arguments': list(args)}) + + +def members(file, needle, kind): + line, col = position(file, needle) + return command('openidea.java.hierarchy.members', uri(file), line, col, kind) + + +def refactor(file, needle, kind, target, choices): + line, col = position(file, needle) + return command('openidea.java.hierarchy.refactor', uri(file), line, col, kind, target, choices) + + +def offset(text, pos): + lines = text.split('\n') + return sum(len(l) + 1 for l in lines[:pos['line']]) + pos['character'] + + +def apply(edit): + """The project after `edit`, compiled; returns the edited sources.""" + per_file = {} + for u, edits in (edit.get('changes') or {}).items(): + per_file.setdefault(u, []).extend(edits) + for change in edit.get('documentChanges') or []: + assert 'kind' not in change, change + per_file.setdefault(change['textDocument']['uri'], []).extend(change['edits']) + out = dict(sources) + for u, edits in per_file.items(): + name = pathlib.Path(urllib.parse.unquote(urllib.parse.urlparse(u).path)).name + text = out[name] + for e in sorted(edits, key=lambda e: offset(text, e['range']['start']), reverse=True): + text = text[:offset(text, e['range']['start'])] + e['newText'] + text[offset(text, e['range']['end']):] + out[name] = text + copy = work / f'compile{seq}' + (copy / 'zoo').mkdir(parents=True) + for name, text in out.items(): + (copy / 'zoo' / name).write_text(text) + javac = subprocess.run(['javac', '-d', str(copy / 'bin'), *map(str, (copy / 'zoo').glob('*.java'))], capture_output=True, text=True) + assert javac.returncode == 0, (javac.stderr, out) + return out + + +def body(text, name): + return [l.strip() for l in text.splitlines() if name in l] + + +try: + init = request('initialize', {'processId': None, 'rootUri': project.as_uri(), 'workspaceFolders': [{'uri': project.as_uri(), 'name': 'zoo'}], 'capabilities': {'workspace': {'workspaceEdit': {'documentChanges': True}}}, 'initializationOptions': {'bundles': [str(bundle)], 'settings': {'java': {'import': {'gradle': {'enabled': False}, 'maven': {'enabled': False}}}}}}) + commands = init['capabilities']['executeCommandProvider']['commands'] + assert 'openidea.java.hierarchy.members' in commands and 'openidea.java.hierarchy.refactor' in commands, commands + send({'jsonrpc': '2.0', 'method': 'initialized', 'params': {}}) + for name, text in sources.items(): + send({'jsonrpc': '2.0', 'method': 'textDocument/didOpen', 'params': {'textDocument': {'uri': uri(name), 'languageId': 'java', 'version': 1, 'text': text}}}) + request('java/buildWorkspace', True) + evidence = {} + + # Members of Dog, with bark() preselected; Animal and Speaker are destinations. + info = members('Dog.java', 'bark', 'pullUp') + evidence['pullUpMembers'] = info + assert info['className'] == 'Dog', info + assert {t['name'] for t in info['targets']} == {'Animal', 'Speaker'}, info['targets'] + rows = {m['label'].split('(')[0].split(':')[0].strip(): m for m in info['members']} + assert rows['bark']['selected'] and not rows['wag']['selected'] and not rows['tail']['selected'], rows + assert rows['bark']['abstractable'] and rows['bark']['kind'] == 'method' and rows['tail']['kind'] == 'field', rows + animal = next(t for t in info['targets'] if t['name'] == 'Animal') + speaker = next(t for t in info['targets'] if t['name'] == 'Speaker') + + # Caret on the class name: nothing preselected. + on_class = members('Dog.java', 'class Dog', 'pullUp') + assert on_class['className'] == 'Dog' and not any(m['selected'] for m in on_class['members']), on_class + + # Pull bark() up to Animal: moved, call sites unchanged, compiles. + result = refactor('Dog.java', 'bark', 'pullUp', animal['id'], [{'id': rows['bark']['id'], 'abstract': False}]) + evidence['pullUp'] = result + assert not [p for p in result['problems'] if p['severity'] in ('error', 'fatal')], result['problems'] + out = apply(result['edit']) + assert 'bark()' in out['Animal.java'] and 'bark()' not in out['Dog.java'], out + assert out['Use.java'] == sources['Use.java'] + + # Make abstract: declared in Animal, kept (and overridden) in Dog. + result = refactor('Dog.java', 'bark', 'pullUp', animal['id'], [{'id': rows['bark']['id'], 'abstract': True}]) + evidence['pullUpAbstract'] = result + out = apply(result['edit']) + assert any('abstract' in l for l in body(out['Animal.java'], 'bark()')), out['Animal.java'] + assert 'Woof' in out['Dog.java'] and '@Override' in out['Dog.java'], out['Dog.java'] + + # To an interface, a method is always declared, never moved. + result = refactor('Dog.java', 'bark', 'pullUp', speaker['id'], [{'id': rows['bark']['id'], 'abstract': False}]) + evidence['pullUpInterface'] = result + out = apply(result['edit']) + assert body(out['Speaker.java'], 'bark()') == ['String bark();'] or body(out['Speaker.java'], 'bark()') == ['public String bark();'], out['Speaker.java'] + assert 'Woof' in out['Dog.java'] and '@Override' in out['Dog.java'], out['Dog.java'] + + # wag() uses Dog's private field: reported before anything changes. + result = refactor('Dog.java', 'wag', 'pullUp', animal['id'], [{'id': rows['wag']['id'], 'abstract': False}]) + evidence['pullUpConflict'] = result + assert any(p['severity'] in ('error', 'fatal') and 'tail' in p['message'] for p in result['problems']), result + + # No superclass in the sources. + refused = members('Lonely.java', 'alone', 'pullUp') + evidence['pullUpRefused'] = refused + assert 'refused' in refused, refused + + # Push describe() down: copied into Dog and Cat, removed from Animal. + info = members('Animal.java', 'describe', 'pushDown') + evidence['pushDownMembers'] = info + assert info['className'] == 'Animal' and {t['name'] for t in info['targets']} == {'Dog', 'Cat'}, info + rows = {m['label'].split('(')[0].split(':')[0].strip(): m for m in info['members']} + assert rows['describe']['selected'] and rows['describe']['abstractable'] and not rows['name']['abstractable'], rows + result = refactor('Animal.java', 'describe', 'pushDown', None, [{'id': rows['describe']['id'], 'abstract': True}]) + evidence['pushDownAbstract'] = result + assert not [p for p in result['problems'] if p['severity'] in ('error', 'fatal')], result['problems'] + out = apply(result['edit']) + assert any('abstract' in l for l in body(out['Animal.java'], 'describe()')), out['Animal.java'] + assert 'I am' in out['Dog.java'] and 'I am' in out['Cat.java'], out + + # Without keeping a declaration, Use's animal.describe() breaks: JDT reports it. + result = refactor('Animal.java', 'describe', 'pushDown', None, [{'id': rows['describe']['id'], 'abstract': False}]) + evidence['pushDownConflict'] = result + assert any(p['severity'] in ('error', 'fatal') for p in result['problems']), result + + (work / 'evidence.json').write_text(json.dumps(evidence, indent=2)) + print('hierarchy: all cases passed') +finally: + proc.terminate() + proc.wait(timeout=20) + print('Evidence:', work, file=sys.stderr) diff --git a/plugins/java-usages/tests/read-write.py b/java/java-usages/tests/read-write.py similarity index 100% rename from plugins/java-usages/tests/read-write.py rename to java/java-usages/tests/read-write.py diff --git a/plugins/java-usages/tests/verify.py b/java/java-usages/tests/verify.py similarity index 100% rename from plugins/java-usages/tests/verify.py rename to java/java-usages/tests/verify.py diff --git a/plugins/kotlin-bridge/MANIFEST.MF b/java/kotlin-bridge/MANIFEST.MF similarity index 100% rename from plugins/kotlin-bridge/MANIFEST.MF rename to java/kotlin-bridge/MANIFEST.MF diff --git a/plugins/kotlin-bridge/README.md b/java/kotlin-bridge/README.md similarity index 94% rename from plugins/kotlin-bridge/README.md rename to java/kotlin-bridge/README.md index a1eb4bf2..afc2ee41 100644 --- a/plugins/kotlin-bridge/README.md +++ b/java/kotlin-bridge/README.md @@ -47,8 +47,8 @@ for the JDT command and Java 8 target for the Maven participant. Use Java 25 to run the build compiler. Archive order and timestamps are deterministic. ```sh -python3 plugins/kotlin-bridge/build.py /path/to/jdtls /path/to/maven -shasum -a 256 plugins/kotlin-bridge/openidea-kotlin-bridge.jar +python3 java/kotlin-bridge/build.py /path/to/jdtls /path/to/maven +shasum -a 256 java/kotlin-bridge/openidea-kotlin-bridge.jar ``` Committed jar SHA-256: @@ -61,7 +61,7 @@ data directory, and includes it in JDT LS initialization bundles. ```sh # `java` on PATH must satisfy JDT LS; JAVA_HOME selects Maven's project JDK. -JAVA_HOME=/path/to/project/jdk python3 plugins/kotlin-bridge/tests/verify.py /path/to/jdtls +JAVA_HOME=/path/to/project/jdk python3 java/kotlin-bridge/tests/verify.py /path/to/jdtls ``` Requires Maven on PATH and access to Kotlin 2.0.10 and the fixture's Maven diff --git a/plugins/kotlin-bridge/build-dependencies.sha256 b/java/kotlin-bridge/build-dependencies.sha256 similarity index 100% rename from plugins/kotlin-bridge/build-dependencies.sha256 rename to java/kotlin-bridge/build-dependencies.sha256 diff --git a/plugins/kotlin-bridge/build.py b/java/kotlin-bridge/build.py similarity index 100% rename from plugins/kotlin-bridge/build.py rename to java/kotlin-bridge/build.py diff --git a/plugins/kotlin-bridge/components.xml b/java/kotlin-bridge/components.xml similarity index 100% rename from plugins/kotlin-bridge/components.xml rename to java/kotlin-bridge/components.xml diff --git a/plugins/kotlin-bridge/maven/ai/openidea/kotlin/IsolatedOutput.java b/java/kotlin-bridge/maven/ai/openidea/kotlin/IsolatedOutput.java similarity index 100% rename from plugins/kotlin-bridge/maven/ai/openidea/kotlin/IsolatedOutput.java rename to java/kotlin-bridge/maven/ai/openidea/kotlin/IsolatedOutput.java diff --git a/plugins/kotlin-bridge/openidea-kotlin-bridge.jar b/java/kotlin-bridge/openidea-kotlin-bridge.jar similarity index 100% rename from plugins/kotlin-bridge/openidea-kotlin-bridge.jar rename to java/kotlin-bridge/openidea-kotlin-bridge.jar diff --git a/plugins/kotlin-bridge/plugin.xml b/java/kotlin-bridge/plugin.xml similarity index 100% rename from plugins/kotlin-bridge/plugin.xml rename to java/kotlin-bridge/plugin.xml diff --git a/plugins/kotlin-bridge/src/ai/openidea/kotlin/KotlinLibraries.java b/java/kotlin-bridge/src/ai/openidea/kotlin/KotlinLibraries.java similarity index 100% rename from plugins/kotlin-bridge/src/ai/openidea/kotlin/KotlinLibraries.java rename to java/kotlin-bridge/src/ai/openidea/kotlin/KotlinLibraries.java diff --git a/plugins/kotlin-bridge/tests/verify.py b/java/kotlin-bridge/tests/verify.py similarity index 100% rename from plugins/kotlin-bridge/tests/verify.py rename to java/kotlin-bridge/tests/verify.py diff --git a/plugins/lombok/LICENSE b/java/lombok/LICENSE similarity index 100% rename from plugins/lombok/LICENSE rename to java/lombok/LICENSE diff --git a/plugins/lombok/README.md b/java/lombok/README.md similarity index 97% rename from plugins/lombok/README.md rename to java/lombok/README.md index 33b513f5..4959509c 100644 --- a/plugins/lombok/README.md +++ b/java/lombok/README.md @@ -52,7 +52,7 @@ git fetch https://github.com/Rawi01/lombok.git eclipse-constructor-call-statemen git cherry-pick 2d535482a913de46c27d06c5b0b294918ede3f9c 3098f4dd78b5349c2f382e7310e64db810dd380b git cherry-pick 7832faa49b35c839088866daf865384c82cd555f 3f3a80b857a969839ab1baba12767aec4965ac32 ant dist -cp dist/lombok.jar /path/to/openidea/plugins/lombok/lombok-1.18.48-openidea.2.jar +cp dist/lombok.jar /path/to/openidea/java/lombok/lombok-1.18.48-openidea.2.jar ``` The changelog context conflicts because the first commit adds a release-note line under the @@ -96,7 +96,7 @@ On macOS arm64 with JDT LS 1.63.0 / Equinox 3.25.0, server OpenJDK 25 and projec programs ran with identical output, and `@NonNull` checks run after the constructor call again. - Real JDT LS `java/buildWorkspace(true)` of the same sources with the agent: `openidea.1` reported Failed with three internal errors and wrote no classes; `openidea.2` Succeeded with all classes. -- `plugins/java-usages/tests/verify.py` and `read-write.py` passed with `openidea.2`. +- `java/java-usages/tests/verify.py` and `read-write.py` passed with `openidea.2`. Windows and Linux runtime validation has not been performed. Rust tests cover source selection for both supported architectures on all three OS variants, offline provisioning, corruption diff --git a/plugins/lombok/SHA256SUMS b/java/lombok/SHA256SUMS similarity index 100% rename from plugins/lombok/SHA256SUMS rename to java/lombok/SHA256SUMS diff --git a/plugins/lombok/lombok-1.18.48-openidea.2.jar b/java/lombok/lombok-1.18.48-openidea.2.jar similarity index 100% rename from plugins/lombok/lombok-1.18.48-openidea.2.jar rename to java/lombok/lombok-1.18.48-openidea.2.jar diff --git a/knip.ts b/knip.ts index 600d5224..572c234a 100644 --- a/knip.ts +++ b/knip.ts @@ -9,6 +9,7 @@ const config: KnipConfig = { "scripts/**/*.ts", "perf/**/*.ts", "e2e/**/*.ts", + "smoke/**/*.ts", ], // Plugins' web pages are projects of their own (own package.json, checked by plugins/build.sh and CI). ignore: ["plugins/**/web-src/**"], diff --git a/languages/c.toml b/languages/c.toml index 06da28f9..436454ed 100644 --- a/languages/c.toml +++ b/languages/c.toml @@ -6,6 +6,14 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = true } ui-mode = "C" tree-sitter = "c" +scratch-template = ''' +#include + +int main(void) { + $END$ + return 0; +} +''' [[language-servers]] name = "clangd" diff --git a/languages/cpp.toml b/languages/cpp.toml index f9e4589c..16802dc1 100644 --- a/languages/cpp.toml +++ b/languages/cpp.toml @@ -6,6 +6,14 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = true } ui-mode = "C++" tree-sitter = "cpp" +scratch-template = ''' +#include + +int main() { + $END$ + return 0; +} +''' [[language-servers]] name = "clangd" diff --git a/languages/csharp.toml b/languages/csharp.toml index fc677e31..ef8d6bd4 100644 --- a/languages/csharp.toml +++ b/languages/csharp.toml @@ -10,7 +10,7 @@ tree-sitter = "csharp" [[language-servers]] name = "csharp-ls" command = "csharp-ls" -root-markers = ["*.sln", "*.slnx", "*.csproj", "global.json", "Directory.Build.props"] +root-markers = ["*.sln", "*.slnx", "*.csproj", "global.json", "Directory.Build.props", "ProjectSettings/ProjectVersion.txt"] install-hint = "dotnet tool install --global csharp-ls" [debugger] diff --git a/languages/go.toml b/languages/go.toml index 6d820dd5..30dbabb6 100644 --- a/languages/go.toml +++ b/languages/go.toml @@ -6,6 +6,13 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = false } ui-mode = "Go" tree-sitter = "go" +scratch-template = ''' +package main + +func main() { + $END$ +} +''' [[language-servers]] name = "gopls" diff --git a/languages/graphql.toml b/languages/graphql.toml index a26bb6ac..7e1c1f43 100644 --- a/languages/graphql.toml +++ b/languages/graphql.toml @@ -10,5 +10,5 @@ tree-sitter = "graphql" name = "graphql-lsp" command = "graphql-lsp" args = ["server", "-m", "stream"] -root-markers = [".graphqlrc", ".graphqlrc.yml", ".graphqlrc.yaml", ".graphqlrc.json", "graphql.config.js", "graphql.config.ts", "graphql.config.yml"] +root-markers = [".graphqlrc", ".graphqlrc.yml", ".graphqlrc.yaml", ".graphqlrc.json", "graphql.config.js", "graphql.config.ts", "graphql.config.yml", "graphql.config.yaml", "graphql.config.json"] install-hint = "npm i -g graphql-language-service-cli" diff --git a/languages/java.toml b/languages/java.toml index 7e7837b0..af9967e9 100644 --- a/languages/java.toml +++ b/languages/java.toml @@ -6,6 +6,13 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = true } ui-mode = "Java" tree-sitter = "java" +scratch-template = ''' +class Scratch { + public static void main(String[] args) { + $END$ + } +} +''' # Microsoft java-debug runs inside jdtls as a bundle (`initializationOptions.bundles`, # added when installed); sessions start with `vscode.java.startDebugSession` and the diff --git a/languages/javascript.toml b/languages/javascript.toml index 5f19df86..70954b50 100644 --- a/languages/javascript.toml +++ b/languages/javascript.toml @@ -17,6 +17,21 @@ args = ["--stdio"] root-markers = ["jsconfig.json", "tsconfig.json", "package.json"] install-hint = "npm i -g @vtsls/language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.typescript.preferences] +useAliasesForRenames = false + +[language-servers.initialization-options.javascript.preferences] +useAliasesForRenames = false + +# Moving a file: the IDE asks for the import updates up front (preview, undo); the server +# doesn't prompt after the move. +[language-servers.initialization-options.typescript.updateImportsOnFileMove] +enabled = "never" + +[language-servers.initialization-options.javascript.updateImportsOnFileMove] +enabled = "never" + [[language-servers]] name = "typescript-language-server" tsserver-project = true @@ -27,6 +42,10 @@ args = ["--stdio"] root-markers = ["jsconfig.json", "tsconfig.json", "package.json"] install-hint = "npm i -g typescript@6 typescript-language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.preferences] +providePrefixAndSuffixTextForRename = false + [[language-servers]] name = "oxlint" command = "oxlint" diff --git a/languages/javascriptreact.toml b/languages/javascriptreact.toml index be5b5ab9..cdab8c31 100644 --- a/languages/javascriptreact.toml +++ b/languages/javascriptreact.toml @@ -16,6 +16,21 @@ args = ["--stdio"] root-markers = ["jsconfig.json", "tsconfig.json", "package.json"] install-hint = "npm i -g @vtsls/language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.typescript.preferences] +useAliasesForRenames = false + +[language-servers.initialization-options.javascript.preferences] +useAliasesForRenames = false + +# Moving a file: the IDE asks for the import updates up front (preview, undo); the server +# doesn't prompt after the move. +[language-servers.initialization-options.typescript.updateImportsOnFileMove] +enabled = "never" + +[language-servers.initialization-options.javascript.updateImportsOnFileMove] +enabled = "never" + [[language-servers]] name = "typescript-language-server" tsserver-project = true @@ -26,6 +41,10 @@ args = ["--stdio"] root-markers = ["jsconfig.json", "tsconfig.json", "package.json"] install-hint = "npm i -g typescript@6 typescript-language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.preferences] +providePrefixAndSuffixTextForRename = false + [[language-servers]] name = "oxlint" command = "oxlint" diff --git a/languages/kotlin.toml b/languages/kotlin.toml index 212bb950..097eafcd 100644 --- a/languages/kotlin.toml +++ b/languages/kotlin.toml @@ -6,6 +6,11 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = true } ui-mode = "Kotlin" tree-sitter = "kotlin" +scratch-template = ''' +fun main() { + $END$ +} +''' [[language-servers]] name = "kotlin-lsp" diff --git a/languages/php.toml b/languages/php.toml index 4b14b8d8..b0fc0f4e 100644 --- a/languages/php.toml +++ b/languages/php.toml @@ -14,6 +14,8 @@ command = "intelephense" args = ["--stdio"] root-markers = ["composer.json", ".phpactor.json", "index.php"] install-hint = "npm i -g intelephense" +# Intelephense applies the PHP version only to documents it parses after reading it. +resync-on-configuration = true [[language-servers]] name = "phpactor" diff --git a/languages/rust.toml b/languages/rust.toml index 61bb1dc1..fcd712e3 100644 --- a/languages/rust.toml +++ b/languages/rust.toml @@ -6,6 +6,11 @@ block-comment = ["/*", "*/"] indent = { tab-size = 4, insert-spaces = true } ui-mode = "Rust" tree-sitter = "rust" +scratch-template = ''' +fn main() { + $END$ +} +''' [[language-servers]] name = "rust-analyzer" diff --git a/languages/typescript.toml b/languages/typescript.toml index 0dedc17a..951c0922 100644 --- a/languages/typescript.toml +++ b/languages/typescript.toml @@ -16,6 +16,21 @@ args = ["--stdio"] root-markers = ["tsconfig.json", "jsconfig.json", "package.json"] install-hint = "npm i -g @vtsls/language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.typescript.preferences] +useAliasesForRenames = false + +[language-servers.initialization-options.javascript.preferences] +useAliasesForRenames = false + +# Moving a file: the IDE asks for the import updates up front (preview, undo); the server +# doesn't prompt after the move. +[language-servers.initialization-options.typescript.updateImportsOnFileMove] +enabled = "never" + +[language-servers.initialization-options.javascript.updateImportsOnFileMove] +enabled = "never" + [[language-servers]] name = "typescript-language-server" tsserver-project = true @@ -26,6 +41,10 @@ args = ["--stdio"] root-markers = ["tsconfig.json", "jsconfig.json", "package.json"] install-hint = "npm i -g typescript@6 typescript-language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.preferences] +providePrefixAndSuffixTextForRename = false + # Angular projects only: templates inline in components (`template: ...`), # and their usages and renames in templates. Last: TypeScript's own hover, # definition and symbols come first. diff --git a/languages/typescriptreact.toml b/languages/typescriptreact.toml index d7ecf0b8..3d4b1377 100644 --- a/languages/typescriptreact.toml +++ b/languages/typescriptreact.toml @@ -16,6 +16,21 @@ args = ["--stdio"] root-markers = ["tsconfig.json", "jsconfig.json", "package.json"] install-hint = "npm i -g @vtsls/language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.typescript.preferences] +useAliasesForRenames = false + +[language-servers.initialization-options.javascript.preferences] +useAliasesForRenames = false + +# Moving a file: the IDE asks for the import updates up front (preview, undo); the server +# doesn't prompt after the move. +[language-servers.initialization-options.typescript.updateImportsOnFileMove] +enabled = "never" + +[language-servers.initialization-options.javascript.updateImportsOnFileMove] +enabled = "never" + [[language-servers]] name = "typescript-language-server" tsserver-project = true @@ -26,6 +41,10 @@ args = ["--stdio"] root-markers = ["tsconfig.json", "jsconfig.json", "package.json"] install-hint = "npm i -g typescript@6 typescript-language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.preferences] +providePrefixAndSuffixTextForRename = false + [[language-servers]] name = "oxlint" command = "oxlint" diff --git a/languages/vue.toml b/languages/vue.toml index 247f0930..e47d985a 100644 --- a/languages/vue.toml +++ b/languages/vue.toml @@ -23,6 +23,21 @@ args = ["--stdio"] root-markers = ["tsconfig.json", "jsconfig.json", "package.json"] install-hint = "npm i -g @vtsls/language-server" +# Rename renames the symbol everywhere: no `import { a as b }` alias in the file renamed from. +[language-servers.initialization-options.typescript.preferences] +useAliasesForRenames = false + +[language-servers.initialization-options.javascript.preferences] +useAliasesForRenames = false + +# Moving a file: the IDE asks for the import updates up front (preview, undo); the server +# doesn't prompt after the move. +[language-servers.initialization-options.typescript.updateImportsOnFileMove] +enabled = "never" + +[language-servers.initialization-options.javascript.updateImportsOnFileMove] +enabled = "never" + [[language-servers]] name = "oxlint" command = "oxlint" diff --git a/package.json b/package.json index 76100246..5ba6e561 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "openidea", "private": true, "description": "An open-source IDE inspired by IntelliJ IDEA", - "version": "0.1.35", + "version": "0.1.36", "license": "Apache-2.0", "type": "module", "scripts": { @@ -12,11 +12,12 @@ "tauri": "node scripts/tauri.ts", "test": "vitest run", "test:watch": "vitest", + "smoke": "vitest run --config smoke/vitest.config.ts", "lint": "oxlint", "format": "oxfmt", "format:check": "oxfmt --check", "knip": "knip", - "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit", + "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit && tsrs -p smoke --noEmit", "gen:bindings": "cargo test -p openidea --test export_bindings", "gen:windows-icons": "node scripts/gen-windows-icons.ts", "gen:angular-grammar": "lezer-generator src/editor/core/cm/angular/angular.grammar --typeScript -o src/editor/core/cm/angular/parser.ts", diff --git a/perf/budgets.json b/perf/budgets.json index f831d10c..783e64ec 100644 --- a/perf/budgets.json +++ b/perf/budgets.json @@ -62,6 +62,8 @@ "text/open_1mb": 2000, "text/apply_keystroke_1mb": 20, "text/apply_100_changes_1mb": 1000, + "text/diff_two_edits_1mb": 10000, + "text/diff_every_line_1mb": 100000, "text/offset_to_pos_round_trip": 5, "text/pos_to_byte": 5, "text/text_for_save_crlf_1mb": 1000, @@ -76,6 +78,9 @@ "index_100k/find_files/ordcon": 10000, "index_100k/find_files/src/user": 10000, "index_100k/find_files/PaymentService": 10000, + "index_100k/find_dirs/util": 5000, + "index_100k/find_dirs/src/ui": 5000, + "index_100k/find_dirs/a": 5000, "index_100k/find_symbols/OCo": 15000, "index_100k/find_symbols/q": 15000, "index_100k/find_symbols/invoiceRepo": 15000, diff --git a/plugins/README.md b/plugins/README.md index 4e994f5e..8a74a1d1 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -44,7 +44,8 @@ The plugin API is defined in [`wit/`](../wit/) (`openidea:plugin@1.0.0`, world The `openidea-plugin` tool scaffolds, builds, checks and links plugins: ```sh -cargo install --path crates/ide-host --bin openidea-plugin # or: cargo run -p ide-host --bin openidea-plugin -- … +cargo install --locked --git https://github.com/l0ng-ai/openidea ide-host --bin openidea-plugin +# in a checkout of this repository: cargo install --path crates/ide-host --bin openidea-plugin openidea-plugin new my-plugin # Rust command plugin (SDK) openidea-plugin new my-theme --template theme # also: icon-theme, language @@ -54,6 +55,17 @@ openidea-plugin build # → dist/plugin.wasm, t openidea-plugin link # load this folder in the IDE ``` +Each release also attaches the tool for Linux x64 +(`openidea-plugin-linux-x86_64.gz`). A plugin is a project of its own: keep it +in its own repository, and list it in the community marketplace when it is +ready (§8.1). + +A Rust plugin made by `new` depends on the SDK of the tool's release, from +this repository's tag (`openidea-plugin = { git = "…", tag = "vX.Y.Z" }`); +change the tag to build against a later API. Inside a checkout of this +repository, or with `--sdk `, it depends on +that SDK by path instead. + `build` runs `cargo build --release --target wasm32-wasip2` (installing the target through rustup if needed), copies the component to the manifest's `wasm` path and runs `check`. `check` validates the package exactly as @@ -79,6 +91,7 @@ authors = ["You "] homepage = "https://example.com" # optional license = "MIT" # optional change-notes = "1.2.0 — …" # shown under What's New +categories = ["tools"] # marketplace categories, see below engines.openidea = "^1.0" # the plugin API this package needs wasm = "dist/plugin.wasm" # omit for data-only plugins activation = ["on-language:markdown"] # optional, see below @@ -94,6 +107,13 @@ category = "Word Tools" # group in Search Everywhere / Keymap when = "editorFocus" # optional: same expressions as built-in actions ``` +`categories` (up to 3) file the plugin in the marketplace: `languages`, +`themes`, `icon-themes`, `linters` (linters and formatters), `vcs`, `ai`, +`frameworks`, `tools`, `other`. Without them, a plugin contributing themes, +icon themes or languages is filed under those, any other under `other`. +Categories an IDE doesn't know, and any after the first three it knows, are +ignored (`check` warns about both). + All paths are relative to the package directory and may not leave it. A manifest is validated as a whole: if one contribution is invalid, nothing is installed or loaded. Contribution types the IDE does not know (a newer API) are @@ -105,23 +125,23 @@ work), but 0.1 code no longer runs: rebuild it against the 1.0 SDK. ### Contribution points -| Point | Declaration | Code needed | -| --------------- | -------------------------------------------------------------------------------------------------- | ------------- | -| `commands` | `id`, `title`, `category?`, `when?` | to run them | -| `themes` | `path` (theme JSON), `editor-scheme?` | no | -| `iconThemes` | `id`, `label`, `path` (icon theme JSON, see §5) | no | -| `languages` | `path` (`languages/.toml` format, see §6) | no | -| `runDetectors` | `id`, `name`, `files`, `command`, `args?`, `test?` (§6.2) | no | -| `providers` | `kinds`, `languages` (editor providers, see _Providers_) | yes | -| `secrets` | `name`, `label`, `description?` (see _Network_) | to use them | -| `menus` | `menu` (`editor.context`, `projectView.context`, `editorTab.context`, `tools`), `command`, `when?` | no | -| `keymaps` | `command`, `key`, `mac?` (`Ctrl+Alt+C`; a chord: `Meta+K Meta+C`) | no | -| `statusBar` | `id`, `text`, `tooltip?`, `command?`, `align?`, `priority?` | no | -| `settings` | `schema` (JSON file), `title?` — one table, see _Settings_ | no | -| `views` | `id`, `title`, `icon?`, `location` (`left`, `right`, `bottom`), `web?` (see _Web views_) | without `web` | -| `customEditors` | `id`, `title`, `patterns` (file names), `priority` (`default`, `option`), `layout?`, `web?` | without `web` | -| `fileIcons` | `id`, `patterns` (file names), `icon` (SVG), `iconDark?` — whatever icon theme is in use | no | -| `fileTemplates` | `id`, `name`, `extension`, `path` (the new file's content; template variables work) — New menu | no | +| Point | Declaration | Code needed | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| `commands` | `id`, `title`, `category?`, `when?` | to run them | +| `themes` | `path` (theme JSON), `editor-scheme?` | no | +| `iconThemes` | `id`, `label`, `path` (icon theme JSON, see §5) | no | +| `languages` | `path` (`languages/.toml` format, see §6) | no | +| `runDetectors` | `id`, `name`, `files`, `command`, `args?`, `test?` (§6.2) | no | +| `providers` | `kinds`, `languages` (editor providers, see _Providers_) | yes | +| `secrets` | `name`, `label`, `description?` (see _Network_) | to use them | +| `menus` | `menu` (`editor.context`, `projectView.context`, `editorTab.context`, `tools`), `command`, `when?` | no | +| `keymaps` | `command`, `key`, `mac?` (`Ctrl+Alt+C`; a chord: `Meta+K Meta+C`) | no | +| `statusBar` | `id`, `text`, `tooltip?`, `command?`, `align?`, `priority?` | no | +| `settings` | `schema` (JSON file), `title?` — one table, see _Settings_ | no | +| `views` | `id`, `title`, `icon?` (an icon name, or an SVG of the package), `location` (`left`, `right`, `bottom`), `web?` (see _Web views_) | without `web` | +| `customEditors` | `id`, `title`, `patterns` (file names), `priority` (`default`, `option`), `layout?`, `web?` | without `web` | +| `fileIcons` | `id`, `patterns` (file names), `icon` (SVG), `iconDark?` — whatever icon theme is in use | no | +| `fileTemplates` | `id`, `name`, `extension`, `path` (the new file's content; template variables work) — New menu | no | Declared contributions are live as soon as the plugin is enabled — commands show in Find Action, menus and the status bar, shortcuts work, themes and icon @@ -204,7 +224,7 @@ edition = "2024" crate-type = ["cdylib", "rlib"] [dependencies] -openidea-plugin = { path = "…/sdk/rust/openidea-plugin" } +openidea-plugin = { git = "https://github.com/l0ng-ai/openidea", tag = "vX.Y.Z" } # or { path = "…/sdk/rust/openidea-plugin" } [profile.release] opt-level = "s" @@ -293,6 +313,7 @@ then run `openidea-plugin check`. | `host::refresh_view(view)` | `views.refresh` | The view (or custom editors) render again; call it when the content changed. | | `host::post_view_message(view, text)` | `views.post-message` | To the script of the view's `html` body. | | `host::open_terminal(title, program, args)` | `views.open-terminal` | A terminal tab in the invoking window running the approved `process:` in the project root: output that keeps coming, interactive sessions. | +| `host::open_location(path, line, column)` | `views.open-location` | Opens a file inside the project in an editor of the invoking window, the caret at `line`/`column` (0-based, UTF-16): a view's links to code. Needs `project.read`. | | `host::setting(key)`, `setting_bool`, `setting_string`, `setting_number` | `settings.get` | The user's value (project over global) or the schema's default. | `run-command` receives an optional string argument: `executeAction("plugin..", "arg")` @@ -313,6 +334,11 @@ Events are batched: a running plugin gets them within a fraction of a second, an idle one collects them for 2 s before it is woken, so frequent events (Key Promoter's actions) don't keep it alive. +Kinds are names (`subscribe` takes a string), so later IDEs add kinds without +a new API: an IDE refuses a kind it doesn't know with `invalid`, and a kind +newer than the plugin's SDK arrives as `Event::Other { kind, data }` (JSON). +The SDK's event enums are `#[non_exhaustive]`; match them with a `_` arm. + ### Providers Declare what the plugin provides in the editor; only plugins whose @@ -330,7 +356,14 @@ Then `register_plugin!(MyPlugin, providers)` and implement the document (path, language, version, text). Results join the language servers' (completion lists, ⌥⏎ quick fixes, ⌘B, gutter icons; a code action carries edits for any files, applied as one change). A completion item's -`kind` (`CompletionKind` in the SDK, a string in the WIT) picks its icon. +`kind` (`CompletionKind` in the SDK, a string in the WIT) picks its icon; it +can be a `snippet` (`$1`, `${1:name}`, `$0`), match on `filter_text` instead +of its label, and bring `additional_edits` elsewhere in the document (an +import). A code action can leave `edits` empty and set `data`: its edits are +then computed by `resolve_code_action` when the user picks it, under the +command budget instead of the 50 ms. Problems (`publish_diagnostics`) can name +the tool that found them (`source`) and be `unnecessary` (faded) or +`deprecated` (struck through). Editor providers run under the edit-path budget: 50 ms, after which the result is dropped, and an interruption at 500 ms. Every plugin providing a kind is asked at the same time, so each has the whole budget. @@ -385,6 +418,9 @@ collapse of items, form fields, table cells, inserted and deleted rows, and `html` messages. A tree is a flat list whose items name their `parent`; an `expandable` item gets its children when it is expanded. A table gets only the rows it shows (`viewport`): report `total_rows` and return those. +Bodies, field types and view events added by later APIs travel as `Other { +kind, data }`: an IDE that doesn't know a body says it needs updating, and +leaves out a field type it doesn't know. Showing a view activates the plugin (`on-view:`). `views` become tool windows; a `customEditors` file (`priority = "default"`) opens in the @@ -585,6 +621,84 @@ attributes only, no scripts, event handlers, external references, `url()` or entities. Apply a theme in Settings › Plugins; see [`official/material-icons`](official/material-icons) for a complete one. +Optional fields: + +```json +{ + "iconDefinitions": { "rust": { "light": "icons/rust.svg", "dark": "icons/rust_dark.svg" } }, + "folderOpen": "folder-open", + "patterns": [ + { "pattern": "^.*\\.spec\\.ts$", "icon": "test-ts", "ignoreCase": true }, + { "pattern": "/fixtures/", "icon": "fixture", "path": true, "group": "tests" }, + { "pattern": "^src$", "icon": "folder-src", "target": "folder", "openIcon": "folder-src-open" } + ], + "overrides": { "chevron-right": "arrow-right", "symbol-class": "class" }, + "groups": [{ "id": "tests", "name": "Tests", "enabled": false }], + "choices": [ + { + "id": "arrows", "name": "Arrows", "default": "plain", + "options": [ + { "id": "plain", "name": "Default" }, + { "id": "plus", "name": "Plus-Minus", "overrides": { "chevron-right": "plus" } } + ] + } + ], + "accent": true, + "settings": [ + { + "title": "Icons", + "items": [ + { "option": "hideFiles", "invert": true, "label": "Show file icons", "hint": "In trees, lists and tabs" }, + { "option": "groups", "columns": 2 }, + { "option": "choice", "choice": "arrows" }, + { "option": "iconSize", "label": "Icon size", "default": 16 } + ] + } + ] +} +``` + +- `iconDefinitions` values may be `{ "light", "dark" }`; the dark SVG is used + in dark themes. Icon ids may not end in `_dark`. +- `patterns` are regular expressions checked in order, before `fileNames` + and `fileExtensions`. `target` is `file` (default) or `folder`; with `path` + a pattern matches the whole path (`/` separators) instead of the name; + `ignoreCase` ignores case. Patterns that could backtrack badly (a repeated + group, too many ways to match one name), lookaround, inline flags and + named groups are refused. +- `openIcon` (folder patterns) and `folderOpen` (other folders) are used for + folders holding a file open in the editor when _Hollow folders_ is on. +- `overrides` replace built-in IDE icons by name (`chevron-right`, + `folder-source`, …); symbol icons (`symbol-`, `class`, `method`, …) + are PSI icons. +- `groups` are packs of patterns the user turns on or off; `enabled` is the + default. +- `choices` are variants the user picks one of; the picked option's + `overrides` win over `overrides`. +- `accent: true`: elements with `data-tint="true"` take the colour the user + picks. +- Elements with `data-themed="true"` take the accent colour of Settings › + Appearance, or the theme's custom accent colour. + +- `settings` describe the theme's page in Settings › Plugins › _theme label_; + a theme without them has no page. Sections have a `title` and `items`; an + item names an `option` the IDE implements, with the theme's `label`, + optional `hint` and, where the option needs one, `default`: + + | `option` | Control | Item fields | + | -------------------------------------------------------------------------- | -------------------- | ---------------------------------------------------- | + | `hideFiles`, `hideFolders`, `hideUiIcons`, `hidePsiIcons`, `hollowFolders` | checkbox | `invert`: checked means off | + | `groups` | a checkbox per group | `columns` (1–4), `label?` (group names are shown) | + | `choice` | the choice's options | `choice` (its id), `label?` (default: its name) | + | `accent`, `monochrome` | checkbox and colour | `default` (`#rrggbb`); `accent` needs `accent: true` | + | `saturation` (0–20), `opacity` (10–100), `iconSize`, `rowHeight` (px) | checkbox and number | `default` within the range | + + `iconSize` is 12–32 and `rowHeight` 16–48. Values are kept per theme in + `appearance.fileIconThemeOptions` and apply while the theme is selected. + +The first icon theme of a plugin is selected when the plugin is installed or +enabled while no icon theme is. + ## 6. Languages `languages/.toml` files use the IDE's own language format; see any file in @@ -650,7 +764,9 @@ granted, and starts the approved program. - **Install:** Settings › Plugins › _Install Plugin from Disk…_ (a package folder) or _Install Plugin from Package File…_ (a `.oiplugin`, a `.zip` with the package at its root or in its only folder, or a folder's `plugin.toml`; - a zip without a `plugin.toml` there, such as an IntelliJ plugin, is refused). The install + a zip without a `plugin.toml` there, such as an IntelliJ plugin, is refused; IntelliJ + theme plugins are imported instead: _Import IntelliJ Theme from File…_ and _Import Theme + from IntelliJ IDE…_, see [`docs/plugins/themes.md`](../docs/plugins/themes.md)). The install page lists every capability and whether the package is signed by a publisher the marketplace trusts; nothing is copied until you accept. - **Versions:** installed plugins live in `/plugins///` @@ -659,13 +775,30 @@ granted, and starts the approved program. the version it replaced automatically (you're told why); _Roll Back to …_ in the plugin's details does it by hand. Data (`storage`) is per plugin, not per version. +- **Updates:** when the daily marketplace check finds newer versions of + installed plugins, the IDE says so once, with _Update All_ (also in the gear + menu and as the _Update All Plugins_ action). An update that asks for + nothing new installs with the grants the installed version has; the install + page is shown only for new capabilities, an unsigned package or another + signer (see _Distribution_). - **Linked plugins:** `openidea-plugin link ` (§1). Listed with a _linked_ tag; capabilities are granted per folder. - **Project plugins:** put (or symlink) the package in `/.openidea/plugins//`. They load only in trusted projects and only after you approve their capabilities. +- **Where:** installing, removing, enabling, granting and reloading plugins + work on the welcome screen and in trusted projects. A project opened in + restricted mode can browse the marketplace and show a package's install + page, but not change which plugin code runs. - Commands appear in Search Everywhere › Actions under their category, and can be bound in Settings › Keymap like any other action. +- **Troubleshooting:** `openidea-plugin check` says why a package would be + refused. Settings › Plugins shows each plugin's state, why it failed if it + did, and its call stats (§3, _Sandbox and limits_). `host::log` lines go to + the IDE log (Help › _Show Log in Finder_ / _Explorer_ / _Files_) tagged with + the plugin id; debug and trace lines are written when the IDE is started + with `RUST_LOG=info,ide_plugin=trace`. `eprintln!` output is logged as + well, and a panic's message is shown with the crash. ## 8. Distribution @@ -678,6 +811,9 @@ files). openidea-plugin keygen acme.key # once; keep the file secret openidea-plugin pack plugins/acme-tools --key acme.key --publisher acme openidea-plugin index market/ --publisher acme --name "ACME" --key acme.key +openidea-plugin check acme.tools-1.0.0.oiplugin # a package as installing would check it +openidea-plugin merge market/index.json https://example.com/index.json --keep acme +openidea-plugin endorse --key acme.key --publisher acme # when rotating keys ``` A **marketplace** is static: an `index.json` and the packages next to it, on @@ -693,8 +829,68 @@ and the publisher's key, then shows the install page. - **Revocation:** the index can revoke plugin versions (or all versions) and publisher keys. The IDE reads the index daily; revoked plugins are disabled with the reason and the user is notified — also after a restart, before - their code could run. A publisher rotates keys by adding the new one; older - keys stay valid until revoked. + their code could run. +- **Key rotation:** a publisher rotates keys by registering the new one with + `index --key new.key --previous-key old.key`: the old key endorses the new + one, and older keys stay valid until revoked (`endorse` prints the + endorsement for an index someone else writes, such as the registry's). An installed plugin updates + silently only to a package signed with the key it was installed with, or + one that key endorsed (directly or through endorsed keys; endorsements by + revoked keys don't count). Another signer — an unendorsed key, or an + unsigned package replacing a signed one — is shown on the install page and + needs the user's confirmation. +- **Official key:** release builds carry the official publisher's public key + (`OPENIDEA_OFFICIAL_PLUGIN_KEYS`, derived from the release's signing key). + `openidea.*` plugins must be signed with it, or with a key it endorsed, + whatever keys an index lists for `openidea`. +- **Format:** fields are only ever added to the index (an IDE ignores those it + doesn't know), and `format` stays `1` unless older IDEs can't read the + change. An IDE reading a newer format still applies its revocations and + `moved`, lists no plugins, and asks the user to update. +- **Moving:** an index with `"moved": ""` sends the IDE to the index at + that URL (relative or absolute; https stays https), so a marketplace can + change hosts while released IDEs keep the address they were built with. + +**Versions.** Each index entry records the plugin API it needs (`engine`: the +manifest's `engines.openidea`, narrowed to the API its component was built +for; an entry without one is offered to no IDE), and a plugin may be listed in +several versions. Settings › Plugins +offers the newest version this IDE can load; when there is none, it shows the +plugin as incompatible and says which API it needs. Entries also carry the +plugin's categories, which Settings › Plugins › Available filters by. Indexing keeps, of each +plugin, the newest version per API requirement, so users of an older IDE keep +the last version built for them. + +`index` with a key registers the publisher and lists the folder's packages +as its plugins (their earlier versions stay as above; plugins no longer in the +folder go), keeping the other publishers'; `--download-base` makes downloads +absolute, so a version stays where it was first published. Without a key, it +lists every package in the folder, each checked against the publishers the +index already has. `merge` takes the +publishers, plugins and revocations of another index (downloads made absolute) +in place of every publisher but `--keep`; `check` also takes a `.oiplugin`. + +### 8.1 Official and community plugins + +The IDE's default marketplace is `plugins-index.json` of the latest release. +It lists two kinds of plugins: + +- **Official** (`plugins/official/`, ids `openidea.*`): maintained in this + repository, built and signed by its release workflow, and kept working as + the API changes. New official plugins are proposed in an issue first; they + cover general needs or show how to use part of the API. +- **Community** (their authors' ids and keys): developed in their authors' + repositories and listed through the registry, + [l0ng-ai/openidea-plugins](https://github.com/l0ng-ai/openidea-plugins). + An author registers a publisher (an id prefix, its public key and the GitHub + accounts that may change it) and adds each plugin version by pull request; + the registry checks the package (digest, signature, `openidea-plugin check`) + and publishes its index, which `plugins-index.yml` merges into the release's + within the hour. Its README has the rules. + +Pull requests that add a plugin to this repository are for official plugins. +A plugin that needs a new extension point brings the extension point here, as +its own pull request, and the plugin to the registry. ## 9. Testing the host @@ -733,7 +929,10 @@ only grows: Extend with new types and new functions (or interfaces). `types.error` is used everywhere and is no exception; new situations use `failed(string)` or `invalid(string)`. Sets that keep growing (completion, token, symbol and - fold kinds, icons) are strings for this reason. + fold kinds, icons, event kinds, action sources, diagnostic tags) are + strings for this reason, and variants that grow (`event`, view `body`, + `view-event`, `field-value`) have an `other(extension)` case, a kind and + JSON data, that carries later cases to older plugins and IDEs. 2. **Host interfaces (imports) may gain functions and interfaces**: older components don't import them. 3. **Plugin interfaces (exports) never gain functions** once released: the @@ -745,6 +944,11 @@ only grows: minor with a message that says to update the IDE or rebuild with the SDK. 5. **One version number**: the `wit/` package version, `ENGINE_VERSION` and `API_VERSION` agree, and the SDK's `wit/` copy is identical to `wit/`. +6. **1.x calls are synchronous and pass documents whole.** Host calls block + the plugin's instance, and providers get the document's text with each + call. WASI 0.3 async components come with 2.0, behind an adapter in the + host that keeps 1.x plugins running; incremental document sync, if + needed, is a new interface. Checked automatically: diff --git a/plugins/official/ai-completion/dist/plugin.wasm b/plugins/official/ai-completion/dist/plugin.wasm index ee11a42e..4fc5e771 100644 Binary files a/plugins/official/ai-completion/dist/plugin.wasm and b/plugins/official/ai-completion/dist/plugin.wasm differ diff --git a/plugins/official/ai-completion/dist/plugin.wasm.sources b/plugins/official/ai-completion/dist/plugin.wasm.sources index 43ce4cdf..d07cfcec 100644 --- a/plugins/official/ai-completion/dist/plugin.wasm.sources +++ b/plugins/official/ai-completion/dist/plugin.wasm.sources @@ -1 +1 @@ -bf264a97dc918f24ac00f0e9e0d0373b68391a96921b44345b466c2a087c5283 +9ad412cb7b853d40467c67305a1126bb87910d179ecfa1c88e13d4a900367c50 diff --git a/plugins/official/ai-completion/plugin.toml b/plugins/official/ai-completion/plugin.toml index 8cf1c563..bf21db21 100644 --- a/plugins/official/ai-completion/plugin.toml +++ b/plugins/official/ai-completion/plugin.toml @@ -5,6 +5,7 @@ description = "Inline code suggestions (ghost text) from a local model (Ollama) authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: local (Ollama) and OpenAI backends; the key stays in the system keychain." +categories = ["ai"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-command:*"] diff --git a/plugins/official/checkstyle/dist/plugin.wasm b/plugins/official/checkstyle/dist/plugin.wasm index b7508504..e719df8d 100644 Binary files a/plugins/official/checkstyle/dist/plugin.wasm and b/plugins/official/checkstyle/dist/plugin.wasm differ diff --git a/plugins/official/checkstyle/dist/plugin.wasm.sources b/plugins/official/checkstyle/dist/plugin.wasm.sources index 6192fb92..f7e63f8f 100644 --- a/plugins/official/checkstyle/dist/plugin.wasm.sources +++ b/plugins/official/checkstyle/dist/plugin.wasm.sources @@ -1 +1 @@ -6095c7e30658b93d79270e6355d42941a4cfe13eeb8bada07acc7ae462194853 +49a3c6bf2511ba137457397203244bdfca3d6e10f62dbc7fde9473dab07f6457 diff --git a/plugins/official/checkstyle/plugin.toml b/plugins/official/checkstyle/plugin.toml index 444157b6..1732ad30 100644 --- a/plugins/official/checkstyle/plugin.toml +++ b/plugins/official/checkstyle/plugin.toml @@ -5,6 +5,7 @@ description = "Checks Java files with Checkstyle when you save them: problems in authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: check on save (a newer save cancels the running check), Google style by default, quick fixes." +categories = ["linters"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-event:file-saved", "on-command:*"] diff --git a/plugins/official/checkstyle/src/lib.rs b/plugins/official/checkstyle/src/lib.rs index 74f17cc4..aebf998e 100644 --- a/plugins/official/checkstyle/src/lib.rs +++ b/plugins/official/checkstyle/src/lib.rs @@ -97,12 +97,8 @@ fn diagnostic(p: &Problem, text: &str) -> Diagnostic { let line_len = text.split('\n').nth(p.line as usize).map_or(0, |l| l.encode_utf16().count() as u32); let start = p.column.unwrap_or(0).min(line_len); let end = if p.column.is_some() && start < line_len { start + 1 } else { line_len }; - Diagnostic { - range: Range::new(Position::new(p.line, start), Position::new(p.line, end.max(start))), - severity: p.severity, - message: p.message.clone(), - code: Some(p.check.clone()), - } + let range = Range::new(Position::new(p.line, start), Position::new(p.line, end.max(start))); + Diagnostic::new(range, p.severity, p.message.clone()).code(p.check.clone()) } /// The project containing `path`. @@ -188,6 +184,7 @@ impl Providers for Checkstyle { kind: Some("quickfix".into()), preferred: true, edits: vec![(doc.path.clone(), vec![edit])], + data: None, }); } } diff --git a/plugins/official/csv-editor/dist/plugin.wasm b/plugins/official/csv-editor/dist/plugin.wasm index 539a1dce..752aba90 100644 Binary files a/plugins/official/csv-editor/dist/plugin.wasm and b/plugins/official/csv-editor/dist/plugin.wasm differ diff --git a/plugins/official/csv-editor/dist/plugin.wasm.sources b/plugins/official/csv-editor/dist/plugin.wasm.sources index d1b11a17..4f6159a0 100644 --- a/plugins/official/csv-editor/dist/plugin.wasm.sources +++ b/plugins/official/csv-editor/dist/plugin.wasm.sources @@ -1 +1 @@ -b96941af697ce7a6c1c8ef1539d34aa7c23d2d41dd1b2ccffdfda5648f90cf56 +c199dc52e00d3943882d532e7e6095f7b2f8d7d6026cf1ccaa685dd6300f2319 diff --git a/plugins/official/csv-editor/plugin.toml b/plugins/official/csv-editor/plugin.toml index 06d920a5..e42c1d7a 100644 --- a/plugins/official/csv-editor/plugin.toml +++ b/plugins/official/csv-editor/plugin.toml @@ -5,6 +5,7 @@ description = "Edit CSV and TSV files as a table. Edits go to the file's text, s authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: table editor for .csv and .tsv with cell editing, inserting and deleting rows; large files load the visible rows only." +categories = ["tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-view:table"] diff --git a/plugins/official/excalidraw/plugin.toml b/plugins/official/excalidraw/plugin.toml index 70bf7ad0..fdfc495b 100644 --- a/plugins/official/excalidraw/plugin.toml +++ b/plugins/official/excalidraw/plugin.toml @@ -5,6 +5,7 @@ description = "Draw and edit Excalidraw diagrams (.excalidraw). The drawing is t authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: the Excalidraw canvas next to (or instead of) the file's text, kept in step both ways; export to PNG and SVG through the save dialog; a file icon and a New › Excalidraw Drawing template. Based on the Excalidraw editor proposed by Deng Liu (#738)." +categories = ["tools"] engines.openidea = "^1.0" [capabilities] diff --git a/plugins/official/git-toolbox/dist/plugin.wasm b/plugins/official/git-toolbox/dist/plugin.wasm index 9479f183..5e26ce4e 100644 Binary files a/plugins/official/git-toolbox/dist/plugin.wasm and b/plugins/official/git-toolbox/dist/plugin.wasm differ diff --git a/plugins/official/git-toolbox/dist/plugin.wasm.sources b/plugins/official/git-toolbox/dist/plugin.wasm.sources index fd509401..0131a6b6 100644 --- a/plugins/official/git-toolbox/dist/plugin.wasm.sources +++ b/plugins/official/git-toolbox/dist/plugin.wasm.sources @@ -1 +1 @@ -ad545605b33c6f4e55a06a376333de3162739ed920b5d7d277942e50a4420083 +6047be5eeaa1330d7aa1f182e88ae2f352f244829ad9a01fdc3a2a081bdca95d diff --git a/plugins/official/git-toolbox/plugin.toml b/plugins/official/git-toolbox/plugin.toml index ff5bbfce..d0f02117 100644 --- a/plugins/official/git-toolbox/plugin.toml +++ b/plugins/official/git-toolbox/plugin.toml @@ -5,6 +5,7 @@ description = "Fetches your repositories in the background and shows the branch authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: auto-fetch every 5 minutes, ahead/behind in the status bar, Fetch Now." +categories = ["vcs"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" # Fetches on schedule (also after being reclaimed while idle) and refreshes diff --git a/plugins/official/graphviz/dist/plugin.wasm b/plugins/official/graphviz/dist/plugin.wasm index fa2c8114..912edbd6 100644 Binary files a/plugins/official/graphviz/dist/plugin.wasm and b/plugins/official/graphviz/dist/plugin.wasm differ diff --git a/plugins/official/graphviz/dist/plugin.wasm.sources b/plugins/official/graphviz/dist/plugin.wasm.sources index 8e261ce0..1657b2d5 100644 --- a/plugins/official/graphviz/dist/plugin.wasm.sources +++ b/plugins/official/graphviz/dist/plugin.wasm.sources @@ -1 +1 @@ -de353be96a6b66629901f55a8027a0f27107a7ac2fb9d891089db1b7fd716299 +7a3ee410d228d1e39155aa042237d222d23d7f5a45e261086cffc45c6ee1ad32 diff --git a/plugins/official/graphviz/plugin.toml b/plugins/official/graphviz/plugin.toml index 04f87a2e..8cf06eb4 100644 --- a/plugins/official/graphviz/plugin.toml +++ b/plugins/official/graphviz/plugin.toml @@ -5,6 +5,7 @@ description = "Graphviz DOT files (.dot, .gv) with a live preview beside the tex authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: live preview (Graphviz compiled to WebAssembly), layout engines, zoom, errors as problems, SVG and PNG export." +categories = ["languages", "tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-view:preview"] diff --git a/plugins/official/graphviz/src/lib.rs b/plugins/official/graphviz/src/lib.rs index cd76c651..c8ab1210 100644 --- a/plugins/official/graphviz/src/lib.rs +++ b/plugins/official/graphviz/src/lib.rs @@ -52,12 +52,9 @@ pub fn diagnostics(text: &str, reports: &[Report]) -> Vec { .map(|r| { let line = r.line.map_or(0, |l| l.saturating_sub(1)).min(lines.len().saturating_sub(1) as u32); let len = lines.get(line as usize).map_or(0, |l| l.encode_utf16().count() as u32); - Diagnostic { - range: Range::new(Position::new(line, 0), Position::new(line, len)), - severity: if r.error { Severity::Error } else { Severity::Warning }, - message: r.message.clone(), - code: Some("graphviz".into()), - } + let range = Range::new(Position::new(line, 0), Position::new(line, len)); + let severity = if r.error { Severity::Error } else { Severity::Warning }; + Diagnostic::new(range, severity, r.message.clone()).code("graphviz") }) .collect() } diff --git a/plugins/official/harbor-night/themes/harbor-night.scheme.json b/plugins/official/harbor-night/themes/harbor-night.scheme.json index 892744ea..41bd2966 100644 --- a/plugins/official/harbor-night/themes/harbor-night.scheme.json +++ b/plugins/official/harbor-night/themes/harbor-night.scheme.json @@ -18,6 +18,7 @@ "--syn-number": "#7fc8f8", "--syn-punct": "#a9b4c2", "--syn-variable": "#d9dfe8", + "--syn-global": "#d9dfe8", "--syn-param": "#d9dfe8", "--syn-property": "#b9a3f0", "--syn-constant": "#e58fa3", diff --git a/plugins/official/key-promoter/dist/plugin.wasm b/plugins/official/key-promoter/dist/plugin.wasm index daefc884..99400b9b 100644 Binary files a/plugins/official/key-promoter/dist/plugin.wasm and b/plugins/official/key-promoter/dist/plugin.wasm differ diff --git a/plugins/official/key-promoter/dist/plugin.wasm.sources b/plugins/official/key-promoter/dist/plugin.wasm.sources index 9965326d..84b4dfaa 100644 --- a/plugins/official/key-promoter/dist/plugin.wasm.sources +++ b/plugins/official/key-promoter/dist/plugin.wasm.sources @@ -1 +1 @@ -345c39f818e4950914bf3fcc795fac9e30ff2d87ad3e66f24c7bc00800f92980 +7658a95eb9fc27468cae4b1d20e3b5ab299fde478e07f69ab0fdb4cc10caf11d diff --git a/plugins/official/key-promoter/plugin.toml b/plugins/official/key-promoter/plugin.toml index 10f032b4..3a430fb9 100644 --- a/plugins/official/key-promoter/plugin.toml +++ b/plugins/official/key-promoter/plugin.toml @@ -5,6 +5,7 @@ description = "Shows the keyboard shortcut of an action you ran with the mouse, authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: shortcut tips, statistics, per-action \"don't show again\"." +categories = ["tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" # Actions arrive in batches (a few seconds apart while the plugin is idle), diff --git a/plugins/official/license-header/dist/plugin.wasm b/plugins/official/license-header/dist/plugin.wasm index 6cd6d293..fdcbdd18 100644 Binary files a/plugins/official/license-header/dist/plugin.wasm and b/plugins/official/license-header/dist/plugin.wasm differ diff --git a/plugins/official/license-header/dist/plugin.wasm.sources b/plugins/official/license-header/dist/plugin.wasm.sources index 01859b4b..e5ee06b9 100644 --- a/plugins/official/license-header/dist/plugin.wasm.sources +++ b/plugins/official/license-header/dist/plugin.wasm.sources @@ -1 +1 @@ -6a4210a3978f058b3526427eb41ef398924e8c011cc4dc53c19f5419d31501f3 +227a30ef0028f167a20825bdd775b15392161e641f6c68030e01d0c79e263bb5 diff --git a/plugins/official/license-header/plugin.toml b/plugins/official/license-header/plugin.toml index af970e2e..c32b4176 100644 --- a/plugins/official/license-header/plugin.toml +++ b/plugins/official/license-header/plugin.toml @@ -5,6 +5,7 @@ description = "Keeps a copyright and SPDX license header at the top of source fi authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: missing headers reported on open and save, Add License Header quick fix and command, project scan in the background." +categories = ["tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-event:file-opened", "on-event:file-saved", "on-command:*"] diff --git a/plugins/official/license-header/src/lib.rs b/plugins/official/license-header/src/lib.rs index 7fd5d789..1da3c9e1 100644 --- a/plugins/official/license-header/src/lib.rs +++ b/plugins/official/license-header/src/lib.rs @@ -129,12 +129,8 @@ pub fn header_edit(path: &str, text: &str, year: i64, owner: &str, license: &str fn missing(text: &str) -> Diagnostic { let line = insert_line(text); let len = text.lines().nth(line as usize).map_or(0, |l| l.encode_utf16().count() as u32); - Diagnostic { - range: Range::new(Position::new(line, 0), Position::new(line, len)), - severity: Severity::Info, - message: "No license header".into(), - code: Some(CODE.into()), - } + Diagnostic::new(Range::new(Position::new(line, 0), Position::new(line, len)), Severity::Info, "No license header") + .code(CODE) } /// Reports `path` when it takes a header and has none, clears it otherwise. @@ -274,6 +270,7 @@ impl Providers for LicenseHeader { kind: Some("quickfix".into()), preferred: true, edits: vec![(doc.path.clone(), vec![edit])], + data: None, }] } } diff --git a/plugins/official/mybatis/dist/plugin.wasm b/plugins/official/mybatis/dist/plugin.wasm index 5ed3800a..b31649e9 100644 Binary files a/plugins/official/mybatis/dist/plugin.wasm and b/plugins/official/mybatis/dist/plugin.wasm differ diff --git a/plugins/official/mybatis/dist/plugin.wasm.sources b/plugins/official/mybatis/dist/plugin.wasm.sources index 0241ff75..34acc267 100644 --- a/plugins/official/mybatis/dist/plugin.wasm.sources +++ b/plugins/official/mybatis/dist/plugin.wasm.sources @@ -1 +1 @@ -f7df58bb0ab2b92370b148de2614174ec1ff9b01c0078b5c20bd331f82298616 +046f769822102de0d4ecec99ad41027cb792941065226716f38a6e64c50ff364 diff --git a/plugins/official/mybatis/plugin.toml b/plugins/official/mybatis/plugin.toml index fac11916..d5a55c69 100644 --- a/plugins/official/mybatis/plugin.toml +++ b/plugins/official/mybatis/plugin.toml @@ -5,6 +5,7 @@ description = "Jumps between MyBatis mapper interfaces and their XML statements authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: navigation both ways, line markers, id and parameter completion." +categories = ["frameworks"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-command:*"] diff --git a/plugins/official/regex-tester/dist/plugin.wasm b/plugins/official/regex-tester/dist/plugin.wasm index 9eed421b..a8e10a84 100644 Binary files a/plugins/official/regex-tester/dist/plugin.wasm and b/plugins/official/regex-tester/dist/plugin.wasm differ diff --git a/plugins/official/regex-tester/dist/plugin.wasm.sources b/plugins/official/regex-tester/dist/plugin.wasm.sources index b9543105..df4b262a 100644 --- a/plugins/official/regex-tester/dist/plugin.wasm.sources +++ b/plugins/official/regex-tester/dist/plugin.wasm.sources @@ -1 +1 @@ -d2e35f3e4ee6b699899ea43fdf96d5ec99f0730c502f94fd6f3246e605be99a7 +0b9c3572390a3caab7d47f9b975439f8dde3feff6b9044579d99b8334ac2a692 diff --git a/plugins/official/regex-tester/plugin.toml b/plugins/official/regex-tester/plugin.toml index a889b96a..22e45435 100644 --- a/plugins/official/regex-tester/plugin.toml +++ b/plugins/official/regex-tester/plugin.toml @@ -5,6 +5,7 @@ description = "Try a regular expression against sample text: matches highlighted authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: live matching with flags, capture groups, replacement preview, the editor's selection as test text." +categories = ["tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" activation = ["on-view:regex", "on-command:*"] diff --git a/plugins/official/solidity/dist/plugin.wasm b/plugins/official/solidity/dist/plugin.wasm index 3cb17fd7..cef826bb 100644 Binary files a/plugins/official/solidity/dist/plugin.wasm and b/plugins/official/solidity/dist/plugin.wasm differ diff --git a/plugins/official/solidity/dist/plugin.wasm.sources b/plugins/official/solidity/dist/plugin.wasm.sources index d7a74f4c..09d61924 100644 --- a/plugins/official/solidity/dist/plugin.wasm.sources +++ b/plugins/official/solidity/dist/plugin.wasm.sources @@ -1 +1 @@ -3893bd8ad06d3e19c0112081c6643b3bf3c67f5a184098432a6fd338426cd682 +0af62bcd4583d126a616cafbf2ddc80f039a3ead2b5dcb7144222b0ebc673432 diff --git a/plugins/official/solidity/plugin.toml b/plugins/official/solidity/plugin.toml index cc03892b..2ef3c864 100644 --- a/plugins/official/solidity/plugin.toml +++ b/plugins/official/solidity/plugin.toml @@ -5,6 +5,7 @@ description = "Solidity support: file detection, highlighting, structure view an authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: .sol detection, highlighting, structure view, folding; language server via @nomicfoundation/solidity-language-server or solc." +categories = ["languages"] engines.openidea = "^1.0" # Highlighting, structure and folding come from this component; it is loaded # when the first Solidity file is opened. diff --git a/plugins/official/string-tools/dist/plugin.wasm b/plugins/official/string-tools/dist/plugin.wasm index 1c33e3e4..3ae03a50 100644 Binary files a/plugins/official/string-tools/dist/plugin.wasm and b/plugins/official/string-tools/dist/plugin.wasm differ diff --git a/plugins/official/string-tools/dist/plugin.wasm.sources b/plugins/official/string-tools/dist/plugin.wasm.sources index 2dbf488f..c511466f 100644 --- a/plugins/official/string-tools/dist/plugin.wasm.sources +++ b/plugins/official/string-tools/dist/plugin.wasm.sources @@ -1 +1 @@ -a88c39d5782369329efa55b12330daa906e5c8d91252bd57c7343648840bac79 +24ef150beefb05622212228e1c44cb487fffe6526776762d713f3ebdf4c02ac0 diff --git a/plugins/official/string-tools/plugin.toml b/plugins/official/string-tools/plugin.toml index 18ff2bb5..9ff53b75 100644 --- a/plugins/official/string-tools/plugin.toml +++ b/plugins/official/string-tools/plugin.toml @@ -5,6 +5,7 @@ description = "Case conversion (camelCase, snake_case, kebab-case…), encoding authors = ["OpenIDEA contributors"] license = "Apache-2.0" change-notes = "0.1.0 — first release: 26 commands in four groups." +categories = ["tools"] engines.openidea = "^1.0" wasm = "dist/plugin.wasm" # The commands are declared below, so Find Action and the keymap list them diff --git a/scripts/theme/import-intellij-theme.ts b/scripts/theme/import-intellij-theme.ts index 4fdd0cf1..d294c1ce 100644 --- a/scripts/theme/import-intellij-theme.ts +++ b/scripts/theme/import-intellij-theme.ts @@ -10,7 +10,8 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, join } from "node:path"; import { parseArgs } from "node:util"; -import { convertIntellijTheme, pluginManifest } from "./intellij.ts"; +import { convertIntellijTheme, pluginManifest } from "../../src/features/plugins/intellijTheme.ts"; +import { stripJsonComments } from "./tokens.ts"; const { values, positionals } = parseArgs({ allowPositionals: true, @@ -25,7 +26,7 @@ if (!themePath || !values.out) { } const converted = convertIntellijTheme( - readFileSync(themePath, "utf8"), + stripJsonComments(readFileSync(themePath, "utf8")), iclsPath ? readFileSync(iclsPath, "utf8") : undefined, values.id, ); @@ -34,7 +35,17 @@ const source = [basename(themePath), iclsPath && basename(iclsPath)].filter(Bool const header = (what: string) => `// ${theme.name}: ${what}, converted from ${source}.\n`; mkdirSync(join(values.out, "themes"), { recursive: true }); -writeFileSync(join(values.out, "plugin.toml"), pluginManifest(converted, values.author ?? "Unknown", source)); +const info = { + id: `theme-${theme.id}`, + name: `${theme.name} Theme`, + version: "1.0.0", + description: `${theme.name}, converted from an IntelliJ theme.`, + authors: [values.author ?? "Unknown"], +}; +writeFileSync( + join(values.out, "plugin.toml"), + pluginManifest(info, [converted], `Converted from ${source} by scripts/theme/import-intellij-theme.ts.`), +); writeFileSync( join(values.out, "themes", `${theme.id}.theme.json`), header("UI theme") + JSON.stringify(theme, null, 2) + "\n", diff --git a/scripts/theme/theme.test.ts b/scripts/theme/theme.test.ts index bf533329..404d9909 100644 --- a/scripts/theme/theme.test.ts +++ b/scripts/theme/theme.test.ts @@ -5,9 +5,10 @@ import { convertIntellijTheme, ICLS_ATTRIBUTES, ICLS_COLORS, + intellijThemePackage, pluginManifest, UI_KEYS, -} from "./intellij.ts"; +} from "../../src/features/plugins/intellijTheme.ts"; import { referenceMarkdown, stripJsonComments, themeTokenGroups, tokenSets, withReference } from "./tokens.ts"; const read = (p: string) => readFileSync(p, "utf8"); @@ -90,6 +91,8 @@ describe("IntelliJ theme import", () => { "Button": { "default": { "startBackground": "accent" } }, "Popup.background": "#181f28", "Label.infoForeground": "#80888F80", + "EditorTabs": { "underlineColor": "#405060", "underlinedBorderColor": "#bd93f9" }, + "CompletionPopup.matchForeground": "#ff79c6", "Tree": { "rowHeight": 24 } } }`; @@ -98,6 +101,7 @@ describe("IntelliJ theme import", () => {