diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 3023d414abe..784172d699e 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -117,6 +117,7 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm build:electron-vite + - run: pnpm verify:cli-runtime # Why: boot the BUILT daemon-entry under plain Node the way production # forks it. v1.4.129-rc.1 shipped a daemon that exited at module load # (leaked electron require) while every other check passed; this fails @@ -158,6 +159,7 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm build:electron-vite + - run: pnpm verify:cli-runtime - run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-mac.e2e.ts tests/e2e/computer-mac-safari.e2e.ts linux: @@ -185,6 +187,7 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm build:electron-vite + - run: pnpm verify:cli-runtime - run: xvfb-run --auto-servernum dbus-run-session -- pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-linux.e2e.ts windows: @@ -204,4 +207,5 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm build:electron-vite + - run: pnpm verify:cli-runtime - run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts diff --git a/.github/workflows/mobile-ios-release.yml b/.github/workflows/mobile-ios-release.yml index b38a79b9267..f759bd3efee 100644 --- a/.github/workflows/mobile-ios-release.yml +++ b/.github/workflows/mobile-ios-release.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: bump_patch_version: - description: 'Bump the iOS marketing version patch number before release. Tick this after a version has shipped to the App Store (the release fails fast if the current version''s train is already closed).' + description: 'Use the first open iOS patch version after the checked-in version, skipping versions already closed on the App Store.' required: false default: false type: boolean diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 2998d1dcce7..e9683904be2 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -30,6 +30,11 @@ jobs: with: node-version-file: package.json + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + - name: Setup pnpm uses: pnpm/action-setup@v6 with: @@ -56,6 +61,9 @@ jobs: - name: Test run: pnpm test + - name: Test iOS release version resolution + run: ruby fastlane/ios_release_version_test.rb + - name: Lint run: pnpm lint diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index 2c48d466305..0142518c4cb 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -1330,7 +1330,8 @@ jobs: } Copy-Item -Path $signedInstaller.FullName -Destination 'dist/orca-windows-setup.exe' -Force - & 'node_modules/app-builder-bin/win/x64/app-builder.exe' blockmap --input 'dist/orca-windows-setup.exe' --output 'dist/orca-windows-setup.exe.blockmap' + node config/scripts/generate-windows-blockmap.mjs 'dist/orca-windows-setup.exe' 'dist/orca-windows-setup.exe.blockmap' + if ($LASTEXITCODE -ne 0) { throw "blockmap generation failed with exit code $LASTEXITCODE" } $installer = Get-Item 'dist/orca-windows-setup.exe' $blockmap = Get-Item 'dist/orca-windows-setup.exe.blockmap' diff --git a/.github/workflows/windows-signing-rehearsal.yml b/.github/workflows/windows-signing-rehearsal.yml index 24b4307a83b..4ca080d803d 100644 --- a/.github/workflows/windows-signing-rehearsal.yml +++ b/.github/workflows/windows-signing-rehearsal.yml @@ -271,7 +271,7 @@ jobs: throw 'Signed Windows installer was not returned by SignPath.' } Copy-Item -Path $signedInstaller.FullName -Destination 'dist/orca-windows-setup.exe' -Force - & 'node_modules/app-builder-bin/win/x64/app-builder.exe' blockmap --input 'dist/orca-windows-setup.exe' --output 'dist/orca-windows-setup.exe.blockmap' + node config/scripts/generate-windows-blockmap.mjs 'dist/orca-windows-setup.exe' 'dist/orca-windows-setup.exe.blockmap' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $installer = Get-Item 'dist/orca-windows-setup.exe' diff --git a/README.md b/README.md index 1a434350bd0..aa4ee905281 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Pair with your desktop app to monitor and steer your agents from your phone. - **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**. - **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements. -- **WeChat:** Groups 1 and 2 are both full — now you can join the third one. +- **WeChat:** All other groups are full, now we're on group 5. WeChat QR code for the Orca community diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 5db7f6ca20e..7ec61073b60 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1390,6 +1390,7 @@ "headed desktop remote-server pairing", "headless remote-server parity", "daemon and relay reconnect", + "remote completion classification across disconnect and reconnect", "terminal exit retirement and restart restore", "mixed-version fallback" ], @@ -1412,22 +1413,39 @@ "local", "daemon", "ssh", + "wsl", "remote-runtime" ], - "coverageNotes": "Deterministic macOS tests cover controller claims, daemon and SSH/relay operation replay, mixed-version selection, runtime ownership, exact provisional handoff, durable terminal retirement, and two independent viewer mirrors. The secondary parity repro runs independent clients against one headless remote Orca runtime over encrypted pairing and a real daemon-backed PTY. A headed Orca desktop server paired to a separate client is the primary live user topology and remains uncollected for this issue. SSH coverage is provider/relay contract and fault-injection coverage only; it does not substitute for paired-server coverage. WSL and live SSH hosts remain gaps.", + "coverageNotes": "Deterministic macOS tests cover controller claims, daemon and SSH/relay operation replay, mixed-version selection, runtime ownership, exact provisional handoff, durable terminal retirement, two independent viewer mirrors, guarded adoption of legacy live PTYs, and completion classification when either the outer remote transport or authoritative host/provider process inspection becomes unreachable. The adoption harness models v1.4.150 agent/setup/shell tabs, current-generation restart and reconnect, exact handle/incarnation/worktree/host checks, topology CAS, competing clients, split-pane/group restoration, WSL ownership, and SSH owner rejection. The secondary parity repro runs independent clients against one headless remote Orca runtime over encrypted pairing and a real daemon-backed PTY. The primary headed topology was exercised with an isolated macOS Orca desktop server, a separate paired Edge client, a real Codex process, injected WebSocket loss, reconnect, and explicit stop; host inspection and visible client state agreed throughout. SSH coverage is provider/relay contract and fault-injection coverage only; it does not substitute for paired-server coverage. Live Windows, Linux, WSL, and SSH hosts remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8878", + "https://github.com/stablyai/orca/issues/9151", "https://github.com/stablyai/orca/issues/9352", "https://github.com/stablyai/orca/pull/9687" ], - "invariant": "For every claim-capable execution route, one provider-session identity has at most one live PTY owner and one canonical host surface across concurrent clients, retries, reconnects, and stale publications. A physical exit retires that exact incarnation durably so stale client state and host restart cannot recreate it. Mixed-version routes select the unchanged legacy request before any authority side effect or execution-owner-local filesystem access.", - "oracle": "Race independent clients and repeated operation IDs, then assert one physical spawn and one canonical PTY/surface; inject exit-before-reply, provider disconnect, conflicting claim scope, and old daemon/relay capabilities; assert safe adoption or explicit failure without a second spawn. After exact exit, assert terminal and tab listings omit the surface, a stale publication cannot restore it, restart cannot resurrect it, and an exact provisional handoff is consumed even when exit wins before the next snapshot.", + "invariant": "For every claim-capable execution route, one provider-session identity has at most one live PTY owner and one canonical host surface across concurrent clients, retries, reconnects, and stale publications. A live orphan may be adopted only when the controller proves its exact handle and incarnation, its worktree and host owner match, no competing visual owner exists, and a host topology CAS wins. A viewer may classify completion only from successful host/provider inspection or explicit lifecycle evidence; transport, handle, or provider unavailability remains unknown and breaks any consecutive-idle proof. A physical exit retires that exact incarnation durably so stale client state and host restart cannot recreate it. Mixed-version routes select the unchanged legacy request before any authority side effect or execution-owner-local filesystem access.", + "oracle": "Race independent clients and repeated operation IDs, then assert one physical spawn and one canonical PTY/surface; inject exit-before-reply, provider disconnect, conflicting claim scope, old daemon/relay capabilities, reused handles, stale incarnations, owner mismatch, and topology revision conflict; assert safe adoption or explicit failure without a second spawn or wrong-process attachment. Restore legacy split panes and groups beside a newer host-owned tab, preserving output, input, resize, titles, tab/leaf identity, active group, and multi-client convergence. For completion, drive a known running agent through outer transport loss, authoritative provider rejection, reconnect, explicit stop, real exit status, and successful hook completion; assert unavailable evidence never dispatches completion and two fresh authoritative idle samples are required after the gap. After exact exit, assert terminal and tab listings omit the surface, a stale publication cannot restore it, restart cannot resurrect it, and an exact provisional handoff is consumed even when exit wins before the next snapshot.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/shared/claimed-agent-pty-owner.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/runtime/orca-runtime-agent-session-operation.test.ts src/main/runtime/remote-agent-session-host-authority.integration.test.ts src/main/runtime/orca-runtime-terminal-retirement.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts src/renderer/src/runtime/web-runtime-session.test.ts src/renderer/src/runtime/web-session-tabs-sync.test.ts", "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/remote-terminal-tab-retirement.unit.test.ts", - "pnpm test:repro:remote-agent-session" + "pnpm test:repro:remote-agent-session", + "Manual headed paired-server journey: isolated Orca desktop host + separate paired web client + real Codex process + 20-second WebSocket fault + reconnect + explicit stop", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/terminal-orphan-owner.test.ts src/main/runtime/terminal-orphan-topology.test.ts src/renderer/src/runtime/web-session-terminal-orphan-recovery.test.ts src/renderer/src/runtime/web-session-terminal-orphan-mixed-version.test.ts src/renderer/src/runtime/web-session-tabs-sync.test.ts --maxWorkers=1", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/pty-process-inspection.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/relay/pty-handler.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/runtime/orca-runtime.test.ts tests/e2e/remote-agent-completion-authority.unit.test.ts src/renderer/src/runtime/runtime-terminal-inspection.test.ts src/renderer/src/components/terminal-pane/agent-completion-coordinator.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/remote-agent-completion-authority.unit.test.ts src/main/providers/pty-process-inspection.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/relay/pty-handler.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/renderer/src/runtime/runtime-terminal-inspection.test.ts src/renderer/src/components/terminal-pane/agent-completion-coordinator.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/lib/codex-session-restart.test.ts" ], "testFiles": [ + "src/main/providers/pty-process-inspection.test.ts", + "src/main/daemon/terminal-host.test.ts", + "src/main/daemon/daemon-pty-router.test.ts", + "src/main/daemon/degraded-daemon-pty-provider.test.ts", + "src/relay/pty-handler.test.ts", + "src/main/runtime/orca-runtime.test.ts", + "tests/e2e/remote-agent-completion-authority.unit.test.ts", + "src/renderer/src/runtime/runtime-terminal-inspection.test.ts", + "src/renderer/src/components/terminal-pane/agent-completion-coordinator.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "src/renderer/src/lib/codex-session-restart.test.ts", "src/shared/claimed-agent-pty-owner.test.ts", "src/main/daemon/daemon-pty-adapter.test.ts", "src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts", @@ -1438,9 +1456,70 @@ "src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts", "src/renderer/src/runtime/web-runtime-session.test.ts", "src/renderer/src/runtime/web-session-tabs-sync.test.ts", - "tests/e2e/remote-terminal-tab-retirement.unit.test.ts" + "tests/e2e/remote-terminal-tab-retirement.unit.test.ts", + "src/main/runtime/orca-runtime.test.ts", + "src/main/runtime/terminal-orphan-owner.test.ts", + "src/main/runtime/terminal-orphan-topology.test.ts", + "src/renderer/src/runtime/web-session-terminal-orphan-recovery.test.ts", + "src/renderer/src/runtime/web-session-terminal-orphan-mixed-version.test.ts" ], "assertionRefs": [ + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "completion-sensitive process inspection preserves authoritative host/provider failures" + ] + }, + { + "file": "src/main/providers/pty-process-inspection.test.ts", + "assertions": [ + "dedicated provider inspection preserves failures and rejects missing PTYs instead of returning idle evidence" + ] + }, + { + "file": "src/main/daemon/daemon-pty-router.test.ts", + "assertions": [ + "completion inspection rejects an unmapped session instead of borrowing the current daemon" + ] + }, + { + "file": "src/main/daemon/degraded-daemon-pty-provider.test.ts", + "assertions": [ + "completion inspection rejects an unmapped session instead of borrowing the local fallback" + ] + }, + { + "file": "src/relay/pty-handler.test.ts", + "assertions": [ + "strict relay inspection rejects a missing PTY" + ] + }, + { + "file": "tests/e2e/remote-agent-completion-authority.unit.test.ts", + "assertions": [ + "transport loss remains unknown through reconnect and cannot dispatch completion", + "returned unavailability or a thrown transport failure interrupts consecutive-idle proof and requires two fresh authoritative idle samples", + "explicit stop, real exit status, and genuine successful completion remain distinct" + ] + }, + { + "file": "src/renderer/src/runtime/runtime-terminal-inspection.test.ts", + "assertions": [ + "direct SSH terminals use strict main-process inspection rather than lax split IPC evidence" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "assertions": [ + "completion polling uses the atomic process-inspection boundary without regressing established lifecycle behavior" + ] + }, + { + "file": "src/renderer/src/lib/codex-session-restart.test.ts", + "assertions": [ + "one unreachable pane cannot suppress restart notices for another authoritatively confirmed Codex pane" + ] + }, { "file": "src/shared/claimed-agent-pty-owner.test.ts", "assertions": [ @@ -1497,9 +1576,52 @@ "one exact exit produces one same-epoch higher-version host publication and one durable persistence flush", "same-epoch stale publications cannot resurrect the retired surface after reconnect" ] + }, + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "v1.4.150-shaped agent, setup, and shell PTYs adopt as one CAS transaction while stale incarnation and competing clients fail safely", + "current-generation restart and disconnect/reconnect preserve output, input, resize, title, tab, leaf, handle, and incarnation identity", + "split-pane and multi-group legacy topology merges beside a newer host-owned terminal without replacing it", + "equivalent Windows and separator-normalized persisted worktree keys canonicalize without duplicate terminal topology", + "connection mismatch, reused handles, SSH ownership mismatch, and stale topology revisions cannot claim a live PTY while WSL ownership succeeds" + ] + }, + { + "file": "src/renderer/src/runtime/web-session-terminal-orphan-recovery.test.ts", + "assertions": [ + "absence stays pending until an exact live orphan adoption settles", + "client pane and group topology is pruned to exact orphan claims and translated to host tab identities", + "a missing split leaf remains recoverable when another leaf in the same tab is already host-owned" + ] + }, + { + "file": "src/renderer/src/runtime/web-session-terminal-orphan-mixed-version.test.ts", + "assertions": [ + "mixed-version inventory without incarnation evidence remains visible but cannot adopt", + "a truncated legacy unfiltered inventory cannot hide a candidate whose liveness is unresolved" + ] } ], "evidenceRuns": [ + { + "date": "2026-07-22", + "runner": "manual", + "platform": "macos", + "command": "Manual headed paired-server journey: isolated Orca desktop host + separate paired web client + real Codex process + 20-second WebSocket fault + reconnect + explicit stop", + "result": "passed", + "durationSeconds": 549, + "summary": "The primary user topology used an isolated headed Orca desktop as the owning server and a separate paired Edge client. Host inspection reported Codex alive before, during, and after a page-scoped WebSocket fault; the client showed no completion toast, reconnected to the same live Codex TUI, and explicit stop restored the shell prompt with no child process." + }, + { + "date": "2026-07-23", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/remote-agent-completion-authority.unit.test.ts src/main/providers/pty-process-inspection.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/relay/pty-handler.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/renderer/src/runtime/runtime-terminal-inspection.test.ts src/renderer/src/components/terminal-pane/agent-completion-coordinator.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/lib/codex-session-restart.test.ts", + "result": "passed", + "durationSeconds": 11.96, + "summary": "Eleven focused files and 870 tests passed on the current-main candidate. The cross-boundary harness fails with the implementation reverted by dispatching process-exit from unavailable remote evidence. Direct SSH uses strict main-process inspection, daemon and relay inspection reject missing or unmapped sessions, the terminal lifecycle suite uses the atomic inspection boundary, and one stale pane cannot suppress restart notices for a separately confirmed Codex pane." + }, { "date": "2026-07-21", "runner": "local", @@ -1510,13 +1632,13 @@ "summary": "Ten focused files and 325 tests passed after the final review fixes, covering claim scope, mixed-version Pi/SSH fallback ordering, operation replay, terminal retirement, causal inventory fencing, exact concurrent handoff confirmation, daemon-generation integration, transport behavior, and remote host integration." }, { - "date": "2026-07-21", + "date": "2026-07-22", "runner": "local", "platform": "macos", "command": "pnpm test:repro:remote-agent-session", "result": "passed", - "durationSeconds": 48.47, - "summary": "The secondary build-backed headless parity harness passed over encrypted WebSocket pairing with independent clients, proving one spawn, retry adoption, durable exit retirement, stale-publication rejection, and no restart resurrection." + "durationSeconds": 48.63, + "summary": "The secondary build-backed headless parity harness passed post-rebase on main@72a2d7bc7 over encrypted WebSocket pairing with independent clients, proving one spawn, retry adoption, durable exit retirement, stale-publication rejection, and no restart resurrection." }, { "date": "2026-07-22", @@ -1556,27 +1678,166 @@ }, "redGreenEvidence": { "status": "partial", - "evidence": "The exact cross-boundary retirement oracle is red on pre-#9687 commit 2a32c5c9a and PR #9053 head d3a1d3047 because the pinned persisted surface remains in the host publication, and green on main@4fce2de49. Duplicate-resume red evidence remains encoded in lower-layer tests; saved CI artifacts are still needed." + "evidence": "Issue #9151 has local red/green evidence: current main dispatches process-exit from a stale remote handle, while the fix preserves unknown liveness and requires fresh consecutive idle evidence after reconnect. The exact retirement oracle is red on pre-#9687 commit 2a32c5c9a and PR #9053 head d3a1d3047 because the pinned persisted surface remains in the host publication, and green on main@4fce2de49. Duplicate-resume red evidence remains encoded in lower-layer tests; saved CI artifacts are still needed." }, "performanceBudget": { "required": true, - "evidence": "Agent-session reconciliation runs only at explicit claim admission, dedupes concurrent provider listing, and adds no polling or renderer output work. Create-operation ledgers are capped globally and per client, expire after 24 hours, and reject rather than evict live replay fences. Capability caches are bounded or connection-scoped, and exact handoffs are consumed by the next authoritative snapshot." + "evidence": "Agent-session reconciliation runs only at explicit claim admission, dedupes concurrent provider listing, and adds no polling or renderer output work. Completion inspection reuses the coordinator's per-pane in-flight guard, global concurrency/rate queue, and existing error backoff; the strict daemon path reduces two foreground RPCs to one. Create-operation ledgers are capped globally and per client, expire after 24 hours, and reject rather than evict live replay fences. Capability caches are bounded or connection-scoped, and exact handoffs are consumed by the next authoritative snapshot." }, "promotionCriteria": [ "Run the focused gate and remote-server repro for at least 100 consecutive passes or 14 days across required CI platforms.", "Attach saved red/green evidence for duplicate remote resume and exit-before-snapshot retirement.", - "Add the primary live journey with a headed Orca desktop server and a separate paired client; use a physical host when OS, ConPTY, update, sleep, firewall, or window lifecycle is causal.", + "Automate the headed Orca desktop-server journey with a separate paired client; add a physical host only when OS, ConPTY, update, sleep, firewall, or window lifecycle is causal.", "Add live SSH/WSL provider evidence before claiming full provider coverage; Docker SSH proves only the SSH provider/relay path." ], "knownGaps": [ - "The primary live topology—a headed Orca desktop server with an isolated profile paired to a separate persistent client—has not been run for this issue. The causal boundary is host membership rather than OS/window/ConPTY/update/sleep behavior, so a physical headed host was not required for the current disposition.", + "The primary headed macOS desktop-server journey is collected but not automated in CI; Windows and Linux window, ConPTY, update, sleep/wake, and firewall behavior remain uncollected.", + "Mixed-version pairings remain conservative only when the completion-aware client and strict-inspection host changes are both present; older peers retain their legacy classification behavior.", "The secondary headless parity harness runs on macOS with a local daemon-backed execution owner and independent short-lived encrypted RPC clients; two persistent viewer-store mirrors and reconnect ordering are joined deterministically in the cross-boundary unit test rather than mounted live.", - "SSH and relay failure ordering is deterministic provider-contract coverage, not a live SSH-host journey or paired-Orca-server proof; WSL has no provider-specific run.", + "SSH and relay failure ordering is deterministic provider-contract coverage, not a live SSH-host journey or paired-Orca-server proof; WSL has no provider-specific run, and Linux and Windows runs remain uncollected.", "Fresh-launch operation replay is memory-backed and intentionally does not survive runtime restart; a durable operation journal is a documented future extension.", "Automatic sleep checkpoints, verified nested-SSH execution namespaces, and multi-process profile coordination remain outside v1." ], "demotionRule": "Keep experimental or demote if the focused gate flakes without a product or harness bug, if a retry can physically spawn twice, if a stale exit/publication can replace or resurrect a terminal, or if mixed-version fallback occurs after an authority side effect." }, + { + "id": "runtime-routing.active-server-preference", + "title": "Active Server changes only through its explicit Advanced control", + "maturity": "experimental", + "protection": "partial", + "owner": "runtime-routing", + "layer": "main-preload-renderer-persistence-contract", + "surfaces": [ + "Advanced Active Server setting", + "saved server Connect and Disconnect", + "remote workspace navigation", + "terminal reveal and create", + "browser and mobile handoff", + "app restart" + ], + "platforms": [ + "macos", + "linux", + "windows", + "mobile" + ], + "providers": [ + "local", + "remote-runtime", + "ssh", + "wsl" + ], + "coveredPlatforms": [ + "macos" + ], + "coveredProviders": [ + "local", + "remote-runtime", + "ssh", + "wsl" + ], + "coverageNotes": "Platform-neutral deterministic tests separate the durable Active Server preference from per-client connection, selected-workspace, browser-session, and execution-host routing. The composed regression models Local desktop -> connect/navigate Windows 2 -> reveal a local terminal -> restart. Multi-client browser host overrides, multi-server profile caches, generic settings IPC rejection, local and remote workspace ownership, and restart reset of transient routing are covered. Live desktop UI runs remain uncollected.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/pull/9687" + ], + "invariant": "Only an explicit user change in Settings > Remote Orca Servers > Advanced > Active Server may mutate activeRuntimeEnvironmentId. Connecting, pairing, disconnecting, selecting or revealing a workspace or terminal, browser/mobile handoff, remote navigation, and reconnect must use transient or target-owner routing and must never rewrite the durable preference. Generic settings mutation cannot bypass the dedicated preference IPC.", + "oracle": "Start with Active Server=Local desktop, connect and navigate Windows 2, then reveal a local terminal and assert it succeeds while the persisted preference remains local before and after restart. Repeat with multiple clients and servers, browser host switches, remote-owned and local-owned workspaces, pairing/connect/disconnect, and generic settings writes. Assert only the dedicated validated preference method changes activeRuntimeEnvironmentId and stale host-operation completions cannot overwrite the newly selected transient host.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/settings.test.ts src/main/ipc/runtime-environments.test.ts src/renderer/src/store/slices/settings.test.ts src/renderer/src/store/slices/browser.test.ts src/renderer/src/components/settings/browser-session-host-selection.test.ts src/renderer/src/components/settings/RuntimeEnvironmentsPane.test.ts src/renderer/src/components/status-bar/SshStatusSegment.test.ts src/renderer/src/components/sidebar/use-add-repo-host-selection.test.ts src/renderer/src/hooks/useIpcEvents.test.ts src/renderer/src/web/web-preload-api.test.ts --maxWorkers=1" + ], + "testFiles": [ + "src/main/ipc/settings.test.ts", + "src/main/ipc/runtime-environments.test.ts", + "src/renderer/src/store/slices/settings.test.ts", + "src/renderer/src/store/slices/browser.test.ts", + "src/renderer/src/components/settings/browser-session-host-selection.test.ts", + "src/renderer/src/components/settings/RuntimeEnvironmentsPane.test.ts", + "src/renderer/src/components/status-bar/SshStatusSegment.test.ts", + "src/renderer/src/components/sidebar/use-add-repo-host-selection.test.ts", + "src/renderer/src/hooks/useIpcEvents.test.ts", + "src/renderer/src/web/web-preload-api.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/ipc/settings.test.ts", + "assertions": [ + "generic settings IPC strips activeRuntimeEnvironmentId while the dedicated validated IPC persists it", + "invalid preference types and unknown server identities cannot mutate the durable preference" + ] + }, + { + "file": "src/renderer/src/hooks/useIpcEvents.test.ts", + "assertions": [ + "Local desktop remains the durable default after transient Windows 2 navigation and a focused local terminal reveal succeeds", + "local and remote terminal create route by target workspace ownership instead of the durable preference" + ] + }, + { + "file": "src/renderer/src/store/slices/browser.test.ts", + "assertions": [ + "multiple clients select different transient browser hosts without changing Active Server", + "restart clears transient browser host override while retaining the durable local preference", + "late profile and import results update only their captured host and cannot overwrite a newer selection" + ] + }, + { + "file": "src/renderer/src/components/settings/browser-session-host-selection.test.ts", + "assertions": [ + "a removed transient server override falls back to an available host instead of leaving browser settings on an invalid option" + ] + }, + { + "file": "src/renderer/src/components/settings/RuntimeEnvironmentsPane.test.ts", + "assertions": [ + "connection status and the Advanced default-host selection are distinct concepts" + ] + }, + { + "file": "src/renderer/src/web/web-preload-api.test.ts", + "assertions": [ + "generic web settings writes cannot mutate Active Server", + "the dedicated web preference setter rejects unknown server identities without corrupting the saved choice" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-22", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/settings.test.ts src/main/ipc/runtime-environments.test.ts src/renderer/src/store/slices/settings.test.ts src/renderer/src/store/slices/browser.test.ts src/renderer/src/components/settings/browser-session-host-selection.test.ts src/renderer/src/components/settings/RuntimeEnvironmentsPane.test.ts src/renderer/src/components/status-bar/SshStatusSegment.test.ts src/renderer/src/components/sidebar/use-add-repo-host-selection.test.ts src/renderer/src/hooks/useIpcEvents.test.ts src/renderer/src/web/web-preload-api.test.ts --maxWorkers=1", + "result": "passed", + "durationSeconds": 6.55, + "summary": "Ten files and 282 tests passed, including the composed Local -> Windows 2 navigation -> local reveal -> restart regression, dedicated-only preference persistence, removed transient-host fallback, multi-client browser routing, late host-operation suppression, and web pairing/preference separation." + } + ], + "runtimeBudget": { + "p95Seconds": 20, + "scope": "focused persistence and routing contract tests" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "New deterministic gate with no soak history." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "The user-observed local terminal reveal failure and unintended Active Server switch are encoded by deterministic routing and persistence tests; a saved intentional-break artifact is not yet attached." + }, + "performanceBudget": { + "required": false, + "evidence": "Preference writes are explicit user actions; transient routing adds no polling and uses existing host/worktree indexes." + }, + "promotionCriteria": [ + "Run the focused gate in soak across macOS, Linux, and Windows.", + "Attach a live Windows Local -> Windows 2 -> local reveal -> restart artifact.", + "Attach saved red/green evidence for generic settings mutation and transient connection routing." + ], + "knownGaps": [ + "The exact journey is deterministic contract coverage, not a packaged Windows UI automation run.", + "Browser/mobile handoff is covered through transient routing state and preload contracts, not a live phone browser session." + ], + "demotionRule": "Demote or block release if any non-Advanced path mutates Active Server, if local reveal depends on the durable default instead of workspace ownership, or if transient host state survives restart." + }, { "id": "terminal-geometry.visible-convergence", "title": "Visible desktop terminals converge across xterm, fit, PTY, shell, and runtime mirror size", @@ -4608,16 +4869,18 @@ "macos" ], "coveredProviders": [], - "coverageNotes": "Local macOS evidence over the runtime-RPC stream budgets on main@1282f5c2d. PR #5824 adds a platform-neutral mobile decision gate proving chat-covered terminal streams pause and resume only after the mounted WebView is ready; live Android restore evidence remains required. The pending stack adds byte-exact 512KB/2MB/256KB/48KB budget assertions; legacy JSON subscribe parity remains undecided.", + "coverageNotes": "Local macOS evidence covers runtime-RPC stream budgets plus paired-renderer parse/discard credit. Deferred credit is shared by local and remote transports, batches ACKs at 192 KiB or 4 ms, grows per-stream windows from 512 KiB to 2 MiB and aggregate windows from 2 MiB to 8 MiB, bounds queued output to 256 KiB per stream, and caps each multiplex connection at 32 active or pending streams for an 8 MiB aggregate pending-output ceiling. Deterministic tests cover replay ordering, stale generations, malformed frames, hidden panes, queue eviction, disposal, send/recovery failure, repeated pending-slot replacement, reconnect, and round-robin fairness. The opt-in benchmark covers 1/20/100 ms RTT and 1/4/8 viewers, exact protocol-frame allocations, scheduler CPU, and measured @xterm/headless parser CPU/retained heap. Live Android restore evidence, browser/WebGL parser measurements, and legacy JSON subscribe parity remain required.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6951", "https://github.com/stablyai/orca/pull/6955", "https://github.com/stablyai/orca/pull/7009" ], - "invariant": "Runtime and mobile terminal subscriptions must cap initial snapshots, live output buffered while snapshots load, chunk sizes, and batch sizes; a terminal covered by native chat must have no live output subscription and must restore from fresh scrollback when revealed, while preserving output order, input locks, resize/driver events, and fallback parity or explicit fallback deprecation.", - "oracle": "The current executable slice asserts mobile initial snapshots downgrade until they fit <=512KB, requested binary snapshots downgrade until they fit <=2MB, binary live output queued while the initial snapshot loads stays <=256KB while preserving the newest tail, large binary output is split into <=48KB frames, output bursts are coalesced before emit, aborts do not register stale listeners, and stale mobile resize re-stream completions are dropped. The mobile native-chat decision test asserts an active stream pauses while covered and resumes only for a ready active terminal. JSON fallback parity and live Android scrollback restoration remain explicit gaps.", + "invariant": "Runtime and mobile terminal subscriptions must cap initial snapshots, live output buffered while snapshots load, chunk sizes, batches, and aggregate in-flight credit. ACK means the renderer parsed the bytes or intentionally discarded them; receipt-time ACK is forbidden. Every replay, stale-generation, malformed-frame, hidden-pane, eviction, disposal, error, and reconnect path must settle credit exactly once so streams neither leak memory nor stall. A terminal covered by native chat must restore from fresh scrollback when revealed, while preserving output order, input locks, resize/driver events, fairness, and fallback parity or explicit fallback deprecation.", + "oracle": "Assert mobile initial snapshots downgrade until they fit <=512KB, requested binary snapshots downgrade until they fit <=2MB, live output queued while snapshots load stays <=256KB per stream, large output splits into <=48KB frames, and output bursts coalesce. Feed paired output through the xterm parse callback and prove ACK is deferred until parse or intentional discard, then inject stale generation, malformed/transformed frames, replay failure, queue eviction, hidden panes, pane disposal, ACK send failure, recovery serialization failure, and reconnect races; assert ordered replay and exactly-once credit settlement. Fill the aggregate window across bulk and interactive streams, ACK once, and prove round-robin progress. Run the opt-in 64 MiB/viewer RTT matrix and enforce bounded 8 MiB aggregate in-flight memory, >7 MiB/s/viewer at 100 ms RTT, and <200 ms completion spread. JSON fallback parity and live Android scrollback restoration remain explicit gaps.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/main/runtime/rpc/terminal-output-batching.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/main/runtime/rpc/terminal-output-batching.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/terminal-pty-ack-gate.test.ts src/renderer/src/lib/pane-manager/terminal-delivery-credit.test.ts src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts src/renderer/src/runtime/runtime-terminal-stream.test.ts --maxWorkers=1", + "ORCA_TERMINAL_PERF_BENCH=1 pnpm exec vitest run --config config/vitest.config.ts --disableConsoleIntercept src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts", "pnpm --dir mobile exec vitest run --root .. mobile/src/session/mobile-native-chat-terminal-stream.test.ts", "pnpm --dir mobile exec vitest run --root .. mobile/src/session/use-mobile-native-chat-terminal-stream.test.ts" ], @@ -4625,6 +4888,12 @@ "src/main/runtime/rpc/terminal-subscribe-buffer.test.ts", "src/main/runtime/rpc/terminal-output-batching.test.ts", "src/main/runtime/rpc/terminal-multiplex.test.ts", + "src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "src/renderer/src/components/terminal-pane/terminal-pty-ack-gate.test.ts", + "src/renderer/src/lib/pane-manager/terminal-delivery-credit.test.ts", + "src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts", + "src/renderer/src/runtime/runtime-terminal-stream.test.ts", "mobile/src/session/mobile-native-chat-terminal-stream.test.ts", "mobile/src/session/use-mobile-native-chat-terminal-stream.test.ts" ], @@ -4649,7 +4918,33 @@ "assertions": [ "requested snapshots fall back smaller when serialized data exceeds the send budget", "oversized live output frames are bounded for subscribed binary streams", - "multibyte live output flushes when encoded bytes reach the batch budget" + "multibyte live output flushes when encoded bytes reach the batch budget", + "adaptive credit grows only after ACK, stays globally bounded, and drains pending streams round-robin", + "send and recovery serialization failures detach once instead of leaking credit or retrying forever", + "32 active or pending slots cap aggregate queued output and repeated pending-slot subscribe cancels its older waiter" + ] + }, + { + "file": "src/renderer/src/lib/pane-manager/terminal-delivery-credit.test.ts", + "assertions": [ + "nested synchronous deliveries restore the outer credit owner", + "unclaimed intentional discards settle automatically while every claimed scheduler child must settle before the parent credits" + ] + }, + { + "file": "src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts", + "assertions": [ + "paired renderer ACK waits for xterm parse completion or explicit discard", + "192 KiB parsed output batches into one ACK while the 4 ms timer releases interactive output", + "malformed frames, malformed transformed output, disposal, late parse, renderer delivery failure, and ACK transport failure release credit or close the owning stream without reordering output" + ] + }, + { + "file": "src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts", + "assertions": [ + "one through eight viewers stay within the 8 MiB aggregate adaptive window", + "the 100 ms RTT model sustains more than 7 MiB/s per viewer with less than 200 ms fairness spread", + "the opt-in benchmark reports RTT throughput, scheduler CPU time, exact protocol frame allocations, completion spread, and measured @xterm/headless parser CPU and retained heap" ] }, { @@ -4686,6 +4981,15 @@ "result": "passed", "durationSeconds": 0.2, "summary": "The focused mobile native-chat suite passed with 3 terminal-stream lifecycle assertions in the staged PR #5824 worktree." + }, + { + "date": "2026-07-22", + "runner": "local", + "platform": "macos", + "command": "ORCA_TERMINAL_PERF_BENCH=1 pnpm exec vitest run --config config/vitest.config.ts --disableConsoleIntercept src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts", + "result": "passed", + "durationSeconds": 0.91, + "summary": "The 1/20/100 ms RTT x 1/4/8 viewer matrix stayed at or below 8 MiB in flight with zero completion spread. At 100 ms it modeled 18.8 MiB/s per viewer for 1-4 viewers and 9.7 MiB/s for 8 viewers. Measured @xterm/headless parsing was 26.7/63.6/95.3 aggregate MiB/s for 1/4/8 viewers, with 84.4/236.5/336.0 ms CPU and 2893/13409/28991 KiB retained heap for 4 MiB per viewer." } ], "runtimeBudget": { @@ -4702,7 +5006,7 @@ }, "performanceBudget": { "required": true, - "evidence": "This gate is the byte and batching budget for runtime/mobile terminal streaming." + "evidence": "Parsed/discarded credit uses 192 KiB/4 ms ACK batching, 512 KiB-to-2 MiB adaptive per-stream windows, a 2 MiB-to-8 MiB aggregate window, <=48 KiB output frames, <=256 KiB queued output per stream, and <=32 streams per connection (8 MiB aggregate pending output). The 64 MiB/viewer model gate requires >7 MiB/s/viewer at 100 ms RTT, <200 ms completion spread, and aggregate in-flight bytes <=8 MiB. The 2026-07-22 run modeled 9.7 MiB/s/viewer at 100 ms with eight viewers and measured @xterm/headless at 95.3 aggregate MiB/s, 336.0 ms parser CPU, and 28991 KiB retained heap for eight 4 MiB viewers." }, "promotionCriteria": [ "Gate binary multiplex first.", @@ -4711,7 +5015,8 @@ ], "knownGaps": [ "The pure mobile decision gate does not yet prove live Android WebView scrollback restore after a chat toggle.", - "Legacy JSON subscribe parity is undecided." + "Legacy JSON subscribe parity is undecided.", + "The parser measurement uses @xterm/headless; browser renderer/WebGL CPU, GPU, and allocation behavior still need packaged-app performance evidence." ], "demotionRule": "Cannot promote while a supported stream path has uncapped snapshot or live-output buffering." }, diff --git a/config/scripts/computer-e2e-workflow.test.mjs b/config/scripts/computer-e2e-workflow.test.mjs index 3ef41583217..560859c5264 100644 --- a/config/scripts/computer-e2e-workflow.test.mjs +++ b/config/scripts/computer-e2e-workflow.test.mjs @@ -177,7 +177,7 @@ describe('computer-use e2e workflow', () => { ) }) - it('builds Electron main output before every computer-use e2e run', () => { + it('builds Electron main output and verifies the CLI before every computer-use e2e run', () => { const workflow = parse( readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') ) @@ -187,6 +187,7 @@ describe('computer-use e2e workflow', () => { .map((step) => step.run) .filter((run) => typeof run === 'string') const buildIndex = runs.indexOf('pnpm build:electron-vite') + const cliVerifyIndex = runs.indexOf('pnpm verify:cli-runtime') const e2eIndexes = runs .map((run, index) => (run.includes('test:e2e:computer') ? index : -1)) .filter((index) => index >= 0) @@ -195,10 +196,17 @@ describe('computer-use e2e workflow', () => { buildIndex, `${jobName} should build out/main before computer e2e` ).toBeGreaterThanOrEqual(0) + expect( + cliVerifyIndex, + `${jobName} should verify the CLI after the main build` + ).toBeGreaterThan(buildIndex) for (const e2eIndex of e2eIndexes) { expect(buildIndex, `${jobName} should build out/main before computer e2e`).toBeLessThan( e2eIndex ) + expect(cliVerifyIndex, `${jobName} should verify the CLI before computer e2e`).toBeLessThan( + e2eIndex + ) } } }) diff --git a/config/scripts/generate-windows-blockmap.mjs b/config/scripts/generate-windows-blockmap.mjs new file mode 100644 index 00000000000..7444a7b43dc --- /dev/null +++ b/config/scripts/generate-windows-blockmap.mjs @@ -0,0 +1,18 @@ +// Regenerate the electron-updater `.blockmap` for a (re-signed) Windows installer. +// electron-builder 26 dropped the app-builder-bin Go binary; blockmap generation +// now lives in app-builder-lib's pure-JS `buildBlockMap`. Mirrors createBlockmap's +// "gzip" format for the standalone NSIS installer blockmap. +import { createRequire } from 'node:module' + +const require = createRequire(import.meta.url) + +const [input, output] = process.argv.slice(2) +if (!input || !output) { + console.error('usage: generate-windows-blockmap.mjs ') + process.exit(1) +} + +const { buildBlockMap } = require('app-builder-lib/out/targets/blockmap/blockmap') + +const info = await buildBlockMap(input, 'gzip', output) +console.log(`blockmap written: ${output} (installer sha512=${info.sha512}, size=${info.size})`) diff --git a/config/scripts/verify-cli-bin.test.mjs b/config/scripts/verify-cli-bin.test.mjs index da988064d05..d26a603a148 100644 --- a/config/scripts/verify-cli-bin.test.mjs +++ b/config/scripts/verify-cli-bin.test.mjs @@ -47,6 +47,14 @@ describe('verifyPackageCliBin', () => { }) }) + it('rejects a compiled CLI with a missing runtime dependency', () => { + const { projectDir } = makeProjectWithCli( + '#!/usr/bin/env node\nrequire("./missing-runtime-dependency")\n' + ) + + expect(() => verifyPackageCliBin({ projectDir, runHelp: true })).toThrow() + }) + it('rejects an empty package bin target', () => { const { projectDir } = makeProjectWithCli('') diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index f8978764ccd..53859a16a32 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -3,6 +3,10 @@ "include": [ "../src/cli/**/*", "../src/shared/**/*", + "../src/main/agent-state-file-reader.ts", + "../src/main/agent-hooks/agent-hook-file-comparison.ts", + "../src/main/agent-hooks/agent-hook-file-limits.ts", + "../src/main/agent-hooks/agent-hook-sftp-text-reader.ts", "../src/main/agent-hooks/hook-stdin-contract.ts", "../src/main/agent-hooks/hook-config-write-path.ts", "../src/main/agent-hooks/hooks-json-read.ts", @@ -19,11 +23,12 @@ "../src/main/codex/codex-app-server-client.ts", "../src/main/codex/codex-app-server-grant-bridge.ts", "../src/main/codex/codex-app-server-grant-envelope.ts", + "../src/main/codex/codex-app-server-grant-json.ts", "../src/main/codex/codex-app-server-session.ts", "../src/main/codex/codex-config-mirror.ts", "../src/main/codex/codex-config-path-reference-rewrite.ts", - "../src/main/codex/codex-config-settings-upsert.ts", "../src/main/codex/codex-home-paths.ts", + "../src/main/codex/codex-host-retry-deadlines.ts", "../src/main/codex/codex-hook-identity.ts", "../src/main/codex/codex-hook-trust-grant.ts", "../src/main/codex/codex-managed-trust-reconciliation.ts", @@ -35,8 +40,8 @@ "../src/main/codex/codex-user-hook-trust-rebase-client.ts", "../src/main/codex/codex-user-hook-trust-rebase.ts", "../src/main/codex/codex-wsl-hook-install-plan.ts", + "../src/main/codex/codex-wsl-reconciliation-generations.ts", "../src/main/codex/config-settings-promotion.ts", - "../src/main/codex/config-toml-key-path.ts", "../src/main/codex/config-toml-line-scan.ts", "../src/main/codex/config-toml-trust.ts", "../src/main/codex/hook-service.ts", @@ -44,12 +49,14 @@ "../src/main/codex-accounts/fs-utils.ts", "../src/main/codex-accounts/wsl-codex-command.ts", "../src/main/codex-cli/command.ts", + "../src/main/codex-cli/nvm-version-directory-discovery.ts", "../src/main/command-code/command-code-managed-script.ts", "../src/main/command-code/hook-service.ts", "../src/main/copilot/hook-service.ts", "../src/main/cursor/hook-service.ts", "../src/main/droid/hook-service.ts", "../src/main/gemini/hook-service.ts", + "../src/main/generated-node-bounded-file-reader.ts", "../src/main/grok/hook-service.ts", "../src/main/devin/hook-settings.ts", "../src/main/devin/hook-service.ts", diff --git a/docs/assets/readme-downloads.svg b/docs/assets/readme-downloads.svg index 50ca9cde0e2..790c0df3df0 100644 --- a/docs/assets/readme-downloads.svg +++ b/docs/assets/readme-downloads.svg @@ -1,5 +1,5 @@ - - downloads: 7.6m + + downloads: 8.0m @@ -15,7 +15,7 @@ downloads downloads - 7.6m - 7.6m + 8.0m + 8.0m diff --git a/docs/assets/wechat-qr.jpg b/docs/assets/wechat-qr.jpg index ed695f996af..7b02a4d72c9 100644 Binary files a/docs/assets/wechat-qr.jpg and b/docs/assets/wechat-qr.jpg differ diff --git a/docs/readme/README.zh-CN.md b/docs/readme/README.zh-CN.md index 595ca2461d1..8121a1fe8da 100644 --- a/docs/readme/README.zh-CN.md +++ b/docs/readme/README.zh-CN.md @@ -235,7 +235,7 @@ yay -S stably-orca-bin - **Discord:** 加入 **[Discord](https://discord.gg/fzjDKHxv8Q)** 社区。 - **Twitter / X:** 关注 **[@orca_build](https://x.com/orca_build)** 获取更新和公告。 -- **微信:** 第一、二个微信群均已满,现在可以加入第三个群。 +- **微信:** 其他群已满,现在加入第 5 群。 Orca 社区微信群二维码 diff --git a/mobile/app.json b/mobile/app.json index bd307ea83c3..6f55551e927 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -18,7 +18,7 @@ "bundleIdentifier": "com.stably.orca.mobile", "buildNumber": "1", "infoPlist": { - "NSLocalNetworkUsageDescription": "Orca connects to the desktop app on your local network.", + "NSLocalNetworkUsageDescription": "Orca connects to the desktop app on your LAN.", "NSMicrophoneUsageDescription": "Allow Orca to record voice dictation and transcribe it on your paired desktop.", "NSPhotoLibraryUsageDescription": "Allow Orca to attach photos from your library to a terminal session on your paired desktop.", "NSAppTransportSecurity": { diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index 71dde856ff0..618545e3f7e 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -86,6 +86,10 @@ import { import { MOBILE_AI_VAULT_CAPABILITY } from '../../../../src/agent-history/agent-history-capability' import type { ConnectionState, RpcFailure, RpcSuccess } from '../../../../src/transport/types' import { headlessActivationNeedsHostRenderer } from '../../../../src/worktree/worktree-activation-result' +import { + LAST_VISITED_WORKTREE_STORAGE_KEY, + serializeLastVisitedWorktreeRecord +} from '../../../../src/worktree/last-visited-worktree-repo' import { useMobileDictation } from '../../../../src/hooks/use-mobile-dictation' import { triggerMediumImpact, @@ -175,13 +179,14 @@ import { } from '../../../../src/session/mobile-terminal-tab-agent' import type { MobileNewTabAgentOption } from '../../../../src/session/mobile-new-tab-agent-options' import { loadMobileNewTabAgentOptions } from '../../../../src/session/mobile-new-tab-agent-loader' -import { useMobileImageAttachment } from '../../../../src/session/use-mobile-image-attachment' +import { useMobileSessionImageAttachments } from '../../../../src/session/use-mobile-session-image-attachments' import { useMobileAttachmentInputLeaseGate } from '../../../../src/session/use-mobile-attachment-input-lease-gate' import { useMobileTerminalPaste } from '../../../../src/session/use-mobile-terminal-paste' import { useTerminalLiveInputModePreference } from '../../../../src/session/use-terminal-live-input-mode-preference' import { MobileTerminalLiveInputStatus } from '../../../../src/session/MobileTerminalLiveInputStatus' import { MobileTerminalInputActions } from '../../../../src/session/MobileTerminalInputActions' import { resolveMobileFileTabDoc } from '../../../../src/files/mobile-file-tab-doc' +import { captureMobileFileMutationOwnership } from '../../../../src/files/mobile-file-mutation-ownership' import { openMobileTerminalFileTap } from '../../../../src/session/mobile-terminal-file-tap-open' import { useLiveWorktreeName } from '../../../../src/session/use-live-worktree-name' import { @@ -208,6 +213,7 @@ import { useMobileNativeChatReadability } from '../../../../src/session/use-mobi import { useMobileNativeChatInputLease } from '../../../../src/session/use-mobile-native-chat-input-lease' import { getMobileTerminalActionSheetActions } from '../../../../src/session/mobile-terminal-action-sheet-actions' import * as nativeChatTerminalStream from '../../../../src/session/mobile-native-chat-terminal-stream' +import { mobileNativeChatScopeKey } from '../../../../src/session/mobile-native-chat-scope-key' import { useMobileNativeChatTerminalStream } from '../../../../src/session/use-mobile-native-chat-terminal-stream' import { subscribeMobileTerminalSafely } from '../../../../src/session/mobile-terminal-stream-subscribe' import { activateMobileSessionTab } from '../../../../src/session/mobile-session-tab-activation' @@ -225,6 +231,8 @@ import { TERMINAL_GESTURE_INPUT_REFILL_PER_SECOND, updateTerminalCwdFromStreamEvent } from '../../../../src/session/mobile-session-route-helpers' +import { MobileSessionFileDocLifecycle } from '../../../../src/session/mobile-session-file-doc-lifecycle' +import { MobileSessionMarkdownDocLifecycle } from '../../../../src/session/mobile-session-markdown-doc-lifecycle' import { resolveMarkdownFloatingActionsBottom } from '../../../../src/session/markdown-floating-actions-layout' import { resolveTabStripScrollOffset } from '../../../../src/session/tab-strip-scroll' import { activateOpenedSourceControlDiffTab } from '../../../../src/session/opened-mobile-session-tab' @@ -895,7 +903,9 @@ export default function SessionScreen() { const tabLayoutsRef = useRef>(new Map()) const [markdownDocs, setMarkdownDocs] = useState>(new Map()) const markdownDocsRef = useRef>(new Map()) + const markdownDocLifecycleRef = useRef(new MobileSessionMarkdownDocLifecycle()) const [fileDocs, setFileDocs] = useState>(new Map()) + const fileDocLifecycleRef = useRef(new MobileSessionFileDocLifecycle()) const [diffComments, setDiffComments] = useState([]) const diffCommentsRef = useRef([]) const [diffCommentBusy, setDiffCommentBusy] = useState(false) @@ -1719,6 +1729,8 @@ export default function SessionScreen() { if (orphanedDraftTabs.length > 0) { nextTabs = [...orphanedDraftTabs, ...nextTabs] } + markdownDocLifecycleRef.current.reconcile(nextTabs, setMarkdownDocs) + fileDocLifecycleRef.current.reconcile(nextTabs, setFileDocs) sessionTabsRef.current = nextTabs // Why: subscribe snapshots often repeat identical payloads; skip re-set to avoid a subscription teardown/replay loop. setSessionTabs((prev) => (mobileSessionTabsEqual(prev, nextTabs) ? prev : nextTabs)) @@ -1836,8 +1848,7 @@ export default function SessionScreen() { if (!client) { return } - setMarkdownDocs((prev) => new Map(prev).set(tab.id, { status: 'loading' })) - try { + await markdownDocLifecycleRef.current.load(tab, setMarkdownDocs, async () => { const response = await client.sendRequest('markdown.readTab', { worktree: `id:${worktreeId}`, tabId: tab.id @@ -1850,19 +1861,16 @@ export default function SessionScreen() { editable?: boolean readOnlyReason?: string } - setMarkdownDocs((prev) => - new Map(prev).set(tab.id, { - status: 'ready', - content: result.content, - localContent: result.content, - baseVersion: result.version, - isDirty: false, - editable: result.editable === true, - stale: result.isDirty, - readOnlyReason: result.readOnlyReason - }) - ) - return + return { + status: 'ready', + content: result.content, + localContent: result.content, + baseVersion: result.version, + isDirty: false, + editable: result.editable === true, + stale: result.isDirty, + readOnlyReason: result.readOnlyReason + } } if (!shouldReadMarkdownFromDiskAfterReadTabFailure(response as RpcFailure)) { throw new Error((response as RpcFailure).error.message) @@ -1880,24 +1888,12 @@ export default function SessionScreen() { truncated: boolean byteLength: number } - setMarkdownDocs((prev) => - new Map(prev).set( - tab.id, - buildMarkdownDiskFallbackDoc({ - content: fileResult.content, - truncated: fileResult.truncated, - tabIsDirty: tab.isDirty - }) - ) - ) - } catch { - setMarkdownDocs((prev) => - new Map(prev).set(tab.id, { - status: 'error', - message: "Couldn't load markdown" - }) - ) - } + return buildMarkdownDiskFallbackDoc({ + content: fileResult.content, + truncated: fileResult.truncated, + tabIsDirty: tab.isDirty + }) + }) }, [client, worktreeId] ) @@ -1907,31 +1903,13 @@ export default function SessionScreen() { if (!client) { return } - setFileDocs((prev) => new Map(prev).set(tab.id, { status: 'loading' })) - try { - const doc = await resolveMobileFileTabDoc(client, { + await fileDocLifecycleRef.current.load(tab, setFileDocs, () => + resolveMobileFileTabDoc(client, { worktreeId, relativePath: tab.relativePath, diffSource: tab.diffSource }) - setFileDocs((prev) => new Map(prev).set(tab.id, doc)) - } catch (err) { - const message = err instanceof Error ? err.message : '' - const previewMessage = - message === 'binary_file' - ? 'Binary preview unavailable' - : message === 'file_too_large' - ? 'File too large for mobile preview' - : tab.diffSource === 'staged' || tab.diffSource === 'unstaged' - ? "Couldn't load diff preview" - : "Couldn't load file preview" - setFileDocs((prev) => - new Map(prev).set(tab.id, { - status: 'error', - message: previewMessage - }) - ) - } + ) }, [client, worktreeId] ) @@ -2246,6 +2224,9 @@ export default function SessionScreen() { }) } finally { markdownSaveInFlightRef.current.delete(tab.id) + if (markdownSaveSeqRef.current.get(tab.id) === saveSeq) { + markdownSaveSeqRef.current.delete(tab.id) + } } }, [client, markdownDocs, showToast, worktreeId] @@ -2518,10 +2499,10 @@ export default function SessionScreen() { useEffect(() => { if (hostId && worktreeId) { - void AsyncStorage.setItem( - 'orca:last-visited-worktree', - JSON.stringify({ hostId, worktreeId }) - ) + const serialized = serializeLastVisitedWorktreeRecord({ hostId, worktreeId }) + if (serialized) { + void AsyncStorage.setItem(LAST_VISITED_WORKTREE_STORAGE_KEY, serialized) + } } }, [hostId, worktreeId]) @@ -2555,6 +2536,10 @@ export default function SessionScreen() { terminalDiagnosticsRef.current.resetRoute() appliedSnapshotMarkerRef.current = { epoch: null, version: -1 } closedTabTombstonesRef.current.clear() + markdownDocLifecycleRef.current.reset() + fileDocLifecycleRef.current.reset() + markdownSaveSeqRef.current.clear() + markdownSaveInFlightRef.current.clear() for (const queued of terminalGestureInputQueuesRef.current.values()) { if (queued.timer) { clearTimeout(queued.timer) @@ -2574,6 +2559,10 @@ export default function SessionScreen() { return () => { sessionTabActionSheetRequestSeqRef.current += 1 sessionTabActionSheetKeyboardHideSubRef.current?.remove() + markdownDocLifecycleRef.current.reset() + fileDocLifecycleRef.current.reset() + markdownSaveSeqRef.current.clear() + markdownSaveInFlightRef.current.clear() clearPendingLiveInputCommit() clearDelayedActionTimers() } @@ -3639,14 +3628,20 @@ export default function SessionScreen() { showToast }) - const { attachImage, isAttaching } = useMobileImageAttachment({ + // Terminal input pastes an attached image straight into the visible terminal; + // native chat instead holds it as a composer chip and rides it along on submit. + const { attachImage, isAttaching, nativeChatImages } = useMobileSessionImageAttachments({ client, activeHandle, + activeHandleRef, canSend, connState, deviceTokenRef, - beforeTerminalSend: flushPendingLiveInputBeforeAttachmentSend, + nativeChatScopeKey: mobileNativeChatScopeKey(hostId, worktreeId, activeSessionTabId), + nativeChatInputLeaseReady, getActiveWorktreeConnectionId, + beforeTerminalSend: flushPendingLiveInputBeforeAttachmentSend, + nativeChatBaseSend: nativeChatController.handleNativeChatSend, showToast, onSuccess: triggerSelection, onError: triggerError @@ -3901,11 +3896,12 @@ export default function SessionScreen() { try { const worktree = `id:${worktreeId}` + const mutationOwnership = await captureMobileFileMutationOwnership(client, worktree) for (let attempt = 1; attempt <= 100; attempt += 1) { const relativePath = attempt === 1 ? 'untitled.md' : `untitled-${attempt}.md` const createResponse = await client.sendRequest( 'files.createFile', - { worktree, relativePath }, + { worktree, relativePath, ...mutationOwnership }, { timeoutMs: 15_000 } ) if (!createResponse.ok) { @@ -4100,7 +4096,23 @@ export default function SessionScreen() { initializedHandlesRef.current.delete(terminalHandle) clearTerminalLiveInputDefault(terminalHandle) } - setSessionTabs((prev) => prev.filter((candidate) => candidate.id !== tab.id)) + if (tab.type === 'file') { + fileDocLifecycleRef.current.close(tab.id, setFileDocs) + } + if (tab.type === 'markdown') { + markdownDocLifecycleRef.current.close(tab.id, (update) => { + setMarkdownDocs((current) => { + const next = update(current) + markdownDocsRef.current = next + return next + }) + }) + markdownSaveSeqRef.current.delete(tab.id) + markdownSaveInFlightRef.current.delete(tab.id) + } + const remainingTabs = sessionTabsRef.current.filter((candidate) => candidate.id !== tab.id) + sessionTabsRef.current = remainingTabs + setSessionTabs(remainingTabs) // Why: tombstone the closed tab and rely on the snapshot, not a blind refetch that often re-added the not-yet-closed tab. closedTabTombstonesRef.current.set(tab.id, Date.now() + 10_000) if (activeSessionTabId === tab.id) { @@ -4379,6 +4391,7 @@ export default function SessionScreen() { hostedChecksSupported: prIsGithubRepo }) const showHeaderMoreButton = showAgentSessionHistoryAction || showChecksAction + const createTabBusy = creating || creatingBrowser || creatingMarkdown return ( @@ -4581,24 +4594,16 @@ export default function SessionScreen() { { setCreateError('') setShowCreateTabDrawer(true) }} > - {creating || creatingBrowser || creatingMarkdown - ? 'Creating...' - : 'Create Tab'} + {createTabBusy ? 'Creating...' : 'Create Tab'} @@ -4715,8 +4720,7 @@ export default function SessionScreen() { ))} void attachImage('library')} - isAttaching={isAttaching} + images={nativeChatImages} onMicPress={handleDictationToggle} micActive={dictation.isRecording} dictationMode={dictationMode} diff --git a/mobile/app/h/[hostId]/tasks.tsx b/mobile/app/h/[hostId]/tasks.tsx index a9d24ae637a..b25a40682ca 100644 --- a/mobile/app/h/[hostId]/tasks.tsx +++ b/mobile/app/h/[hostId]/tasks.tsx @@ -137,9 +137,15 @@ import { clearMobileTaskCopyFeedbackTimer, scheduleMobileTaskCopyFeedbackReset } from '../../../src/tasks/mobile-task-copy-feedback-timer' +import { + createMobileItemPrFileContentScope, + createMobileProjectPrFileContentScope, + useMobilePrFileContentCache +} from '../../../src/tasks/use-mobile-pr-file-content-cache' import type { BaseRefSearchResult, GitHubOwnerRepo, + GitHubPRFileContents, PersistedTrustedOrcaHooks, SparsePreset, TuiAgent @@ -358,13 +364,6 @@ type GitHubDetailCheck = { url?: string | null } -type GitHubPRFileContents = { - original: string - modified: string - originalIsBinary: boolean - modifiedIsBinary: boolean -} - type DetailPayload = | { provider: 'github' @@ -2263,8 +2262,6 @@ export default function MobileTasksScreen() { const [itemReviewersDraft, setItemReviewersDraft] = useState('') const [itemReplyDrafts, setItemReplyDrafts] = useState>({}) const [expandedPrFilePath, setExpandedPrFilePath] = useState(null) - const [prFileContents, setPrFileContents] = useState>({}) - const [prFileLoadingPath, setPrFileLoadingPath] = useState(null) const [prFileCommentDrafts, setPrFileCommentDrafts] = useState>({}) const [copiedLinkKey, setCopiedLinkKey] = useState(null) const copiedLinkResetTimerRef = useRef | null>(null) @@ -2602,6 +2599,23 @@ export default function MobileTasksScreen() { () => (projectRowItem ? findProjectRowRepo(projectRowItem) : null), [findProjectRowRepo, projectRowItem] ) + const itemPrFileContentScope = createMobileItemPrFileContentScope(actionItem, detailPayload) + const projectPrFileContentScope = createMobileProjectPrFileContentScope( + projectRowItem, + projectRowHostedRepo, + projectRowDetail, + projectRowItem ? projectRowGitHubRepository(projectRowItem, activeGitHubProjectHost) : null + ) + const activePrFileContentScope = projectRowItem + ? projectPrFileContentScope + : itemPrFileContentScope + const { + clear: clearPrFileContents, + contents: prFileContents, + load: loadPrFileContent, + loadingPath: prFileLoadingPath + } = useMobilePrFileContentCache(activePrFileContentScope) + const itemReviewerCandidates = useMemo(() => { if (!actionItem || actionItem.provider !== 'github' || actionItem.source.type !== 'pr') { return [] @@ -4141,8 +4155,7 @@ export default function MobileTasksScreen() { setItemReviewersDraft('') setItemReplyDrafts({}) setExpandedPrFilePath(null) - setPrFileContents({}) - setPrFileLoadingPath(null) + clearPrFileContents() setPrFileCommentDrafts({}) setExpandedResolvedCommentGroups(new Set()) return @@ -4157,11 +4170,10 @@ export default function MobileTasksScreen() { setItemReviewersDraft('') setItemReplyDrafts({}) setExpandedPrFilePath(null) - setPrFileContents({}) - setPrFileLoadingPath(null) + clearPrFileContents() setPrFileCommentDrafts({}) setExpandedResolvedCommentGroups(new Set()) - }, [actionItem]) + }, [actionItem, clearPrFileContents]) useEffect(() => { if (!detailPayload) { @@ -4458,8 +4470,7 @@ export default function MobileTasksScreen() { setProjectEditingCommentDraft('') setProjectReviewersDraft('') setExpandedPrFilePath(null) - setPrFileContents({}) - setPrFileLoadingPath(null) + clearPrFileContents() setPrFileCommentDrafts({}) setProjectFieldDrafts({}) return @@ -4474,8 +4485,7 @@ export default function MobileTasksScreen() { setProjectEditingCommentDraft('') setProjectReviewersDraft('') setExpandedPrFilePath(null) - setPrFileContents({}) - setPrFileLoadingPath(null) + clearPrFileContents() setPrFileCommentDrafts({}) setProjectFieldDrafts( Object.fromEntries( @@ -4579,6 +4589,7 @@ export default function MobileTasksScreen() { } }, [ activeGitHubProjectHost, + clearPrFileContents, client, githubProjectTable, projectRowDetailRefreshSeq, @@ -6549,9 +6560,6 @@ export default function MobileTasksScreen() { return } setExpandedPrFilePath(file.path) - if (prFileContents[file.path]) { - return - } const repo = findProjectRowRepo(row) if ( !client || @@ -6560,49 +6568,45 @@ export default function MobileTasksScreen() { !row.content.number || projectRowDetail?.provider !== 'github' || !projectRowDetail.headSha || - !projectRowDetail.baseSha + !projectRowDetail.baseSha || + !projectPrFileContentScope ) { setProjectRowDetailError('Unable to load file contents for this pull request.') return } - setPrFileLoadingPath(file.path) - setProjectRowDetailError('') - try { - const response = await client.sendRequest( - 'github.prFileContents', - { - repo: `id:${repo.id}`, - prNumber: row.content.number, - prRepo: projectRowGitHubRepository(row, activeGitHubProjectHost), - path: file.path, - oldPath: file.oldPath, - status: file.status ?? 'modified', - headSha: projectRowDetail.headSha, - baseSha: projectRowDetail.baseSha - }, - { timeoutMs: 30_000 } - ) - if (!isSuccess(response)) { - throw new Error(response.error.message) - } - setPrFileContents((current) => ({ - ...current, - [file.path]: response.result as GitHubPRFileContents - })) - } catch (err) { - setProjectRowDetailError( - err instanceof Error ? err.message : 'Failed to load file contents' - ) - } finally { - setPrFileLoadingPath(null) - } + await loadPrFileContent( + projectPrFileContentScope, + file, + async () => { + const response = await client.sendRequest( + 'github.prFileContents', + { + repo: `id:${repo.id}`, + prNumber: row.content.number, + prRepo: projectRowGitHubRepository(row, activeGitHubProjectHost), + path: file.path, + oldPath: file.oldPath, + status: file.status ?? 'modified', + headSha: projectRowDetail.headSha, + baseSha: projectRowDetail.baseSha + }, + { timeoutMs: 30_000 } + ) + if (!isSuccess(response)) { + throw new Error(response.error.message) + } + return response.result + }, + setProjectRowDetailError + ) }, [ activeGitHubProjectHost, client, expandedPrFilePath, findProjectRowRepo, - prFileContents, + loadPrFileContent, + projectPrFileContentScope, projectRowDetail ] ) @@ -7526,49 +7530,43 @@ export default function MobileTasksScreen() { return } setExpandedPrFilePath(file.path) - if (prFileContents[file.path]) { - return - } if ( !client || item.source.type !== 'pr' || detailPayload?.provider !== 'github' || !detailPayload.headSha || - !detailPayload.baseSha + !detailPayload.baseSha || + !itemPrFileContentScope ) { setError('Unable to load file contents for this pull request.') return } - setPrFileLoadingPath(file.path) - setError('') - try { - const response = await client.sendRequest( - 'github.prFileContents', - { - repo: `id:${item.source.repoId}`, - prNumber: item.source.number, - path: file.path, - oldPath: file.oldPath, - status: file.status ?? 'modified', - headSha: detailPayload.headSha, - baseSha: detailPayload.baseSha - }, - { timeoutMs: 30_000 } - ) - if (!isSuccess(response)) { - throw new Error(response.error.message) - } - setPrFileContents((current) => ({ - ...current, - [file.path]: response.result as GitHubPRFileContents - })) - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to load file contents') - } finally { - setPrFileLoadingPath(null) - } + await loadPrFileContent( + itemPrFileContentScope, + file, + async () => { + const response = await client.sendRequest( + 'github.prFileContents', + { + repo: `id:${item.source.repoId}`, + prNumber: item.source.number, + path: file.path, + oldPath: file.oldPath, + status: file.status ?? 'modified', + headSha: detailPayload.headSha, + baseSha: detailPayload.baseSha + }, + { timeoutMs: 30_000 } + ) + if (!isSuccess(response)) { + throw new Error(response.error.message) + } + return response.result + }, + setError + ) }, - [client, detailPayload, expandedPrFilePath, prFileContents] + [client, detailPayload, expandedPrFilePath, itemPrFileContentScope, loadPrFileContent] ) const addGitHubFileReviewComment = useCallback( @@ -12322,7 +12320,7 @@ export default function MobileTasksScreen() { ) : prFileContents[file.path] ? ( @@ -13268,7 +13266,7 @@ export default function MobileTasksScreen() { ) : prFileContents[file.path] ? ( diff --git a/mobile/app/index.tsx b/mobile/app/index.tsx index b7132849845..6110a5a3c09 100644 --- a/mobile/app/index.tsx +++ b/mobile/app/index.tsx @@ -27,6 +27,10 @@ import AsyncStorage from '@react-native-async-storage/async-storage' import { loadHosts } from '../src/transport/host-store' import { removeHostAndCloseClient } from '../src/transport/host-removal-lifecycle' import { pickResumeWorktree } from '../src/worktree/resume-worktree' +import { + LAST_VISITED_WORKTREE_STORAGE_KEY, + readLastVisitedWorktreeRecord +} from '../src/worktree/last-visited-worktree-repo' import type { RpcClient } from '../src/transport/rpc-client' import { sendSingleFlightRequest } from '../src/transport/request-single-flight' import { @@ -394,13 +398,14 @@ export default function HomeScreen() { router.replace(mobileOnboardingDestination(onboardingSteps)) } }) - void AsyncStorage.getItem('orca:last-visited-worktree').then((raw) => { + void AsyncStorage.getItem(LAST_VISITED_WORKTREE_STORAGE_KEY).then((raw) => { if (stale || !raw) { return } - try { - setLastVisited(JSON.parse(raw)) - } catch {} + const record = readLastVisitedWorktreeRecord(raw) + if (record) { + setLastVisited(record) + } }) for (const entry of allClientsRef.current) { if (entry.client.getState() === 'connected') { diff --git a/mobile/fastlane/Fastfile b/mobile/fastlane/Fastfile index a8ccf9b8c49..3ceb91c0d97 100644 --- a/mobile/fastlane/Fastfile +++ b/mobile/fastlane/Fastfile @@ -16,6 +16,7 @@ require "base64" require "json" +require_relative "ios_release_version" default_platform(:ios) @@ -66,25 +67,6 @@ def current_mobile_version(config) config.fetch("expo").fetch("version") end -def truthy_option?(value) - %w[1 true yes on].include?(value.to_s.strip.downcase) -end - -def bump_patch_version(version) - match = version.match(/\A(\d+)\.(\d+)\.(\d+)\z/) - UI.user_error!("Cannot bump non-semver mobile version '#{version}'") unless match - - "#{match[1]}.#{match[2]}.#{match[3].to_i + 1}" -end - -def resolve_requested_version(options, config) - requested = options[:version].to_s.strip - return requested unless requested.empty? - - current_version = current_mobile_version(config) - truthy_option?(options[:bump_patch]) ? bump_patch_version(current_version) : current_version -end - # Returns true when `version`'s App Store train is closed to new build uploads. # `app` is a Spaceship::ConnectAPI::App the caller looks up (nil if lookup # failed). On a nil app or any API error, degrade to "open": we then proceed as @@ -113,13 +95,7 @@ platform :ios do lane :prepare_release_version do |options| api_key = app_store_connect_api_key_from_env config = load_mobile_app_config - version = resolve_requested_version(options, config) - # Fail fast (seconds) if the resolved version's App Store train is already - # closed: Apple would otherwise reject the upload ~20 min later with 90186. - # Bumping the version is left to the human (the bump_patch input or a - # "Prepare mobile X" commit) so the marketing version stays a deliberate, - # release-notes-bearing decision rather than something CI invents. app = begin Spaceship::ConnectAPI::App.find(BUNDLE_ID) @@ -127,11 +103,31 @@ platform :ios do UI.error("Could not look up App Store app #{BUNDLE_ID} (#{error.message}); skipping closed-train check.") nil end + version = + begin + IosReleaseVersion.resolve( + requested: options[:version], + bump_patch: options[:bump_patch], + current_version: current_mobile_version(config), + train_closed: ->(candidate) { version_train_closed?(app, candidate) }, + ) + rescue ArgumentError => error + UI.user_error!(error.message) + end + + # Explicit and checked-in versions still fail fast when closed. Patch bumps + # skip closed trains above because workflow-only releases can outpace Git. if version_train_closed?(app, version) + retry_guidance = + if IosReleaseVersion.truthy?(options[:bump_patch]) + "Use a higher release_version or land a \"Prepare mobile \" commit." + else + "Re-dispatch with bump_patch_version: true (or a higher release_version), " \ + "or land a \"Prepare mobile \" commit." + end UI.user_error!( "iOS version #{version} is already submitted/released on the App Store and cannot accept " \ - "new builds. Re-dispatch with bump_patch_version: true (or a higher release_version), " \ - "or land a \"Prepare mobile \" commit.", + "new builds. #{retry_guidance}", ) end diff --git a/mobile/fastlane/ios_release_version.rb b/mobile/fastlane/ios_release_version.rb new file mode 100644 index 00000000000..2b4fda40e06 --- /dev/null +++ b/mobile/fastlane/ios_release_version.rb @@ -0,0 +1,24 @@ +module IosReleaseVersion + module_function + + def truthy?(value) + %w[1 true yes on].include?(value.to_s.strip.downcase) + end + + def bump_patch(version) + match = version.match(/\A(\d+)\.(\d+)\.(\d+)\z/) + raise ArgumentError, "Cannot bump non-semver mobile version '#{version}'" unless match + + "#{match[1]}.#{match[2]}.#{match[3].to_i + 1}" + end + + def resolve(requested:, bump_patch:, current_version:, train_closed:) + exact_version = requested.to_s.strip + return exact_version unless exact_version.empty? + return current_version unless truthy?(bump_patch) + + candidate = bump_patch(current_version) + candidate = bump_patch(candidate) while train_closed.call(candidate) + candidate + end +end diff --git a/mobile/fastlane/ios_release_version_test.rb b/mobile/fastlane/ios_release_version_test.rb new file mode 100644 index 00000000000..795e6748275 --- /dev/null +++ b/mobile/fastlane/ios_release_version_test.rb @@ -0,0 +1,48 @@ +require "minitest/autorun" +require_relative "ios_release_version" + +class IosReleaseVersionTest < Minitest::Test + def test_exact_version_wins_without_checking_trains + checked_versions = [] + + version = IosReleaseVersion.resolve( + requested: " 0.0.40 ", + bump_patch: true, + current_version: "0.0.32", + train_closed: ->(candidate) { checked_versions << candidate }, + ) + + assert_equal("0.0.40", version) + assert_empty(checked_versions) + end + + def test_uses_current_version_without_a_patch_bump + version = IosReleaseVersion.resolve( + requested: "", + bump_patch: false, + current_version: "0.0.32", + train_closed: ->(_) { flunk("should not check trains") }, + ) + + assert_equal("0.0.32", version) + end + + def test_skips_closed_patch_versions_from_a_stale_repo_version + closed_versions = %w[0.0.33 0.0.34] + + version = IosReleaseVersion.resolve( + requested: "", + bump_patch: true, + current_version: "0.0.32", + train_closed: ->(candidate) { closed_versions.include?(candidate) }, + ) + + assert_equal("0.0.35", version) + end + + def test_rejects_non_semver_versions + error = assert_raises(ArgumentError) { IosReleaseVersion.bump_patch("0.0") } + + assert_equal("Cannot bump non-semver mobile version '0.0'", error.message) + end +end diff --git a/mobile/scripts/bounded-process-line-reader.mjs b/mobile/scripts/bounded-process-line-reader.mjs new file mode 100644 index 00000000000..a209f03eb0a --- /dev/null +++ b/mobile/scripts/bounded-process-line-reader.mjs @@ -0,0 +1,103 @@ +import { Buffer } from 'node:buffer' + +export const PROCESS_LINE_MAX_BYTES = 64 * 1024 +export const PROCESS_OUTPUT_TAIL_MAX_CODE_UNITS = 64 * 1024 +const TRUNCATED_LINE_SUFFIX = '… [line truncated]' + +export function appendProcessOutputTail( + current, + line, + maxCodeUnits = PROCESS_OUTPUT_TAIL_MAX_CODE_UNITS +) { + if (!Number.isSafeInteger(maxCodeUnits) || maxCodeUnits < 0) { + throw new RangeError('Process output tail limit must be a non-negative safe integer') + } + if (maxCodeUnits === 0) { + return '' + } + const appended = `${current}${line}\n` + return appended.length <= maxCodeUnits ? appended : appended.slice(-maxCodeUnits) +} + +export function attachBoundedProcessLineReader( + stream, + onLine, + maxLineBytes = PROCESS_LINE_MAX_BYTES +) { + if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 0) { + throw new RangeError('Process line limit must be a non-negative safe integer') + } + + const retained = Buffer.allocUnsafe(maxLineBytes) + let retainedBytes = 0 + let truncated = false + let swallowLineFeed = false + let closed = false + + const append = (bytes, start, end) => { + if (start >= end) { + return + } + const available = maxLineBytes - retainedBytes + const copied = Math.min(available, end - start) + if (copied > 0) { + bytes.copy(retained, retainedBytes, start, start + copied) + retainedBytes += copied + } + truncated ||= copied < end - start + } + + const emit = () => { + const line = retained.subarray(0, retainedBytes).toString('utf8') + onLine(truncated ? `${line}${TRUNCATED_LINE_SUFFIX}` : line) + retainedBytes = 0 + truncated = false + } + + const onData = (chunk) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + let start = 0 + if (swallowLineFeed) { + swallowLineFeed = false + if (bytes[0] === 0x0a) { + start = 1 + } + } + + for (let index = start; index < bytes.length; index += 1) { + const value = bytes[index] + if (value !== 0x0a && value !== 0x0d) { + continue + } + append(bytes, start, index) + emit() + if (value === 0x0d && index + 1 < bytes.length && bytes[index + 1] === 0x0a) { + index += 1 + } else if (value === 0x0d && index + 1 === bytes.length) { + swallowLineFeed = true + } + start = index + 1 + } + append(bytes, start, bytes.length) + } + + const detach = () => { + if (closed) { + return + } + closed = true + stream.off('data', onData) + stream.off('end', onEnd) + } + + const onEnd = () => { + if (retainedBytes > 0 || truncated) { + emit() + } + detach() + } + + stream.on('data', onData) + stream.on('end', onEnd) + return detach +} diff --git a/mobile/scripts/bounded-response-body.mjs b/mobile/scripts/bounded-response-body.mjs new file mode 100644 index 00000000000..cef6099d2bb --- /dev/null +++ b/mobile/scripts/bounded-response-body.mjs @@ -0,0 +1,36 @@ +export async function responseBodyIncludesWithinLimit(response, needle, maxBytes) { + const declaredLength = response.headers.get('content-length') + if (declaredLength && /^\d+$/.test(declaredLength) && Number(declaredLength) > maxBytes) { + await response.body?.cancel().catch(() => undefined) + return false + } + if (!response.body) { + return false + } + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let observedBytes = 0 + let suffix = '' + try { + while (true) { + const { done, value } = await reader.read() + if (done) { + return `${suffix}${decoder.decode()}`.includes(needle) + } + observedBytes += value.byteLength + if (observedBytes > maxBytes) { + await reader.cancel().catch(() => undefined) + return false + } + const candidate = suffix + decoder.decode(value, { stream: true }) + if (candidate.includes(needle)) { + await reader.cancel().catch(() => undefined) + return true + } + suffix = candidate.slice(-Math.max(0, needle.length - 1)) + } + } finally { + reader.releaseLock() + } +} diff --git a/mobile/scripts/start-emulator-pairing-runtime.mjs b/mobile/scripts/start-emulator-pairing-runtime.mjs index 02078825db4..3e23c54093b 100644 --- a/mobile/scripts/start-emulator-pairing-runtime.mjs +++ b/mobile/scripts/start-emulator-pairing-runtime.mjs @@ -3,7 +3,10 @@ import { mkdirSync, mkdtempSync } from 'node:fs' import os from 'node:os' import path from 'node:path' import process from 'node:process' -import readline from 'node:readline' +import { + appendProcessOutputTail, + attachBoundedProcessLineReader +} from './bounded-process-line-reader.mjs' function primaryLanIp(lanIpCandidates) { return lanIpCandidates()[0] || '127.0.0.1' @@ -66,15 +69,15 @@ async function waitForPairingRuntime({ child, userData, pairingAddress, logSucce let stderr = '' let resolved = false let exited = false - let rl = null - let rlErr = null + let closeStdout = () => {} + let closeStderr = () => {} const stop = () => { if (!exited) { child.kill('SIGTERM') } - rl?.close() - rlErr?.close() + closeStdout() + closeStderr() child.stdout?.destroy() child.stderr?.destroy() } @@ -119,15 +122,17 @@ async function waitForPairingRuntime({ child, userData, pairingAddress, logSucce reject(error) } - rl = readline.createInterface({ input: child.stdout }) - rl.on('line', (line) => { - output += line + '\n' + closeStdout = attachBoundedProcessLineReader(child.stdout, (line) => { + if (!resolved) { + output = appendProcessOutputTail(output, line) + } handleRuntimeLine(line, finishResolve) }) - rlErr = readline.createInterface({ input: child.stderr }) - rlErr.on('line', (line) => { - stderr += line + '\n' + closeStderr = attachBoundedProcessLineReader(child.stderr, (line) => { + if (!resolved) { + stderr = appendProcessOutputTail(stderr, line) + } }) child.on('error', (error) => { diff --git a/mobile/scripts/start-emulator.mjs b/mobile/scripts/start-emulator.mjs index d6489b530ee..8c5e11f944c 100755 --- a/mobile/scripts/start-emulator.mjs +++ b/mobile/scripts/start-emulator.mjs @@ -22,16 +22,21 @@ import os from 'node:os' import { promisify } from 'node:util' import path from 'node:path' import process from 'node:process' -import readline from 'node:readline' import { registerWorktreeForPairingRuntime, startHeadlessPairingRuntime } from './start-emulator-pairing-runtime.mjs' +import { + appendProcessOutputTail, + attachBoundedProcessLineReader +} from './bounded-process-line-reader.mjs' +import { responseBodyIncludesWithinLimit } from './bounded-response-body.mjs' import { ensureMobileExpoCli, getMobileExpoExecutablePath } from './mobile-expo-cli.mjs' const execFileAsync = promisify(execFile) const DEFAULT_METRO_PORT = 8081 const METRO_PORT_SEARCH_LIMIT = 100 +const METRO_STATUS_MAX_BYTES = 64 * 1024 // Parse CLI arguments const args = process.argv.slice(2) @@ -113,6 +118,22 @@ function logInfo(message) { log(`[info] ${message}`, 'yellow') } +function createBackpressuredLineWriter(source, target, color) { + let waitingForDrain = false + return (line) => { + const accepted = target.write(color + line + colors.reset + '\n') + if (accepted || waitingForDrain) { + return + } + waitingForDrain = true + source.pause() + target.once('drain', () => { + waitingForDrain = false + source.resume() + }) + } +} + function assertIosSimulatorPlatform() { if (process.platform !== 'darwin') { throw new Error('iOS Simulator automation requires macOS and Xcode.') @@ -382,8 +403,10 @@ async function startMetro(worktree) { let url = null let resolved = false let exited = false - let rl = null - let rlErr = null + let closeStdout = () => {} + let closeStderr = () => {} + const writeStdoutLine = createBackpressuredLineWriter(metro.stdout, process.stdout, colors.dim) + const writeStderrLine = createBackpressuredLineWriter(metro.stderr, process.stderr, colors.red) const metroResult = () => ({ process: metro, @@ -391,8 +414,8 @@ async function startMetro(worktree) { output, isExited: () => exited, closeOutput: () => { - rl?.close() - rlErr?.close() + closeStdout() + closeStderr() metro.stdin?.destroy() metro.stdout?.destroy() metro.stderr?.destroy() @@ -400,10 +423,11 @@ async function startMetro(worktree) { }) // Parse Metro output for the development URL - rl = readline.createInterface({ input: metro.stdout }) - rl.on('line', (line) => { - output += line + '\n' - process.stdout.write(colors.dim + line + colors.reset + '\n') + closeStdout = attachBoundedProcessLineReader(metro.stdout, (line) => { + if (!resolved) { + output = appendProcessOutputTail(output, line) + } + writeStdoutLine(line) // Look for "Waiting on" message from Metro // When Metro says "Waiting on http://localhost:8081", we need to construct the dev-client URL @@ -446,10 +470,11 @@ async function startMetro(worktree) { }) // Also check stderr - rlErr = readline.createInterface({ input: metro.stderr }) - rlErr.on('line', (line) => { - output += line + '\n' - process.stderr.write(colors.red + line + colors.reset + '\n') + closeStderr = attachBoundedProcessLineReader(metro.stderr, (line) => { + if (!resolved) { + output = appendProcessOutputTail(output, line) + } + writeStderrLine(line) }) metro.on('error', (error) => { @@ -547,7 +572,11 @@ async function verifyMetro(url) { try { const response = await fetch(statusUrl, { signal: controller.signal }) - return (await response.text()).includes('packager-status:running') + return await responseBodyIncludesWithinLimit( + response, + 'packager-status:running', + METRO_STATUS_MAX_BYTES + ) } catch { return false } finally { diff --git a/mobile/src/browser/MobileBrowserPane.tsx b/mobile/src/browser/MobileBrowserPane.tsx index 683b972bfa8..bc516a4787a 100644 --- a/mobile/src/browser/MobileBrowserPane.tsx +++ b/mobile/src/browser/MobileBrowserPane.tsx @@ -1,7 +1,4 @@ /* oxlint-disable react-doctor/no-adjust-state-on-prop-change -- Why: mobile browser state mirrors a remote desktop screencast session and CDP dialogs, which are external systems that cannot be derived during render. */ -// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' — Metro -// can't resolve Node's builtin in a React Native bundle. -import { Buffer } from 'buffer' import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { ActivityIndicator, @@ -42,6 +39,10 @@ import { getInitialMobileBrowserViewMode, saveMobileBrowserViewMode } from './mobile-browser-view-mode-state' +import { + createMobileBrowserFrameDataUri, + MobileBrowserFrameCache +} from './mobile-browser-frame-cache' import { clampBrowserZoomState, computeBrowserFrameGeometry, @@ -108,14 +109,7 @@ const TOUCH_CLICK_RADIUS_DIP = 14 const MIN_ZOOM = 1 const MAX_ZOOM = 3.5 const DEFAULT_ZOOM: BrowserZoomState = { scale: 1, offsetX: 0, offsetY: 0 } -const BROWSER_FRAME_CACHE_LIMIT = 4 - -type BrowserFrameCacheEntry = { - uri: string - metadata: BrowserScreencastFrameMetadata -} - -const browserFrameCache = new Map() +const browserFrameCache = new MobileBrowserFrameCache() type BrowserPageParams = { worktree: string @@ -143,7 +137,7 @@ export function MobileBrowserPane({ getInitialMobileBrowserViewMode(worktreeId, tab.browserPageId) ) const cacheKey = makeBrowserFrameCacheKey(worktreeId, tab.browserPageId, browserViewMode) - const cachedInitialFrame = peekCachedBrowserFrame(cacheKey) + const cachedInitialFrame = browserFrameCache.peek(cacheKey) const [addressValue, setAddressValue] = useState(displayBrowserUrl(tab.url)) const [addressFocused, setAddressFocused] = useState(false) const [addressSyncState, setAddressSyncState] = useState({ @@ -229,7 +223,7 @@ export function MobileBrowserPane({ const subscription = AppState.addEventListener('change', (nextState) => { const active = nextState === 'active' if (!active) { - clearCachedBrowserFramesForWorktree(worktreeId) + browserFrameCache.clearWorktree(worktreeId) } setAppActive(active) }) @@ -280,8 +274,14 @@ export function MobileBrowserPane({ frameMetadataRef.current = frame.metadata setFrameMetadata(frame.metadata) } - const nextFrameUri = createBrowserFrameDataUri(frame) - cacheBrowserFrame(frameCacheKey, { uri: nextFrameUri, metadata: frame.metadata }) + const nextFrameUri = createMobileBrowserFrameDataUri(frame) + if (!nextFrameUri) { + busyRef.current = false + setBusy(false) + setError('Browser frame is too large to display safely.') + return + } + browserFrameCache.set(frameCacheKey, { uri: nextFrameUri, metadata: frame.metadata }) if (!frameMountedRef.current) { frameUriRef.current = nextFrameUri frameMountedRef.current = true @@ -389,7 +389,7 @@ export function MobileBrowserPane({ const sameStream = Boolean(cacheKey) && lastStreamCacheKeyRef.current === cacheKey lastStreamCacheKeyRef.current = cacheKey if (!sameStream || !frameUriRef.current) { - const cachedFrame = getCachedBrowserFrame(cacheKey) + const cachedFrame = browserFrameCache.get(cacheKey) if (cachedFrame) { frameUriRef.current = cachedFrame.uri frameMountedRef.current = true @@ -1303,10 +1303,6 @@ function buttonColor(enabled: boolean): string { return enabled ? colors.textSecondary : colors.textMuted } -function createBrowserFrameDataUri(frame: BrowserScreencastFrame): string { - return `data:image/${frame.format};base64,${Buffer.from(frame.image).toString('base64')}` -} - function makeBrowserFrameCacheKey( worktreeId: string, browserPageId: string | null, @@ -1315,47 +1311,6 @@ function makeBrowserFrameCacheKey( return browserPageId ? `${worktreeId}:${browserPageId}:${viewMode}` : null } -function clearCachedBrowserFramesForWorktree(worktreeId: string): void { - const prefix = `${worktreeId}:` - for (const key of browserFrameCache.keys()) { - if (key.startsWith(prefix)) { - browserFrameCache.delete(key) - } - } -} - -function getCachedBrowserFrame(cacheKey: string | null): BrowserFrameCacheEntry | null { - if (!cacheKey) { - return null - } - const cached = browserFrameCache.get(cacheKey) - if (!cached) { - return null - } - browserFrameCache.delete(cacheKey) - browserFrameCache.set(cacheKey, cached) - return cached -} - -function peekCachedBrowserFrame(cacheKey: string | null): BrowserFrameCacheEntry | null { - return cacheKey ? (browserFrameCache.get(cacheKey) ?? null) : null -} - -function cacheBrowserFrame(cacheKey: string | null, entry: BrowserFrameCacheEntry): void { - if (!cacheKey) { - return - } - browserFrameCache.delete(cacheKey) - browserFrameCache.set(cacheKey, entry) - while (browserFrameCache.size > BROWSER_FRAME_CACHE_LIMIT) { - const oldestKey = browserFrameCache.keys().next().value - if (typeof oldestKey !== 'string') { - break - } - browserFrameCache.delete(oldestKey) - } -} - function updateBrowserLayerVisibility( layers: [View | null, View | null], visible: FrameLayer diff --git a/mobile/src/browser/mobile-browser-frame-cache.test.ts b/mobile/src/browser/mobile-browser-frame-cache.test.ts new file mode 100644 index 00000000000..79280dd1055 --- /dev/null +++ b/mobile/src/browser/mobile-browser-frame-cache.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest' +import { + BrowserScreencastOpcode, + type BrowserScreencastFrame +} from '../transport/browser-screencast-protocol' +import { + createMobileBrowserFrameDataUri, + MOBILE_BROWSER_FRAME_MAX_IMAGE_BYTES, + MobileBrowserFrameCache +} from './mobile-browser-frame-cache' + +function entry(uri: string) { + return { uri, metadata: {} } +} + +function frame(imageBytes: number): BrowserScreencastFrame { + return { + opcode: BrowserScreencastOpcode.Frame, + seq: 1, + format: 'jpeg', + metadata: {}, + image: new Uint8Array(imageBytes) + } +} + +describe('MobileBrowserFrameCache', () => { + it('preserves LRU behavior below the count and retained-character limits', () => { + const cache = new MobileBrowserFrameCache(2, 100) + cache.set('first', entry('frame-1')) + cache.set('second', entry('frame-2')) + + expect(cache.get('first')).toEqual(entry('frame-1')) + cache.set('third', entry('frame-3')) + + expect(cache.peek('second')).toBeNull() + expect(cache.evidence().keysOldestFirst).toEqual(['first', 'third']) + }) + + it('accepts the exact aggregate budget and evicts oldest at one character over', () => { + const cache = new MobileBrowserFrameCache(4, 20) + expect(cache.set('a', entry('x'.repeat(9)))).toBe(true) + expect(cache.set('b', entry('x'.repeat(9)))).toBe(true) + expect(cache.evidence()).toMatchObject({ entryCount: 2, retainedCharacters: 20 }) + + expect(cache.set('c', entry(''))).toBe(true) + expect(cache.peek('a')).toBeNull() + expect(cache.evidence()).toMatchObject({ entryCount: 2, retainedCharacters: 11 }) + }) + + it('does not retain one entry larger than the full budget', () => { + const cache = new MobileBrowserFrameCache(4, 10) + + expect(cache.set('key', entry('x'.repeat(8)))).toBe(false) + expect(cache.evidence()).toEqual({ + entryCount: 0, + retainedCharacters: 0, + keysOldestFirst: [] + }) + }) +}) + +describe('createMobileBrowserFrameDataUri', () => { + it('accepts the exact image-byte limit and rejects one byte over before base64 expansion', () => { + expect(MOBILE_BROWSER_FRAME_MAX_IMAGE_BYTES).toBe(8 * 1024 * 1024) + expect(createMobileBrowserFrameDataUri(frame(4), 4)).toBe('data:image/jpeg;base64,AAAAAA==') + expect(createMobileBrowserFrameDataUri(frame(5), 4)).toBeNull() + }) +}) diff --git a/mobile/src/browser/mobile-browser-frame-cache.ts b/mobile/src/browser/mobile-browser-frame-cache.ts new file mode 100644 index 00000000000..0d856c546b3 --- /dev/null +++ b/mobile/src/browser/mobile-browser-frame-cache.ts @@ -0,0 +1,134 @@ +import { Buffer } from 'buffer' +import type { + BrowserScreencastFrame, + BrowserScreencastFrameMetadata +} from '../transport/browser-screencast-protocol' + +export const MOBILE_BROWSER_FRAME_MAX_IMAGE_BYTES = 8 * 1024 * 1024 +export const MOBILE_BROWSER_FRAME_CACHE_MAX_ENTRIES = 4 +export const MOBILE_BROWSER_FRAME_CACHE_MAX_RETAINED_CHARACTERS = 16 * 1024 * 1024 + +export type MobileBrowserFrameCacheEntry = { + uri: string + metadata: BrowserScreencastFrameMetadata +} + +export type MobileBrowserFrameCacheEvidence = { + entryCount: number + retainedCharacters: number + keysOldestFirst: string[] +} + +type RetainedFrame = { + entry: MobileBrowserFrameCacheEntry + retainedCharacters: number +} + +export class MobileBrowserFrameCache { + private readonly entries = new Map() + private retainedCharacters = 0 + + constructor( + private readonly maxEntries = MOBILE_BROWSER_FRAME_CACHE_MAX_ENTRIES, + private readonly maxRetainedCharacters = MOBILE_BROWSER_FRAME_CACHE_MAX_RETAINED_CHARACTERS + ) { + if ( + !Number.isInteger(maxEntries) || + maxEntries < 1 || + !Number.isSafeInteger(maxRetainedCharacters) || + maxRetainedCharacters < 1 + ) { + throw new Error('Mobile browser frame cache limits must be positive integers') + } + } + + get(key: string | null): MobileBrowserFrameCacheEntry | null { + if (!key) { + return null + } + const retained = this.entries.get(key) + if (!retained) { + return null + } + this.entries.delete(key) + this.entries.set(key, retained) + return retained.entry + } + + peek(key: string | null): MobileBrowserFrameCacheEntry | null { + return key ? (this.entries.get(key)?.entry ?? null) : null + } + + set(key: string | null, entry: MobileBrowserFrameCacheEntry): boolean { + if (!key) { + return false + } + const retainedCharacters = key.length + entry.uri.length + const previous = this.entries.get(key) + if (previous) { + this.retainedCharacters -= previous.retainedCharacters + this.entries.delete(key) + } + if (retainedCharacters > this.maxRetainedCharacters) { + return false + } + this.entries.set(key, { entry, retainedCharacters }) + this.retainedCharacters += retainedCharacters + this.evictOverflow() + return this.entries.has(key) + } + + clearWorktree(worktreeId: string): void { + const prefix = `${worktreeId}:` + for (const key of this.entries.keys()) { + if (key.startsWith(prefix)) { + this.delete(key) + } + } + } + + clear(): void { + this.entries.clear() + this.retainedCharacters = 0 + } + + evidence(): MobileBrowserFrameCacheEvidence { + return { + entryCount: this.entries.size, + retainedCharacters: this.retainedCharacters, + keysOldestFirst: [...this.entries.keys()] + } + } + + private delete(key: string): void { + const retained = this.entries.get(key) + if (!retained) { + return + } + this.retainedCharacters -= retained.retainedCharacters + this.entries.delete(key) + } + + private evictOverflow(): void { + while ( + this.entries.size > this.maxEntries || + this.retainedCharacters > this.maxRetainedCharacters + ) { + const oldestKey = this.entries.keys().next().value + if (typeof oldestKey !== 'string') { + return + } + this.delete(oldestKey) + } + } +} + +export function createMobileBrowserFrameDataUri( + frame: BrowserScreencastFrame, + maxImageBytes = MOBILE_BROWSER_FRAME_MAX_IMAGE_BYTES +): string | null { + if (frame.image.byteLength > maxImageBytes) { + return null + } + return `data:image/${frame.format};base64,${Buffer.from(frame.image).toString('base64')}` +} diff --git a/mobile/src/browser/mobile-browser-view-mode-state.test.ts b/mobile/src/browser/mobile-browser-view-mode-state.test.ts index 20ba4844de5..448412fff5e 100644 --- a/mobile/src/browser/mobile-browser-view-mode-state.test.ts +++ b/mobile/src/browser/mobile-browser-view-mode-state.test.ts @@ -1,5 +1,7 @@ import { beforeEach, describe, expect, it } from 'vitest' import { + BROWSER_VIEW_MODE_PAGE_KEY_MAX_CHARACTERS, + BROWSER_VIEW_MODE_STATE_LIMIT, clearMobileBrowserViewModeState, getInitialMobileBrowserViewMode, saveMobileBrowserViewMode @@ -22,4 +24,28 @@ describe('mobile browser view mode state', () => { expect(getInitialMobileBrowserViewMode('worktree-1', 'page-2')).toBe('web') expect(getInitialMobileBrowserViewMode('worktree-2', 'page-1')).toBe('web') }) + + it('retains the exact LRU count and evicts the oldest page at one over', () => { + for (let index = 0; index < BROWSER_VIEW_MODE_STATE_LIMIT; index += 1) { + saveMobileBrowserViewMode('worktree', `page-${index}`, 'mobile') + } + expect(getInitialMobileBrowserViewMode('worktree', 'page-0')).toBe('mobile') + + saveMobileBrowserViewMode('worktree', 'one-over', 'mobile') + + expect(getInitialMobileBrowserViewMode('worktree', 'page-0')).toBe('web') + expect(getInitialMobileBrowserViewMode('worktree', 'one-over')).toBe('mobile') + }) + + it('accepts the exact page-key character limit and rejects one over', () => { + const worktreeId = 'w' + const exactPageId = 'p'.repeat(BROWSER_VIEW_MODE_PAGE_KEY_MAX_CHARACTERS - 2) + const oversizedPageId = `${exactPageId}p` + + saveMobileBrowserViewMode(worktreeId, exactPageId, 'mobile') + saveMobileBrowserViewMode(worktreeId, oversizedPageId, 'mobile') + + expect(getInitialMobileBrowserViewMode(worktreeId, exactPageId)).toBe('mobile') + expect(getInitialMobileBrowserViewMode(worktreeId, oversizedPageId)).toBe('web') + }) }) diff --git a/mobile/src/browser/mobile-browser-view-mode-state.ts b/mobile/src/browser/mobile-browser-view-mode-state.ts index 35923d61a81..da9a906f5e2 100644 --- a/mobile/src/browser/mobile-browser-view-mode-state.ts +++ b/mobile/src/browser/mobile-browser-view-mode-state.ts @@ -1,6 +1,7 @@ import type { MobileBrowserViewMode } from './browser-screencast-request' -const BROWSER_VIEW_MODE_STATE_LIMIT = 40 +export const BROWSER_VIEW_MODE_STATE_LIMIT = 40 +export const BROWSER_VIEW_MODE_PAGE_KEY_MAX_CHARACTERS = 4_096 const browserViewModeByPageKey = new Map() export function getInitialMobileBrowserViewMode( @@ -42,5 +43,9 @@ function makeBrowserViewModePageKey( worktreeId: string, browserPageId: string | null ): string | null { - return browserPageId ? `${worktreeId}:${browserPageId}` : null + if (!browserPageId) { + return null + } + const key = `${worktreeId}:${browserPageId}` + return key.length <= BROWSER_VIEW_MODE_PAGE_KEY_MAX_CHARACTERS ? key : null } diff --git a/mobile/src/cache/home-snapshot-cache.test.ts b/mobile/src/cache/home-snapshot-cache.test.ts new file mode 100644 index 00000000000..4c2485b741b --- /dev/null +++ b/mobile/src/cache/home-snapshot-cache.test.ts @@ -0,0 +1,80 @@ +import AsyncStorage from '@react-native-async-storage/async-storage' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + HOME_SNAPSHOT_MAX_SERIALIZED_BYTES, + loadHomeSnapshot, + resetHomeSnapshotCacheForTests, + saveHomeSnapshot, + type HomeSnapshot +} from './home-snapshot-cache' + +vi.mock('@react-native-async-storage/async-storage', () => ({ + default: { + getItem: vi.fn(), + setItem: vi.fn() + } +})) + +function snapshotWithSerializedBytes(serializedBytes: number): HomeSnapshot { + const base = { + worktreeInfo: {}, + accountsByHost: {}, + savedAt: 1, + padding: '' + } + const baseBytes = JSON.stringify(base).length + return { + ...base, + padding: 'x'.repeat(serializedBytes - baseBytes) + } as unknown as HomeSnapshot +} + +describe('home snapshot cache', () => { + beforeEach(() => { + vi.useFakeTimers() + resetHomeSnapshotCacheForTests() + vi.mocked(AsyncStorage.getItem).mockReset() + vi.mocked(AsyncStorage.setItem).mockReset().mockResolvedValue(undefined) + }) + + afterEach(() => { + resetHomeSnapshotCacheForTests() + vi.useRealTimers() + }) + + it('persists and reloads normal snapshots unchanged', async () => { + const snapshot = snapshotWithSerializedBytes(128) + saveHomeSnapshot(snapshot) + await vi.advanceTimersByTimeAsync(250) + + const raw = vi.mocked(AsyncStorage.setItem).mock.calls[0]?.[1] + expect(raw).toBe(JSON.stringify(snapshot)) + + resetHomeSnapshotCacheForTests() + vi.mocked(AsyncStorage.getItem).mockResolvedValue(raw ?? null) + await expect(loadHomeSnapshot()).resolves.toEqual(snapshot) + }) + + it('accepts the exact serialized-byte cap and rejects one byte over', async () => { + const exact = snapshotWithSerializedBytes(HOME_SNAPSHOT_MAX_SERIALIZED_BYTES) + saveHomeSnapshot(exact) + await vi.advanceTimersByTimeAsync(250) + expect(AsyncStorage.setItem).toHaveBeenCalledOnce() + expect(vi.mocked(AsyncStorage.setItem).mock.calls[0]?.[1]).toHaveLength( + HOME_SNAPSHOT_MAX_SERIALIZED_BYTES + ) + + vi.mocked(AsyncStorage.setItem).mockClear() + saveHomeSnapshot(snapshotWithSerializedBytes(HOME_SNAPSHOT_MAX_SERIALIZED_BYTES + 1)) + await vi.advanceTimersByTimeAsync(250) + expect(AsyncStorage.setItem).not.toHaveBeenCalled() + }) + + it('rejects an oversized durable payload before JSON parsing', async () => { + vi.mocked(AsyncStorage.getItem).mockResolvedValue( + `"${'x'.repeat(HOME_SNAPSHOT_MAX_SERIALIZED_BYTES)}"` + ) + + await expect(loadHomeSnapshot()).resolves.toBeNull() + }) +}) diff --git a/mobile/src/cache/home-snapshot-cache.ts b/mobile/src/cache/home-snapshot-cache.ts index a591bb01d70..4d906887c04 100644 --- a/mobile/src/cache/home-snapshot-cache.ts +++ b/mobile/src/cache/home-snapshot-cache.ts @@ -5,8 +5,10 @@ // WebSocket reconnects and the first responses come back. import AsyncStorage from '@react-native-async-storage/async-storage' import type { AccountsSnapshot } from '../components/AccountUsage' +import { stringifyMobileOutboundJson } from '../transport/mobile-outbound-json' const STORAGE_KEY = 'orca:home-snapshot:v1' +export const HOME_SNAPSHOT_MAX_SERIALIZED_BYTES = 2 * 1024 * 1024 type WorktreeSummary = { worktreeId: string @@ -42,6 +44,12 @@ export async function loadHomeSnapshot(): Promise { if (!raw) { return null } + if ( + raw.length > HOME_SNAPSHOT_MAX_SERIALIZED_BYTES || + utf8ByteLengthExceeds(raw, HOME_SNAPSHOT_MAX_SERIALIZED_BYTES) + ) { + return null + } const parsed = JSON.parse(raw) as HomeSnapshot if ( typeof parsed !== 'object' || @@ -61,12 +69,53 @@ export async function loadHomeSnapshot(): Promise { // Why: throttle writes so a flurry of streamed account-snapshot updates // (one per provider fetch finishing) doesn't hammer AsyncStorage. export function saveHomeSnapshot(snapshot: HomeSnapshot): void { + let serialized: string + try { + serialized = stringifyMobileOutboundJson(snapshot, HOME_SNAPSHOT_MAX_SERIALIZED_BYTES) + } catch { + return + } memoryCache = snapshot if (writeTimer) { clearTimeout(writeTimer) } writeTimer = setTimeout(() => { writeTimer = null - void AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(snapshot)).catch(() => {}) + void AsyncStorage.setItem(STORAGE_KEY, serialized).catch(() => {}) }, 250) } + +function utf8ByteLengthExceeds(value: string, limit: number): boolean { + let bytes = 0 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if (code <= 0x7f) { + bytes += 1 + } else if (code <= 0x7ff) { + bytes += 2 + } else if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4 + index += 1 + } else { + bytes += 3 + } + } else { + bytes += 3 + } + if (bytes > limit) { + return true + } + } + return false +} + +/** Test-only: clear the process-lifetime snapshot and delayed write. */ +export function resetHomeSnapshotCacheForTests(): void { + memoryCache = null + if (writeTimer) { + clearTimeout(writeTimer) + writeTimer = null + } +} diff --git a/mobile/src/cache/mobile-rpc-list-cache.test.ts b/mobile/src/cache/mobile-rpc-list-cache.test.ts new file mode 100644 index 00000000000..50eeccf027e --- /dev/null +++ b/mobile/src/cache/mobile-rpc-list-cache.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest' +import { MobileRpcListCache } from './mobile-rpc-list-cache' + +describe('MobileRpcListCache', () => { + it('preserves values below the limits and expires them at the existing age boundary', () => { + const cache = new MobileRpcListCache(100, 2, 3, 100) + const values = [{ id: 'repo' }] + + expect(cache.set('host', values, 1_000)).toBe(true) + expect(cache.get('host', 1_100)).toBe(values) + expect(cache.get('host', 1_101)).toBeNull() + }) + + it('accepts the exact item cap and rejects one over', () => { + const cache = new MobileRpcListCache(100, 2, 3, 1_000) + const exact = [1, 2, 3] + + expect(cache.set('host', exact)).toBe(true) + expect(cache.get('host')).toBe(exact) + expect(cache.set('host', [...exact, 4])).toBe(false) + expect(cache.get('host')).toBeNull() + }) + + it('accepts the exact aggregate byte budget and evicts oldest at one over', () => { + const probe = new MobileRpcListCache(100, 3, 3, 1_000) + probe.set('first', ['a']) + const firstBytes = probe.evidence().retainedBytes + probe.clear() + + const cache = new MobileRpcListCache(100, 3, 3, firstBytes * 2) + expect(cache.set('first', ['a'])).toBe(true) + expect(cache.set('other', ['a'])).toBe(true) + expect(cache.evidence().retainedBytes).toBe(firstBytes * 2) + + expect(cache.set('x', [])).toBe(true) + expect(cache.get('first')).toBeNull() + expect(cache.evidence().retainedBytes).toBeLessThanOrEqual(firstBytes * 2) + }) + + it('rejects a single payload larger than the full byte budget', () => { + const cache = new MobileRpcListCache(100, 2, 3, 20) + + expect(cache.set('host', ['a'.repeat(100)])).toBe(false) + expect(cache.evidence()).toEqual({ + entryCount: 0, + retainedBytes: 0, + keysOldestFirst: [] + }) + }) +}) diff --git a/mobile/src/cache/mobile-rpc-list-cache.ts b/mobile/src/cache/mobile-rpc-list-cache.ts new file mode 100644 index 00000000000..33619a0fed6 --- /dev/null +++ b/mobile/src/cache/mobile-rpc-list-cache.ts @@ -0,0 +1,126 @@ +import { stringifyMobileOutboundJson } from '../transport/mobile-outbound-json' + +export type MobileRpcListCacheEvidence = { + entryCount: number + retainedBytes: number + keysOldestFirst: string[] +} + +type CacheEntry = { + values: unknown[] + at: number + retainedBytes: number +} + +export class MobileRpcListCache { + private readonly entries = new Map() + private retainedBytes = 0 + + constructor( + private readonly maxAgeMs: number, + private readonly maxEntries: number, + private readonly maxItemsPerEntry: number, + private readonly maxRetainedBytes: number + ) { + if ( + !Number.isFinite(maxAgeMs) || + maxAgeMs < 0 || + !Number.isInteger(maxEntries) || + maxEntries < 1 || + !Number.isInteger(maxItemsPerEntry) || + maxItemsPerEntry < 1 || + !Number.isSafeInteger(maxRetainedBytes) || + maxRetainedBytes < 1 + ) { + throw new Error('Mobile RPC list cache limits must be positive') + } + } + + set(key: string, values: unknown[], now = Date.now()): boolean { + this.delete(key) + if (values.length > this.maxItemsPerEntry) { + return false + } + let serialized: string + try { + serialized = stringifyMobileOutboundJson({ key, values }, this.maxRetainedBytes) + } catch { + return false + } + const retainedBytes = utf8ByteLength(serialized) + if (retainedBytes > this.maxRetainedBytes) { + return false + } + this.entries.set(key, { values, at: now, retainedBytes }) + this.retainedBytes += retainedBytes + this.evictOverflow() + return this.entries.has(key) + } + + get(key: string, now = Date.now()): unknown[] | null { + const entry = this.entries.get(key) + if (!entry) { + return null + } + if (now - entry.at > this.maxAgeMs) { + this.delete(key) + return null + } + return entry.values + } + + clear(): void { + this.entries.clear() + this.retainedBytes = 0 + } + + evidence(): MobileRpcListCacheEvidence { + return { + entryCount: this.entries.size, + retainedBytes: this.retainedBytes, + keysOldestFirst: [...this.entries.keys()] + } + } + + private delete(key: string): void { + const entry = this.entries.get(key) + if (!entry) { + return + } + this.retainedBytes -= entry.retainedBytes + this.entries.delete(key) + } + + private evictOverflow(): void { + while (this.entries.size > this.maxEntries || this.retainedBytes > this.maxRetainedBytes) { + const oldestKey = this.entries.keys().next().value + if (typeof oldestKey !== 'string') { + return + } + this.delete(oldestKey) + } + } +} + +function utf8ByteLength(value: string): number { + let bytes = 0 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if (code <= 0x7f) { + bytes += 1 + } else if (code <= 0x7ff) { + bytes += 2 + } else if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4 + index += 1 + } else { + bytes += 3 + } + } else { + bytes += 3 + } + } + return bytes +} diff --git a/mobile/src/cache/repo-cache.test.ts b/mobile/src/cache/repo-cache.test.ts index 156fa507619..adef8d02fcc 100644 --- a/mobile/src/cache/repo-cache.test.ts +++ b/mobile/src/cache/repo-cache.test.ts @@ -1,8 +1,17 @@ -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' -import { getCachedRepos, setCachedRepos } from './repo-cache' +import { + getCachedRepos, + MOBILE_REPO_CACHE_MAX_ITEMS_PER_HOST, + resetRepoCacheForTests, + setCachedRepos +} from './repo-cache' describe('repo cache', () => { + beforeEach(() => { + resetRepoCacheForTests() + }) + it('returns recent host-scoped repos', () => { const repos = [{ id: 'repo-1' }] @@ -23,4 +32,13 @@ describe('repo cache', () => { vi.useRealTimers() } }) + + it('retains the exact per-host item cap and rejects one over', () => { + const exact = Array.from({ length: MOBILE_REPO_CACHE_MAX_ITEMS_PER_HOST }, () => null) + setCachedRepos('host', exact) + expect(getCachedRepos('host')).toBe(exact) + + setCachedRepos('host', [...exact, null]) + expect(getCachedRepos('host')).toBeNull() + }) }) diff --git a/mobile/src/cache/repo-cache.ts b/mobile/src/cache/repo-cache.ts index ecb485c4d67..e5e407aee06 100644 --- a/mobile/src/cache/repo-cache.ts +++ b/mobile/src/cache/repo-cache.ts @@ -2,35 +2,29 @@ // host-scoped cache lets workspace creation open from the last known list while // a fresh repo.list refresh happens in the background. -type CachedRepos = { - repos: unknown[] - at: number -} - -const cache = new Map() +import { MobileRpcListCache } from './mobile-rpc-list-cache' const MAX_AGE_MS = 60_000 -const MAX_ENTRIES = 20 +export const MOBILE_REPO_CACHE_MAX_ENTRIES = 20 +export const MOBILE_REPO_CACHE_MAX_ITEMS_PER_HOST = 10_000 +export const MOBILE_REPO_CACHE_MAX_RETAINED_BYTES = 16 * 1024 * 1024 + +const cache = new MobileRpcListCache( + MAX_AGE_MS, + MOBILE_REPO_CACHE_MAX_ENTRIES, + MOBILE_REPO_CACHE_MAX_ITEMS_PER_HOST, + MOBILE_REPO_CACHE_MAX_RETAINED_BYTES +) export function setCachedRepos(hostId: string, repos: unknown[]): void { - cache.delete(hostId) - cache.set(hostId, { repos, at: Date.now() }) - if (cache.size > MAX_ENTRIES) { - const oldest = cache.keys().next().value - if (oldest) { - cache.delete(oldest) - } - } + cache.set(hostId, repos) } export function getCachedRepos(hostId: string): unknown[] | null { - const entry = cache.get(hostId) - if (!entry) { - return null - } - if (Date.now() - entry.at > MAX_AGE_MS) { - cache.delete(hostId) - return null - } - return entry.repos + return cache.get(hostId) +} + +/** Test-only: clear process-lifetime cache state between cases. */ +export function resetRepoCacheForTests(): void { + cache.clear() } diff --git a/mobile/src/cache/worktree-cache.test.ts b/mobile/src/cache/worktree-cache.test.ts index f167c6c547d..5412627516c 100644 --- a/mobile/src/cache/worktree-cache.test.ts +++ b/mobile/src/cache/worktree-cache.test.ts @@ -1,10 +1,19 @@ -import { describe, expect, it } from 'vitest' -import { setCachedWorktrees, getCachedWorktrees } from './worktree-cache' +import { beforeEach, describe, expect, it } from 'vitest' +import { + getCachedWorktrees, + MOBILE_WORKTREE_CACHE_MAX_ITEMS_PER_HOST, + resetWorktreeCacheForTests, + setCachedWorktrees +} from './worktree-cache' // Why: AC #8498 guarantees a reconnect refetch writes through the // same cache path the host detail screen seeds from, so a reconnect can't // serve a stale snapshot. This unit pins the write-through contract. describe('worktree-cache write-through', () => { + beforeEach(() => { + resetWorktreeCacheForTests() + }) + it('returns the most-recently written snapshot, not a stale one', () => { const hostId = 'host-write-through' const stale = [{ worktreeId: 'a', name: 'stale' }] @@ -40,4 +49,13 @@ describe('worktree-cache write-through', () => { // A fresh screen mount reads the cache — must see the connected set. expect(getCachedWorktrees(hostId)).toEqual(reconnected) }) + + it('retains the exact per-host item cap and rejects one over', () => { + const exact = Array.from({ length: MOBILE_WORKTREE_CACHE_MAX_ITEMS_PER_HOST }, () => null) + setCachedWorktrees('host', exact) + expect(getCachedWorktrees('host')).toBe(exact) + + setCachedWorktrees('host', [...exact, null]) + expect(getCachedWorktrees('host')).toBeNull() + }) }) diff --git a/mobile/src/cache/worktree-cache.ts b/mobile/src/cache/worktree-cache.ts index 0b05f78b260..b95d97f8549 100644 --- a/mobile/src/cache/worktree-cache.ts +++ b/mobile/src/cache/worktree-cache.ts @@ -2,38 +2,29 @@ // so the host detail page can render instantly on navigation instead of // waiting for a fresh RPC connection + fetch cycle. -type CachedWorktrees = { - worktrees: unknown[] - at: number -} - -const cache = new Map() +import { MobileRpcListCache } from './mobile-rpc-list-cache' const MAX_AGE_MS = 30_000 -const MAX_ENTRIES = 20 +export const MOBILE_WORKTREE_CACHE_MAX_ENTRIES = 20 +export const MOBILE_WORKTREE_CACHE_MAX_ITEMS_PER_HOST = 10_000 +export const MOBILE_WORKTREE_CACHE_MAX_RETAINED_BYTES = 16 * 1024 * 1024 + +const cache = new MobileRpcListCache( + MAX_AGE_MS, + MOBILE_WORKTREE_CACHE_MAX_ENTRIES, + MOBILE_WORKTREE_CACHE_MAX_ITEMS_PER_HOST, + MOBILE_WORKTREE_CACHE_MAX_RETAINED_BYTES +) export function setCachedWorktrees(hostId: string, worktrees: unknown[]): void { - // Why: Map.set on an existing key does not move it to the end of iteration - // order. Delete first so the re-inserted key becomes the newest entry, - // giving us true LRU eviction when the cap is hit. - cache.delete(hostId) - cache.set(hostId, { worktrees, at: Date.now() }) - if (cache.size > MAX_ENTRIES) { - const oldest = cache.keys().next().value - if (oldest) { - cache.delete(oldest) - } - } + cache.set(hostId, worktrees) } export function getCachedWorktrees(hostId: string): unknown[] | null { - const entry = cache.get(hostId) - if (!entry) { - return null - } - if (Date.now() - entry.at > MAX_AGE_MS) { - cache.delete(hostId) - return null - } - return entry.worktrees + return cache.get(hostId) +} + +/** Test-only: clear process-lifetime cache state between cases. */ +export function resetWorktreeCacheForTests(): void { + cache.clear() } diff --git a/mobile/src/components/BottomDrawer.tsx b/mobile/src/components/BottomDrawer.tsx index 138e39ddc0e..030e38bcd3f 100644 --- a/mobile/src/components/BottomDrawer.tsx +++ b/mobile/src/components/BottomDrawer.tsx @@ -1,41 +1,6 @@ -import { type ReactNode, useCallback, useEffect, useState } from 'react' -import { - View, - Pressable, - StyleSheet, - Platform, - useWindowDimensions, - ScrollView, - Keyboard, - BackHandler, - Modal -} from 'react-native' -import { useSafeAreaInsets } from 'react-native-safe-area-context' -import { Gesture, GestureDetector, GestureHandlerRootView } from 'react-native-gesture-handler' -import Animated, { - useSharedValue, - useAnimatedStyle, - useAnimatedScrollHandler, - withSpring, - withTiming, - runOnJS, - interpolate, - Extrapolation -} from 'react-native-reanimated' -import { colors, spacing } from '../theme/mobile-theme' +import { type ReactNode, useState } from 'react' import { resolveBottomDrawerMounted } from './bottom-drawer-mount-state' -import { useInsideBottomDrawerModalHost } from './bottom-drawer-modal-host' -import { useResponsiveLayout } from '../layout/responsive-layout' - -const DISMISS_THRESHOLD = 80 -const SPRING_CONFIG = { damping: 28, stiffness: 400 } -// Why: negative translateY (pulling up) is damped with a rubber-band factor -// so the drawer resists upward dragging — a subtle polish touch that signals -// the drawer cannot expand further. -const RUBBER_BAND_FACTOR = 0.25 -const SHOW_DURATION = 180 -export const BOTTOM_DRAWER_HIDE_DURATION_MS = 150 -const TOP_SCROLL_EPSILON = 1 +import { MountedBottomDrawer } from './mounted-bottom-drawer' type Props = { visible: boolean @@ -44,6 +9,13 @@ type Props = { children: ReactNode dragContentToDismiss?: boolean contentScrollable?: boolean + // Why: smart-source (and similar) need a stable outer frame so a docked + // TextInput can sit above the keyboard while results reflow in flex space + // above it — content-sized sheets make that field ride every list change. + fillAvailable?: boolean + // Why: pin an outer content-sized sheet under an inner fill picker without + // letting it take touches, draw a second backdrop, or keyboard-lift. + interactive?: boolean zIndex?: number } @@ -54,6 +26,8 @@ export function BottomDrawer({ children, dragContentToDismiss = true, contentScrollable = true, + fillAvailable = false, + interactive = true, zIndex }: Props) { const [mounted, setMounted] = useState(visible) @@ -81,365 +55,11 @@ export function BottomDrawer({ }} dragContentToDismiss={dragContentToDismiss} contentScrollable={contentScrollable} + fillAvailable={fillAvailable} + interactive={interactive} zIndex={zIndex} > {children} ) } - -type MountedBottomDrawerProps = Props & { - onHidden: () => void -} - -function MountedBottomDrawer({ - visible, - onClose, - onHidden, - children, - dragContentToDismiss = true, - contentScrollable = true, - zIndex = 1000 -}: MountedBottomDrawerProps) { - const translateY = useSharedValue(0) - const progress = useSharedValue(0) - const keyboardOffset = useSharedValue(0) - const scrollOffsetY = useSharedValue(0) - const contentDragStartY = useSharedValue(0) - const contentDragCanDismiss = useSharedValue(false) - const { height: screenHeight } = useWindowDimensions() - const insets = useSafeAreaInsets() - // Why: on wide/tablet canvases a full-width sheet looks stretched; cap it and - // center it horizontally. Vertical bottom-anchoring (and all the drag/keyboard - // transforms below) is unchanged, so phone behavior stays identical. - const { isWideLayout, modalMaxWidth } = useResponsiveLayout() - const insideModalHost = useInsideBottomDrawerModalHost() - - useEffect(() => { - if (visible) { - translateY.value = 0 - scrollOffsetY.value = 0 - progress.value = withTiming(1, { duration: SHOW_DURATION }) - } else { - Keyboard.dismiss() - progress.value = withTiming(0, { duration: BOTTOM_DRAWER_HIDE_DURATION_MS }, (finished) => { - if (finished) { - runOnJS(onHidden)() - } - }) - } - }, [onHidden, visible]) - - // Why: KeyboardAvoidingView and useAnimatedKeyboard are both unreliable - // inside Modal (iOS ignores KAV; Android needs adjustNothing for - // useAnimatedKeyboard). Keyboard event listeners work on both platforms - // and give us the exact height to shift the drawer by. - useEffect(() => { - if (!visible) { - return - } - - const showEvent = Platform.OS === 'ios' ? 'keyboardWillShow' : 'keyboardDidShow' - const hideEvent = Platform.OS === 'ios' ? 'keyboardWillHide' : 'keyboardDidHide' - - const onShow = Keyboard.addListener(showEvent, (e) => { - const height = e.endCoordinates.height - insets.bottom - keyboardOffset.value = withTiming(Math.max(height, 0), { duration: e.duration || 250 }) - }) - const onHide = Keyboard.addListener(hideEvent, (e) => { - keyboardOffset.value = withTiming(0, { duration: e.duration || 250 }) - }) - - return () => { - onShow.remove() - onHide.remove() - keyboardOffset.value = 0 - } - }, [visible, insets.bottom]) - - const dismiss = useCallback(() => { - Keyboard.dismiss() - progress.value = withTiming(0, { duration: BOTTOM_DRAWER_HIDE_DURATION_MS }, (finished) => { - if (finished) { - runOnJS(onClose)() - } - }) - }, [onClose, progress]) - - useEffect(() => { - if (!visible) { - return - } - - const sub = BackHandler.addEventListener('hardwareBackPress', () => { - dismiss() - return true - }) - return () => sub.remove() - }, [visible, dismiss]) - - const scrollHandler = useAnimatedScrollHandler((event) => { - scrollOffsetY.value = Math.max(event.contentOffset.y, 0) - }) - - const scrollGesture = Gesture.Native() - const handlePanGesture = Gesture.Pan() - .activeOffsetY([-8, 8]) - .simultaneousWithExternalGesture(scrollGesture) - .onUpdate((e) => { - if (e.translationY > 0) { - translateY.value = e.translationY - } else { - translateY.value = e.translationY * RUBBER_BAND_FACTOR - } - }) - .onEnd((e) => { - if (e.translationY > DISMISS_THRESHOLD || e.velocityY > 500) { - const velocity = Math.max(e.velocityY, 800) - const remaining = screenHeight - e.translationY - const duration = Math.min(Math.max((remaining / velocity) * 1000, 120), 300) - translateY.value = withTiming(screenHeight, { duration }) - progress.value = withTiming(0, { duration }, () => { - runOnJS(onClose)() - }) - } else { - translateY.value = withSpring(0, SPRING_CONFIG) - } - }) - const contentPanGesture = Gesture.Pan() - .activeOffsetY([-8, 8]) - .simultaneousWithExternalGesture(scrollGesture) - .onBegin(() => { - contentDragStartY.value = 0 - contentDragCanDismiss.value = scrollOffsetY.value <= TOP_SCROLL_EPSILON - }) - .onUpdate((e) => { - // Why: action-sheet content can be taller than the drawer; downward drags - // should scroll back to the top before they start dismissing the sheet. - if (scrollOffsetY.value > TOP_SCROLL_EPSILON) { - contentDragCanDismiss.value = false - contentDragStartY.value = 0 - if (translateY.value !== 0) { - translateY.value = withSpring(0, SPRING_CONFIG) - } - return - } - - if (!contentDragCanDismiss.value) { - contentDragCanDismiss.value = true - contentDragStartY.value = e.translationY - } - - const translationY = e.translationY - contentDragStartY.value - if (translationY > 0) { - translateY.value = translationY - } else { - translateY.value = translationY * RUBBER_BAND_FACTOR - } - }) - .onEnd((e) => { - if (!contentDragCanDismiss.value || scrollOffsetY.value > TOP_SCROLL_EPSILON) { - return - } - - const translationY = e.translationY - contentDragStartY.value - if (translationY > DISMISS_THRESHOLD || e.velocityY > 500) { - const velocity = Math.max(e.velocityY, 800) - const remaining = screenHeight - translationY - const duration = Math.min(Math.max((remaining / velocity) * 1000, 120), 300) - translateY.value = withTiming(screenHeight, { duration }) - progress.value = withTiming(0, { duration }, () => { - runOnJS(onClose)() - }) - } else { - translateY.value = withSpring(0, SPRING_CONFIG) - } - }) - - const drawerStyle = useAnimatedStyle(() => ({ - transform: [ - { - translateY: - interpolate(progress.value, [0, 1], [screenHeight, 0], Extrapolation.CLAMP) + - translateY.value - - keyboardOffset.value - } - ] - })) - - const backdropStyle = useAnimatedStyle(() => { - const dragFade = interpolate(translateY.value, [0, 300], [1, 0], Extrapolation.CLAMP) - return { opacity: progress.value * dragFade } - }) - - // Why: the sheet renders through a full-screen native window (its own Modal - // below, or the shared BottomDrawerModalHost) so it always covers the viewport - // — even when mounted deep inside a ScrollView, where a plain absolute overlay - // anchors to the scrolled content and clips the sheet. Show/hide is driven by - // `progress` (animationType "none") so the reanimated exit animation runs before - // the parent unmounts us. - const overlay = ( - - - - - - - - - {!contentScrollable ? ( - <> - - - - - - {children} - - ) : dragContentToDismiss ? ( - <> - - - - - - - - - - {children} - - - - - - ) : ( - <> - - - - - - - {children} - - - )} - - - - - - ) - - // Why: inside a BottomDrawerModalHost the host owns the single native Modal; - // rendering our own would stack modals and reintroduce the iOS present/dismiss - // race the host exists to avoid. The host handles the Android back button. - if (insideModalHost) { - return overlay - } - - return ( - - {overlay} - - ) -} - -const styles = StyleSheet.create({ - overlay: { - ...StyleSheet.absoluteFillObject, - zIndex: 1000 - }, - root: { - flex: 1 - }, - backdrop: { - ...StyleSheet.absoluteFillObject, - backgroundColor: 'rgba(0,0,0,0.5)' - }, - anchor: { - flex: 1, - justifyContent: 'flex-end' - }, - anchorWide: { - alignItems: 'center' - }, - drawer: { - backgroundColor: colors.bgBase, - borderTopLeftRadius: 16, - borderTopRightRadius: 16, - paddingHorizontal: spacing.md, - ...Platform.select({ - ios: { - shadowColor: '#000', - shadowOffset: { width: 0, height: -2 }, - shadowOpacity: 0.2, - shadowRadius: 10 - }, - android: { elevation: 8 } - }) - }, - handle: { - alignSelf: 'center', - width: 36, - height: 4, - borderRadius: 2, - backgroundColor: colors.textMuted, - opacity: 0.4 - }, - handleHitArea: { - alignItems: 'center', - paddingTop: spacing.sm, - paddingBottom: spacing.md - }, - staticContent: { - minHeight: 0 - }, - bottomExtension: { - position: 'absolute', - bottom: -500, - left: 0, - right: 0, - height: 500, - backgroundColor: colors.bgBase - } -}) diff --git a/mobile/src/components/CustomKeyModal.tsx b/mobile/src/components/CustomKeyModal.tsx index 2777e310dc8..6f81da63880 100644 --- a/mobile/src/components/CustomKeyModal.tsx +++ b/mobile/src/components/CustomKeyModal.tsx @@ -1,7 +1,6 @@ import { useCallback, useMemo, useState } from 'react' import { View, Text, Pressable, TextInput, StyleSheet, Switch } from 'react-native' import { ChevronLeft } from 'lucide-react-native' -import AsyncStorage from '@react-native-async-storage/async-storage' import { colors, spacing, radii, typography } from '../theme/mobile-theme' import { BottomDrawer } from './BottomDrawer' import { @@ -11,15 +10,19 @@ import { type TerminalShortcutModifier, type TerminalShortcutSpecialKey } from '../terminal/terminal-accessory-keys' +import { + CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS, + CUSTOM_ACCESSORY_KEY_MAX_LABEL_CHARACTERS, + loadCustomKeys, + saveCustomKeys, + type CustomKey +} from '../terminal/custom-accessory-key-store' -const CUSTOM_ACCESSORY_KEYS_STORAGE_KEY = 'orca:custom-accessory-keys' - -export type CustomKey = { - id: string - label: string - bytes: string - enter: boolean -} +export { + loadCustomKeys, + saveCustomKeys, + type CustomKey +} from '../terminal/custom-accessory-key-store' type Step = 'choose-type' | 'shortcut-combo' | 'special-keys' | 'text-macro' @@ -64,19 +67,6 @@ type Props = { onManageShortcuts?: () => void } -export async function loadCustomKeys(): Promise { - try { - const raw = await AsyncStorage.getItem(CUSTOM_ACCESSORY_KEYS_STORAGE_KEY) - return raw ? (JSON.parse(raw) as CustomKey[]) : [] - } catch { - return [] - } -} - -export async function saveCustomKeys(keys: CustomKey[]): Promise { - await AsyncStorage.setItem(CUSTOM_ACCESSORY_KEYS_STORAGE_KEY, JSON.stringify(keys)) -} - export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortcuts }: Props) { const [step, setStep] = useState('choose-type') const [shortcutKey, setShortcutKey] = useState('c') @@ -104,8 +94,7 @@ export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortc async (key: Omit) => { const existing = await loadCustomKeys() const newKey: CustomKey = { ...key, id: `custom-${Date.now()}` } - const updated = [...existing, newKey] - await saveCustomKeys(updated) + const updated = await saveCustomKeys([...existing, newKey]) onKeysChanged(updated) onClose() }, @@ -372,6 +361,7 @@ export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortc placeholderTextColor={colors.textMuted} autoCapitalize="none" autoCorrect={false} + maxLength={CUSTOM_ACCESSORY_KEY_MAX_LABEL_CHARACTERS} /> Command Press Enter diff --git a/mobile/src/components/NewWorktreeModal.tsx b/mobile/src/components/NewWorktreeModal.tsx index 61c9b2e711b..60dffab0d2a 100644 --- a/mobile/src/components/NewWorktreeModal.tsx +++ b/mobile/src/components/NewWorktreeModal.tsx @@ -14,8 +14,9 @@ import { ChevronDown, ChevronUp } from 'lucide-react-native' import type { RpcClient } from '../transport/rpc-client' import type { RpcResponse, RpcSuccess } from '../transport/types' import { colors, spacing, radii, typography } from '../theme/mobile-theme' -import { BottomDrawer, BOTTOM_DRAWER_HIDE_DURATION_MS } from './BottomDrawer' +import { BottomDrawer } from './BottomDrawer' import { BottomDrawerModalHost } from './bottom-drawer-modal-host' +import { useNewWorktreeDrawerNavigation } from './use-new-worktree-drawer-navigation' import { PickerListDrawer } from './PickerListDrawer' import { MobileAgentIcon } from './MobileAgentIcon' import { getSuggestedCreatureName } from './worktree-name-suggestion' @@ -111,12 +112,6 @@ type CreateOptions = { approvedSetupContentHash?: string } -type NewWorktreeDrawerView = 'form' | 'transition' | 'source' | 'repo' | 'agent' | 'trust' - -// Why: iOS cannot reliably present a second native modal until the first drawer's -// exit commits; one extra frame keeps transitions sequential on slower devices. -const NEW_WORKTREE_DRAWER_TRANSITION_MS = BOTTOM_DRAWER_HIDE_DURATION_MS + 16 - function repoColor(name: string): string { const palette = ['#f97316', '#8b5cf6', '#06b6d4', '#ec4899', '#84cc16', '#f59e0b', '#6366f1'] let hash = 0 @@ -196,8 +191,8 @@ function NewWorktreeModalContent({ const [initialRepos] = useState(() => (hostId ? (getCachedRepos(hostId) as Repo[] | null) : null)) const [repos, setRepos] = useState(initialRepos ?? []) const [selectedRepo, setSelectedRepo] = useState(null) - const [drawerView, setDrawerView] = useState('form') - const drawerTransitionTimerRef = useRef | null>(null) + const { drawerView, formSheetVisible, formSheetInteractive, transitionDrawer, openSourceDrawer } = + useNewWorktreeDrawerNavigation(visible) const createInFlightRef = useRef(false) const setupTrustActionInFlightRef = useRef(false) const [selectedAgentState, setSelectedAgent] = useState(AGENT_OPTIONS[0]!) @@ -232,29 +227,6 @@ function NewWorktreeModalContent({ [existingWorktrees, selectedRepo] ) - useEffect(() => { - return () => { - if (drawerTransitionTimerRef.current) { - clearTimeout(drawerTransitionTimerRef.current) - } - } - }, []) - - function transitionDrawer(nextView: Exclude): void { - if (drawerTransitionTimerRef.current) { - clearTimeout(drawerTransitionTimerRef.current) - } - setDrawerView('transition') - drawerTransitionTimerRef.current = setTimeout(() => { - drawerTransitionTimerRef.current = null - setDrawerView(nextView) - }, NEW_WORKTREE_DRAWER_TRANSITION_MS) - } - - // The Smart source picker owns the workspace name AND the linked-source - // selection: typing names the workspace and drives source search, and picking - // a source resolves the base/branch/push metadata (matching desktop). The - // creature-name fallback is only computed lazily at submit for a blank name. const composer = useMobileComposerSource({ client, selectedRepoId: selectedRepo?.id ?? null, @@ -830,7 +802,7 @@ function NewWorktreeModalContent({ } }} > - + Create Workspace @@ -876,8 +848,9 @@ function NewWorktreeModalContent({ composer={composer} label={selectedRepoIsGit ? "Name or 'Create From'" : 'Workspace name'} disabled={sshGate.requiresConnection} + interactive={formSheetInteractive} onBeforeOpen={() => setError('')} - onOpenDrawer={() => transitionDrawer('source')} + onOpenDrawer={openSourceDrawer} /> {composer.forkPushWarning ? ( diff --git a/mobile/src/components/PickerListDrawer.tsx b/mobile/src/components/PickerListDrawer.tsx index 018832cdf98..e908c98f449 100644 --- a/mobile/src/components/PickerListDrawer.tsx +++ b/mobile/src/components/PickerListDrawer.tsx @@ -3,7 +3,8 @@ import { FlatList, Pressable, StyleSheet, Text, View } from 'react-native' import { Check } from 'lucide-react-native' import { colors, spacing, typography } from '../theme/mobile-theme' -import { BottomDrawer, BOTTOM_DRAWER_HIDE_DURATION_MS } from './BottomDrawer' +import { BottomDrawer } from './BottomDrawer' +import { BOTTOM_DRAWER_HIDE_DURATION_MS } from './bottom-drawer-constants' type Props = { visible: boolean diff --git a/mobile/src/components/SmartWorkspaceSourceDrawer.tsx b/mobile/src/components/SmartWorkspaceSourceDrawer.tsx index c5f1a6c47db..df12d5aae62 100644 --- a/mobile/src/components/SmartWorkspaceSourceDrawer.tsx +++ b/mobile/src/components/SmartWorkspaceSourceDrawer.tsx @@ -2,8 +2,8 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { ActivityIndicator, FlatList, + InteractionManager, Pressable, - StyleSheet, Text, TextInput, View @@ -25,11 +25,16 @@ import { } from '../tasks/smart-source-paste-intent' import { useSmartWorkspaceSource } from '../tasks/use-smart-workspace-source' import type { MobileComposerSource } from '../tasks/use-mobile-composer-source' -import { colors, radii, spacing, typography } from '../theme/mobile-theme' -import { BottomDrawer, BOTTOM_DRAWER_HIDE_DURATION_MS } from './BottomDrawer' +import { colors } from '../theme/mobile-theme' +import { BottomDrawer } from './BottomDrawer' +import { smartWorkspaceSourceDrawerStyles as styles } from './smart-workspace-source-drawer-styles' import { SmartSourceModeIcon } from './SmartSourceModeIcon' import { SmartWorkspaceSourceRow } from './SmartWorkspaceSourceRow' +// Why: match MobileSearchField — native autoFocus alone often fails to raise +// the soft keyboard when the drawer is mid-present animation. +const SOURCE_INPUT_FOCUS_DELAY_MS = 120 + type Props = { visible: boolean client: RpcClient | null @@ -58,6 +63,7 @@ export function SmartWorkspaceSourceDrawer({ const availableModes = useMemo(() => resolveAvailableSmartModes(availability), [availability]) const [mode, setMode] = useState(() => resolveDefaultSmartMode(availability)) const [mrStateFilter, setMrStateFilter] = useState('opened') + const inputRef = useRef(null) // Why: read latest availability inside the open effect without making it a // reactive dep (the object is recreated each render), so re-seeding happens // only on open, not on every availability recompute. @@ -71,6 +77,26 @@ export function SmartWorkspaceSourceDrawer({ } }, [visible]) + // Why: focus after open interactions settle so the keyboard appears and the + // caret lands in the docked field (same value as the form via composer.name). + useEffect(() => { + if (!visible) { + return + } + let timeout: ReturnType | undefined + const task = InteractionManager.runAfterInteractions(() => { + timeout = setTimeout(() => { + inputRef.current?.focus() + }, SOURCE_INPUT_FOCUS_DELAY_MS) + }) + return () => { + task.cancel() + if (timeout) { + clearTimeout(timeout) + } + } + }, [visible]) + // Snap the chosen mode back into the available set if availability changes. const effectiveMode = availableModes.includes(mode) ? mode : (availableModes[0] ?? 'text') @@ -100,10 +126,6 @@ export function SmartWorkspaceSourceDrawer({ repos }) - function closeSoon(): void { - setTimeout(onClose, BOTTOM_DRAWER_HIDE_DURATION_MS) - } - function handleSelectRow(row: SourceRow): void { switch (row.kind) { case 'use-name': @@ -154,272 +176,146 @@ export function SmartWorkspaceSourceDrawer({ const showEmpty = !loading && !error && !needsGitHubRemote && effectiveMode !== 'text' && rows.length === 0 + const modeTabs = SMART_MODE_OPTIONS.filter((option: SmartModeOption) => + availableModes.includes(option.id) + ) + return ( - - Name or 'Create From' - - Done - - - - - - - {SMART_MODE_OPTIONS.filter((option: SmartModeOption) => - availableModes.includes(option.id) - ).map((option) => { - const selected = option.id === effectiveMode - const tint = selected ? colors.textPrimary : colors.textSecondary - return ( - setMode(option.id)} - > - - - {option.label} - - - ) - })} - - - {effectiveMode === 'gitlab' ? ( - - {MR_STATE_FILTER_OPTIONS.map((option) => { - const selected = option.id === mrStateFilter - return ( - setMrStateFilter(option.id)} - > - - {option.label} - - - ) - })} + {/* Why: column with results flex:1 + dock flex-shrink:0 at the end. + Fill sheet height + marginBottom place this column on the keyboard + top; dock must stay a non-flex sibling so FlatList cannot clip it. */} + + + Name or 'Create From' + + Done + - ) : null} - {crossRepoPrompt ? ( - - - This item lives in {crossRepoPrompt.link.slug.owner}/{crossRepoPrompt.link.slug.repo}. - - - - Cancel - - void handleAcceptCrossRepo()}> - - Switch to {crossRepoPrompt.matchingRepo.displayName} + + {crossRepoPrompt ? ( + + + This item lives in {crossRepoPrompt.link.slug.owner}/ + {crossRepoPrompt.link.slug.repo}. - - - - ) : null} - - {!sshReady && effectiveMode !== 'text' && effectiveMode !== 'linear' ? ( - Connect the repository to search sources. - ) : needsGitHubRemote ? ( - - This SSH repo needs a GitHub remote to list issues and PRs. - - ) : error ? ( - {error} - ) : null} - - row.value} - style={styles.list} - keyboardShouldPersistTaps="handled" - nestedScrollEnabled - ListFooterComponent={ - loading ? ( - - + + + Cancel + + void handleAcceptCrossRepo()} + > + + Switch to {crossRepoPrompt.matchingRepo.displayName} + + + - ) : showEmpty ? ( - {emptyHint || 'No results found.'} - ) : null - } - renderItem={({ item }) => ( - handleSelectRow(item)} /> - )} - /> + ) : null} + + {!sshReady && effectiveMode !== 'text' && effectiveMode !== 'linear' ? ( + Connect the repository to search sources. + ) : needsGitHubRemote ? ( + + This SSH repo needs a GitHub remote to list issues and PRs. + + ) : error ? ( + {error} + ) : null} + + row.value} + style={styles.list} + contentContainerStyle={styles.listContent} + keyboardShouldPersistTaps="handled" + keyboardDismissMode="none" + nestedScrollEnabled + ListFooterComponent={ + loading ? ( + + + + ) : showEmpty ? ( + {emptyHint || 'No results found.'} + ) : rows.length === 0 && effectiveMode === 'text' ? ( + Type a workspace name in the field below. + ) : null + } + renderItem={({ item }) => ( + handleSelectRow(item)} /> + )} + /> + + + + {effectiveMode === 'gitlab' ? ( + + {MR_STATE_FILTER_OPTIONS.map((option) => { + const selected = option.id === mrStateFilter + return ( + setMrStateFilter(option.id)} + > + + {option.label} + + + ) + })} + + ) : null} + + + {modeTabs.map((option) => { + const selected = option.id === effectiveMode + const tint = selected ? colors.textPrimary : colors.textSecondary + return ( + setMode(option.id)} + > + + + {option.label} + + + ) + })} + + + + + ) } - -const styles = StyleSheet.create({ - header: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - paddingHorizontal: spacing.xs, - paddingBottom: spacing.sm - }, - title: { - fontSize: 15, - fontWeight: '600', - color: colors.textPrimary - }, - done: { - fontSize: typography.bodySize, - fontWeight: '600', - color: colors.accentBlue - }, - search: { - backgroundColor: colors.bgRaised, - color: colors.textPrimary, - borderRadius: radii.input, - paddingHorizontal: spacing.md, - paddingVertical: spacing.sm, - fontSize: typography.bodySize, - borderWidth: 1, - borderColor: colors.borderSubtle, - marginBottom: spacing.sm - }, - tabRow: { - flexDirection: 'row', - flexWrap: 'wrap', - gap: spacing.xs, - marginBottom: spacing.sm - }, - tab: { - flexDirection: 'row', - alignItems: 'center', - gap: spacing.xs, - paddingHorizontal: spacing.sm + 2, - paddingVertical: spacing.xs + 2, - borderRadius: radii.button, - borderWidth: 1, - borderColor: colors.borderSubtle - }, - tabSelected: { - backgroundColor: colors.bgPanel, - borderColor: colors.textSecondary - }, - tabText: { - fontSize: 13, - color: colors.textSecondary - }, - tabTextSelected: { - color: colors.textPrimary, - fontWeight: '600' - }, - chipRow: { - flexDirection: 'row', - gap: spacing.xs, - marginBottom: spacing.sm - }, - chip: { - paddingHorizontal: spacing.md, - paddingVertical: spacing.xs, - borderRadius: radii.button, - borderWidth: 1, - borderColor: colors.borderSubtle - }, - chipSelected: { - backgroundColor: colors.bgPanel, - borderColor: colors.textSecondary - }, - chipText: { - fontSize: 12, - color: colors.textSecondary - }, - chipTextSelected: { - color: colors.textPrimary, - fontWeight: '600' - }, - crossRepo: { - backgroundColor: colors.bgRaised, - borderRadius: radii.input, - borderWidth: 1, - borderColor: colors.borderSubtle, - padding: spacing.md, - marginBottom: spacing.sm, - gap: spacing.sm - }, - crossRepoText: { - fontSize: 13, - color: colors.textSecondary - }, - crossRepoActions: { - flexDirection: 'row', - justifyContent: 'flex-end', - gap: spacing.sm - }, - crossRepoDismiss: { - paddingHorizontal: spacing.md, - paddingVertical: spacing.xs + 2, - borderRadius: radii.button, - borderWidth: 1, - borderColor: colors.borderSubtle - }, - crossRepoDismissText: { - fontSize: 13, - color: colors.textSecondary - }, - crossRepoSwitch: { - paddingHorizontal: spacing.md, - paddingVertical: spacing.xs + 2, - borderRadius: radii.button, - backgroundColor: colors.bgPanel, - borderWidth: 1, - borderColor: colors.textSecondary - }, - crossRepoSwitchText: { - fontSize: 13, - fontWeight: '600', - color: colors.textPrimary - }, - notice: { - fontSize: 12, - color: colors.textMuted, - paddingHorizontal: spacing.xs, - paddingBottom: spacing.sm - }, - errorNotice: { - fontSize: 12, - color: colors.statusRed, - paddingHorizontal: spacing.xs, - paddingBottom: spacing.sm - }, - list: { - backgroundColor: colors.bgPanel, - borderRadius: radii.card, - overflow: 'hidden', - maxHeight: 420, - flexGrow: 0 - }, - loading: { - paddingVertical: spacing.lg, - alignItems: 'center' - }, - empty: { - paddingVertical: spacing.lg, - textAlign: 'center', - color: colors.textMuted, - fontSize: 13 - } -}) diff --git a/mobile/src/components/SmartWorkspaceSourceField.tsx b/mobile/src/components/SmartWorkspaceSourceField.tsx index eb08ddca66a..9b8972b16be 100644 --- a/mobile/src/components/SmartWorkspaceSourceField.tsx +++ b/mobile/src/components/SmartWorkspaceSourceField.tsx @@ -1,4 +1,4 @@ -import { Linking, Pressable, StyleSheet, Text, View } from 'react-native' +import { Linking, Pressable, StyleSheet, Text, TextInput, View } from 'react-native' import { CircleDot, ExternalLink, @@ -16,6 +16,10 @@ type Props = { composer: MobileComposerSource label: string disabled?: boolean + // Why: only the active form view may focus this field. While the source drawer + // is open/closing this stays non-focusable so the drawer's dismiss (which + // restores native focus back here) can't re-fire onFocus and reopen the drawer. + interactive: boolean onBeforeOpen?: () => void onOpenDrawer: () => void } @@ -40,6 +44,7 @@ export function SmartWorkspaceSourceField({ composer, label, disabled, + interactive, onBeforeOpen, onOpenDrawer }: Props) { @@ -77,18 +82,24 @@ export function SmartWorkspaceSourceField({ ) : ( - - - {composer.name || 'Type a name or search a source'} - - + value={composer.name} + onChangeText={composer.setName} + onFocus={openDrawer} + editable={!disabled && interactive} + placeholder="Type a name or search a source" + placeholderTextColor={colors.textMuted} + autoCapitalize="none" + autoCorrect={false} + // Why: form field is a portal into the picker; return should not + // submit the create form while the drawer is about to open. + blurOnSubmit={false} + showSoftInputOnFocus={false} + /> )} ) @@ -114,17 +125,12 @@ const styles = StyleSheet.create({ paddingHorizontal: spacing.md, paddingVertical: spacing.sm + 2, borderWidth: 1, - borderColor: colors.borderSubtle - }, - disabled: { - opacity: 0.55 - }, - inputText: { + borderColor: colors.borderSubtle, fontSize: typography.bodySize, color: colors.textPrimary }, - inputPlaceholder: { - color: colors.textMuted + disabled: { + opacity: 0.55 }, pill: { flexDirection: 'row', diff --git a/mobile/src/components/TerminalShortcutSettings.tsx b/mobile/src/components/TerminalShortcutSettings.tsx index 18d882aa3fb..66bf28046c5 100644 --- a/mobile/src/components/TerminalShortcutSettings.tsx +++ b/mobile/src/components/TerminalShortcutSettings.tsx @@ -117,7 +117,7 @@ export function TerminalShortcutSettings({ pendingCustomKeysWritesRef.current += 1 customKeysWriteChainRef.current = customKeysWriteChainRef.current .catch(() => {}) - .then(() => saveCustomKeys(next)) + .then(() => saveCustomKeys(next).then(() => undefined)) .catch(() => {}) .finally(() => { pendingCustomKeysWritesRef.current -= 1 diff --git a/mobile/src/components/bottom-drawer-constants.ts b/mobile/src/components/bottom-drawer-constants.ts new file mode 100644 index 00000000000..d021d3d4b78 --- /dev/null +++ b/mobile/src/components/bottom-drawer-constants.ts @@ -0,0 +1 @@ +export const BOTTOM_DRAWER_HIDE_DURATION_MS = 150 diff --git a/mobile/src/components/bottom-drawer-fill-height.test.ts b/mobile/src/components/bottom-drawer-fill-height.test.ts new file mode 100644 index 00000000000..84a0498d768 --- /dev/null +++ b/mobile/src/components/bottom-drawer-fill-height.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { resolveBottomDrawerFillHeight } from './bottom-drawer-fill-height' + +describe('resolveBottomDrawerFillHeight', () => { + it('fills the space under the safe top when the keyboard is closed', () => { + expect( + resolveBottomDrawerFillHeight({ + screenHeight: 844, + topInset: 54, + keyboardInset: 0, + topGap: 16 + }) + ).toBe(844 - 54 - 16) + }) + + it('shrinks by the keyboard inset so the sheet top stays under the status bar', () => { + expect( + resolveBottomDrawerFillHeight({ + screenHeight: 844, + topInset: 54, + keyboardInset: 292, + topGap: 16 + }) + ).toBe(844 - 54 - 16 - 292) + }) + + it('never expands past the space above the keyboard on tiny viewports', () => { + expect( + resolveBottomDrawerFillHeight({ + screenHeight: 400, + topInset: 50, + keyboardInset: 300, + topGap: 16 + }) + ).toBe(34) + }) + + it('pairs with marginBottom=keyboardInset so the sheet sits on the keyboard top', () => { + // screen 844, top 54, gap 16, keyboard 292 → height 482; marginBottom 292 + // bottom edge at 844-292=552; top edge at 552-482=70 (= 54+16) + const keyboardInset = 292 + const height = resolveBottomDrawerFillHeight({ + screenHeight: 844, + topInset: 54, + keyboardInset, + topGap: 16 + }) + const topEdge = 844 - keyboardInset - height + expect(height).toBe(482) + expect(topEdge).toBe(54 + 16) + }) + + it('keeps the top edge under the status bar when the keyboard is large', () => { + const screenHeight = 400 + const topInset = 50 + const topGap = 16 + const keyboardInset = 300 + const height = resolveBottomDrawerFillHeight({ + screenHeight, + topInset, + keyboardInset, + topGap + }) + const topEdge = screenHeight - keyboardInset - height + expect(topEdge).toBe(topInset + topGap) + }) +}) diff --git a/mobile/src/components/bottom-drawer-fill-height.ts b/mobile/src/components/bottom-drawer-fill-height.ts new file mode 100644 index 00000000000..0627b33fa97 --- /dev/null +++ b/mobile/src/components/bottom-drawer-fill-height.ts @@ -0,0 +1,19 @@ +// Why: fill-mode sheets need a stable outer height so docked chrome (e.g. the +// smart-source TextInput) does not ride result-list reflow. Height shrinks by +// the keyboard inset; the sheet is also lifted with marginBottom equal to that +// inset so the bottom edge sits on the keyboard top (height shrink alone still +// leaves the dock in the keyboard footprint). + +export function resolveBottomDrawerFillHeight(input: { + screenHeight: number + topInset: number + keyboardInset: number + topGap?: number +}): number { + const topGap = input.topGap ?? 16 + const keyboardInset = Math.max(0, input.keyboardInset) + // Never exceed the space under the status-bar gap and above the keyboard — + // a hard minHeight here would grow the sheet upward under the status bar + // while marginBottom still equals the full keyboard inset. + return Math.max(0, input.screenHeight - input.topInset - topGap - keyboardInset) +} diff --git a/mobile/src/components/bottom-drawer-keyboard-inset.test.ts b/mobile/src/components/bottom-drawer-keyboard-inset.test.ts new file mode 100644 index 00000000000..2b8ba36d972 --- /dev/null +++ b/mobile/src/components/bottom-drawer-keyboard-inset.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' +import { resolveBottomDrawerKeyboardInset } from './bottom-drawer-keyboard-inset' + +describe('resolveBottomDrawerKeyboardInset', () => { + it('uses the full keyboard frame for fill sheets on iOS and Android', () => { + expect( + resolveBottomDrawerKeyboardInset({ + keyboardHeight: 336, + bottomInset: 34, + fillAvailable: true, + platform: 'ios' + }) + ).toBe(336) + expect( + resolveBottomDrawerKeyboardInset({ + keyboardHeight: 300, + bottomInset: 48, + fillAvailable: true, + platform: 'android' + }) + ).toBe(300) + }) + + it('subtracts the home-indicator inset only for iOS content-sized sheets', () => { + expect( + resolveBottomDrawerKeyboardInset({ + keyboardHeight: 336, + bottomInset: 34, + fillAvailable: false, + platform: 'ios' + }) + ).toBe(302) + }) + + it('uses the full IME height for Android content-sized sheets', () => { + // Why: Android keyboard height does not include the nav bar (session terminal lift). + expect( + resolveBottomDrawerKeyboardInset({ + keyboardHeight: 300, + bottomInset: 48, + fillAvailable: false, + platform: 'android' + }) + ).toBe(300) + }) + + it('never returns a negative inset', () => { + expect( + resolveBottomDrawerKeyboardInset({ + keyboardHeight: 20, + bottomInset: 34, + fillAvailable: false, + platform: 'ios' + }) + ).toBe(0) + }) +}) diff --git a/mobile/src/components/bottom-drawer-keyboard-inset.ts b/mobile/src/components/bottom-drawer-keyboard-inset.ts new file mode 100644 index 00000000000..618d07bd196 --- /dev/null +++ b/mobile/src/components/bottom-drawer-keyboard-inset.ts @@ -0,0 +1,26 @@ +// Why: iOS keyboard frame height includes the home-indicator region; Android +// IME height does not include the system nav bar. That split is already used +// by the session terminal keyboard lift — keep fill/content-sized drawers on +// the same contract so OEM/Android and iPhone behave consistently. +// +// Fill sheets dock chrome to the *true keyboard top* via marginBottom + height +// shrink. They always use the raw frame height (subtracting safe-bottom on iOS +// parks the TextInput under the keys). Content-sized sheets keep the legacy +// translate path: iOS subtracts safe-bottom (padding already covers it), +// Android uses the full IME height. + +export function resolveBottomDrawerKeyboardInset(input: { + keyboardHeight: number + bottomInset: number + fillAvailable: boolean + platform: 'ios' | 'android' | 'windows' | 'macos' | 'web' +}): number { + const keyboardHeight = Math.max(0, input.keyboardHeight) + if (input.fillAvailable) { + return keyboardHeight + } + if (input.platform === 'ios') { + return Math.max(0, keyboardHeight - Math.max(0, input.bottomInset)) + } + return keyboardHeight +} diff --git a/mobile/src/components/bottom-drawer-styles.ts b/mobile/src/components/bottom-drawer-styles.ts new file mode 100644 index 00000000000..ce74ac5a50e --- /dev/null +++ b/mobile/src/components/bottom-drawer-styles.ts @@ -0,0 +1,74 @@ +import { Platform, StyleSheet } from 'react-native' +import { colors, spacing } from '../theme/mobile-theme' + +export const bottomDrawerStyles = StyleSheet.create({ + overlay: { + ...StyleSheet.absoluteFillObject, + zIndex: 1000 + }, + root: { + flex: 1 + }, + backdrop: { + ...StyleSheet.absoluteFillObject, + backgroundColor: 'rgba(0,0,0,0.5)' + }, + backdropPressable: { + ...StyleSheet.absoluteFillObject + }, + anchor: { + flex: 1, + justifyContent: 'flex-end' + }, + anchorWide: { + alignItems: 'center' + }, + drawer: { + backgroundColor: colors.bgBase, + borderTopLeftRadius: 16, + borderTopRightRadius: 16, + paddingHorizontal: spacing.md, + ...Platform.select({ + ios: { + shadowColor: '#000', + shadowOffset: { width: 0, height: -2 }, + shadowOpacity: 0.2, + shadowRadius: 10 + }, + android: { elevation: 8 } + }) + }, + drawerFill: { + // Why: flex children (results + dock) need a column height budget; without + // this, fill height alone still leaves staticContent height content-sized. + overflow: 'hidden', + flexDirection: 'column' + }, + handle: { + alignSelf: 'center', + width: 36, + height: 4, + borderRadius: 2, + backgroundColor: colors.textMuted, + opacity: 0.4 + }, + handleHitArea: { + alignItems: 'center', + paddingTop: spacing.sm, + paddingBottom: spacing.md + }, + staticContent: { + minHeight: 0 + }, + staticContentFill: { + flex: 1 + }, + bottomExtension: { + position: 'absolute', + bottom: -500, + left: 0, + right: 0, + height: 500, + backgroundColor: colors.bgBase + } +}) diff --git a/mobile/src/components/mounted-bottom-drawer.tsx b/mobile/src/components/mounted-bottom-drawer.tsx new file mode 100644 index 00000000000..3331d7298d8 --- /dev/null +++ b/mobile/src/components/mounted-bottom-drawer.tsx @@ -0,0 +1,400 @@ +import { type ReactNode, useCallback, useEffect, useState } from 'react' +import { + View, + Pressable, + useWindowDimensions, + ScrollView, + Keyboard, + BackHandler, + Modal, + Platform +} from 'react-native' +import { useSafeAreaInsets } from 'react-native-safe-area-context' +import { Gesture, GestureDetector, GestureHandlerRootView } from 'react-native-gesture-handler' +import Animated, { + useSharedValue, + useAnimatedStyle, + useAnimatedScrollHandler, + withSpring, + withTiming, + runOnJS, + interpolate, + Extrapolation +} from 'react-native-reanimated' +import { spacing } from '../theme/mobile-theme' +import { resolveBottomDrawerFillHeight } from './bottom-drawer-fill-height' +import { resolveBottomDrawerKeyboardInset } from './bottom-drawer-keyboard-inset' +import { BOTTOM_DRAWER_HIDE_DURATION_MS } from './bottom-drawer-constants' +import { bottomDrawerStyles as styles } from './bottom-drawer-styles' +import { useInsideBottomDrawerModalHost } from './bottom-drawer-modal-host' +import { useResponsiveLayout } from '../layout/responsive-layout' + +const DISMISS_THRESHOLD = 80 +const SPRING_CONFIG = { damping: 28, stiffness: 400 } +// Why: negative translateY (pulling up) is damped with a rubber-band factor +// so the drawer resists upward dragging — a subtle polish touch that signals +// the drawer cannot expand further. +const RUBBER_BAND_FACTOR = 0.25 +const SHOW_DURATION = 180 +const TOP_SCROLL_EPSILON = 1 + +export type MountedBottomDrawerProps = { + visible: boolean + onClose: () => void + onHidden: () => void + children: ReactNode + dragContentToDismiss?: boolean + contentScrollable?: boolean + fillAvailable?: boolean + // Why: outer sheets pinned under an inner fill picker stay laid out (size + // preserved) but must not take touches, stack backdrops, or keyboard-lift. + interactive?: boolean + zIndex?: number +} + +export function MountedBottomDrawer({ + visible, + onClose, + onHidden, + children, + dragContentToDismiss = true, + contentScrollable = true, + fillAvailable = false, + interactive = true, + zIndex = 1000 +}: MountedBottomDrawerProps) { + const translateY = useSharedValue(0) + const progress = useSharedValue(0) + const keyboardOffset = useSharedValue(0) + const scrollOffsetY = useSharedValue(0) + const contentDragStartY = useSharedValue(0) + const contentDragCanDismiss = useSharedValue(false) + // Why: fill mode needs the keyboard inset in React layout (not only the + // reanimated translate) so height shrinks as the sheet lifts and the top + // edge stays under the status bar. + const [keyboardInset, setKeyboardInset] = useState(0) + const { height: screenHeight } = useWindowDimensions() + const insets = useSafeAreaInsets() + // Why: on wide/tablet canvases a full-width sheet looks stretched; cap it and + // center it horizontally. Vertical bottom-anchoring (and all the drag/keyboard + // transforms below) is unchanged, so phone behavior stays identical. + const { isWideLayout, modalMaxWidth } = useResponsiveLayout() + const insideModalHost = useInsideBottomDrawerModalHost() + const fillHeight = fillAvailable + ? resolveBottomDrawerFillHeight({ + screenHeight, + topInset: insets.top, + keyboardInset, + topGap: spacing.lg + }) + : undefined + + useEffect(() => { + if (visible) { + translateY.value = 0 + scrollOffsetY.value = 0 + progress.value = withTiming(1, { duration: SHOW_DURATION }) + } else { + Keyboard.dismiss() + setKeyboardInset(0) + progress.value = withTiming(0, { duration: BOTTOM_DRAWER_HIDE_DURATION_MS }, (finished) => { + if (finished) { + runOnJS(onHidden)() + } + }) + } + }, [onHidden, visible]) + + // Why: KeyboardAvoidingView and useAnimatedKeyboard are both unreliable + // inside Modal (iOS ignores KAV; Android needs adjustNothing for + // useAnimatedKeyboard). Keyboard event listeners work on both platforms + // and give us the exact height to shift the drawer by. + useEffect(() => { + // Pinned-under sheets stay visible for size but must not ride the keyboard — + // only the top interactive sheet owns inset/lift. + if (!visible || !interactive) { + keyboardOffset.value = 0 + setKeyboardInset(0) + return + } + + function applyKeyboardHeight(keyboardHeight: number, duration = 0): void { + const inset = resolveBottomDrawerKeyboardInset({ + keyboardHeight, + bottomInset: insets.bottom, + fillAvailable, + platform: Platform.OS + }) + setKeyboardInset(inset) + if (duration > 0) { + keyboardOffset.value = withTiming(inset, { duration }) + } else { + keyboardOffset.value = inset + } + } + + // Why: fill sheets dock to the true keyboard top; autoFocus can raise the + // keyboard before listeners attach. Seed only in fill mode so content-sized + // outer sheets do not inherit a stale metrics height after an inner dismiss. + if (fillAvailable) { + const existing = Keyboard.metrics() + if (existing != null && existing.height > 0) { + applyKeyboardHeight(existing.height) + } + } + + const showEvent = Platform.OS === 'ios' ? 'keyboardWillShow' : 'keyboardDidShow' + const hideEvent = Platform.OS === 'ios' ? 'keyboardWillHide' : 'keyboardDidHide' + + const onShow = Keyboard.addListener(showEvent, (e) => { + applyKeyboardHeight(e.endCoordinates.height, e.duration || 250) + }) + const onHide = Keyboard.addListener(hideEvent, (e) => { + setKeyboardInset(0) + keyboardOffset.value = withTiming(0, { duration: e.duration || 250 }) + }) + + return () => { + onShow.remove() + onHide.remove() + keyboardOffset.value = 0 + setKeyboardInset(0) + } + }, [visible, interactive, insets.bottom, fillAvailable]) + + const dismiss = useCallback(() => { + Keyboard.dismiss() + progress.value = withTiming(0, { duration: BOTTOM_DRAWER_HIDE_DURATION_MS }, (finished) => { + if (finished) { + runOnJS(onClose)() + } + }) + }, [onClose, progress]) + + useEffect(() => { + if (!visible || !interactive) { + return + } + + const sub = BackHandler.addEventListener('hardwareBackPress', () => { + dismiss() + return true + }) + return () => sub.remove() + }, [visible, interactive, dismiss]) + + const scrollHandler = useAnimatedScrollHandler((event) => { + scrollOffsetY.value = Math.max(event.contentOffset.y, 0) + }) + + const scrollGesture = Gesture.Native() + const handlePanGesture = Gesture.Pan() + .activeOffsetY([-8, 8]) + .simultaneousWithExternalGesture(scrollGesture) + .onUpdate((e) => { + if (e.translationY > 0) { + translateY.value = e.translationY + } else { + translateY.value = e.translationY * RUBBER_BAND_FACTOR + } + }) + .onEnd((e) => { + if (e.translationY > DISMISS_THRESHOLD || e.velocityY > 500) { + const velocity = Math.max(e.velocityY, 800) + const remaining = screenHeight - e.translationY + const duration = Math.min(Math.max((remaining / velocity) * 1000, 120), 300) + translateY.value = withTiming(screenHeight, { duration }) + progress.value = withTiming(0, { duration }, () => { + runOnJS(onClose)() + }) + } else { + translateY.value = withSpring(0, SPRING_CONFIG) + } + }) + const contentPanGesture = Gesture.Pan() + .activeOffsetY([-8, 8]) + .simultaneousWithExternalGesture(scrollGesture) + .onBegin(() => { + contentDragStartY.value = 0 + contentDragCanDismiss.value = scrollOffsetY.value <= TOP_SCROLL_EPSILON + }) + .onUpdate((e) => { + // Why: action-sheet content can be taller than the drawer; downward drags + // should scroll back to the top before they start dismissing the sheet. + if (scrollOffsetY.value > TOP_SCROLL_EPSILON) { + contentDragCanDismiss.value = false + contentDragStartY.value = 0 + if (translateY.value !== 0) { + translateY.value = withSpring(0, SPRING_CONFIG) + } + return + } + + if (!contentDragCanDismiss.value) { + contentDragCanDismiss.value = true + contentDragStartY.value = e.translationY + } + + const translationY = e.translationY - contentDragStartY.value + if (translationY > 0) { + translateY.value = translationY + } else { + translateY.value = translationY * RUBBER_BAND_FACTOR + } + }) + .onEnd((e) => { + if (!contentDragCanDismiss.value || scrollOffsetY.value > TOP_SCROLL_EPSILON) { + return + } + + const translationY = e.translationY - contentDragStartY.value + if (translationY > DISMISS_THRESHOLD || e.velocityY > 500) { + const velocity = Math.max(e.velocityY, 800) + const remaining = screenHeight - translationY + const duration = Math.min(Math.max((remaining / velocity) * 1000, 120), 300) + translateY.value = withTiming(screenHeight, { duration }) + progress.value = withTiming(0, { duration }, () => { + runOnJS(onClose)() + }) + } else { + translateY.value = withSpring(0, SPRING_CONFIG) + } + }) + + const drawerStyle = useAnimatedStyle(() => { + // Why: fill mode already shrinks height by the keyboard inset and lifts via + // marginBottom (layout). Also subtracting keyboardOffset here would double- + // count and park the dock under the keys (input hidden). + const keyboardShift = fillAvailable ? 0 : keyboardOffset.value + return { + transform: [ + { + translateY: + interpolate(progress.value, [0, 1], [screenHeight, 0], Extrapolation.CLAMP) + + translateY.value - + keyboardShift + } + ] + } + }) + + const backdropStyle = useAnimatedStyle(() => { + const dragFade = interpolate(translateY.value, [0, 300], [1, 0], Extrapolation.CLAMP) + return { opacity: progress.value * dragFade } + }) + + // Why: the sheet renders through a full-screen native window (its own Modal + // below, or the shared BottomDrawerModalHost) so it always covers the viewport + // — even when mounted deep inside a ScrollView, where a plain absolute overlay + // anchors to the scrolled content and clips the sheet. Show/hide is driven by + // `progress` (animationType "none") so the reanimated exit animation runs before + // the parent unmounts us. + const handle = ( + + + + + + ) + + const body = !contentScrollable ? ( + <> + {handle} + + {children} + + + ) : dragContentToDismiss ? ( + <> + {handle} + + + + + {children} + + + + + + ) : ( + <> + {handle} + + {children} + + + ) + + const overlay = ( + + + + {interactive ? : null} + + + + 0 ? spacing.sm : insets.bottom + spacing.lg + }, + drawerStyle + ]} + > + {body} + + + + + + ) + + // Why: inside a BottomDrawerModalHost the host owns the single native Modal; + // rendering our own would stack modals and reintroduce the iOS present/dismiss + // race the host exists to avoid. The host handles the Android back button. + if (insideModalHost) { + return overlay + } + + return ( + + {overlay} + + ) +} diff --git a/mobile/src/components/new-worktree-form-sheet-visibility.test.ts b/mobile/src/components/new-worktree-form-sheet-visibility.test.ts new file mode 100644 index 00000000000..450097b74b0 --- /dev/null +++ b/mobile/src/components/new-worktree-form-sheet-visibility.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { resolveNewWorktreeFormSheetVisible } from './new-worktree-form-sheet-visibility' + +describe('resolveNewWorktreeFormSheetVisible', () => { + it('keeps the form under the source picker and its close transition', () => { + expect( + resolveNewWorktreeFormSheetVisible({ + modalVisible: true, + drawerView: 'source', + formPinnedUnderSource: true + }) + ).toBe(true) + expect( + resolveNewWorktreeFormSheetVisible({ + modalVisible: true, + drawerView: 'transition', + formPinnedUnderSource: true + }) + ).toBe(true) + }) + + it('hides the form for sequential repo/agent transitions', () => { + expect( + resolveNewWorktreeFormSheetVisible({ + modalVisible: true, + drawerView: 'transition', + formPinnedUnderSource: false + }) + ).toBe(false) + expect( + resolveNewWorktreeFormSheetVisible({ + modalVisible: true, + drawerView: 'repo', + formPinnedUnderSource: false + }) + ).toBe(false) + }) +}) diff --git a/mobile/src/components/new-worktree-form-sheet-visibility.ts b/mobile/src/components/new-worktree-form-sheet-visibility.ts new file mode 100644 index 00000000000..57ad63c0ef6 --- /dev/null +++ b/mobile/src/components/new-worktree-form-sheet-visibility.ts @@ -0,0 +1,16 @@ +// Why: pin the create form under the fill-height name picker (and during that +// picker's close transition) so dismiss reveals the original content height. + +export function resolveNewWorktreeFormSheetVisible(input: { + modalVisible: boolean + drawerView: string + formPinnedUnderSource: boolean +}): boolean { + if (!input.modalVisible) { + return false + } + if (input.drawerView === 'form' || input.drawerView === 'source') { + return true + } + return input.drawerView === 'transition' && input.formPinnedUnderSource +} diff --git a/mobile/src/components/pr-sidebar/CommentMarkdown.tsx b/mobile/src/components/pr-sidebar/CommentMarkdown.tsx index 0fcc49008c2..236f653f81b 100644 --- a/mobile/src/components/pr-sidebar/CommentMarkdown.tsx +++ b/mobile/src/components/pr-sidebar/CommentMarkdown.tsx @@ -167,7 +167,10 @@ function TableBlock({ block: Extract base: number }) { - const columnCount = Math.max(block.headers.length, ...block.rows.map((r) => r.length), 1) + let columnCount = Math.max(block.headers.length, 1) + for (const row of block.rows) { + columnCount = Math.max(columnCount, row.length) + } const columns = Array.from({ length: columnCount }, (_, c) => c) return ( { it('classifies headings, fenced code, quotes, lists, hr, and paragraphs', () => { @@ -100,6 +100,42 @@ describe('parseMarkdownBlocks tables', () => { ]) }) + it('accepts the exact table column limit and falls back one column later', () => { + const header = Array.from({ length: COMMENT_MARKDOWN_LIMITS.tableColumns }, (_, index) => + String(index) + ).join('|') + const delimiter = Array.from( + { length: COMMENT_MARKDOWN_LIMITS.tableColumns }, + () => '---' + ).join('|') + + expect(parseMarkdownBlocks(`${header}\n${delimiter}`)[0]).toMatchObject({ + kind: 'table', + headers: { length: COMMENT_MARKDOWN_LIMITS.tableColumns } + }) + expect(parseMarkdownBlocks(`${header}|extra\n${delimiter}|---`)[0]).toMatchObject({ + kind: 'paragraph' + }) + }) + + it('falls back to a bounded preview before splitting oversized input', () => { + const exact = 'x'.repeat(COMMENT_MARKDOWN_LIMITS.sourceCodeUnits) + const content = 'x'.repeat(COMMENT_MARKDOWN_LIMITS.sourceCodeUnits + 1) + const [block] = parseMarkdownBlocks(content) + + expect(parseMarkdownBlocks(exact)).toEqual([{ kind: 'paragraph', text: exact }]) + expect(block).toMatchObject({ kind: 'paragraph' }) + expect(block?.kind === 'paragraph' ? block.text.length : 0).toBeLessThan( + COMMENT_MARKDOWN_LIMITS.fallbackCodeUnits + 100 + ) + }) + + it('checks normalized break tags before allocating line arrays', () => { + const content = '
'.repeat(COMMENT_MARKDOWN_LIMITS.lines + 1) + + expect(parseMarkdownBlocks(content)[0]).toMatchObject({ kind: 'paragraph' }) + }) + it('reads per-column alignment from the delimiter row', () => { const md = ['| L | C | R |', '| :--- | :---: | ---: |', '| a | b | c |'].join('\n') const block = parseMarkdownBlocks(md)[0] diff --git a/mobile/src/components/pr-sidebar/markdown-blocks.ts b/mobile/src/components/pr-sidebar/markdown-blocks.ts index d5866585b68..dd597383914 100644 --- a/mobile/src/components/pr-sidebar/markdown-blocks.ts +++ b/mobile/src/components/pr-sidebar/markdown-blocks.ts @@ -40,6 +40,20 @@ const ORDERED = /^\s*\d+[.)]\s+(.*)$/ const HTML_BLOCK = /<(details|blockquote)\b[^>]*>([\s\S]*?)<\/\1>/i const SUMMARY = /]*>([\s\S]*?)<\/summary>/i +export const COMMENT_MARKDOWN_LIMITS = { + sourceCodeUnits: 1024 * 1024, + lines: 10_000, + detailsDepth: 64, + tableColumns: 256, + tableRows: 2000, + tableCells: 50_000, + fallbackCodeUnits: 64 * 1024 +} as const + +const LIMIT_FALLBACK_SUFFIX = '\n\n[Comment preview truncated for safety]' + +class CommentMarkdownLimitError extends Error {} + // Removes residual HTML tags from rendered text so stray // etc. don't // show literally. Conservative: only matches `` / `` shapes, so a bare // "a < b" in prose is left alone. @@ -48,9 +62,56 @@ export function stripHtmlTags(text: string): string { } export function parseMarkdownBlocks(content: string): MarkdownBlock[] { + if (!isCommentMarkdownWithinPreparseLimits(content)) { + return commentMarkdownLimitFallback(content) + } // Drop HTML comments and normalize
before block parsing. const cleaned = content.replace(//g, '').replace(//gi, '\n') - return parseSegment(cleaned) + if (!isCommentMarkdownWithinPreparseLimits(cleaned)) { + return commentMarkdownLimitFallback(content) + } + try { + return parseSegment(cleaned) + } catch (error) { + if (error instanceof CommentMarkdownLimitError) { + return commentMarkdownLimitFallback(content) + } + throw error + } +} + +function isCommentMarkdownWithinPreparseLimits(content: string): boolean { + if (content.length > COMMENT_MARKDOWN_LIMITS.sourceCodeUnits) { + return false + } + let lines = 1 + let detailsDepth = 0 + const detailsPattern = /<\/?details\b/gi + for (let index = 0; index < content.length; index += 1) { + if (content.charCodeAt(index) !== 10) { + continue + } + lines += 1 + if (lines > COMMENT_MARKDOWN_LIMITS.lines) { + return false + } + } + for (const match of content.matchAll(detailsPattern)) { + if (match[0][1] === '/') { + detailsDepth = Math.max(0, detailsDepth - 1) + continue + } + detailsDepth += 1 + if (detailsDepth > COMMENT_MARKDOWN_LIMITS.detailsDepth) { + return false + } + } + return true +} + +function commentMarkdownLimitFallback(content: string): MarkdownBlock[] { + const preview = content.slice(0, COMMENT_MARKDOWN_LIMITS.fallbackCodeUnits) + return [{ kind: 'paragraph', text: `${preview}${LIMIT_FALLBACK_SUFFIX}` }] } // Splits a segment at top-level
/
regions (preserving order), @@ -120,8 +181,17 @@ function parseLines(content: string): MarkdownBlock[] { const align = parseAlignRow(lines[i + 1]) i += 2 const rows: string[][] = [] + let tableCells = headers.length while (i < lines.length && lines[i].includes('|') && lines[i].trim() !== '') { - rows.push(splitTableRow(lines[i])) + if (rows.length >= COMMENT_MARKDOWN_LIMITS.tableRows) { + throw new CommentMarkdownLimitError('Table row limit exceeded') + } + const row = splitTableRow(lines[i]) + tableCells += row.length + if (tableCells > COMMENT_MARKDOWN_LIMITS.tableCells) { + throw new CommentMarkdownLimitError('Table cell limit exceeded') + } + rows.push(row) i += 1 } blocks.push({ kind: 'table', headers, rows, align }) @@ -207,12 +277,18 @@ function splitTableRow(line: string): string[] { continue } if (ch === '|') { + if (cells.length >= COMMENT_MARKDOWN_LIMITS.tableColumns) { + throw new CommentMarkdownLimitError('Table column limit exceeded') + } cells.push(cell.trim()) cell = '' continue } cell += ch } + if (cells.length >= COMMENT_MARKDOWN_LIMITS.tableColumns) { + throw new CommentMarkdownLimitError('Table column limit exceeded') + } cells.push(cell.trim()) return cells } diff --git a/mobile/src/components/smart-workspace-source-drawer-styles.ts b/mobile/src/components/smart-workspace-source-drawer-styles.ts new file mode 100644 index 00000000000..3adc4aab8cb --- /dev/null +++ b/mobile/src/components/smart-workspace-source-drawer-styles.ts @@ -0,0 +1,179 @@ +import { StyleSheet } from 'react-native' +import { colors, radii, spacing, typography } from '../theme/mobile-theme' + +export const smartWorkspaceSourceDrawerStyles = StyleSheet.create({ + root: { + flex: 1, + minHeight: 0 + }, + header: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + paddingHorizontal: spacing.xs, + paddingBottom: spacing.sm, + flexShrink: 0 + }, + title: { + fontSize: 15, + fontWeight: '600', + color: colors.textPrimary + }, + done: { + fontSize: typography.bodySize, + fontWeight: '600', + color: colors.accentBlue + }, + results: { + flex: 1, + minHeight: 0 + }, + list: { + flex: 1, + backgroundColor: colors.bgPanel, + borderTopLeftRadius: radii.card, + borderTopRightRadius: radii.card, + overflow: 'hidden' + }, + listContent: { + flexGrow: 1, + paddingBottom: spacing.sm + }, + // Why: pin the dock to the sheet bottom so a flex-greedy FlatList cannot + // push the TextInput out of the fill frame (and under the keyboard). + dock: { + flexShrink: 0, + borderTopWidth: StyleSheet.hairlineWidth, + borderTopColor: colors.borderSubtle, + backgroundColor: colors.bgBase, + paddingTop: spacing.sm, + paddingBottom: spacing.sm, + gap: spacing.sm, + zIndex: 2 + }, + search: { + backgroundColor: colors.bgRaised, + color: colors.textPrimary, + borderRadius: radii.input, + paddingHorizontal: spacing.md, + paddingVertical: spacing.sm + 2, + fontSize: typography.bodySize, + borderWidth: 1, + borderColor: colors.borderSubtle + }, + tabRow: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: spacing.xs + }, + tab: { + flexDirection: 'row', + alignItems: 'center', + gap: spacing.xs, + paddingHorizontal: spacing.sm + 2, + paddingVertical: spacing.xs + 2, + borderRadius: radii.button, + borderWidth: 1, + borderColor: colors.borderSubtle + }, + tabSelected: { + backgroundColor: colors.bgPanel, + borderColor: colors.textSecondary + }, + tabText: { + fontSize: 13, + color: colors.textSecondary + }, + tabTextSelected: { + color: colors.textPrimary, + fontWeight: '600' + }, + chipRow: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: spacing.xs + }, + chip: { + paddingHorizontal: spacing.md, + paddingVertical: spacing.xs, + borderRadius: radii.button, + borderWidth: 1, + borderColor: colors.borderSubtle + }, + chipSelected: { + backgroundColor: colors.bgPanel, + borderColor: colors.textSecondary + }, + chipText: { + fontSize: 12, + color: colors.textSecondary + }, + chipTextSelected: { + color: colors.textPrimary, + fontWeight: '600' + }, + crossRepo: { + backgroundColor: colors.bgRaised, + borderRadius: radii.input, + borderWidth: 1, + borderColor: colors.borderSubtle, + padding: spacing.md, + marginBottom: spacing.sm, + gap: spacing.sm + }, + crossRepoText: { + fontSize: 13, + color: colors.textSecondary + }, + crossRepoActions: { + flexDirection: 'row', + justifyContent: 'flex-end', + gap: spacing.sm + }, + crossRepoDismiss: { + paddingHorizontal: spacing.md, + paddingVertical: spacing.xs + 2, + borderRadius: radii.button, + borderWidth: 1, + borderColor: colors.borderSubtle + }, + crossRepoDismissText: { + fontSize: 13, + color: colors.textSecondary + }, + crossRepoSwitch: { + paddingHorizontal: spacing.md, + paddingVertical: spacing.xs + 2, + borderRadius: radii.button, + backgroundColor: colors.bgPanel, + borderWidth: 1, + borderColor: colors.textSecondary + }, + crossRepoSwitchText: { + fontSize: 13, + fontWeight: '600', + color: colors.textPrimary + }, + notice: { + fontSize: 12, + color: colors.textMuted, + paddingHorizontal: spacing.xs, + paddingBottom: spacing.sm + }, + errorNotice: { + fontSize: 12, + color: colors.statusRed, + paddingHorizontal: spacing.xs, + paddingBottom: spacing.sm + }, + loading: { + paddingVertical: spacing.lg, + alignItems: 'center' + }, + empty: { + paddingVertical: spacing.lg, + textAlign: 'center', + color: colors.textMuted, + fontSize: 13 + } +}) diff --git a/mobile/src/components/use-new-worktree-drawer-navigation.ts b/mobile/src/components/use-new-worktree-drawer-navigation.ts new file mode 100644 index 00000000000..c21a6dee4f1 --- /dev/null +++ b/mobile/src/components/use-new-worktree-drawer-navigation.ts @@ -0,0 +1,80 @@ +import { useEffect, useRef, useState } from 'react' +import { BOTTOM_DRAWER_HIDE_DURATION_MS } from './bottom-drawer-constants' +import { resolveNewWorktreeFormSheetVisible } from './new-worktree-form-sheet-visibility' + +export type NewWorktreeDrawerView = 'form' | 'transition' | 'source' | 'repo' | 'agent' | 'trust' + +// Why: iOS cannot reliably present a second native modal until the first drawer's +// exit commits; one extra frame keeps transitions sequential on slower devices. +const NEW_WORKTREE_DRAWER_TRANSITION_MS = BOTTOM_DRAWER_HIDE_DURATION_MS + 16 + +export function useNewWorktreeDrawerNavigation(modalVisible: boolean): { + drawerView: NewWorktreeDrawerView + formSheetVisible: boolean + formSheetInteractive: boolean + transitionDrawer: (nextView: Exclude) => void + openSourceDrawer: () => void +} { + const [drawerView, setDrawerView] = useState('form') + const formPinnedUnderSourceRef = useRef(false) + const drawerTransitionTimerRef = useRef | null>(null) + + // Why: cancel any queued transition and reset when the modal closes, so a + // timer can't land after close and leave a stale drawer/pin for the next open. + useEffect(() => { + if (modalVisible) { + return + } + if (drawerTransitionTimerRef.current) { + clearTimeout(drawerTransitionTimerRef.current) + drawerTransitionTimerRef.current = null + } + formPinnedUnderSourceRef.current = false + setDrawerView('form') + }, [modalVisible]) + + useEffect(() => { + return () => { + if (drawerTransitionTimerRef.current) { + clearTimeout(drawerTransitionTimerRef.current) + } + } + }, []) + + function transitionDrawer(nextView: Exclude): void { + if (drawerTransitionTimerRef.current) { + clearTimeout(drawerTransitionTimerRef.current) + } + setDrawerView('transition') + drawerTransitionTimerRef.current = setTimeout(() => { + drawerTransitionTimerRef.current = null + if (nextView === 'form') { + formPinnedUnderSourceRef.current = false + } + setDrawerView(nextView) + }, NEW_WORKTREE_DRAWER_TRANSITION_MS) + } + + function openSourceDrawer(): void { + // Why: same-beat open; pin form under fill picker so outer content height + // is preserved when the name dialog dismisses. + if (drawerTransitionTimerRef.current) { + clearTimeout(drawerTransitionTimerRef.current) + } + drawerTransitionTimerRef.current = null + formPinnedUnderSourceRef.current = true + setDrawerView('source') + } + + return { + drawerView, + formSheetVisible: resolveNewWorktreeFormSheetVisible({ + modalVisible, + drawerView, + formPinnedUnderSource: formPinnedUnderSourceRef.current + }), + formSheetInteractive: drawerView === 'form', + transitionDrawer, + openSourceDrawer + } +} diff --git a/mobile/src/diagnostics/troubleshoot-common-issues.tsx b/mobile/src/diagnostics/troubleshoot-common-issues.tsx index 5dc91eb65fc..b794ad004d5 100644 --- a/mobile/src/diagnostics/troubleshoot-common-issues.tsx +++ b/mobile/src/diagnostics/troubleshoot-common-issues.tsx @@ -14,7 +14,7 @@ export const troubleshootCommonIssues: TroubleshootSection[] = [ icon: , title: 'Different WiFi Networks', steps: [ - 'Both devices must be on the same local network (unless connected through Tailscale).', + 'Both devices must be on the same LAN (unless connected through Tailscale).', 'Ethernet and WiFi must share the same subnet.', 'Try reconnecting WiFi on both devices.' ] diff --git a/mobile/src/dictation/dictation-setup-poll-controller.test.ts b/mobile/src/dictation/dictation-setup-poll-controller.test.ts index 5ca010759d6..c1eacd0c999 100644 --- a/mobile/src/dictation/dictation-setup-poll-controller.test.ts +++ b/mobile/src/dictation/dictation-setup-poll-controller.test.ts @@ -132,6 +132,27 @@ describe('DictationSetupPollController', () => { poller.dispose() }) + it('shares one completion while repeated manual refreshes are pending', async () => { + const requests = [deferred(), deferred()] + const refresh = vi.fn(() => requests[refresh.mock.calls.length - 1].promise) + const poller = new DictationSetupPollController(refresh, POLL_INTERVAL_MS) + poller.setPolling(true) + poller.setVisible(true) + poller.setForeground(true) + + const completions = Array.from({ length: 1_000 }, () => poller.refreshNow()) + expect(new Set(completions).size).toBe(1) + expect(refresh).toHaveBeenCalledOnce() + + requests[0].resolve(true) + await flushPromises() + expect(refresh).toHaveBeenCalledTimes(2) + + requests[1].resolve(false) + await expect(completions[0]).resolves.toBeUndefined() + poller.dispose() + }) + it('refreshes immediately when visibility or foreground eligibility resumes', async () => { const refresh = vi.fn().mockResolvedValue(true) const poller = new DictationSetupPollController(refresh, POLL_INTERVAL_MS) diff --git a/mobile/src/dictation/dictation-setup-poll-controller.ts b/mobile/src/dictation/dictation-setup-poll-controller.ts index cb927f161f9..de74fdb4f8e 100644 --- a/mobile/src/dictation/dictation-setup-poll-controller.ts +++ b/mobile/src/dictation/dictation-setup-poll-controller.ts @@ -11,7 +11,7 @@ export class DictationSetupPollController { private timer: ReturnType | null = null private inFlight = false private immediateRefreshPending = false - private refreshWaiters: Array<() => void> = [] + private refreshCompletion: { promise: Promise; resolve: () => void } | null = null private disposed = false // Why: an explicit setPolling is a newer lifecycle intent than a read that was already on the wire. // Bumped on every setPolling so an in-flight refresh resolving after an explicit stop/start can be @@ -40,17 +40,22 @@ export class DictationSetupPollController { if (this.disposed || !this.isEligible()) { return Promise.resolve() } - return new Promise((resolve) => { - this.refreshWaiters.push(resolve) - this.requestRefresh(true) - }) + if (!this.refreshCompletion) { + let resolve!: () => void + const promise = new Promise((nextResolve) => { + resolve = nextResolve + }) + this.refreshCompletion = { promise, resolve } + } + this.requestRefresh(true) + return this.refreshCompletion.promise } dispose(): void { this.disposed = true this.immediateRefreshPending = false this.clearTimer() - this.resolveRefreshWaiters() + this.resolveRefreshCompletion() } private update(next: Partial): void { @@ -113,7 +118,7 @@ export class DictationSetupPollController { this.state.polling = shouldContinue } if (this.disposed || !this.isEligible()) { - this.resolveRefreshWaiters() + this.resolveRefreshCompletion() return } if (this.immediateRefreshPending) { @@ -121,7 +126,7 @@ export class DictationSetupPollController { this.requestRefresh(true) return } - this.resolveRefreshWaiters() + this.resolveRefreshCompletion() if (this.state.polling) { this.scheduleRefresh() } @@ -144,10 +149,9 @@ export class DictationSetupPollController { } } - private resolveRefreshWaiters(): void { - const waiters = this.refreshWaiters.splice(0) - for (const resolve of waiters) { - resolve() - } + private resolveRefreshCompletion(): void { + const completion = this.refreshCompletion + this.refreshCompletion = null + completion?.resolve() } } diff --git a/mobile/src/files/MobileFileExplorerPanel.test.ts b/mobile/src/files/MobileFileExplorerPanel.test.ts index 0d003e9f456..86719d7c1cd 100644 --- a/mobile/src/files/MobileFileExplorerPanel.test.ts +++ b/mobile/src/files/MobileFileExplorerPanel.test.ts @@ -2,6 +2,7 @@ import { createElement } from 'react' import { act, create, type ReactTestRenderer } from 'react-test-renderer' import { afterEach, describe, expect, it, vi } from 'vitest' import { MobileFileExplorerPanel } from './MobileFileExplorerPanel' +import { LEGACY_MOBILE_FILE_PATH_MAX_BYTES } from './file-list-fallback' import type { MobileDirEntry } from './file-tree' import type { RpcResponse } from '../transport/types' @@ -206,6 +207,18 @@ describe('MobileFileExplorerPanel', () => { expect(client.sendRequest).toHaveBeenCalledTimes(2) }) + it('rejects an extreme directory listing instead of silently truncating it', async () => { + const client = createMockClient({ + '': Array.from({ length: 10_001 }, (_, index) => entry(`file-${index}.txt`)) + }) + mockTransport.client = client + + const renderer = await renderExplorer() + + expect(renderedText(renderer)).toContain('This folder is too large to show safely on mobile') + expect(renderedText(renderer)).toContain('10,000 items') + }) + it('keeps the loaded tree visible during a transient disconnect', async () => { const client = createMockClient({ '': [entry('src', true), entry('README.md')] @@ -366,6 +379,43 @@ describe('MobileFileExplorerPanel', () => { expect(renderedText(renderer)).toContain('Showing first 5000') }) + it('reports a bounded-memory error when a legacy path exceeds the fallback cap', async () => { + const legacyClient: MockClient = { + sendRequest: vi.fn(async (method: string): Promise => { + if (method === 'files.readDir') { + return { + id: 'response-id', + ok: false, + error: { code: 'method_not_found', message: 'Unknown method' }, + _meta: { runtimeId: 'runtime-id' } + } + } + return { + id: 'response-id', + ok: true, + result: { + files: [ + { + relativePath: 'x'.repeat(LEGACY_MOBILE_FILE_PATH_MAX_BYTES + 1), + basename: 'oversized', + kind: 'text' + } + ], + totalCount: 1, + truncated: false + }, + _meta: { runtimeId: 'runtime-id' } + } + }) + } + mockTransport.client = legacyClient + + const renderer = await renderExplorer() + + expect(renderedText(renderer)).toContain('legacy file list is too large') + expect(renderedText(renderer)).toContain('Update Orca Desktop') + }) + it('reports the files.list failure when the fallback itself fails', async () => { const legacyClient: MockClient = { sendRequest: vi.fn(async (method: string): Promise => { diff --git a/mobile/src/files/MobileFileExplorerPanel.tsx b/mobile/src/files/MobileFileExplorerPanel.tsx index 7e67bf81b5f..d34c4ffd07a 100644 --- a/mobile/src/files/MobileFileExplorerPanel.tsx +++ b/mobile/src/files/MobileFileExplorerPanel.tsx @@ -16,14 +16,15 @@ import { flattenDirectoryCache, getDirectoryCacheState, type DirectoryCache, - type FileExplorerRow, - type MobileDirEntry + type DirectoryState, + type FileExplorerRow } from './file-tree' import type { RpcSuccess } from '../transport/types' import { colors } from '../theme/mobile-theme' import { beginDirectoryLoad, createDirectoryLoadRevisions, + forgetDirectoryLoadBranches, isCurrentDirectoryLoad, resetDirectoryLoadRevisions, type DirectoryLoadRevisions @@ -36,6 +37,12 @@ import { import { fileExplorerStyles as styles } from './mobile-file-explorer-styles' import { MobileFileExplorerRow } from './mobile-file-explorer-row' import { navigateToMobileFilePreview } from './mobile-file-preview-navigation' +import { + MOBILE_DIRECTORY_CACHE_LIMIT_MESSAGE, + parseBoundedMobileDirectoryEntries, + removeEvictedExpandedPaths, + retainMobileDirectoryState +} from './mobile-directory-cache-retention' export function MobileFileExplorerPanel(props: { hostId: string @@ -54,13 +61,48 @@ export function MobileFileExplorerPanel(props: { const directoryLoadRevisionsRef = useRef(createDirectoryLoadRevisions()) const pendingDirectoryRetriesRef = useRef>(new Set()) const directoryCacheRef = useRef({}) + const directoryCacheAccessRef = useRef(0) const [directoryCache, setDirectoryCache] = useState({}) const [expanded, setExpanded] = useState>(() => new Set()) + const expandedRef = useRef(expanded) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) const [legacyListTruncated, setLegacyListTruncated] = useState(false) const worktreeLabel = getWorktreeLabel(name, worktreeId) + const commitDirectoryState = useCallback( + (relativePath: string, state: DirectoryState): boolean => { + const retained = retainMobileDirectoryState( + directoryCacheRef.current, + relativePath, + { ...state, lastAccess: ++directoryCacheAccessRef.current }, + expandedRef.current + ) + if (!retained.admitted) { + return false + } + directoryCacheRef.current = retained.cache + setDirectoryCache(retained.cache) + if (retained.evictedPaths.length > 0) { + forgetDirectoryLoadBranches(directoryLoadRevisionsRef.current, retained.evictedPaths) + for (const pendingPath of pendingDirectoryRetriesRef.current) { + if ( + retained.evictedPaths.some( + (evicted) => pendingPath === evicted || pendingPath.startsWith(`${evicted}/`) + ) + ) { + pendingDirectoryRetriesRef.current.delete(pendingPath) + } + } + const nextExpanded = removeEvictedExpandedPaths(expandedRef.current, retained.evictedPaths) + expandedRef.current = nextExpanded + setExpanded(nextExpanded) + } + return true + }, + [] + ) + const loadDirectory = useCallback( async (relativePath: string) => { const scope = scopeRef.current @@ -77,13 +119,10 @@ export function MobileFileExplorerPanel(props: { // Why: transient reconnects should not blank an already browsable tree. setError(hasLoadedRoot ? null : message) } else { - setDirectoryCache((prev) => ({ - ...prev, - [relativePath]: { - entries: getDirectoryCacheState(prev, relativePath)?.entries ?? [], - error: message - } - })) + commitDirectoryState(relativePath, { + entries: getDirectoryCacheState(directoryCacheRef.current, relativePath)?.entries ?? [], + error: message + }) } return } @@ -98,13 +137,16 @@ export function MobileFileExplorerPanel(props: { } setError(null) } - setDirectoryCache((prev) => ({ - ...prev, - [relativePath]: { - entries: getDirectoryCacheState(prev, relativePath)?.entries ?? [], - loading: true - } - })) + const admittedLoading = commitDirectoryState(relativePath, { + entries: getDirectoryCacheState(directoryCacheRef.current, relativePath)?.entries ?? [], + loading: true + }) + if (!admittedLoading) { + forgetDirectoryLoadBranches(directoryLoadRevisionsRef.current, [relativePath]) + setLoading(false) + setError(MOBILE_DIRECTORY_CACHE_LIMIT_MESSAGE) + return + } try { const response = await client.sendRequest('files.readDir', { @@ -133,7 +175,9 @@ export function MobileFileExplorerPanel(props: { return } const legacyResult = (legacy as RpcSuccess).result as LegacyFilesListResult - setDirectoryCache(directoryCacheFromFileList(legacyResult.files)) + const legacyCache = directoryCacheFromFileList(legacyResult.files) + directoryCacheRef.current = legacyCache + setDirectoryCache(legacyCache) // Why: the capped list silently omits files past the cap — keep // the legacy explorer's "Showing first 5000" note. setLegacyListTruncated(legacyResult.truncated) @@ -150,14 +194,13 @@ export function MobileFileExplorerPanel(props: { ) { return } - const entries = (response as RpcSuccess).result as MobileDirEntry[] + const entries = parseBoundedMobileDirectoryEntries((response as RpcSuccess).result) if (rootLoad) { setLegacyListTruncated(false) } - setDirectoryCache((prev) => ({ - ...prev, - [relativePath]: { entries } - })) + if (!commitDirectoryState(relativePath, { entries })) { + throw new Error(MOBILE_DIRECTORY_CACHE_LIMIT_MESSAGE) + } } catch (err) { if ( !isCurrentDirectoryLoad(directoryLoadRevisionsRef.current, scopeRef.current, loadToken) @@ -170,13 +213,10 @@ export function MobileFileExplorerPanel(props: { // only a cold load surfaces the full-screen error. setError(hadLoadedRoot ? null : message) } else { - setDirectoryCache((prev) => ({ - ...prev, - [relativePath]: { - entries: getDirectoryCacheState(prev, relativePath)?.entries ?? [], - error: message - } - })) + commitDirectoryState(relativePath, { + entries: getDirectoryCacheState(directoryCacheRef.current, relativePath)?.entries ?? [], + error: message + }) } } finally { if ( @@ -187,7 +227,7 @@ export function MobileFileExplorerPanel(props: { } } }, - [client, connState, worktreeId] + [client, commitDirectoryState, connState, worktreeId] ) useEffect(() => { @@ -195,17 +235,15 @@ export function MobileFileExplorerPanel(props: { resetDirectoryLoadRevisions(directoryLoadRevisionsRef.current) pendingDirectoryRetriesRef.current.clear() directoryCacheRef.current = {} + directoryCacheAccessRef.current = 0 setDirectoryCache({}) - setExpanded(new Set()) + expandedRef.current = new Set() + setExpanded(expandedRef.current) setLoading(true) setError(null) setLegacyListTruncated(false) }, [scope]) - useEffect(() => { - directoryCacheRef.current = directoryCache - }, [directoryCache]) - useEffect(() => { void loadDirectory('') }, [hostId, loadDirectory]) @@ -228,21 +266,21 @@ export function MobileFileExplorerPanel(props: { const toggleDirectory = useCallback( (relativePath: string) => { - setExpanded((prev) => { - const next = new Set(prev) - if (next.has(relativePath)) { - next.delete(relativePath) - } else { - next.add(relativePath) - } - return next - }) + const wasExpanded = expandedRef.current.has(relativePath) + const nextExpanded = new Set(expandedRef.current) + if (wasExpanded) { + nextExpanded.delete(relativePath) + } else { + nextExpanded.add(relativePath) + } + expandedRef.current = nextExpanded + setExpanded(nextExpanded) const state = getDirectoryCacheState(directoryCache, relativePath) - if (!expanded.has(relativePath) && !state?.loading && (!state?.entries || state.error)) { + if (!wasExpanded && !state?.loading && (!state?.entries || state.error)) { void loadDirectory(relativePath) } }, - [directoryCache, expanded, loadDirectory] + [directoryCache, loadDirectory] ) const retryDirectory = useCallback( diff --git a/mobile/src/files/directory-load-revisions.test.ts b/mobile/src/files/directory-load-revisions.test.ts index 86ebfc3c0e8..b36f88c7022 100644 --- a/mobile/src/files/directory-load-revisions.test.ts +++ b/mobile/src/files/directory-load-revisions.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { beginDirectoryLoad, createDirectoryLoadRevisions, + forgetDirectoryLoadBranches, isCurrentDirectoryLoad, resetDirectoryLoadRevisions, type DirectoryLoadRevisions @@ -41,4 +42,17 @@ describe('directory-load-revisions', () => { expect(isCurrentDirectoryLoad(revisions, 'host-a:worktree-a', load)).toBe(true) }) + + it('invalidates evicted branches without staling unrelated loads', () => { + const revisions = createDirectoryLoadRevisions() + const src = beginDirectoryLoad(revisions, 'host-a:worktree-a', 'src') + const nested = beginDirectoryLoad(revisions, 'host-a:worktree-a', 'src/lib') + const docs = beginDirectoryLoad(revisions, 'host-a:worktree-a', 'docs') + + forgetDirectoryLoadBranches(revisions, ['src']) + + expect(isCurrentDirectoryLoad(revisions, 'host-a:worktree-a', src)).toBe(false) + expect(isCurrentDirectoryLoad(revisions, 'host-a:worktree-a', nested)).toBe(false) + expect(isCurrentDirectoryLoad(revisions, 'host-a:worktree-a', docs)).toBe(true) + }) }) diff --git a/mobile/src/files/directory-load-revisions.ts b/mobile/src/files/directory-load-revisions.ts index 04153092bbe..584d2bc0aa9 100644 --- a/mobile/src/files/directory-load-revisions.ts +++ b/mobile/src/files/directory-load-revisions.ts @@ -19,6 +19,21 @@ export function resetDirectoryLoadRevisions(revisions: DirectoryLoadRevisions): revisions.revisionsByPath.clear() } +export function forgetDirectoryLoadBranches( + revisions: DirectoryLoadRevisions, + relativePaths: readonly string[] +): void { + for (const loadedPath of revisions.revisionsByPath.keys()) { + if ( + relativePaths.some( + (relativePath) => loadedPath === relativePath || loadedPath.startsWith(`${relativePath}/`) + ) + ) { + revisions.revisionsByPath.delete(loadedPath) + } + } +} + export function beginDirectoryLoad( revisions: DirectoryLoadRevisions, scope: string, diff --git a/mobile/src/files/file-list-fallback.test.ts b/mobile/src/files/file-list-fallback.test.ts index a69affc5fe8..ad6fc7eacec 100644 --- a/mobile/src/files/file-list-fallback.test.ts +++ b/mobile/src/files/file-list-fallback.test.ts @@ -1,7 +1,34 @@ import { describe, expect, it } from 'vitest' -import { directoryCacheFromFileList, isMobileMethodUnavailableError } from './file-list-fallback' +import { + directoryCacheFromFileList, + isMobileMethodUnavailableError, + LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES, + LEGACY_MOBILE_FILE_CACHE_MAX_ENTRIES, + LEGACY_MOBILE_FILE_CACHE_MAX_RETAINED_BYTES, + LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE, + LEGACY_MOBILE_FILE_LIST_MAX_FILES, + LEGACY_MOBILE_FILE_PATH_MAX_BYTES, + LEGACY_MOBILE_FILE_PATH_MAX_DEPTH, + type LegacyMobileFileEntry +} from './file-list-fallback' import { getDirectoryCacheState } from './file-tree' +const RETAINED_NODE_ESTIMATE_BYTES = 64 + +function file(relativePath: string): LegacyMobileFileEntry { + return { relativePath, basename: relativePath, kind: 'text' } +} + +function directoryChainPath(prefix: string, directories: number): string { + const names = Array.from({ length: directories }, (_, index) => (index === 0 ? prefix : 'd')) + return `${names.join('/')}/file` +} + +function fixedLengthUniqueName(index: number, length: number): string { + const suffix = `-${index.toString(36)}` + return `${'x'.repeat(length - suffix.length)}${suffix}` +} + describe('isMobileMethodUnavailableError', () => { it('detects old-desktop allowlist and missing-method failures', () => { expect(isMobileMethodUnavailableError('forbidden', undefined)).toBe(true) @@ -20,9 +47,9 @@ describe('isMobileMethodUnavailableError', () => { describe('directoryCacheFromFileList', () => { it('synthesizes every ancestor directory from flat paths', () => { const cache = directoryCacheFromFileList([ - { relativePath: 'src/lib/util.ts', basename: 'util.ts', kind: 'text' }, - { relativePath: 'src/app.ts', basename: 'app.ts', kind: 'text' }, - { relativePath: 'README.md', basename: 'README.md', kind: 'text' } + file('src/lib/util.ts'), + file('src/app.ts'), + file('README.md') ]) expect(cache['']?.entries).toEqual( expect.arrayContaining([ @@ -39,11 +66,27 @@ describe('directoryCacheFromFileList', () => { expect(cache['src/lib']?.entries).toEqual([{ name: 'util.ts', isDirectory: false }]) }) - it('keeps a name a directory when it appears as both file and dir prefix', () => { + it('preserves empty-segment filtering and first-seen entry order', () => { const cache = directoryCacheFromFileList([ - { relativePath: 'src', basename: 'src', kind: 'text' }, - { relativePath: 'src/app.ts', basename: 'app.ts', kind: 'text' } + file('//src///lib//util.ts//'), + file('/README.md'), + file('src/app.ts'), + file('///') ]) + + expect(cache['']?.entries).toEqual([ + { name: 'src', isDirectory: true }, + { name: 'README.md', isDirectory: false } + ]) + expect(cache['src']?.entries).toEqual([ + { name: 'lib', isDirectory: true }, + { name: 'app.ts', isDirectory: false } + ]) + expect(cache['src/lib']?.entries).toEqual([{ name: 'util.ts', isDirectory: false }]) + }) + + it('keeps a name a directory when it appears as both file and dir prefix', () => { + const cache = directoryCacheFromFileList([file('src'), file('src/app.ts')]) expect(cache['']?.entries).toEqual([{ name: 'src', isDirectory: true }]) }) @@ -53,13 +96,112 @@ describe('directoryCacheFromFileList', () => { }) it('stores a __proto__ directory as an own key instead of mutating the prototype', () => { - const cache = directoryCacheFromFileList([ - { relativePath: '__proto__/pollute.js', basename: 'pollute.js', kind: 'text' } - ]) + const cache = directoryCacheFromFileList([file('__proto__/pollute.js')]) expect(Object.getPrototypeOf(cache)).toBe(Object.prototype) expect(cache['']?.entries).toEqual([{ name: '__proto__', isDirectory: true }]) expect(getDirectoryCacheState(cache, '__proto__')?.entries).toEqual([ { name: 'pollute.js', isDirectory: false } ]) }) + + it('accepts a delimiter-heavy path at the exact byte cap without retaining empty segments', () => { + const cache = directoryCacheFromFileList([file('/'.repeat(LEGACY_MOBILE_FILE_PATH_MAX_BYTES))]) + expect(cache['']?.entries).toEqual([]) + }) + + it('measures the per-file path cap in UTF-8 bytes and rejects one byte over', () => { + const exactPath = 'é'.repeat(LEGACY_MOBILE_FILE_PATH_MAX_BYTES / 2) + expect(directoryCacheFromFileList([file(exactPath)])['']?.entries).toEqual([ + { name: exactPath, isDirectory: false } + ]) + expect(() => directoryCacheFromFileList([file(`${exactPath}a`)])).toThrow( + LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE + ) + }) + + it('accepts the exact path depth and rejects the next segment', () => { + const exactPath = Array.from({ length: LEGACY_MOBILE_FILE_PATH_MAX_DEPTH }, () => 'd').join('/') + const parentPath = exactPath.slice(0, exactPath.lastIndexOf('/')) + expect( + getDirectoryCacheState(directoryCacheFromFileList([file(exactPath)]), parentPath) + ).toEqual({ + entries: [{ name: 'd', isDirectory: false }] + }) + expect(() => directoryCacheFromFileList([file(`${exactPath}/overflow`)])).toThrow( + LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE + ) + }) + + it('rejects a response above the desktop files.list record cap', () => { + const files = Array.from({ length: LEGACY_MOBILE_FILE_LIST_MAX_FILES + 1 }, () => file('')) + expect(() => directoryCacheFromFileList(files)).toThrow(LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE) + }) + + it('accepts the exact directory cap and rejects one additional directory', () => { + const fullChains = Math.floor((LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES - 1) / 255) + const remainingDirectories = LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES - 1 - fullChains * 255 + const files = Array.from({ length: fullChains }, (_, index) => + file(directoryChainPath(`root-${index}`, 255)) + ) + if (remainingDirectories > 0) { + files.push(file(directoryChainPath('tail', remainingDirectories))) + } + + expect(Object.keys(directoryCacheFromFileList(files))).toHaveLength( + LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES + ) + expect(() => + directoryCacheFromFileList([...files, file(directoryChainPath('overflow', 1))]) + ).toThrow(LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE) + }) + + it('accepts the exact aggregate entry cap and rejects one additional entry', () => { + const exactFiles = Array.from({ length: LEGACY_MOBILE_FILE_LIST_MAX_FILES }, (_, index) => + file(`root-${index}/a/b/file`) + ) + const overflowFiles = exactFiles.map((entry, index) => + index === 0 ? file('root-0/a/b/c/file') : entry + ) + + const cache = directoryCacheFromFileList(exactFiles) + const entryCount = Object.values(cache).reduce( + (total, state) => total + (state?.entries.length ?? 0), + 0 + ) + expect(entryCount).toBe(LEGACY_MOBILE_FILE_CACHE_MAX_ENTRIES) + expect(() => directoryCacheFromFileList(overflowFiles)).toThrow( + LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE + ) + }) + + it('accepts the exact retained-byte budget and rejects two bytes over', () => { + const namesBytes = + LEGACY_MOBILE_FILE_CACHE_MAX_RETAINED_BYTES - + RETAINED_NODE_ESTIMATE_BYTES - + LEGACY_MOBILE_FILE_LIST_MAX_FILES * RETAINED_NODE_ESTIMATE_BYTES + const namesCharacters = namesBytes / 2 + const baseLength = Math.floor(namesCharacters / LEGACY_MOBILE_FILE_LIST_MAX_FILES) + const longerNames = namesCharacters % LEGACY_MOBILE_FILE_LIST_MAX_FILES + const exactFiles = Array.from({ length: LEGACY_MOBILE_FILE_LIST_MAX_FILES }, (_, index) => { + const length = baseLength + (index < longerNames ? 1 : 0) + return file(fixedLengthUniqueName(index, length)) + }) + const overflowFiles = exactFiles.map((entry, index) => + index === 0 ? file(`${entry.relativePath}x`) : entry + ) + + expect(directoryCacheFromFileList(exactFiles)['']?.entries).toHaveLength( + LEGACY_MOBILE_FILE_LIST_MAX_FILES + ) + expect(() => directoryCacheFromFileList(overflowFiles)).toThrow( + LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE + ) + }) + + it('rejects malformed legacy response shapes with a clear error', () => { + expect(() => directoryCacheFromFileList({ files: [] })).toThrow('invalid legacy file list') + expect(() => directoryCacheFromFileList([{ relativePath: 42 }])).toThrow( + 'invalid legacy file list' + ) + }) }) diff --git a/mobile/src/files/file-list-fallback.ts b/mobile/src/files/file-list-fallback.ts index ef3041029af..5c2dde0f30e 100644 --- a/mobile/src/files/file-list-fallback.ts +++ b/mobile/src/files/file-list-fallback.ts @@ -1,8 +1,22 @@ // Fallback for desktops that predate files.readDir in the mobile RPC // allowlist: synthesize the lazy directory cache from the flat, capped // files.list result so the Files tab stays browsable against old desktops. +import { getUtf8ByteLengthForCodePoint } from '../../../src/shared/utf8-byte-limits' import type { DirectoryCache, MobileDirEntry } from './file-tree' +// Why: each flat legacy path can amplify into many retained ancestor records. +export const LEGACY_MOBILE_FILE_LIST_MAX_FILES = 5_000 +export const LEGACY_MOBILE_FILE_PATH_MAX_BYTES = 16 * 1024 +export const LEGACY_MOBILE_FILE_PATH_MAX_DEPTH = 256 +export const LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES = 16_384 +export const LEGACY_MOBILE_FILE_CACHE_MAX_ENTRIES = 20_000 +export const LEGACY_MOBILE_FILE_CACHE_MAX_RETAINED_BYTES = 16 * 1024 * 1024 +export const LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE = + 'This legacy file list is too large to show safely on mobile. Update Orca Desktop to browse it folder by folder.' + +const DIRECTORY_RETAINED_BYTES = 64 +const ENTRY_RETAINED_BYTES = 64 + export type LegacyMobileFileEntry = { relativePath: string basename: string @@ -15,6 +29,13 @@ export type LegacyFilesListResult = { truncated: boolean } +type LegacyFileCacheBuildState = { + childrenByDir: Map> + directories: number + entries: number + retainedBytes: number +} + // Same detection shape as isMobileGitUnavailable in mobile-git-status.ts: // 'forbidden' = method exists but is not mobile-allowlisted on the old // desktop; 'method_not_found' = desktop predates the method entirely. @@ -29,45 +50,156 @@ export function isMobileMethodUnavailableError( ) } -export function directoryCacheFromFileList(files: LegacyMobileFileEntry[]): DirectoryCache { - const childrenByDir = new Map>() - const ensureDir = (path: string): Map => { - let children = childrenByDir.get(path) - if (!children) { - children = new Map() - childrenByDir.set(path, children) - } - return children +export function directoryCacheFromFileList(files: unknown): DirectoryCache { + if (!Array.isArray(files)) { + throw new Error('Desktop returned an invalid legacy file list.') } - ensureDir('') + if (files.length > LEGACY_MOBILE_FILE_LIST_MAX_FILES) { + throwLegacyFileListLimitError() + } + + const state: LegacyFileCacheBuildState = { + childrenByDir: new Map(), + directories: 0, + entries: 0, + retainedBytes: 0 + } + ensureDirectory(state, '') for (const file of files) { - const parts = file.relativePath.split('/').filter(Boolean) - let parentPath = '' - parts.forEach((name, index) => { - const isDirectory = index < parts.length - 1 - const children = ensureDir(parentPath) - children.set(name, children.get(name) === true || isDirectory) - parentPath = parentPath ? `${parentPath}/${name}` : name - if (isDirectory) { - ensureDir(parentPath) + const relativePath = getLegacyRelativePath(file) + if (relativePath === null) { + throw new Error('Desktop returned an invalid legacy file list.') + } + addFilePath(state, relativePath, measureBoundedPathDepth(relativePath)) + } + return createDirectoryCache(state.childrenByDir) +} + +function getLegacyRelativePath(value: unknown): string | null { + if (value === null || typeof value !== 'object') { + return null + } + const relativePath = (value as { relativePath?: unknown }).relativePath + return typeof relativePath === 'string' ? relativePath : null +} + +function measureBoundedPathDepth(relativePath: string): number { + let bytes = 0 + let depth = 0 + let insideSegment = false + for (let index = 0; index < relativePath.length; index += 1) { + const codePoint = relativePath.codePointAt(index) ?? 0 + bytes += getUtf8ByteLengthForCodePoint(codePoint) + if (bytes > LEGACY_MOBILE_FILE_PATH_MAX_BYTES) { + throwLegacyFileListLimitError() + } + if (codePoint === 47) { + if (insideSegment) { + depth += 1 + assertPathDepth(depth) } + insideSegment = false + } else { + insideSegment = true + } + if (codePoint > 0xffff) { + index += 1 + } + } + if (insideSegment) { + depth += 1 + assertPathDepth(depth) + } + return depth +} + +function assertPathDepth(depth: number): void { + if (depth > LEGACY_MOBILE_FILE_PATH_MAX_DEPTH) { + throwLegacyFileListLimitError() + } +} + +function addFilePath(state: LegacyFileCacheBuildState, relativePath: string, depth: number): void { + let parentPath = '' + let segmentStart = 0 + let segmentIndex = 0 + for (let cursor = 0; cursor <= relativePath.length; cursor += 1) { + if (cursor < relativePath.length && relativePath.charCodeAt(cursor) !== 47) { + continue + } + if (cursor > segmentStart) { + segmentIndex += 1 + const name = relativePath.slice(segmentStart, cursor) + const isDirectory = segmentIndex < depth + addDirectoryEntry(state, parentPath, name, isDirectory) + if (isDirectory) { + parentPath = parentPath ? `${parentPath}/${name}` : name + ensureDirectory(state, parentPath) + } + } + segmentStart = cursor + 1 + } +} + +function ensureDirectory(state: LegacyFileCacheBuildState, path: string): void { + if (state.childrenByDir.has(path)) { + return + } + const retainedBytes = path.length * 2 + DIRECTORY_RETAINED_BYTES + if ( + state.directories >= LEGACY_MOBILE_FILE_CACHE_MAX_DIRECTORIES || + state.retainedBytes > LEGACY_MOBILE_FILE_CACHE_MAX_RETAINED_BYTES - retainedBytes + ) { + throwLegacyFileListLimitError() + } + state.childrenByDir.set(path, new Map()) + state.directories += 1 + state.retainedBytes += retainedBytes +} + +function addDirectoryEntry( + state: LegacyFileCacheBuildState, + parentPath: string, + name: string, + isDirectory: boolean +): void { + const children = state.childrenByDir.get(parentPath) + if (!children) { + throw new Error('Legacy file cache builder lost its parent directory.') + } + const existing = children.get(name) + if (existing) { + existing.isDirectory ||= isDirectory + return + } + const retainedBytes = name.length * 2 + ENTRY_RETAINED_BYTES + if ( + state.entries >= LEGACY_MOBILE_FILE_CACHE_MAX_ENTRIES || + state.retainedBytes > LEGACY_MOBILE_FILE_CACHE_MAX_RETAINED_BYTES - retainedBytes + ) { + throwLegacyFileListLimitError() + } + children.set(name, { name, isDirectory }) + state.entries += 1 + state.retainedBytes += retainedBytes +} + +function createDirectoryCache( + childrenByDir: ReadonlyMap> +): DirectoryCache { + const cache: DirectoryCache = {} + for (const [path, children] of childrenByDir) { + // Why: assignment to a '__proto__' path would invoke its legacy setter. + Object.defineProperty(cache, path, { + configurable: true, + enumerable: true, + value: { entries: Array.from(children.values()) }, + writable: true }) } - // Why: plain `cache[path] = ...` with a '__proto__' path segment mutates the - // object's prototype instead of storing the directory; fromEntries always - // creates own keys. - return Object.fromEntries( - Array.from(childrenByDir, ([path, children]) => [ - path, - { - entries: Array.from( - children, - ([name, isDirectory]): MobileDirEntry => ({ - name, - isDirectory - }) - ) - } - ]) - ) + return cache +} + +function throwLegacyFileListLimitError(): never { + throw new Error(LEGACY_MOBILE_FILE_LIST_LIMIT_MESSAGE) } diff --git a/mobile/src/files/file-tree.ts b/mobile/src/files/file-tree.ts index a9283cfa457..8a8a2170e07 100644 --- a/mobile/src/files/file-tree.ts +++ b/mobile/src/files/file-tree.ts @@ -22,6 +22,7 @@ export type DirectoryState = { entries: MobileDirEntry[] loading?: boolean error?: string + lastAccess?: number } export type DirectoryCache = Record diff --git a/mobile/src/files/mobile-diff-image-preview.test.ts b/mobile/src/files/mobile-diff-image-preview.test.ts index e63f532b28d..a7ae1cdc185 100644 --- a/mobile/src/files/mobile-diff-image-preview.test.ts +++ b/mobile/src/files/mobile-diff-image-preview.test.ts @@ -1,44 +1,57 @@ import { describe, expect, it } from 'vitest' import { mobileDiffImageDataUri } from './mobile-diff-image-preview' +function pngBase64(width = 1): string { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(1, 20) + return bytes.toString('base64') +} + describe('mobileDiffImageDataUri', () => { it('renders a modified image diff from the post-change bytes', () => { + const modifiedContent = pngBase64(2) expect( mobileDiffImageDataUri({ kind: 'binary', originalContent: 'b2xk', - modifiedContent: 'bmV3', + modifiedContent, isImage: true, mimeType: 'image/png' }) - ).toBe('data:image/png;base64,bmV3') + ).toBe(`data:image/png;base64,${modifiedContent}`) }) it('renders an added image diff (no original) from the modified bytes', () => { + const modifiedContent = pngBase64() expect( mobileDiffImageDataUri({ kind: 'binary', originalContent: '', - modifiedContent: 'bmV3', + modifiedContent, isImage: true, mimeType: 'image/png' }) - ).toBe('data:image/png;base64,bmV3') + ).toBe(`data:image/png;base64,${modifiedContent}`) }) it('falls back to the original bytes for a proven deletion (modifiedDeleted)', () => { + const originalContent = pngBase64() expect( mobileDiffImageDataUri({ kind: 'binary', - originalContent: 'b2xk', + originalContent, originalIsBinary: true, modifiedContent: '', modifiedIsBinary: false, modifiedDeleted: true, isImage: true, - mimeType: 'image/jpeg' + mimeType: 'image/png' }) - ).toBe('data:image/jpeg;base64,b2xk') + ).toBe(`data:image/png;base64,${originalContent}`) }) // The reviewer's read-failure case: a relay/SSH read returns an empty modified diff --git a/mobile/src/files/mobile-directory-cache-retention.test.ts b/mobile/src/files/mobile-directory-cache-retention.test.ts new file mode 100644 index 00000000000..421d995492d --- /dev/null +++ b/mobile/src/files/mobile-directory-cache-retention.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import type { DirectoryCache, DirectoryState } from './file-tree' +import { + MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES, + parseBoundedMobileDirectoryEntries, + removeEvictedExpandedPaths, + retainMobileDirectoryState +} from './mobile-directory-cache-retention' + +function state(name: string, lastAccess: number): DirectoryState { + return { entries: [{ name, isDirectory: false }], lastAccess } +} + +describe('mobile directory cache retention', () => { + it('keeps the cache bounded across an unlimited sequence of visited directories', () => { + let cache: DirectoryCache = { '': state('root', 0) } + + for (let index = 1; index <= MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES * 3; index++) { + const result = retainMobileDirectoryState( + cache, + `dir-${index}`, + state(`file-${index}`, index), + new Set() + ) + expect(result.admitted).toBe(true) + cache = result.cache + } + + expect(Object.keys(cache)).toHaveLength(MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES) + expect(cache['']).toBeDefined() + expect(cache['dir-1']).toBeUndefined() + expect(cache[`dir-${MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES * 3}`]).toBeDefined() + }) + + it('evicts a collapsed branch before an older expanded branch', () => { + const cache: DirectoryCache = { + '': state('root', 0), + expanded: state('expanded', 1), + collapsed: state('collapsed', 2) + } + + const result = retainMobileDirectoryState( + cache, + 'new', + state('new', 3), + new Set(['expanded']), + { directories: 3, entries: 100, retainedBytes: 10_000 } + ) + + expect(result.cache.expanded).toBeDefined() + expect(result.cache.collapsed).toBeUndefined() + expect(result.evictedPaths).toEqual(['collapsed']) + }) + + it('evicts old content when aggregate entry retention reaches its cap', () => { + const cache: DirectoryCache = { + '': state('root', 0), + old: state('old', 1) + } + + const result = retainMobileDirectoryState(cache, 'new', state('new', 2), new Set(), { + directories: 10, + entries: 2, + retainedBytes: 10_000 + }) + + expect(result.cache.old).toBeUndefined() + expect(result.cache.new).toBeDefined() + }) + + it('evicts old content when aggregate retained bytes reach their cap', () => { + const cache: DirectoryCache = { + '': state('root', 0), + old: state('x'.repeat(100), 1) + } + + const result = retainMobileDirectoryState(cache, 'new', state('new', 2), new Set(), { + directories: 10, + entries: 100, + retainedBytes: 300 + }) + + expect(result.cache.old).toBeUndefined() + expect(result.cache.new).toBeDefined() + }) + + it('collapses expanded descendants whose cached branch was evicted', () => { + expect([...removeEvictedExpandedPaths(new Set(['src', 'src/lib', 'docs']), ['src'])]).toEqual([ + 'docs' + ]) + }) + + it('rejects malformed listings instead of retaining untrusted response shapes', () => { + expect(() => parseBoundedMobileDirectoryEntries([{ name: 'src' }])).toThrow( + 'invalid folder listing' + ) + }) +}) diff --git a/mobile/src/files/mobile-directory-cache-retention.ts b/mobile/src/files/mobile-directory-cache-retention.ts new file mode 100644 index 00000000000..b6ec4b9748b --- /dev/null +++ b/mobile/src/files/mobile-directory-cache-retention.ts @@ -0,0 +1,156 @@ +import { + assertMobileFileDirectoryWithinLimit, + estimateMobileDirectoryEntryBytes, + MOBILE_FILE_DIRECTORY_LIMIT_MESSAGE, + MOBILE_FILE_DIRECTORY_MAX_ENTRIES +} from '../../../src/shared/mobile-file-directory-limit' +import type { DirectoryCache, DirectoryState, MobileDirEntry } from './file-tree' + +// Why: old collapsed branches can reload on demand once explorer metadata reaches a phone-safe ceiling. +export const MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES = 128 +export const MOBILE_DIRECTORY_CACHE_MAX_ENTRIES = 25_000 +export const MOBILE_DIRECTORY_CACHE_MAX_RETAINED_BYTES = 16 * 1024 * 1024 +export const MOBILE_DIRECTORY_CACHE_LIMIT_MESSAGE = + 'Too many folders are open to load this folder safely. Close another folder and retry.' + +type CacheLimits = { + directories: number + entries: number + retainedBytes: number +} + +type RetentionResult = { + cache: DirectoryCache + evictedPaths: string[] + admitted: boolean +} + +const DEFAULT_LIMITS: CacheLimits = { + directories: MOBILE_DIRECTORY_CACHE_MAX_DIRECTORIES, + entries: MOBILE_DIRECTORY_CACHE_MAX_ENTRIES, + retainedBytes: MOBILE_DIRECTORY_CACHE_MAX_RETAINED_BYTES +} + +export function parseBoundedMobileDirectoryEntries(value: unknown): MobileDirEntry[] { + if (!Array.isArray(value)) { + throw new Error('Desktop returned an invalid folder listing.') + } + if (value.length > MOBILE_FILE_DIRECTORY_MAX_ENTRIES) { + throw new Error(MOBILE_FILE_DIRECTORY_LIMIT_MESSAGE) + } + for (const entry of value) { + if (!isMobileDirectoryEntry(entry)) { + throw new Error('Desktop returned an invalid folder listing.') + } + } + const entries = value as MobileDirEntry[] + assertMobileFileDirectoryWithinLimit(entries) + return entries +} + +export function retainMobileDirectoryState( + cache: DirectoryCache, + relativePath: string, + state: DirectoryState, + expandedPaths: ReadonlySet, + limits: CacheLimits = DEFAULT_LIMITS +): RetentionResult { + const next: DirectoryCache = { ...cache, [relativePath]: state } + const evictedPaths: string[] = [] + const essentialPaths = directoryAncestors(relativePath) + + while (cacheExceedsLimits(next, limits)) { + const victim = selectEvictionPath(next, expandedPaths, essentialPaths) + if (victim === null) { + return { cache, evictedPaths: [], admitted: false } + } + for (const path of Object.keys(next)) { + if (path === victim || path.startsWith(`${victim}/`)) { + delete next[path] + evictedPaths.push(path) + } + } + } + return { cache: next, evictedPaths, admitted: true } +} + +export function removeEvictedExpandedPaths( + expandedPaths: ReadonlySet, + evictedPaths: readonly string[] +): Set { + if (evictedPaths.length === 0) { + return new Set(expandedPaths) + } + return new Set( + [...expandedPaths].filter( + (expanded) => + !evictedPaths.some((evicted) => expanded === evicted || expanded.startsWith(`${evicted}/`)) + ) + ) +} + +function isMobileDirectoryEntry(value: unknown): value is MobileDirEntry { + if (!value || typeof value !== 'object') { + return false + } + const entry = value as Record + return ( + typeof entry.name === 'string' && + typeof entry.isDirectory === 'boolean' && + (entry.isSymlink === undefined || typeof entry.isSymlink === 'boolean') + ) +} + +function directoryAncestors(relativePath: string): Set { + const ancestors = new Set(['', relativePath]) + let cursor = relativePath + while (cursor.includes('/')) { + cursor = cursor.slice(0, cursor.lastIndexOf('/')) + ancestors.add(cursor) + } + return ancestors +} + +function selectEvictionPath( + cache: DirectoryCache, + expandedPaths: ReadonlySet, + essentialPaths: ReadonlySet +): string | null { + const candidates = Object.keys(cache) + .filter((path) => !essentialPaths.has(path)) + .sort((left, right) => accessOrder(cache, left) - accessOrder(cache, right)) + return candidates.find((path) => !expandedPaths.has(path)) ?? candidates[0] ?? null +} + +function accessOrder(cache: DirectoryCache, path: string): number { + return cache[path]?.lastAccess ?? 0 +} + +function cacheExceedsLimits(cache: DirectoryCache, limits: CacheLimits): boolean { + const paths = Object.keys(cache) + if (paths.length > limits.directories) { + return true + } + let entries = 0 + let retainedBytes = 0 + for (const path of paths) { + const state = cache[path] + if (!state) { + continue + } + entries += state.entries.length + retainedBytes += estimateDirectoryStateBytes(path, state) + if (entries > limits.entries || retainedBytes > limits.retainedBytes) { + return true + } + } + return false +} + +function estimateDirectoryStateBytes(path: string, state: DirectoryState): number { + let bytes = path.length * 2 + (state.error?.length ?? 0) * 2 + 64 + for (const entry of state.entries) { + bytes += estimateMobileDirectoryEntryBytes(entry) + } + return bytes +} diff --git a/mobile/src/files/mobile-file-mutation-ownership.test.ts b/mobile/src/files/mobile-file-mutation-ownership.test.ts new file mode 100644 index 00000000000..f3e98dca9c6 --- /dev/null +++ b/mobile/src/files/mobile-file-mutation-ownership.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshConnectionState } from '../../../src/shared/ssh-types' +import { + FILE_MUTATION_OWNERSHIP_RUNTIME_CAPABILITY, + FILE_MUTATION_OWNERSHIP_UPDATE_REQUIRED_MESSAGE +} from '../../../src/shared/protocol-version' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcResponse } from '../transport/types' +import { + buildMobileFileMutationOwnership, + captureMobileFileMutationOwnership +} from './mobile-file-mutation-ownership' + +function success(result: unknown): RpcResponse { + return { id: 'rpc-1', ok: true, result, _meta: { runtimeId: 'runtime-1' } } +} + +function clientWithResponses(responses: RpcResponse[]): { + client: Pick + sendRequest: ReturnType +} { + const sendRequest = vi.fn(async () => { + const response = responses.shift() + if (!response) { + throw new Error('Unexpected RPC request') + } + return response + }) + return { client: { sendRequest }, sendRequest } +} + +function sshState(targetId: string, connectionGeneration: number | undefined): SshConnectionState { + return { + targetId, + status: 'connected', + error: null, + reconnectAttempt: 0, + connectionGeneration + } +} + +describe('mobile file mutation ownership', () => { + it.each([undefined, 'local', 'runtime:environment-1'])( + 'binds %s worktrees to the runtime-local file host', + (hostId) => { + expect(buildMobileFileMutationOwnership(hostId)).toEqual({ + expectedExecutionHostId: 'local' + }) + } + ) + + it('binds SSH worktrees to the target and live connection generation', () => { + expect( + buildMobileFileMutationOwnership('ssh:target%20one', sshState('target one', 17)) + ).toEqual({ + expectedExecutionHostId: 'ssh:target%20one', + expectedSshTargetId: 'target one', + expectedSshConnectionGeneration: 17 + }) + }) + + it.each([ + ['a malformed owner', 'not-an-execution-host', null], + ['a missing SSH state', 'ssh:target-1', null], + ['a mismatched SSH target', 'ssh:target-1', sshState('target-2', 4)], + ['a missing SSH generation', 'ssh:target-1', sshState('target-1', undefined)] + ])('rejects %s', (_name, hostId, state) => { + expect(() => buildMobileFileMutationOwnership(hostId, state)).toThrow( + "Couldn't verify the SSH connection" + ) + }) + + it('captures local ownership only after verifying the runtime capability', async () => { + const { client, sendRequest } = clientWithResponses([ + success({ capabilities: [FILE_MUTATION_OWNERSHIP_RUNTIME_CAPABILITY] }), + success({ worktree: { hostId: 'local' } }) + ]) + + await expect(captureMobileFileMutationOwnership(client, 'id:worktree-1')).resolves.toEqual({ + expectedExecutionHostId: 'local' + }) + expect(sendRequest.mock.calls).toEqual([ + ['status.get', undefined, { timeoutMs: 15_000 }], + ['worktree.show', { worktree: 'id:worktree-1' }, { timeoutMs: 15_000 }] + ]) + }) + + it('captures SSH generation from the HUB before building mutation params', async () => { + const state = sshState('target-1', 9) + const { client, sendRequest } = clientWithResponses([ + success({ capabilities: [FILE_MUTATION_OWNERSHIP_RUNTIME_CAPABILITY] }), + success({ worktree: { hostId: 'ssh:target-1' } }), + success({ state }) + ]) + + await expect(captureMobileFileMutationOwnership(client, 'id:worktree-1')).resolves.toEqual({ + expectedExecutionHostId: 'ssh:target-1', + expectedSshTargetId: 'target-1', + expectedSshConnectionGeneration: 9 + }) + expect(sendRequest.mock.calls[2]).toEqual([ + 'ssh.getState', + { targetId: 'target-1' }, + { timeoutMs: 15_000 } + ]) + }) + + it('refuses older runtimes before reading or mutating workspace files', async () => { + const { client, sendRequest } = clientWithResponses([success({ capabilities: [] })]) + + await expect(captureMobileFileMutationOwnership(client, 'id:worktree-1')).rejects.toThrow( + FILE_MUTATION_OWNERSHIP_UPDATE_REQUIRED_MESSAGE + ) + expect(sendRequest).toHaveBeenCalledTimes(1) + }) +}) diff --git a/mobile/src/files/mobile-file-mutation-ownership.ts b/mobile/src/files/mobile-file-mutation-ownership.ts new file mode 100644 index 00000000000..e5a1cbbeb65 --- /dev/null +++ b/mobile/src/files/mobile-file-mutation-ownership.ts @@ -0,0 +1,81 @@ +import { parseExecutionHostId } from '../../../src/shared/execution-host' +import { assertFileMutationOwnershipCapability } from '../../../src/shared/file-mutation-ownership' +import type { RuntimeStatus } from '../../../src/shared/runtime-types' +import type { SshConnectionState, SshMutationExpectation } from '../../../src/shared/ssh-types' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcFailure, RpcSuccess } from '../transport/types' + +const FILE_MUTATION_TIMEOUT_MS = 15_000 +const SSH_OWNER_CHANGED_MESSAGE = + "Couldn't verify the SSH connection. Reconnect the host and try again." + +export type MobileFileMutationOwnership = SshMutationExpectation & { + expectedExecutionHostId: 'local' | `ssh:${string}` +} + +export function buildMobileFileMutationOwnership( + worktreeHostId: string | null | undefined, + sshState: SshConnectionState | null = null +): MobileFileMutationOwnership { + const host = parseExecutionHostId(worktreeHostId) + if (worktreeHostId !== undefined && !host) { + throw new Error(SSH_OWNER_CHANGED_MESSAGE) + } + if (!host || host.kind === 'local' || host.kind === 'runtime') { + return { expectedExecutionHostId: 'local' } + } + if (sshState?.targetId !== host.targetId || sshState.connectionGeneration === undefined) { + throw new Error(SSH_OWNER_CHANGED_MESSAGE) + } + return { + expectedExecutionHostId: host.id, + expectedSshTargetId: host.targetId, + expectedSshConnectionGeneration: sshState.connectionGeneration + } +} + +export async function captureMobileFileMutationOwnership( + client: Pick, + worktree: string +): Promise { + const status = await requestResult>( + client, + 'status.get', + undefined + ) + assertFileMutationOwnershipCapability(status) + + const result = await requestResult<{ worktree?: { hostId?: string | null } }>( + client, + 'worktree.show', + { worktree } + ) + if (!result.worktree) { + throw new Error(SSH_OWNER_CHANGED_MESSAGE) + } + + const host = parseExecutionHostId(result.worktree.hostId) + const sshState = + host?.kind === 'ssh' + ? ( + await requestResult<{ state: SshConnectionState | null }>(client, 'ssh.getState', { + targetId: host.targetId + }) + ).state + : null + return buildMobileFileMutationOwnership(result.worktree.hostId, sshState) +} + +async function requestResult( + client: Pick, + method: string, + params: unknown +): Promise { + const response = await client.sendRequest(method, params, { + timeoutMs: FILE_MUTATION_TIMEOUT_MS + }) + if (!response.ok) { + throw new Error((response as RpcFailure).error.message) + } + return (response as RpcSuccess).result as TResult +} diff --git a/mobile/src/files/mobile-file-preview-request.test.ts b/mobile/src/files/mobile-file-preview-request.test.ts index f7f77fabd19..185962bad72 100644 --- a/mobile/src/files/mobile-file-preview-request.test.ts +++ b/mobile/src/files/mobile-file-preview-request.test.ts @@ -28,6 +28,16 @@ function clientWithResponses(responses: RpcResponse[]) { } } +function pngBase64(width = 1, height = 1): string { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(height, 20) + return bytes.toString('base64') +} + describe('mobile-file-preview-request', () => { it('selects readPreview for raster images and read for text-like files', () => { expect(createMobileFilePreviewRequest('wt-1', 'assets/logo.png')).toEqual({ @@ -45,14 +55,13 @@ describe('mobile-file-preview-request', () => { }) it('loads images through readPreview and never calls files.open', async () => { - const client = clientWith( - ok({ content: 'aW1hZ2U=', isBinary: true, isImage: true, mimeType: 'image/png' }) - ) + const content = pngBase64() + const client = clientWith(ok({ content, isBinary: true, isImage: true, mimeType: 'image/png' })) await expect(loadMobileFilePreview(client, 'wt-1', 'assets/logo.png')).resolves.toEqual({ status: 'ready', kind: 'image', - dataUri: 'data:image/png;base64,aW1hZ2U=' + dataUri: `data:image/png;base64,${content}` }) expect(client.sendRequest).toHaveBeenCalledWith('files.readPreview', { worktree: 'id:wt-1', @@ -527,9 +536,9 @@ describe('mobile-file-preview-request', () => { }) it.each([ - ['missing isBinary', { content: 'aW1hZ2U=', isImage: true, mimeType: 'image/png' }], - ['missing isImage', { content: 'aW1hZ2U=', isBinary: true, mimeType: 'image/png' }], - ['missing mimeType', { content: 'aW1hZ2U=', isBinary: true, isImage: true }], + ['missing isBinary', { content: pngBase64(), isImage: true, mimeType: 'image/png' }], + ['missing isImage', { content: pngBase64(), isBinary: true, mimeType: 'image/png' }], + ['missing mimeType', { content: pngBase64(), isBinary: true, isImage: true }], ['empty content', { content: '', isBinary: true, isImage: true, mimeType: 'image/png' }] ])('rejects invalid image preview results: %s', (_label, result) => { expect(normalizeMobileFilePreviewResponse('assets/logo.png', ok(result))).toEqual({ @@ -539,6 +548,24 @@ describe('mobile-file-preview-request', () => { }) }) + it('rejects an oversized raster response before React Native receives a data URI', () => { + expect( + normalizeMobileFilePreviewResponse( + 'assets/logo.png', + ok({ + content: pngBase64(32_769, 1), + isBinary: true, + isImage: true, + mimeType: 'image/png' + }) + ) + ).toEqual({ + status: 'error', + message: 'Binary preview unavailable', + reconnect: false + }) + }) + it('normalizes markdown, html, text, empty, and truncated reads', () => { expect( normalizeMobileFilePreviewResponse( diff --git a/mobile/src/files/mobile-file-preview-response.ts b/mobile/src/files/mobile-file-preview-response.ts index 0b7b1b23be8..8f57a92edf7 100644 --- a/mobile/src/files/mobile-file-preview-response.ts +++ b/mobile/src/files/mobile-file-preview-response.ts @@ -2,6 +2,7 @@ import { classifyMobileArtifact } from '../session/mobile-artifact-kind' import type { RpcFailure, RpcResponse, RpcSuccess } from '../transport/types' import { isMarkdownPath } from './file-tree' import { isTerminalArtifactGrantError } from './terminal-artifact-grant-error' +import { buildImageDataUri } from '../../../src/shared/image-data-uri' export type MobileFilePreviewTextKind = 'html' | 'markdown' | 'text' @@ -117,10 +118,14 @@ function normalizeImagePreviewResult(result: unknown): MobileFilePreviewResult { ) { return previewError('binary_file') } + const dataUri = buildImageDataUri(preview.mimeType, preview.content) + if (!dataUri) { + return previewError('binary_file') + } return { status: 'ready', kind: 'image', - dataUri: `data:${preview.mimeType};base64,${preview.content}` + dataUri } } diff --git a/mobile/src/files/mobile-file-tab-doc.test.ts b/mobile/src/files/mobile-file-tab-doc.test.ts index a9570e6cf44..cfe5578a7e0 100644 --- a/mobile/src/files/mobile-file-tab-doc.test.ts +++ b/mobile/src/files/mobile-file-tab-doc.test.ts @@ -31,6 +31,16 @@ function clientOf(byMethod: Record): { const WT = { worktreeId: 'wt1' } +function pngBase64(width = 1): string { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(1, 20) + return bytes.toString('base64') +} + describe('resolveMobileFileTabDoc', () => { it('renders a staged text diff', async () => { const client = clientOf({ @@ -46,11 +56,12 @@ describe('resolveMobileFileTabDoc', () => { }) it('renders an unstaged image diff from the modified bytes', async () => { + const modifiedContent = pngBase64() const client = clientOf({ 'git.diff': ok({ kind: 'binary', originalContent: 'b2xk', - modifiedContent: 'bmV3', + modifiedContent, modifiedIsBinary: true, isImage: true, mimeType: 'image/png' @@ -61,7 +72,11 @@ describe('resolveMobileFileTabDoc', () => { relativePath: 'm1.png', diffSource: 'unstaged' }) - expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' }) + expect(doc).toEqual({ + status: 'ready', + kind: 'image', + dataUri: `data:image/png;base64,${modifiedContent}` + }) }) it('throws binary_file for an image modify whose bytes are empty (no stale fallback)', async () => { @@ -88,11 +103,16 @@ describe('resolveMobileFileTabDoc', () => { }) it('renders a live image preview via files.readPreview', async () => { + const content = pngBase64() const client = clientOf({ - 'files.readPreview': ok({ content: 'bmV3', isImage: true, mimeType: 'image/png' }) + 'files.readPreview': ok({ content, isImage: true, mimeType: 'image/png' }) }) const doc = await resolveMobileFileTabDoc(client, { ...WT, relativePath: 'logo.png' }) - expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' }) + expect(doc).toEqual({ + status: 'ready', + kind: 'image', + dataUri: `data:image/png;base64,${content}` + }) expect(client.calls).toEqual(['files.readPreview']) }) diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.test.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.test.ts new file mode 100644 index 00000000000..69fa92f0265 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' +import { + enqueueMobileDictationAudioChunk, + MOBILE_DICTATION_MAX_PENDING_CHUNKS +} from './mobile-dictation-audio-chunk' +import { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' + +describe('enqueueMobileDictationAudioChunk', () => { + it('accepts the exact pending-promise cap and rejects one over even for empty chunks', () => { + const sendRequest = vi.fn(() => new Promise(() => undefined)) + const failActiveDictation = vi.fn() + const pendingChunks = new Set>() + const queue = { + pendingChunks, + pendingAudioBudget: new MobileDictationPendingAudioBudget(), + shouldReleaseBudget: () => true, + failActiveDictation + } + const event = { data: new Uint8Array() } + + for (let index = 0; index < MOBILE_DICTATION_MAX_PENDING_CHUNKS; index += 1) { + enqueueMobileDictationAudioChunk( + { sendRequest } as unknown as RpcClient, + 'dictation', + event, + queue + ) + } + expect(pendingChunks).toHaveLength(MOBILE_DICTATION_MAX_PENDING_CHUNKS) + expect(failActiveDictation).not.toHaveBeenCalled() + + enqueueMobileDictationAudioChunk( + { sendRequest } as unknown as RpcClient, + 'dictation', + event, + queue + ) + expect(pendingChunks).toHaveLength(MOBILE_DICTATION_MAX_PENDING_CHUNKS) + expect(failActiveDictation).toHaveBeenCalledOnce() + }) +}) diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index 792292e6c16..947deee0e61 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -14,12 +14,21 @@ type MobileDictationAudioChunkQueue = { failActiveDictation: (dictationId: string, err: unknown) => void } +export const MOBILE_DICTATION_MAX_PENDING_CHUNKS = 256 + export function enqueueMobileDictationAudioChunk( client: RpcClient, dictationId: string, event: MicrophoneDataEvent, queue: MobileDictationAudioChunkQueue ): void { + if (queue.pendingChunks.size >= MOBILE_DICTATION_MAX_PENDING_CHUNKS) { + queue.failActiveDictation( + dictationId, + new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) + ) + return + } const raw = event.data const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw) const byteLength = bytes.byteLength diff --git a/mobile/src/hooks/mobile-dictation-keep-awake.test.ts b/mobile/src/hooks/mobile-dictation-keep-awake.test.ts index 362f3a0cf68..4bb2803def4 100644 --- a/mobile/src/hooks/mobile-dictation-keep-awake.test.ts +++ b/mobile/src/hooks/mobile-dictation-keep-awake.test.ts @@ -11,9 +11,11 @@ vi.mock('expo-keep-awake', () => ({ })) import { + MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS, MOBILE_DICTATION_KEEP_AWAKE_NATIVE_TIMEOUT_MS, MobileDictationKeepAwakeOwner, - drainMobileDictationKeepAwakeCleanup + drainMobileDictationKeepAwakeCleanup, + resetMobileDictationKeepAwakeForTests } from './mobile-dictation-keep-awake' function deferred(): { @@ -36,10 +38,33 @@ function deferred(): { describe('MobileDictationKeepAwakeOwner', () => { beforeEach(() => { + resetMobileDictationKeepAwakeForTests() keepAwake.activate.mockReset().mockResolvedValue(undefined) keepAwake.deactivate.mockReset().mockResolvedValue(undefined) }) + it('accepts the exact tracked-tag cap, rejects one over, and recovers after release', async () => { + const owners = Array.from( + { length: MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS }, + () => new MobileDictationKeepAwakeOwner() + ) + await Promise.all(owners.map((owner, index) => owner.acquire(`dictation-${index}`))) + expect(keepAwake.activate).toHaveBeenCalledTimes(MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS) + + const oneOver = new MobileDictationKeepAwakeOwner() + await expect(oneOver.acquire('one-over')).rejects.toThrow( + 'Too many dictation keep-awake operations are pending' + ) + expect(keepAwake.activate).toHaveBeenCalledTimes(MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS) + + await owners[0]!.release('dictation-0') + await expect(oneOver.reacquire('one-over')).resolves.toBeUndefined() + await Promise.all([ + ...owners.slice(1).map((owner, index) => owner.release(`dictation-${index + 1}`)), + oneOver.release('one-over') + ]) + }) + it('retries a failed native deactivation after the hook owner is replaced', async () => { const firstOwner = new MobileDictationKeepAwakeOwner() diff --git a/mobile/src/hooks/mobile-dictation-keep-awake.ts b/mobile/src/hooks/mobile-dictation-keep-awake.ts index f94e3fb785a..08f93ac98be 100644 --- a/mobile/src/hooks/mobile-dictation-keep-awake.ts +++ b/mobile/src/hooks/mobile-dictation-keep-awake.ts @@ -5,6 +5,7 @@ const MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX = 'orca-mobile-dictation' // Native keep-awake promises can be lost during Activity teardown; a bounded // wait keeps the serialized queue below from wedging dictation until restart. export const MOBILE_DICTATION_KEEP_AWAKE_NATIVE_TIMEOUT_MS = 10_000 +export const MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS = 128 let nextOwnerId = 0 let keepAwakeOperation: Promise = Promise.resolve() @@ -51,7 +52,34 @@ function withNativeCallTimeout(nativeCall: Promise): Promise { }) } +function trackedTagCount(): number { + let count = activeTags.size + for (const tag of pendingCleanupTags) { + if (!activeTags.has(tag)) { + count += 1 + } + } + for (const tag of pendingActivations.keys()) { + if (!activeTags.has(tag) && !pendingCleanupTags.has(tag)) { + count += 1 + } + } + return count +} + +function assertTrackedTagCapacity(tag: string): void { + if ( + !activeTags.has(tag) && + !pendingCleanupTags.has(tag) && + !pendingActivations.has(tag) && + trackedTagCount() >= MOBILE_DICTATION_KEEP_AWAKE_MAX_TRACKED_TAGS + ) { + throw new Error('Too many dictation keep-awake operations are pending') + } +} + async function activateTrackedTag(tag: string, isStillWanted: () => boolean): Promise { + assertTrackedTagCapacity(tag) const nativeActivation = activateKeepAwakeAsync(tag) try { await withNativeCallTimeout(nativeActivation) @@ -234,3 +262,12 @@ export function createMobileDictationKeepAwakeOwner(): MobileDictationKeepAwakeO export function drainMobileDictationKeepAwakeCleanup(): Promise { return enqueueKeepAwakeOperation(cleanupPendingTags) } + +/** Test-only: drop retained native-operation bookkeeping between cases. */ +export function resetMobileDictationKeepAwakeForTests(): void { + nextOwnerId = 0 + keepAwakeOperation = Promise.resolve() + activeTags.clear() + pendingCleanupTags.clear() + pendingActivations.clear() +} diff --git a/mobile/src/notifications/mobile-notification-delivery.ts b/mobile/src/notifications/mobile-notification-delivery.ts new file mode 100644 index 00000000000..60bc572e291 --- /dev/null +++ b/mobile/src/notifications/mobile-notification-delivery.ts @@ -0,0 +1,203 @@ +import * as Notifications from 'expo-notifications' +import { Platform } from 'react-native' +import { loadPushNotificationsEnabled } from '../storage/preferences' +import { buildLocalNotificationData, type DesktopNotificationSource } from './notification-routing' +import { MobileNotificationDeliveryLedger } from './mobile-notification-retention' +import { + MobileScheduledNotificationRegistry, + type MobileScheduledNotificationState +} from './mobile-scheduled-notification-registry' + +export type NotificationEvent = { + type: 'notification' + source: DesktopNotificationSource + title: string + body: string + worktreeId?: string + notificationId?: string + notificationSeq?: number +} + +export type DismissNotificationEvent = { + type: 'dismiss' + notificationId: string + notificationSeq?: number +} + +const scheduledNotifications = new MobileScheduledNotificationRegistry() +const notificationDeliveryLedger = new MobileNotificationDeliveryLedger() + +function getStoredNotificationKey(hostId: string, notificationId: string): string { + return `${encodeURIComponent(hostId)}:${encodeURIComponent(notificationId)}` +} + +/** Test-only: override the cap (pass no arg to restore the default). */ +export function setScheduledNotificationsMaxForTests(max?: number): void { + scheduledNotifications.resetForTests(max) + notificationDeliveryLedger.resetForTests(max) +} + +export type NotificationPermissionState = { + granted: boolean + status: string + canAskAgain: boolean + authorizationReflectsUserChoice: boolean +} + +export async function getNotificationPermissionState(): Promise { + const { status, canAskAgain } = await Notifications.getPermissionsAsync() + return { + granted: status === 'granted', + status, + canAskAgain, + // Why: Android <33 has no runtime notification permission, so "granted" is capability, not user consent. + authorizationReflectsUserChoice: + status === 'granted' && (Platform.OS !== 'android' || Number(Platform.Version) >= 33) + } +} + +// Why: re-read OS state every call — users can change it in Settings while Orca is backgrounded. +export async function ensureNotificationPermissions(): Promise { + const existing = await getNotificationPermissionState() + if (existing.granted) { + return true + } + const { status } = await Notifications.requestPermissionsAsync() + return status === 'granted' +} + +export function configureNotificationChannel(): void { + if (Platform.OS === 'android') { + void Notifications.setNotificationChannelAsync('orca-desktop', { + name: 'Desktop Notifications', + importance: Notifications.AndroidImportance.HIGH, + vibrationPattern: [0, 250], + lightColor: '#6366f1' + }) + } +} + +async function showLocalNotification(event: NotificationEvent, hostId: string): Promise { + const storedKey = event.notificationId + ? getStoredNotificationKey(hostId, event.notificationId) + : null + if (!storedKey) { + if (!(await loadPushNotificationsEnabled()) || !(await ensureNotificationPermissions())) { + return + } + await scheduleLocalNotification(event, hostId) + return + } + + let state = scheduledNotifications.get(storedKey) + let evictedIdentifiers: string[] = [] + if (state?.pending) { + return + } + if (!state) { + const reservation = scheduledNotifications.reserve(storedKey) + if (!reservation) { + return + } + state = reservation.state + evictedIdentifiers = reservation.evictedIdentifiers + } + const notificationState = state + const pending = scheduleTrackedNotification(event, hostId, notificationState, evictedIdentifiers) + notificationState.pending = pending + + try { + const scheduledIdentifier = await pending + if (!scheduledIdentifier) { + if (!notificationState.identifier) { + scheduledNotifications.delete(storedKey) + } + return + } + if (notificationState.dismissAfterSchedule) { + notificationState.dismissAfterSchedule = false + scheduledNotifications.delete(storedKey) + await Notifications.dismissNotificationAsync(scheduledIdentifier).catch(() => {}) + return + } + if (!scheduledNotifications.retainIdentifier(notificationState, scheduledIdentifier)) { + scheduledNotifications.delete(storedKey) + await Notifications.dismissNotificationAsync(scheduledIdentifier).catch(() => {}) + } + } finally { + if (notificationState.pending === pending) { + notificationState.pending = undefined + notificationState.dismissAfterSchedule = false + } + } +} + +async function scheduleTrackedNotification( + event: NotificationEvent, + hostId: string, + state: MobileScheduledNotificationState, + evictedIdentifiers: string[] +): Promise { + // Why: retaining the native id makes a later desktop dismiss work; eviction must close it before slot reuse. + for (const identifier of evictedIdentifiers) { + await Notifications.dismissNotificationAsync(identifier).catch(() => {}) + } + if (!(await loadPushNotificationsEnabled()) || !(await ensureNotificationPermissions())) { + return null + } + if (state.identifier) { + await Notifications.dismissNotificationAsync(state.identifier).catch(() => {}) + scheduledNotifications.clearIdentifier(state) + } + return scheduleLocalNotification(event, hostId) +} + +function scheduleLocalNotification(event: NotificationEvent, hostId: string): Promise { + return Notifications.scheduleNotificationAsync({ + content: { + title: event.title, + body: event.body, + data: buildLocalNotificationData(event, hostId), + ...(Platform.OS === 'android' ? { channelId: 'orca-desktop' } : {}) + }, + trigger: null + }) +} + +async function dismissLocalNotification( + event: DismissNotificationEvent, + hostId: string +): Promise { + const storedKey = getStoredNotificationKey(hostId, event.notificationId) + const state = scheduledNotifications.get(storedKey) + if (!state) { + return + } + if (state.pending) { + // Why: dismiss can arrive while the OS is still scheduling; defer it so no stale banner survives. + state.dismissAfterSchedule = true + return + } + if (!state.identifier) { + return + } + scheduledNotifications.delete(storedKey) + await Notifications.dismissNotificationAsync(state.identifier).catch(() => {}) +} + +export function startLocalNotificationDelivery( + event: NotificationEvent | DismissNotificationEvent, + hostId: string +): Promise | null { + const releaseDelivery = notificationDeliveryLedger.claim(event, hostId) + if (!releaseDelivery) { + return null + } + const delivery = + event.type === 'notification' + ? showLocalNotification(event, hostId) + : dismissLocalNotification(event, hostId) + return delivery.finally(() => { + releaseDelivery() + }) +} diff --git a/mobile/src/notifications/mobile-notification-retention.test.ts b/mobile/src/notifications/mobile-notification-retention.test.ts new file mode 100644 index 00000000000..a2fb3e52a22 --- /dev/null +++ b/mobile/src/notifications/mobile-notification-retention.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'vitest' +import { + MOBILE_NOTIFICATION_ACTIVE_MAX_BYTES, + MOBILE_NOTIFICATION_EVENT_MAX_BYTES, + MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES, + MOBILE_NOTIFICATION_ID_MAX_BYTES, + MobileNotificationDeliveryLedger, + measureMobileNotificationDeliveryBytes +} from './mobile-notification-retention' +import { + MOBILE_NATIVE_NOTIFICATION_ID_MAX_BYTES, + MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES, + MobileScheduledNotificationRegistry +} from './mobile-scheduled-notification-registry' + +describe('mobile notification retention', () => { + it('accepts an exact-limit event and rejects one more byte', () => { + const baseEvent = { + type: 'notification', + source: 'test', + title: '', + body: '', + notificationId: 'notification-1' + } + const baseBytes = measureMobileNotificationDeliveryBytes(baseEvent, 'host-1')! + const exactEvent = { + ...baseEvent, + body: 'x'.repeat(MOBILE_NOTIFICATION_EVENT_MAX_BYTES - baseBytes) + } + + expect(measureMobileNotificationDeliveryBytes(exactEvent, 'host-1')).toBe( + MOBILE_NOTIFICATION_EVENT_MAX_BYTES + ) + expect( + measureMobileNotificationDeliveryBytes( + { ...exactEvent, body: `${exactEvent.body}x` }, + 'host-1' + ) + ).toBeNull() + }) + + it('bounds host and notification identifiers before URI key expansion', () => { + const event = { + type: 'dismiss', + notificationId: 'n'.repeat(MOBILE_NOTIFICATION_ID_MAX_BYTES) + } + + expect( + measureMobileNotificationDeliveryBytes( + event, + 'h'.repeat(MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES) + ) + ).not.toBeNull() + expect( + measureMobileNotificationDeliveryBytes( + { ...event, notificationId: `${event.notificationId}n` }, + 'host-1' + ) + ).toBeNull() + expect( + measureMobileNotificationDeliveryBytes( + event, + 'h'.repeat(MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES + 1) + ) + ).toBeNull() + }) + + it('caps aggregate active bytes and restores capacity on release', () => { + const ledger = new MobileNotificationDeliveryLedger() + const baseEvent = { type: 'notification', title: '', body: '' } + const baseBytes = measureMobileNotificationDeliveryBytes(baseEvent, 'host-1')! + const exactEvent = { + ...baseEvent, + body: 'x'.repeat(MOBILE_NOTIFICATION_EVENT_MAX_BYTES - baseBytes) + } + const exactClaims = MOBILE_NOTIFICATION_ACTIVE_MAX_BYTES / MOBILE_NOTIFICATION_EVENT_MAX_BYTES + const releases = Array.from({ length: exactClaims }, () => ledger.claim(exactEvent, 'host-1')) + + expect(releases.every(Boolean)).toBe(true) + expect(ledger.claim(exactEvent, 'host-1')).toBeNull() + releases[0]?.() + expect(ledger.claim(exactEvent, 'host-1')).not.toBeNull() + }) + + it('rejects a scheduled key that alone exceeds the aggregate budget', () => { + const registry = new MobileScheduledNotificationRegistry() + const maximumKeyCharacters = (MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES - 64) / 2 + + expect(registry.reserve('x'.repeat(maximumKeyCharacters))).not.toBeNull() + registry.resetForTests() + expect(registry.reserve('x'.repeat(maximumKeyCharacters + 1))).toBeNull() + }) + + it('retains exact-limit native ids and rejects oversized values', () => { + const registry = new MobileScheduledNotificationRegistry() + const exactState = registry.reserve('exact')!.state + const oversizedState = registry.reserve('oversized')!.state + + expect( + registry.retainIdentifier(exactState, 'x'.repeat(MOBILE_NATIVE_NOTIFICATION_ID_MAX_BYTES)) + ).toBe(true) + expect( + registry.retainIdentifier( + oversizedState, + 'x'.repeat(MOBILE_NATIVE_NOTIFICATION_ID_MAX_BYTES + 1) + ) + ).toBe(false) + expect(registry.getRetainedBytesForTests()).toBeLessThanOrEqual( + MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES + ) + }) +}) diff --git a/mobile/src/notifications/mobile-notification-retention.ts b/mobile/src/notifications/mobile-notification-retention.ts new file mode 100644 index 00000000000..c68f11b66b5 --- /dev/null +++ b/mobile/src/notifications/mobile-notification-retention.ts @@ -0,0 +1,95 @@ +import { measureUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' + +export const MOBILE_NOTIFICATION_EVENT_MAX_BYTES = 256 * 1024 +export const MOBILE_NOTIFICATION_ACTIVE_MAX_BYTES = 4 * 1024 * 1024 +export const MOBILE_NOTIFICATION_ACTIVE_MAX_ENTRIES = 256 +export const MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES = 8 * 1024 +export const MOBILE_NOTIFICATION_ID_MAX_BYTES = 8 * 1024 +export const MOBILE_NOTIFICATION_WORKTREE_ID_MAX_BYTES = 16 * 1024 + +type MobileNotificationRetentionEvent = { + type: string + source?: string + title?: string + body?: string + worktreeId?: string + notificationId?: string +} + +function boundedStringBytes(value: unknown, maxBytes: number): number | null { + if (value === undefined) { + return 0 + } + if (typeof value !== 'string') { + return null + } + const measured = measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }) + return measured.exceededLimit ? null : measured.byteLength +} + +export function isMobileNotificationHostIdRetainable(hostId: string): boolean { + return boundedStringBytes(hostId, MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES) !== null +} + +export function measureMobileNotificationDeliveryBytes( + event: MobileNotificationRetentionEvent, + hostId: string +): number | null { + const hostBytes = boundedStringBytes(hostId, MOBILE_NOTIFICATION_HOST_ID_MAX_BYTES) + const notificationIdBytes = boundedStringBytes( + event.notificationId, + MOBILE_NOTIFICATION_ID_MAX_BYTES + ) + const worktreeIdBytes = boundedStringBytes( + event.worktreeId, + MOBILE_NOTIFICATION_WORKTREE_ID_MAX_BYTES + ) + if (hostBytes === null || notificationIdBytes === null || worktreeIdBytes === null) { + return null + } + + let retainedBytes = 256 + hostBytes + notificationIdBytes + worktreeIdBytes + for (const value of [event.type, event.source, event.title, event.body]) { + const remaining = MOBILE_NOTIFICATION_EVENT_MAX_BYTES - retainedBytes + const valueBytes = boundedStringBytes(value, remaining) + if (valueBytes === null) { + return null + } + retainedBytes += valueBytes + } + return retainedBytes <= MOBILE_NOTIFICATION_EVENT_MAX_BYTES ? retainedBytes : null +} + +export class MobileNotificationDeliveryLedger { + private activeEntries = 0 + private activeBytes = 0 + private maxEntries = MOBILE_NOTIFICATION_ACTIVE_MAX_ENTRIES + + claim(event: MobileNotificationRetentionEvent, hostId: string): (() => void) | null { + const retainedBytes = measureMobileNotificationDeliveryBytes(event, hostId) + if ( + retainedBytes === null || + this.activeEntries >= this.maxEntries || + this.activeBytes + retainedBytes > MOBILE_NOTIFICATION_ACTIVE_MAX_BYTES + ) { + return null + } + this.activeEntries += 1 + this.activeBytes += retainedBytes + let released = false + return () => { + if (released) { + return + } + released = true + this.activeEntries = Math.max(0, this.activeEntries - 1) + this.activeBytes = Math.max(0, this.activeBytes - retainedBytes) + } + } + + resetForTests(maxEntries?: number): void { + this.activeEntries = 0 + this.activeBytes = 0 + this.maxEntries = maxEntries ?? MOBILE_NOTIFICATION_ACTIVE_MAX_ENTRIES + } +} diff --git a/mobile/src/notifications/mobile-notifications.test.ts b/mobile/src/notifications/mobile-notifications.test.ts index b1f7c89578e..5fce317b915 100644 --- a/mobile/src/notifications/mobile-notifications.test.ts +++ b/mobile/src/notifications/mobile-notifications.test.ts @@ -313,9 +313,11 @@ describe('subscribeToDesktopNotifications', () => { // Why: notificationId is unique per completion, so the map grew unbounded when // the desktop never sent a dismiss (the remote-mobile case). It is now capped. - it('evicts the oldest scheduled entry once the cap is exceeded', async () => { + it('dismisses the oldest scheduled entry when reusing its bounded slot', async () => { setScheduledNotificationsMaxForTests(1) try { + vi.mocked(Notifications.scheduleNotificationAsync).mockReset() + vi.mocked(Notifications.dismissNotificationAsync).mockReset() vi.mocked(loadPushNotificationsEnabled).mockResolvedValue(true) vi.mocked(Notifications.getPermissionsAsync).mockResolvedValue({ status: 'granted', @@ -341,10 +343,12 @@ describe('subscribeToDesktopNotifications', () => { onEvent?.({ type: 'notification', title: 't', body: 'b', notificationId: 'agent:new' }) await flushAsync() - // The older entry was evicted by the cap: dismissing it is a no-op... + expect(Notifications.dismissNotificationAsync).toHaveBeenCalledWith('scheduled-old') + + // The older entry was already dismissed during eviction, so a later desktop dismiss is a no-op... onEvent?.({ type: 'dismiss', notificationId: 'agent:old' }) await flushAsync() - expect(Notifications.dismissNotificationAsync).not.toHaveBeenCalledWith('scheduled-old') + expect(Notifications.dismissNotificationAsync).toHaveBeenCalledTimes(1) // ...while the most-recent entry is retained and still dismissable. onEvent?.({ type: 'dismiss', notificationId: 'agent:new' }) @@ -354,6 +358,105 @@ describe('subscribeToDesktopNotifications', () => { setScheduledNotificationsMaxForTests() } }) + + it('keeps overload gaps replayable and advances after reconnect catches up', async () => { + setScheduledNotificationsMaxForTests(1) + const firstSchedule = makeDeferred() + try { + vi.mocked(Notifications.scheduleNotificationAsync).mockReset() + vi.mocked(Notifications.dismissNotificationAsync).mockReset() + vi.mocked(loadPushNotificationsEnabled).mockResolvedValue(true) + vi.mocked(Notifications.getPermissionsAsync).mockResolvedValue({ + status: 'granted', + canAskAgain: true + } as never) + vi.mocked(Notifications.scheduleNotificationAsync) + .mockReturnValueOnce(firstSchedule.promise) + .mockResolvedValueOnce('scheduled-after-drain') + .mockResolvedValueOnce('scheduled-replayed') + vi.mocked(Notifications.dismissNotificationAsync).mockResolvedValue(undefined) + let onEvent: ((data: unknown) => void) | null = null + const client = { + subscribe: vi.fn((_method, _params, callback: (data: unknown) => void) => { + onEvent = callback + return vi.fn() + }), + getState: vi.fn(() => 'connected'), + sendRequest: vi.fn(async (method: string) => { + if (method === 'notifications.getMissedSince') { + return { + ok: true, + result: { + notifications: [ + { + type: 'notification', + title: 'two', + body: 'two', + notificationId: 'two', + notificationSeq: 2 + }, + { + type: 'notification', + title: 'three', + body: 'three', + notificationId: 'three', + notificationSeq: 3 + } + ] + } + } as never + } + return { ok: true, result: undefined } as never + }) + } as unknown as RpcClient + + subscribeToDesktopNotifications(client, 'host-hung') + onEvent?.({ type: 'ready', subscriptionId: 'sub-1' }) + onEvent?.({ + type: 'notification', + title: 'one', + body: 'one', + notificationId: 'one', + notificationSeq: 1 + }) + await flushAsync() + onEvent?.({ + type: 'notification', + title: 'two', + body: 'two', + notificationId: 'two', + notificationSeq: 2 + }) + await flushAsync() + expect(Notifications.scheduleNotificationAsync).toHaveBeenCalledOnce() + expect(AsyncStorage.setItem).toHaveBeenCalledTimes(1) + + firstSchedule.resolve('scheduled-first') + await flushAsync() + onEvent?.({ + type: 'notification', + title: 'three', + body: 'three', + notificationId: 'three', + notificationSeq: 3 + }) + await flushAsync() + expect(Notifications.scheduleNotificationAsync).toHaveBeenCalledTimes(2) + expect(AsyncStorage.setItem).toHaveBeenCalledTimes(1) + + onEvent?.({ type: 'ready', subscriptionId: 'sub-2' }) + await flushAsync() + await flushAsync() + expect(Notifications.scheduleNotificationAsync).toHaveBeenCalledTimes(3) + expect(AsyncStorage.setItem).toHaveBeenLastCalledWith( + 'orca:mobileNotificationsLastSeq:host-hung', + '3' + ) + } finally { + firstSchedule.resolve('scheduled-first') + setScheduledNotificationsMaxForTests() + } + }) }) // Why: #8129 catch-up. On a reconnect the live stream re-emits `ready`; the diff --git a/mobile/src/notifications/mobile-notifications.ts b/mobile/src/notifications/mobile-notifications.ts index 7e1444d9b0c..938c5c56ce0 100644 --- a/mobile/src/notifications/mobile-notifications.ts +++ b/mobile/src/notifications/mobile-notifications.ts @@ -1,8 +1,14 @@ -import * as Notifications from 'expo-notifications' -import { Platform } from 'react-native' import type { RpcClient } from '../transport/rpc-client' -import { loadPushNotificationsEnabled } from '../storage/preferences' -import { buildLocalNotificationData, type DesktopNotificationSource } from './notification-routing' +import { + configureNotificationChannel, + startLocalNotificationDelivery, + type DismissNotificationEvent, + type NotificationEvent +} from './mobile-notification-delivery' +import { + isMobileNotificationHostIdRetainable, + measureMobileNotificationDeliveryBytes +} from './mobile-notification-retention' import { createSeenNotificationGuard, loadLastSeenSeq, @@ -10,249 +16,81 @@ import { seenKeyForEvent } from './notification-reconnect-catchup' -type NotificationEvent = { - type: 'notification' - source: DesktopNotificationSource - title: string - body: string - worktreeId?: string - notificationId?: string - // Desktop-assigned seq for reconnect catch-up (#8129); optional since older runtimes may omit it. - notificationSeq?: number -} - -type DismissNotificationEvent = { - type: 'dismiss' - notificationId: string - notificationSeq?: number -} +export { + ensureNotificationPermissions, + getNotificationPermissionState, + setScheduledNotificationsMaxForTests +} from './mobile-notification-delivery' +export type { NotificationPermissionState } from './mobile-notification-delivery' type SubscribeResult = { type: 'ready' subscriptionId: string } -type ScheduledNotificationState = { - identifier?: string - pending?: Promise - dismissAfterSchedule?: boolean -} - -const scheduledNotificationsByHostAndNotificationId = new Map() - -// Why: keys never repeat and are only freed on desktop dismiss (which remote users often miss), so bound the map to stop unbounded growth. -const MAX_SCHEDULED_NOTIFICATIONS = 256 -let maxScheduledNotifications = MAX_SCHEDULED_NOTIFICATIONS - -function getStoredNotificationKey(hostId: string, notificationId: string): string { - return `${encodeURIComponent(hostId)}:${encodeURIComponent(notificationId)}` -} - -// Evict oldest settled entries (never mid-schedule); Map iteration is insertion order so the first match is oldest. -function boundScheduledNotifications(): void { - while (scheduledNotificationsByHostAndNotificationId.size > maxScheduledNotifications) { - let evicted = false - for (const [key, state] of scheduledNotificationsByHostAndNotificationId) { - if (!state.pending) { - scheduledNotificationsByHostAndNotificationId.delete(key) - evicted = true - break - } - } - if (!evicted) { - break - } - } -} - -/** Test-only: override the cap (pass no arg to restore the default). */ -export function setScheduledNotificationsMaxForTests(max?: number): void { - maxScheduledNotifications = max ?? MAX_SCHEDULED_NOTIFICATIONS -} - -export type NotificationPermissionState = { - granted: boolean - status: string - canAskAgain: boolean - authorizationReflectsUserChoice: boolean -} - -export async function getNotificationPermissionState(): Promise { - const { status, canAskAgain } = await Notifications.getPermissionsAsync() - return { - granted: status === 'granted', - status, - canAskAgain, - // Why: Android <33 has no runtime notification permission, so "granted" is capability, not user consent. - authorizationReflectsUserChoice: - status === 'granted' && (Platform.OS !== 'android' || Number(Platform.Version) >= 33) - } -} - -// Why: re-read OS state every call — users can change it in Settings while Orca is backgrounded. -export async function ensureNotificationPermissions(): Promise { - const existing = await getNotificationPermissionState() - if (existing.granted) { - return true - } - - const { status } = await Notifications.requestPermissionsAsync() - return status === 'granted' -} - -function configureNotificationChannel(): void { - if (Platform.OS === 'android') { - void Notifications.setNotificationChannelAsync('orca-desktop', { - name: 'Desktop Notifications', - importance: Notifications.AndroidImportance.HIGH, - vibrationPattern: [0, 250], - lightColor: '#6366f1' - }) - } -} - -async function showLocalNotification(event: NotificationEvent, hostId: string): Promise { - const storedKey = event.notificationId - ? getStoredNotificationKey(hostId, event.notificationId) - : null - - if (!storedKey) { - const enabled = await loadPushNotificationsEnabled() - if (!enabled) { - return - } - - const granted = await ensureNotificationPermissions() - if (!granted) { - return - } - - await Notifications.scheduleNotificationAsync({ - content: { - title: event.title, - body: event.body, - data: buildLocalNotificationData(event, hostId), - ...(Platform.OS === 'android' ? { channelId: 'orca-desktop' } : {}) - }, - trigger: null - }) - return - } - - let state = scheduledNotificationsByHostAndNotificationId.get(storedKey) - if (state?.pending) { - return - } - if (!state) { - state = {} - scheduledNotificationsByHostAndNotificationId.set(storedKey, state) - } - const notificationState = state - - const pending = (async () => { - const enabled = await loadPushNotificationsEnabled() - if (!enabled) { - return null - } - - const granted = await ensureNotificationPermissions() - if (!granted) { - return null - } - - if (notificationState.identifier) { - await Notifications.dismissNotificationAsync(notificationState.identifier).catch(() => {}) - notificationState.identifier = undefined - } - - return Notifications.scheduleNotificationAsync({ - content: { - title: event.title, - body: event.body, - data: buildLocalNotificationData(event, hostId), - ...(Platform.OS === 'android' ? { channelId: 'orca-desktop' } : {}) - }, - trigger: null - }) - })() - notificationState.pending = pending - - try { - const scheduledIdentifier = await pending - if (!scheduledIdentifier) { - if (!notificationState.identifier) { - scheduledNotificationsByHostAndNotificationId.delete(storedKey) - } - return - } - if (notificationState.dismissAfterSchedule) { - notificationState.dismissAfterSchedule = false - scheduledNotificationsByHostAndNotificationId.delete(storedKey) - await Notifications.dismissNotificationAsync(scheduledIdentifier).catch(() => {}) - return - } - notificationState.identifier = scheduledIdentifier - boundScheduledNotifications() - } finally { - if (notificationState.pending === pending) { - notificationState.pending = undefined - notificationState.dismissAfterSchedule = false - } - } -} - -async function dismissLocalNotification( - event: DismissNotificationEvent, - hostId: string -): Promise { - if (!event.notificationId) { - return - } - const storedKey = getStoredNotificationKey(hostId, event.notificationId) - const state = scheduledNotificationsByHostAndNotificationId.get(storedKey) - if (!state) { - return - } - if (state.pending) { - // Why: dismiss can arrive while the OS is still scheduling; defer it so no stale banner survives. - state.dismissAfterSchedule = true - return - } - if (!state.identifier) { - return - } - scheduledNotificationsByHostAndNotificationId.delete(storedKey) - await Notifications.dismissNotificationAsync(state.identifier).catch(() => {}) -} - // Per-connection subscription; a reconnect `ready` triggers watermarked catch-up (#8129) so already-pushed events aren't re-sent. export function subscribeToDesktopNotifications(client: RpcClient, hostId: string): () => void { + if (!isMobileNotificationHostIdRetainable(hostId)) { + return () => {} + } configureNotificationChannel() let subscriptionId: string | null = null let disposed = false // Highest seq delivered (live or replay) this connection; persisted per-host so cold start resumes from the right cut. let lastDeliveredSeq = 0 + let watermarkBlockedThroughSeq: number | null = null // Why: defense-in-depth dedup for replayed events if the desktop's bounded buffer evicted across a reconnect boundary. const seenReplay = createSeenNotificationGuard() - function deliverLive( - type: 'notification' | 'dismiss', - event: NotificationEvent | DismissNotificationEvent - ): Promise { - if (event.notificationSeq != null && event.notificationSeq > lastDeliveredSeq) { - lastDeliveredSeq = event.notificationSeq - void saveLastSeenSeq(hostId, lastDeliveredSeq) + function eventSequence(event: NotificationEvent | DismissNotificationEvent): number | null { + return typeof event.notificationSeq === 'number' && Number.isSafeInteger(event.notificationSeq) + ? event.notificationSeq + : null + } + + function advanceDeliveredWatermark( + event: NotificationEvent | DismissNotificationEvent, + replay: boolean + ): void { + const seq = eventSequence(event) + if (seq == null || seq <= lastDeliveredSeq) { + return } - // Why (#8129): mark seen on the live path too, so a later replay of an already-pushed id dedups instead of double-pushing. + if (watermarkBlockedThroughSeq !== null) { + if (!replay) { + watermarkBlockedThroughSeq = Math.max(watermarkBlockedThroughSeq, seq) + return + } + if (seq < watermarkBlockedThroughSeq) { + return + } + watermarkBlockedThroughSeq = null + } + lastDeliveredSeq = seq + void saveLastSeenSeq(hostId, lastDeliveredSeq) + } + + function deliverLive( + event: NotificationEvent | DismissNotificationEvent, + replay = false + ): Promise { + const delivery = startLocalNotificationDelivery(event, hostId) + if (!delivery) { + const seq = eventSequence(event) + if (seq !== null && seq > lastDeliveredSeq) { + // Why: advancing past a dropped event would make reconnect catch-up permanently skip it. + watermarkBlockedThroughSeq = Math.max(watermarkBlockedThroughSeq ?? seq, seq) + } + return Promise.resolve() + } + // Why (#8129): only accepted work is seen; overload drops remain eligible for reconnect catch-up. const key = seenKeyForEvent(event) if (key) { seenReplay.add(key) } - if (type === 'notification') { - return showLocalNotification(event as NotificationEvent, hostId) - } - return dismissLocalNotification(event as DismissNotificationEvent, hostId) + advanceDeliveredWatermark(event, replay) + return delivery } // Why: desktop cuts by seq > lastSeenSeq, so re-fetching from the watermark is idempotent (seenReplay guards residual overlap). @@ -272,17 +110,19 @@ export function subscribeToDesktopNotifications(client: RpcClient, hostId: strin .catch(() => []) for (const raw of missed) { const event = raw as NotificationEvent | DismissNotificationEvent - const key = seenKeyForEvent(event) - if (key && seenReplay.has(key)) { + if (measureMobileNotificationDeliveryBytes(event, hostId) === null) { + await deliverLive(event, true) continue } - if (key) { - seenReplay.add(key) + const key = seenKeyForEvent(event) + if (key && seenReplay.has(key)) { + advanceDeliveredWatermark(event, true) + continue } if (event.type === 'notification') { - await deliverLive('notification', event) + await deliverLive(event, true) } else if (event.type === 'dismiss') { - await deliverLive('dismiss', event) + await deliverLive(event, true) } } } @@ -331,9 +171,9 @@ export function subscribeToDesktopNotifications(client: RpcClient, hostId: strin return } if (event.type === 'notification') { - void deliverLive('notification', event as NotificationEvent) + void deliverLive(event as NotificationEvent) } else if (event.type === 'dismiss') { - void deliverLive('dismiss', event as DismissNotificationEvent) + void deliverLive(event as DismissNotificationEvent) } }) diff --git a/mobile/src/notifications/mobile-scheduled-notification-registry.ts b/mobile/src/notifications/mobile-scheduled-notification-registry.ts new file mode 100644 index 00000000000..27f662b0453 --- /dev/null +++ b/mobile/src/notifications/mobile-scheduled-notification-registry.ts @@ -0,0 +1,127 @@ +import { measureUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' + +export const MOBILE_SCHEDULED_NOTIFICATION_MAX_ENTRIES = 256 +export const MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES = 2 * 1024 * 1024 +export const MOBILE_NATIVE_NOTIFICATION_ID_MAX_BYTES = 8 * 1024 + +export type MobileScheduledNotificationState = { + identifier?: string + pending?: Promise + dismissAfterSchedule?: boolean + retainedKeyBytes: number + retainedIdentifierBytes: number + retained: boolean +} + +export type MobileScheduledNotificationReservation = { + state: MobileScheduledNotificationState + evictedIdentifiers: string[] +} + +function retainedStringBytes(value: string): number { + return 64 + value.length * 2 +} + +export class MobileScheduledNotificationRegistry { + private readonly entries = new Map() + private retainedBytes = 0 + private maxEntries = MOBILE_SCHEDULED_NOTIFICATION_MAX_ENTRIES + + get(key: string): MobileScheduledNotificationState | undefined { + return this.entries.get(key) + } + + reserve(key: string): MobileScheduledNotificationReservation | null { + const retainedKeyBytes = retainedStringBytes(key) + if (retainedKeyBytes > MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES) { + return null + } + const evictedIdentifiers: string[] = [] + while ( + this.entries.size >= this.maxEntries || + this.retainedBytes + retainedKeyBytes > MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES + ) { + const settled = this.findOldestSettled() + if (!settled) { + return null + } + this.delete(settled[0]) + if (settled[1].identifier) { + evictedIdentifiers.push(settled[1].identifier) + } + } + const state: MobileScheduledNotificationState = { + retainedKeyBytes, + retainedIdentifierBytes: 0, + retained: true + } + this.entries.set(key, state) + this.retainedBytes += retainedKeyBytes + return { state, evictedIdentifiers } + } + + delete(key: string): boolean { + const state = this.entries.get(key) + if (!state) { + return false + } + this.entries.delete(key) + this.retainedBytes -= state.retainedKeyBytes + state.retainedIdentifierBytes + state.retained = false + return true + } + + clearIdentifier(state: MobileScheduledNotificationState): void { + if (!state.identifier) { + return + } + if (state.retained) { + this.retainedBytes -= state.retainedIdentifierBytes + } + state.identifier = undefined + state.retainedIdentifierBytes = 0 + } + + retainIdentifier(state: MobileScheduledNotificationState, identifier: string): boolean { + const measurement = measureUtf8ByteLength(identifier, { + stopAfterBytes: MOBILE_NATIVE_NOTIFICATION_ID_MAX_BYTES + }) + if (measurement.exceededLimit || !state.retained) { + return false + } + const identifierBytes = retainedStringBytes(identifier) + if ( + this.retainedBytes - state.retainedIdentifierBytes + identifierBytes > + MOBILE_SCHEDULED_NOTIFICATION_MAX_RETAINED_BYTES + ) { + return false + } + this.clearIdentifier(state) + state.identifier = identifier + state.retainedIdentifierBytes = identifierBytes + this.retainedBytes += identifierBytes + return true + } + + resetForTests(maxEntries?: number): void { + for (const state of this.entries.values()) { + state.retained = false + } + this.entries.clear() + this.retainedBytes = 0 + this.maxEntries = maxEntries ?? MOBILE_SCHEDULED_NOTIFICATION_MAX_ENTRIES + } + + getRetainedBytesForTests(): number { + return this.retainedBytes + } + + private findOldestSettled(): [string, MobileScheduledNotificationState] | null { + for (const entry of this.entries) { + if (!entry[1].pending) { + return entry + } + } + return null + } +} diff --git a/mobile/src/notifications/notification-reconnect-catchup.ts b/mobile/src/notifications/notification-reconnect-catchup.ts index d40af93a090..22172cbfdbf 100644 --- a/mobile/src/notifications/notification-reconnect-catchup.ts +++ b/mobile/src/notifications/notification-reconnect-catchup.ts @@ -1,4 +1,6 @@ import AsyncStorage from '@react-native-async-storage/async-storage' +import { measureUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' +import { MOBILE_NOTIFICATION_ID_MAX_BYTES } from './mobile-notification-retention' // Why: the reconnect catch-up watermark + dedup helpers for #8129, extracted // from mobile-notifications.ts so that file stays under its max-lines budget. @@ -84,14 +86,23 @@ export function seenKeyForEvent(event: { notificationSeq?: number }): string | null { const id = event.notificationId - if (id != null && event.notificationSeq != null) { - return `id:${id}#${event.notificationSeq}` + const seq = + typeof event.notificationSeq === 'number' && Number.isSafeInteger(event.notificationSeq) + ? event.notificationSeq + : null + const retainedId = + typeof id === 'string' && + !measureUtf8ByteLength(id, { stopAfterBytes: MOBILE_NOTIFICATION_ID_MAX_BYTES }).exceededLimit + ? id + : null + if (retainedId !== null && seq !== null) { + return `id:${retainedId}#${seq}` } - if (id != null) { - return `id:${id}` + if (retainedId !== null) { + return `id:${retainedId}` } - if (event.notificationSeq != null) { - return `seq:${event.notificationSeq}` + if (seq !== null) { + return `seq:${seq}` } return null } diff --git a/mobile/src/session/MobileNativeChatComposer.test.ts b/mobile/src/session/MobileNativeChatComposer.test.ts index ef2c1012603..36006d2f633 100644 --- a/mobile/src/session/MobileNativeChatComposer.test.ts +++ b/mobile/src/session/MobileNativeChatComposer.test.ts @@ -7,6 +7,7 @@ vi.mock('react-native', async () => { const React = await import('react') return { ActivityIndicator: 'ActivityIndicator', + Image: 'Image', Pressable: 'Pressable', ScrollView: ({ children, ...props }: { children?: unknown }) => React.createElement('ScrollView', props, children), @@ -24,7 +25,8 @@ vi.mock('lucide-react-native', () => ({ ArrowUp: 'ArrowUp', ImagePlus: 'ImagePlus', Mic: 'Mic', - Square: 'Square' + Square: 'Square', + X: 'X' })) function suppressRendererWarning(): () => void { @@ -112,6 +114,61 @@ describe('MobileNativeChatComposer', () => { expect(onSend).not.toHaveBeenCalled() }) + it('renders a removable thumbnail for each pending image attachment', async () => { + const onRemoveAttachment = vi.fn() + const restore = suppressRendererWarning() + try { + await act(async () => { + renderer = create( + createElement(MobileNativeChatComposer, { + value: '', + onChangeText: vi.fn(), + onSend: vi.fn().mockResolvedValue(true), + attachments: [ + { id: 'img-1', path: '/tmp/a.png', previewUri: 'file:///a.png' }, + { id: 'img-2', path: '/tmp/b.png', previewUri: 'file:///b.png' } + ], + onRemoveAttachment + }) + ) + }) + } finally { + restore() + } + const thumbs = renderer!.root.findAll((node) => node.type === 'Image') as Array<{ + props: { source: { uri: string } } + }> + expect(thumbs.map((t) => t.props.source.uri)).toEqual(['file:///a.png', 'file:///b.png']) + + const remove = renderer!.root.findAll( + (node) => node.type === 'Pressable' && node.props.accessibilityLabel === 'Remove image' + ) as Array<{ props: { onPress: () => void } }> + remove[1].props.onPress() + expect(onRemoveAttachment).toHaveBeenCalledWith('img-2') + }) + + it('enables send with an attached image even when the text is empty', async () => { + const onSend = vi.fn().mockResolvedValue(true) + const restore = suppressRendererWarning() + try { + await act(async () => { + renderer = create( + createElement(MobileNativeChatComposer, { + value: '', + onChangeText: vi.fn(), + onSend, + attachments: [{ id: 'img-1', path: '/tmp/a.png', previewUri: 'file:///a.png' }] + }) + ) + }) + } finally { + restore() + } + expect(sendButton().props).toMatchObject({ disabled: false }) + await act(async () => sendButton().props.onPress()) + expect(onSend).toHaveBeenCalledWith('') + }) + it('moves the caret to the insert point after an autocomplete pick, then releases control', async () => { const restore = suppressRendererWarning() try { diff --git a/mobile/src/session/MobileNativeChatComposer.tsx b/mobile/src/session/MobileNativeChatComposer.tsx index ba02fe3d44b..58508be77b5 100644 --- a/mobile/src/session/MobileNativeChatComposer.tsx +++ b/mobile/src/session/MobileNativeChatComposer.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { ActivityIndicator, + Image, Pressable, ScrollView, StyleSheet, @@ -8,13 +9,14 @@ import { TextInput, View } from 'react-native' -import { ArrowUp, ImagePlus, Mic, Square } from 'lucide-react-native' +import { ArrowUp, ImagePlus, Mic, Square, X } from 'lucide-react-native' import { colors, radii, spacing, typography } from '../theme/mobile-theme' import { applyAutocomplete, detectAutocompleteTrigger, rankSuggestions } from './mobile-native-chat-autocomplete' +import type { PendingNativeChatImage } from './mobile-native-chat-image-attachment' // Common agent slash commands offered as autocomplete; sending them is just text // to the agent's terminal, so the set is intentionally provider-agnostic. @@ -30,6 +32,7 @@ const SLASH_COMMANDS = [ ] const NO_FILE_PATHS: string[] = [] +const NO_ATTACHMENTS: PendingNativeChatImage[] = [] type Props = { /** Controlled composer text — owned by the parent so dictation can write to it. */ @@ -37,6 +40,10 @@ type Props = { onChangeText: (text: string) => void onSend: (text: string) => Promise onAttachImage?: () => void + /** Images picked-and-uploaded but not yet sent — shown as removable thumbnails + * and ridden along on the next send (desktop native-chat parity). */ + attachments?: PendingNativeChatImage[] + onRemoveAttachment?: (id: string) => void isAttaching?: boolean onMicPress?: () => void micActive?: boolean @@ -55,6 +62,8 @@ export function MobileNativeChatComposer({ onChangeText, onSend, onAttachImage, + attachments = NO_ATTACHMENTS, + onRemoveAttachment, isAttaching = false, onMicPress, micActive = false, @@ -76,7 +85,10 @@ export function MobileNativeChatComposer({ const sendingRef = useRef(false) const [sending, setSending] = useState(false) const trimmed = value.trim() - const canSend = trimmed.length > 0 && !disabled && !sending && !isAttaching + // An attached image alone is a valid send (desktop parity), so the image rides + // along even when the user sends no accompanying text. + const canSend = + (trimmed.length > 0 || attachments.length > 0) && !disabled && !sending && !isAttaching const trigger = useMemo(() => detectAutocompleteTrigger(value, cursor), [value, cursor]) const suggestions = useMemo(() => { @@ -145,6 +157,35 @@ export function MobileNativeChatComposer({ ) : null} + {attachments.length > 0 ? ( + + {attachments.map((attachment) => ( + + + {onRemoveAttachment ? ( + onRemoveAttachment(attachment.id)} + hitSlop={8} + > + + + ) : null} + + ))} + + ) : null} {onAttachImage ? ( { + const React = await import('react') + return { + Image: 'Image', + Pressable: 'Pressable', + Text: ({ children, ...props }: { children?: unknown }) => + React.createElement('Text', props, children), + View: ({ children, ...props }: { children?: unknown }) => + React.createElement('View', props, children), + StyleSheet: { create: (styles: unknown) => styles, hairlineWidth: 1 } + } +}) +vi.mock('expo-clipboard', () => ({ setStringAsync: vi.fn() })) +vi.mock('lucide-react-native', () => ({ + ArrowUp: 'ArrowUp', + ChevronDown: 'ChevronDown', + Copy: 'Copy', + SquareChevronRight: 'SquareChevronRight' +})) +vi.mock('../components/MobileMarkdown', () => ({ MobileMarkdown: 'MobileMarkdown' })) + +import { MobileNativeChatMessage } from './MobileNativeChatMessage' + +function userMessage(blocks: NativeChatMessage['blocks']): NativeChatMessage { + return { id: 'u1', role: 'user', blocks, timestamp: null, source: 'transcript' } +} + +describe('MobileNativeChatMessage image-ref rendering', () => { + let renderer: ReactTestRenderer | null = null + + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + }) + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + }) + + function render(message: NativeChatMessage): ReactTestRenderer { + const original = console.error + const spy = vi.spyOn(console, 'error').mockImplementation((...a) => { + if (typeof a[0] === 'string' && a[0].includes('react-test-renderer is deprecated')) { + return + } + original(...a) + }) + try { + act(() => { + renderer = create(createElement(MobileNativeChatMessage, { message })) + }) + } finally { + spy.mockRestore() + } + return renderer! + } + + it('renders a loadable preview URI as an image thumbnail', () => { + const tree = render(userMessage([{ type: 'image-ref', url: 'file:///a.jpg', alt: 'a photo' }])) + const image = tree.root.findByType('Image' as never) + expect(image.props.source).toEqual({ uri: 'file:///a.jpg' }) + expect(image.props.accessibilityLabel).toBe('a photo') + }) + + it('prefers the url over the path when both are present', () => { + const tree = render( + userMessage([{ type: 'image-ref', url: 'file:///local.jpg', path: '/tmp/host.png' }]) + ) + expect(tree.root.findByType('Image' as never).props.source).toEqual({ + uri: 'file:///local.jpg' + }) + }) + + it('falls back to a text placeholder for a bare host path', () => { + // A host temp path (e.g. on an SSH host) is not loadable on the device. + const tree = render(userMessage([{ type: 'image-ref', path: '/tmp/host.png' }])) + expect(tree.root.findAllByType('Image' as never)).toHaveLength(0) + const texts = tree.root + .findAllByType('Text' as never) + .map((node) => String(node.children.join(''))) + expect(texts.some((text) => text.includes('/tmp/host.png'))).toBe(true) + }) +}) diff --git a/mobile/src/session/MobileNativeChatMessage.tsx b/mobile/src/session/MobileNativeChatMessage.tsx index eadfd2043f8..d8458f1b889 100644 --- a/mobile/src/session/MobileNativeChatMessage.tsx +++ b/mobile/src/session/MobileNativeChatMessage.tsx @@ -1,5 +1,5 @@ import { memo, useEffect, useRef, useState } from 'react' -import { Pressable, Text, View } from 'react-native' +import { Image, Pressable, Text, View } from 'react-native' import * as Clipboard from 'expo-clipboard' import { ArrowUp, ChevronDown, Copy, SquareChevronRight } from 'lucide-react-native' import type { NativeChatBlock, NativeChatMessage } from '../../../src/shared/native-chat-types' @@ -13,6 +13,7 @@ import { type ToolPair } from './mobile-native-chat-blocks' import { diffFromText, diffFromToolCall, type DiffLine } from './mobile-native-chat-diff' +import { isRenderableImageUri } from './mobile-native-chat-image-preview' import { MAX_TOOL_RESULT_CHARS, styles, TEXT_SIZE } from './mobile-native-chat-message-styles' import { nativeChatMessageText } from './mobile-native-chat-message-text' import { @@ -160,6 +161,19 @@ function Prose({ ) } if (isImageRefBlock(block)) { + // A local preview (composer echo) or real URL renders as a thumbnail; a bare + // host path (not loadable on the device) falls back to a text placeholder. + const uri = block.url ?? block.path + if (isRenderableImageUri(uri)) { + return ( + + ) + } return ( 🖼 {block.alt ?? block.path ?? block.url ?? 'image'} diff --git a/mobile/src/session/MobileNativeChatOverlay.tsx b/mobile/src/session/MobileNativeChatOverlay.tsx index 6acf2d305f6..b1fcfd4c368 100644 --- a/mobile/src/session/MobileNativeChatOverlay.tsx +++ b/mobile/src/session/MobileNativeChatOverlay.tsx @@ -1,11 +1,13 @@ import { StyleSheet, View } from 'react-native' import { MobileNativeChatView, type MobileNativeChatInputLockReason } from './MobileNativeChatView' +import type { MobileNativeChatImageAttachments } from './use-mobile-native-chat-image-attachments' import type { MobileNativeChatController } from './use-mobile-native-chat-controller' type Props = { controller: MobileNativeChatController - onAttachImage: () => void - isAttaching: boolean + /** Native-chat image attachments: picking adds a composer chip, and sending + * rides the pending images along with the message text (desktop parity). */ + images: MobileNativeChatImageAttachments onMicPress: () => void micActive: boolean dictationMode: 'toggle' | 'hold' @@ -19,8 +21,7 @@ type Props = { * view toggles while the native surface owns the visible composer. */ export function MobileNativeChatOverlay({ controller, - onAttachImage, - isAttaching, + images, onMicPress, micActive, dictationMode, @@ -54,12 +55,14 @@ export function MobileNativeChatOverlay({ hasMore={session.hasMore} loadingEarlier={session.loadingEarlier} onLoadEarlier={session.loadEarlier} - onSend={controller.handleNativeChatSend} + onSend={images.sendNativeChat} pending={controller.chatPending} composerText={controller.chatComposerText} onComposerTextChange={controller.setChatComposerText} - onAttachImage={onAttachImage} - isAttaching={isAttaching} + onAttachImage={() => void images.attachImage('library')} + attachments={images.attachments} + onRemoveAttachment={images.removeAttachment} + isAttaching={images.isAttaching} onMicPress={onMicPress} micActive={micActive} dictationMode={dictationMode} diff --git a/mobile/src/session/MobileNativeChatView.tsx b/mobile/src/session/MobileNativeChatView.tsx index 1cfe37e5d4d..f2a23910a26 100644 --- a/mobile/src/session/MobileNativeChatView.tsx +++ b/mobile/src/session/MobileNativeChatView.tsx @@ -17,11 +17,13 @@ import { styles } from './mobile-native-chat-view-styles' import { buildMobileNativeChatTransientData, foldMobileNativeChatMessages, - mobileNativeChatEmptyState + mobileNativeChatEmptyState, + type MobileNativeChatPendingItem } from './mobile-native-chat-render-data' import { useMobileNativeChatAskDismiss } from './use-mobile-native-chat-ask-dismiss' import { useMobileNativeChatPinchGesture } from './use-mobile-native-chat-pinch-gesture' import { MobileAgentWorkingIndicator } from './MobileAgentWorkingIndicator' +import type { PendingNativeChatImage } from './mobile-native-chat-image-attachment' import { MobileNativeChatComposer } from './MobileNativeChatComposer' import { MobileNativeChatMessage } from './MobileNativeChatMessage' import { MobileNativeChatAsk } from './MobileNativeChatAsk' @@ -53,11 +55,15 @@ type Props = { onLoadEarlier?: () => void onSend: (text: string) => Promise /** Optimistic queued sends (owned by the route so they survive view switches). */ - pending: Array<{ id: string; text: string }> + /** Optimistic user echoes, including any ridden-along image preview URIs. */ + pending: MobileNativeChatPendingItem[] /** Controlled composer text (owned by the route so dictation can write to it). */ composerText: string onComposerTextChange: (text: string) => void onAttachImage?: () => void + /** Pending image attachments shown as composer thumbnails until the next send. */ + attachments?: PendingNativeChatImage[] + onRemoveAttachment?: (id: string) => void isAttaching?: boolean onMicPress?: () => void micActive?: boolean @@ -103,6 +109,8 @@ export function MobileNativeChatView({ composerText, onComposerTextChange, onAttachImage, + attachments, + onRemoveAttachment, isAttaching, onMicPress, micActive, @@ -403,6 +411,8 @@ export function MobileNativeChatView({ onChangeText={onComposerTextChange} onSend={handleSend} onAttachImage={onAttachImage} + attachments={attachments} + onRemoveAttachment={onRemoveAttachment} isAttaching={isAttaching} onMicPress={onMicPress} micActive={micActive} diff --git a/mobile/src/session/mobile-image-base64-accumulator.test.ts b/mobile/src/session/mobile-image-base64-accumulator.test.ts new file mode 100644 index 00000000000..d6da669b7d3 --- /dev/null +++ b/mobile/src/session/mobile-image-base64-accumulator.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { MobileImageBase64Accumulator } from './mobile-image-base64-accumulator' + +describe('MobileImageBase64Accumulator', () => { + it('preserves bytes delivered as 100,000 one-byte fragments', () => { + const accumulator = new MobileImageBase64Accumulator() + const expected = Buffer.alloc(100_000) + + for (let index = 0; index < expected.byteLength; index += 1) { + const value = index % 251 + expected[index] = value + accumulator.append(Uint8Array.of(value)) + } + + expect(accumulator.finish()).toBe(expected.toString('base64')) + }) +}) diff --git a/mobile/src/session/mobile-image-base64-accumulator.ts b/mobile/src/session/mobile-image-base64-accumulator.ts new file mode 100644 index 00000000000..079a4d89a0e --- /dev/null +++ b/mobile/src/session/mobile-image-base64-accumulator.ts @@ -0,0 +1,40 @@ +import { Buffer } from 'buffer' + +const MOBILE_IMAGE_BASE64_CHUNK_BYTES = 256 * 1024 - 1 + +export class MobileImageBase64Accumulator { + private readonly staging = new Uint8Array(MOBILE_IMAGE_BASE64_CHUNK_BYTES) + private readonly encodedChunks: string[] = [] + private stagingLength = 0 + + append(bytes: Uint8Array): void { + let offset = 0 + while (offset < bytes.byteLength) { + const copied = Math.min( + this.staging.byteLength - this.stagingLength, + bytes.byteLength - offset + ) + this.staging.set(bytes.subarray(offset, offset + copied), this.stagingLength) + this.stagingLength += copied + offset += copied + if (this.stagingLength === this.staging.byteLength) { + this.flushStaging() + } + } + } + + finish(): string { + this.flushStaging() + return this.encodedChunks.join('') + } + + private flushStaging(): void { + if (this.stagingLength === 0) { + return + } + this.encodedChunks.push( + Buffer.from(this.staging.subarray(0, this.stagingLength)).toString('base64') + ) + this.stagingLength = 0 + } +} diff --git a/mobile/src/session/mobile-image-source-picker.test.ts b/mobile/src/session/mobile-image-source-picker.test.ts index 3e35a7bed61..1a582bde3fe 100644 --- a/mobile/src/session/mobile-image-source-picker.test.ts +++ b/mobile/src/session/mobile-image-source-picker.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { CLIPBOARD_IMAGE_MAX_SOURCE_BYTES } from '../../../src/shared/clipboard-image' vi.mock('expo-image-picker', () => ({ requestMediaLibraryPermissionsAsync: vi.fn(), @@ -7,6 +8,9 @@ vi.mock('expo-image-picker', () => ({ vi.mock('expo-document-picker', () => ({ getDocumentAsync: vi.fn() })) +vi.mock('expo-file-system', () => ({ + File: vi.fn() +})) import { ImageLibraryPermissionError, pickMobileImage } from './mobile-image-source-picker' @@ -15,17 +19,48 @@ const granted = { granted: true } as Awaited< > const denied = { granted: false } as typeof granted +function fileFactory( + chunks: Uint8Array[], + options?: { fileSize?: number; handleSize?: number | null; readError?: Error } +) { + const close = vi.fn() + const readBytes = vi.fn(() => { + if (options?.readError) { + throw options.readError + } + return chunks.shift() ?? new Uint8Array() + }) + const open = vi.fn(() => ({ + size: options?.handleSize ?? options?.fileSize ?? 0, + readBytes, + close + })) + const createFile = vi.fn(() => ({ size: options?.fileSize ?? 0, open })) + return { close, createFile, open, readBytes } +} + describe('pickMobileImage', () => { it('returns base64 from the photo library', async () => { + const bytes = new Uint8Array([0, 1, 2, 3]) + const file = fileFactory([bytes]) + const launchLibrary = vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///x.jpg', fileSize: bytes.length }] + }) const result = await pickMobileImage('library', { requestLibraryPermission: vi.fn().mockResolvedValue(granted), - launchLibrary: vi.fn().mockResolvedValue({ - canceled: false, - assets: [{ uri: 'file:///x.jpg', base64: 'AAAA' }] - }) + launchLibrary, + createFile: file.createFile }) - expect(result).toEqual({ base64: 'AAAA' }) + expect(result).toEqual({ + base64: Buffer.from(bytes).toString('base64'), + uri: 'file:///x.jpg' + }) + expect(launchLibrary).toHaveBeenCalledWith( + expect.objectContaining({ base64: false, allowsMultipleSelection: false }) + ) + expect(file.close).toHaveBeenCalledTimes(1) }) it('throws when photo library permission is denied', async () => { @@ -48,19 +83,25 @@ describe('pickMobileImage', () => { it('reads a picked file URI into base64 for the files source', async () => { const bytes = new Uint8Array([1, 2, 3, 4]) - const fetchSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValue(new Response(bytes.buffer, { headers: { 'content-type': 'image/png' } })) - - const result = await pickMobileImage('files', { - launchFiles: vi.fn().mockResolvedValue({ - canceled: false, - assets: [{ uri: 'file:///doc.png' }] - }) + const file = fileFactory([bytes]) + const launchFiles = vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///doc.png', size: bytes.length }] }) - expect(result).toEqual({ base64: Buffer.from(bytes).toString('base64') }) - fetchSpy.mockRestore() + const result = await pickMobileImage('files', { + launchFiles, + createFile: file.createFile + }) + + expect(result).toEqual({ + base64: Buffer.from(bytes).toString('base64'), + uri: 'file:///doc.png' + }) + expect(launchFiles).toHaveBeenCalledWith( + expect.objectContaining({ copyToCacheDirectory: true }) + ) + expect(file.close).toHaveBeenCalledTimes(1) }) it('returns null when the files picker is cancelled', async () => { @@ -70,4 +111,72 @@ describe('pickMobileImage', () => { expect(result).toBeNull() }) + + it('rejects a declared oversized asset before opening it', async () => { + const file = fileFactory([], { fileSize: 1 }) + await expect( + pickMobileImage('files', { + launchFiles: vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///huge.png', size: CLIPBOARD_IMAGE_MAX_SOURCE_BYTES + 1 }] + }), + createFile: file.createFile + }) + ).rejects.toThrow('Clipboard image is too large') + expect(file.createFile).not.toHaveBeenCalled() + expect(file.open).not.toHaveBeenCalled() + }) + + it('does not let stale size metadata bypass the bounded read', async () => { + const close = vi.fn() + const readBytes = vi.fn((length: number) => new Uint8Array(length)) + const createFile = vi.fn(() => ({ + size: 1, + open: () => ({ size: 1, readBytes, close }) + })) + + await expect( + pickMobileImage('library', { + requestLibraryPermission: vi.fn().mockResolvedValue(granted), + launchLibrary: vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///grew.png', fileSize: 1 }] + }), + createFile + }) + ).rejects.toThrow('Clipboard image is too large') + expect(readBytes).toHaveBeenLastCalledWith(1) + expect(close).toHaveBeenCalledTimes(1) + }) + + it('closes the file handle when a read fails', async () => { + const file = fileFactory([], { fileSize: 4, readError: new Error('read failed') }) + await expect( + pickMobileImage('files', { + launchFiles: vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///broken.png', size: 4 }] + }), + createFile: file.createFile + }) + ).rejects.toThrow('read failed') + expect(file.close).toHaveBeenCalledTimes(1) + }) + + it('preserves bytes across chunk boundaries that are not base64 aligned', async () => { + const chunks = [new Uint8Array([1]), new Uint8Array([2, 3]), new Uint8Array([4, 5])] + const file = fileFactory([...chunks], { fileSize: 5, handleSize: 5 }) + const result = await pickMobileImage('files', { + launchFiles: vi.fn().mockResolvedValue({ + canceled: false, + assets: [{ uri: 'file:///chunked.png', size: 5 }] + }), + createFile: file.createFile + }) + expect(result).toEqual({ + base64: Buffer.from([1, 2, 3, 4, 5]).toString('base64'), + uri: 'file:///chunked.png' + }) + expect(file.close).toHaveBeenCalledTimes(1) + }) }) diff --git a/mobile/src/session/mobile-image-source-picker.ts b/mobile/src/session/mobile-image-source-picker.ts index 8bc824965fc..bf9c529cc25 100644 --- a/mobile/src/session/mobile-image-source-picker.ts +++ b/mobile/src/session/mobile-image-source-picker.ts @@ -1,14 +1,21 @@ -// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' — Metro -// can't resolve Node's builtin in a React Native bundle. -import { Buffer } from 'buffer' import * as DocumentPicker from 'expo-document-picker' +import { File as FsFile } from 'expo-file-system' import * as ImagePicker from 'expo-image-picker' +import { + CLIPBOARD_IMAGE_MAX_SOURCE_BYTES, + assertClipboardImageBase64LengthWithinLimit, + assertClipboardImageByteLengthWithinLimit +} from '../../../src/shared/clipboard-image' +import { MobileImageBase64Accumulator } from './mobile-image-base64-accumulator' export type MobileImageSource = 'library' | 'files' export type PickedMobileImage = { // Raw base64 (no data: prefix); fed straight into the existing upload pipeline. readonly base64: string + // Local file URI of the picked asset — used only to render a composer preview + // thumbnail (the host upload uses `base64`); absent when the source can't supply one. + readonly uri?: string } export class ImageLibraryPermissionError extends Error { @@ -18,18 +25,68 @@ export class ImageLibraryPermissionError extends Error { } } -// Why: expo-document-picker returns a file URI, not base64. Read it through -// fetch + Buffer so we match the base64 contract the upload pipeline expects -// without pulling in expo-file-system. -async function readUriAsBase64(uri: string): Promise { - const response = await fetch(uri) - const bytes = new Uint8Array(await response.arrayBuffer()) - return Buffer.from(bytes).toString('base64') +const MOBILE_IMAGE_READ_CHUNK_BYTES = 256 * 1024 + +type MobileImageFileHandle = { + readonly size: number | null + readBytes(length: number): Uint8Array + close(): void +} + +type MobileImageFile = { + readonly size: number + open(): MobileImageFileHandle +} + +export type MobileImageFileFactory = (uri: string) => MobileImageFile + +function defaultMobileImageFileFactory(uri: string): MobileImageFile { + return new FsFile(uri) +} + +async function readUriAsBase64( + uri: string, + declaredSize: number | undefined, + createFile: MobileImageFileFactory +): Promise { + if (typeof declaredSize === 'number' && Number.isFinite(declaredSize)) { + assertClipboardImageByteLengthWithinLimit(declaredSize) + } + + const file = createFile(uri) + assertClipboardImageByteLengthWithinLimit(file.size) + const handle = file.open() + try { + if (handle.size !== null) { + assertClipboardImageByteLengthWithinLimit(handle.size) + } + const accumulator = new MobileImageBase64Accumulator() + let bytesRead = 0 + while (bytesRead <= CLIPBOARD_IMAGE_MAX_SOURCE_BYTES) { + const requested = Math.min( + MOBILE_IMAGE_READ_CHUNK_BYTES, + CLIPBOARD_IMAGE_MAX_SOURCE_BYTES - bytesRead + 1 + ) + const bytes = handle.readBytes(requested) + if (bytes.byteLength === 0) { + break + } + bytesRead += bytes.byteLength + assertClipboardImageByteLengthWithinLimit(bytesRead) + accumulator.append(bytes) + } + const base64 = accumulator.finish() + assertClipboardImageBase64LengthWithinLimit(base64.length) + return base64 + } finally { + handle.close() + } } async function pickFromLibrary( requestPermission: typeof ImagePicker.requestMediaLibraryPermissionsAsync = ImagePicker.requestMediaLibraryPermissionsAsync, - launch: typeof ImagePicker.launchImageLibraryAsync = ImagePicker.launchImageLibraryAsync + launch: typeof ImagePicker.launchImageLibraryAsync = ImagePicker.launchImageLibraryAsync, + createFile: MobileImageFileFactory = defaultMobileImageFileFactory ): Promise { const permission = await requestPermission() // Why: `granted` covers full + limited iOS access; only a hard denial blocks us. @@ -38,7 +95,7 @@ async function pickFromLibrary( } const result = await launch({ mediaTypes: ['images'], - base64: true, + base64: false, allowsMultipleSelection: false, quality: 1 }) @@ -46,15 +103,16 @@ async function pickFromLibrary( return null } const asset = result.assets[0] - const base64 = asset?.base64 ?? (asset?.uri ? await readUriAsBase64(asset.uri) : null) + const base64 = asset?.uri ? await readUriAsBase64(asset.uri, asset.fileSize, createFile) : null if (!base64) { return null } - return { base64 } + return { base64, ...(asset?.uri ? { uri: asset.uri } : {}) } } async function pickFromFiles( - launch: typeof DocumentPicker.getDocumentAsync = DocumentPicker.getDocumentAsync + launch: typeof DocumentPicker.getDocumentAsync = DocumentPicker.getDocumentAsync, + createFile: MobileImageFileFactory = defaultMobileImageFileFactory ): Promise { const result = await launch({ type: 'image/*', @@ -68,7 +126,8 @@ async function pickFromFiles( if (!asset?.uri) { return null } - return { base64: await readUriAsBase64(asset.uri) } + const base64 = await readUriAsBase64(asset.uri, asset.size, createFile) + return base64 ? { base64, uri: asset.uri } : null } export async function pickMobileImage( @@ -77,10 +136,11 @@ export async function pickMobileImage( readonly requestLibraryPermission?: typeof ImagePicker.requestMediaLibraryPermissionsAsync readonly launchLibrary?: typeof ImagePicker.launchImageLibraryAsync readonly launchFiles?: typeof DocumentPicker.getDocumentAsync + readonly createFile?: MobileImageFileFactory } ): Promise { if (source === 'library') { - return pickFromLibrary(deps?.requestLibraryPermission, deps?.launchLibrary) + return pickFromLibrary(deps?.requestLibraryPermission, deps?.launchLibrary, deps?.createFile) } - return pickFromFiles(deps?.launchFiles) + return pickFromFiles(deps?.launchFiles, deps?.createFile) } diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts new file mode 100644 index 00000000000..1af010ff503 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -0,0 +1,103 @@ +import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { + isImageSourceUserTurn, + stripImagePromptMarker +} from './mobile-native-chat-image-transcript-markers' + +/** An ack-lost ('unknown' outcome) send held until its transcript echo lands or + * the deadline surfaces the uncertainty. */ +export type UnconfirmedSend = { + draftKey: string + pendingKey: string | null + text: string + normalizedText: string + baselineTailMessageId: string | null + deadline: ReturnType | null +} + +export function normalizedUserText(message: NativeChatMessage): string | null { + if (message.role !== 'user') { + return null + } + const text = message.blocks + .filter((block) => block.type === 'text') + .map((block) => (block.type === 'text' ? block.text : '')) + .join('') + // Claude echoes a captioned image send as `[Image #1] caption` — the sent + // text must still match its echo, so strip the marker before comparing. + const stripped = stripImagePromptMarker(text).trim() + return stripped || null +} + +export function countUserTextOccurrences( + messages: readonly NativeChatMessage[], + text: string +): number { + let count = 0 + for (const message of messages) { + if (normalizedUserText(message) === text) { + count++ + } + } + return count +} + +/** Number of `[Image: source: …]` echo turns strictly after `tailId` (or the + * whole transcript when the tail was paginated out). An image-only send has no + * caption to match, so it reconciles by ordinal against this count — counting + * only image echoes keeps an unrelated text send's echo from clearing it. */ +export function countImageSourceTurnsAfter( + messages: readonly NativeChatMessage[], + tailId: string | null +): number { + const tailIndex = tailId ? messages.findIndex((message) => message.id === tailId) : -1 + let count = 0 + for (let i = tailIndex + 1; i < messages.length; i++) { + const message = messages[i] + if (message && isImageSourceUserTurn(message)) { + count++ + } + } + return count +} + +export function findLandedUnconfirmedSends( + messages: readonly NativeChatMessage[], + entries: readonly UnconfirmedSend[] +): UnconfirmedSend[] { + // Why: pagination prepends old equal text; only unclaimed matches after each + // captured tail prove new echoes. User turns are keyed by text; an image echo + // (`[Image: source: …]` or no text) keys under '' so an empty-text send can + // claim it. + const messageIndexById = new Map() + const userMessagesByText = new Map>() + for (const [index, message] of messages.entries()) { + messageIndexById.set(message.id, index) + if (message.role !== 'user') { + continue + } + const key = isImageSourceUserTurn(message) ? '' : (normalizedUserText(message) ?? '') + const current = userMessagesByText.get(key) ?? [] + current.push({ id: message.id, index }) + userMessagesByText.set(key, current) + } + + const claimedMessageIds = new Set() + const landed: UnconfirmedSend[] = [] + for (const entry of entries) { + const tailIndex = entry.baselineTailMessageId + ? messageIndexById.get(entry.baselineTailMessageId) + : -1 + if (tailIndex === undefined) { + continue + } + const echo = userMessagesByText + .get(entry.normalizedText) + ?.find((message) => message.index > tailIndex && !claimedMessageIds.has(message.id)) + if (echo) { + claimedMessageIds.add(echo.id) + landed.push(entry) + } + } + return landed +} diff --git a/mobile/src/session/mobile-native-chat-image-attachment.test.ts b/mobile/src/session/mobile-native-chat-image-attachment.test.ts new file mode 100644 index 00000000000..57c3f67bd1c --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-attachment.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcResponse, RpcSuccess } from '../transport/types' +import { uploadMobileNativeChatImage } from './mobile-native-chat-image-attachment' + +function ok(id: string, result: unknown): RpcSuccess { + return { id, ok: true, result, _meta: { runtimeId: 'runtime-1' } } +} + +function methodNotFound(id: string): RpcResponse { + return { + id, + ok: false, + error: { code: 'method_not_found', message: 'no' }, + _meta: { runtimeId: 'r' } + } +} + +function clientWithResponses(responses: RpcResponse[]): Pick & { + calls: { method: string; params: unknown }[] +} { + const calls: { method: string; params: unknown }[] = [] + return { + calls, + sendRequest: vi.fn(async (method: string, params?: unknown) => { + calls.push({ method, params }) + const response = responses.shift() + if (!response) { + throw new Error(`unexpected request: ${method}`) + } + return response + }) + } +} + +describe('uploadMobileNativeChatImage', () => { + it('uploads the picked image and returns its host path + local preview uri, without any terminal.send', async () => { + const client = clientWithResponses([ + methodNotFound('start'), + ok('save', '/tmp/orca-attach.png') + ]) + + const result = await uploadMobileNativeChatImage('library', { + client, + getConnectionId: async () => 'conn-7', + pickImage: vi.fn().mockResolvedValue({ base64: 'AAAA', uri: 'file:///photo.jpg' }) + }) + + expect(result).toEqual({ path: '/tmp/orca-attach.png', previewUri: 'file:///photo.jpg' }) + // Native chat defers the paste to submit — nothing is sent to the terminal here. + expect(client.calls.some((call) => call.method === 'terminal.send')).toBe(false) + const saveCall = client.calls.find((c) => c.method === 'clipboard.saveImageAsTempFile') + expect(saveCall?.params).toMatchObject({ connectionId: 'conn-7' }) + }) + + it('returns null when the picker is cancelled and uploads nothing', async () => { + const client = clientWithResponses([]) + + const result = await uploadMobileNativeChatImage('library', { + client, + getConnectionId: async () => null, + pickImage: vi.fn().mockResolvedValue(null) + }) + + expect(result).toBeNull() + expect(client.calls).toEqual([]) + }) + + it('falls back to an inline data uri for the preview when the picker omits a uri', async () => { + const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/x.png')]) + + const result = await uploadMobileNativeChatImage('files', { + client, + getConnectionId: async () => null, + pickImage: vi.fn().mockResolvedValue({ base64: 'BBBB' }) + }) + + expect(result).toEqual({ path: '/tmp/x.png', previewUri: 'data:image/png;base64,BBBB' }) + }) + + it('signals upload start only after a real image is picked', async () => { + const onUploadStart = vi.fn() + const cancelledClient = clientWithResponses([]) + await uploadMobileNativeChatImage('library', { + client: cancelledClient, + getConnectionId: async () => null, + pickImage: vi.fn().mockResolvedValue(null), + onUploadStart + }) + expect(onUploadStart).not.toHaveBeenCalled() + + const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/y.png')]) + await uploadMobileNativeChatImage('library', { + client, + getConnectionId: async () => null, + pickImage: vi.fn().mockResolvedValue({ base64: 'CCCC', uri: 'file:///y.jpg' }), + onUploadStart + }) + expect(onUploadStart).toHaveBeenCalledTimes(1) + }) +}) diff --git a/mobile/src/session/mobile-native-chat-image-attachment.ts b/mobile/src/session/mobile-native-chat-image-attachment.ts new file mode 100644 index 00000000000..f81b9d2ad98 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-attachment.ts @@ -0,0 +1,46 @@ +import type { RpcClient } from '../transport/rpc-client' +import { saveMobileClipboardImageAsTempFile } from './mobile-clipboard-image' +// Type-only import so this module (and its unit test) stays free of the expo/ +// react-native picker chain; the concrete `pickImage` is injected by the hook. +import type { MobileImageSource, PickedMobileImage } from './mobile-image-source-picker' + +/** A picked-and-uploaded image held in the native-chat composer until submit. + * `path` is the host temp file pasted into the agent on send; `previewUri` is a + * local URI used only to render the composer thumbnail. */ +export type PendingNativeChatImage = { + readonly id: string + readonly path: string + readonly previewUri: string +} + +export type UploadNativeChatImageDeps = { + readonly client: Pick + readonly getConnectionId: () => Promise + // Injected so this module stays free of expo/react-native imports (unit-testable). + readonly pickImage: (source: MobileImageSource) => Promise + // Fired once the user has picked an image and the host upload is about to start — + // lets the UI show the attach spinner only for the transfer, not the picker. + readonly onUploadStart?: () => void +} + +/** Picks an image and uploads it to the host, returning the host path + a local + * preview URI — but does NOT paste it into the terminal. Unlike the terminal + * attach flow, native chat holds the image as a composer chip and rides it along + * on submit (desktop parity), so the chip and the agent input never diverge. + * Returns null when the user cancels the picker. */ +export async function uploadMobileNativeChatImage( + source: MobileImageSource, + { client, getConnectionId, pickImage, onUploadStart }: UploadNativeChatImageDeps +): Promise | null> { + const picked = await pickImage(source) + if (!picked) { + return null + } + onUploadStart?.() + const connectionId = await getConnectionId() + const path = await saveMobileClipboardImageAsTempFile(client, picked.base64, { connectionId }) + // Prefer the picker's local URI for the thumbnail; fall back to an inline data + // URI when the source omitted one (RN renders both). + const previewUri = picked.uri ?? `data:image/png;base64,${picked.base64}` + return { path, previewUri } +} diff --git a/mobile/src/session/mobile-native-chat-image-preview.test.ts b/mobile/src/session/mobile-native-chat-image-preview.test.ts new file mode 100644 index 00000000000..64ec62d57e3 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-preview.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { isRenderableImageUri } from './mobile-native-chat-image-preview' + +describe('isRenderableImageUri', () => { + it('accepts local previews and real URLs the device can load', () => { + for (const uri of [ + 'file:///var/mobile/a.jpg', + 'data:image/png;base64,AAAA', + 'content://media/1', + 'blob:abc', + 'http://host/a.png', + 'https://host/a.png' + ]) { + expect(isRenderableImageUri(uri)).toBe(true) + } + }) + + it('rejects bare host paths (not loadable on the device) and empty values', () => { + for (const uri of [ + '/tmp/orca-attach.png', + 'C:\\tmp\\a.png', + 'orca-attach.png', + '', + undefined + ]) { + expect(isRenderableImageUri(uri)).toBe(false) + } + }) +}) diff --git a/mobile/src/session/mobile-native-chat-image-preview.ts b/mobile/src/session/mobile-native-chat-image-preview.ts new file mode 100644 index 00000000000..8062ac5b83f --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-preview.ts @@ -0,0 +1,9 @@ +// A URI RN can actually load: a local composer/echo preview (file://, +// data:, content://, blob:) or a real remote URL. A bare host path from the +// transcript (e.g. /tmp/x.png on an SSH host) is not loadable on the device, so +// it stays a text placeholder instead of a broken image. +const RENDERABLE_IMAGE_URI = /^(file:|data:|https?:|content:|blob:)/i + +export function isRenderableImageUri(uri: string | undefined): uri is string { + return typeof uri === 'string' && RENDERABLE_IMAGE_URI.test(uri) +} diff --git a/mobile/src/session/mobile-native-chat-image-send.test.ts b/mobile/src/session/mobile-native-chat-image-send.test.ts new file mode 100644 index 00000000000..368cd6a8781 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-send.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcResponse, RpcSuccess } from '../transport/types' +import { pasteMobileNativeChatImagePaths } from './mobile-native-chat-image-send' + +function sendResult(accepted: boolean, id = 'send'): RpcSuccess { + return { id, ok: true, result: { send: { accepted } }, _meta: { runtimeId: 'r' } } +} + +function clientWithResponses(responses: RpcResponse[]): Pick & { + calls: { method: string; params: Record }[] +} { + const calls: { method: string; params: Record }[] = [] + return { + calls, + sendRequest: vi.fn(async (method: string, params?: unknown) => { + calls.push({ method, params: params as Record }) + const response = responses.shift() + if (!response) { + throw new Error(`unexpected request: ${method}`) + } + return response + }) + } +} + +describe('pasteMobileNativeChatImagePaths', () => { + it('clears the input line, then pastes each path as a bracketed, non-submitting terminal.send with the mobile client tag', async () => { + const client = clientWithResponses([sendResult(true), sendResult(true), sendResult(true)]) + + const ok = await pasteMobileNativeChatImagePaths({ + client, + terminal: 'term-1', + deviceToken: 'device-9', + imagePaths: ['/tmp/a.png', '/tmp/b.png'] + }) + + expect(ok).toBe(true) + expect(client.calls).toHaveLength(3) + // Leading Ctrl+U clears any stale input so a retry can't duplicate the image. + expect(client.calls[0]).toEqual({ + method: 'terminal.send', + params: { + terminal: 'term-1', + text: '\x15', + enter: false, + client: { id: 'device-9', type: 'mobile' } + } + }) + expect(client.calls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~') + expect(client.calls[2]?.params.text).toBe('\x1b[200~/tmp/b.png\x1b[201~') + }) + + it('stops and reports failure as soon as a paste is rejected', async () => { + // Clear accepted, first image paste rejected. + const client = clientWithResponses([sendResult(true), sendResult(false)]) + + const ok = await pasteMobileNativeChatImagePaths({ + client, + terminal: 'term-1', + deviceToken: null, + imagePaths: ['/tmp/a.png', '/tmp/b.png'] + }) + + expect(ok).toBe(false) + // Never attempts the second path after the first is rejected. + expect(client.calls).toHaveLength(2) + expect(client.calls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~') + expect(client.calls[0]?.params).not.toHaveProperty('client') + }) +}) diff --git a/mobile/src/session/mobile-native-chat-image-send.ts b/mobile/src/session/mobile-native-chat-image-send.ts new file mode 100644 index 00000000000..6d163b41fc3 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-send.ts @@ -0,0 +1,54 @@ +import type { RpcClient } from '../transport/rpc-client' +import { buildMobileImagePastePayload } from './mobile-clipboard-image' +import { isTerminalSendRpcAccepted } from '../terminal/terminal-send-rpc-response' + +// Give the agent TUI a beat to register each bracketed image paste before the +// message text + Enter arrive, so the image attaches instead of being treated as +// part of the prompt body (mirrors desktop's NATIVE_CHAT_IMAGE_ATTACHMENT_SETTLE_MS). +export const MOBILE_NATIVE_CHAT_IMAGE_SETTLE_MS = 300 + +// Ctrl+U kills the agent's unsubmitted input line. Sent before pasting so a retry +// after a rejected body/Enter can't leave a stale image paste that then rides along +// with (and duplicates) the next attempt — matches desktop clearUnsubmittedAgentInput. +const MOBILE_NATIVE_CHAT_CLEAR_UNSUBMITTED_INPUT = '\x15' + +type MobileTerminalClient = { id: string; type: 'mobile' } + +type PasteImagesArgs = { + readonly client: Pick + readonly terminal: string + readonly deviceToken: string | null + readonly imagePaths: readonly string[] +} + +/** Clears the agent's unsubmitted input line, then pastes each uploaded image + * path into the terminal as a bracketed paste (no Enter) — the same payload + * desktop native chat rides along on submit. The leading clear keeps a retry + * idempotent after a failed body/Enter. Returns false as soon as the host rejects + * one, so the caller can abort before Enter. */ +export async function pasteMobileNativeChatImagePaths({ + client, + terminal, + deviceToken, + imagePaths +}: PasteImagesArgs): Promise { + const mobileClient: MobileTerminalClient | null = deviceToken + ? { id: deviceToken, type: 'mobile' } + : null + const clientField = mobileClient ? { client: mobileClient } : {} + for (const text of [ + MOBILE_NATIVE_CHAT_CLEAR_UNSUBMITTED_INPUT, + ...imagePaths.map(buildMobileImagePastePayload) + ]) { + const response = await client.sendRequest('terminal.send', { + terminal, + text, + enter: false, + ...clientField + }) + if (!isTerminalSendRpcAccepted(response)) { + return false + } + } + return true +} diff --git a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts new file mode 100644 index 00000000000..2c1ecc291ff --- /dev/null +++ b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts @@ -0,0 +1,21 @@ +// Single-sources the marker logic (pure functions over shared types): +// Claude records an attached image as `[Image: source: /path]` (+ `[Image #N]` +// prefix on the caption turn), and both render and echo reconciliation must +// agree with desktop on how those marker turns are interpreted. +export { + imageSourcePathFromText, + normalizeImageTranscriptMessages, + stripImagePromptMarker +} from '../../../src/shared/native-chat-image-transcript-markers' +import { imageSourcePathFromText } from '../../../src/shared/native-chat-image-transcript-markers' +import { isTextBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' + +/** A raw (un-normalized) transcript user turn that is an image-source marker — + * the echo shape of an image riding along on a send. */ +export function isImageSourceUserTurn(message: NativeChatMessage): boolean { + if (message.role !== 'user' || message.blocks.length !== 1) { + return false + } + const block = message.blocks[0] + return block !== undefined && isTextBlock(block) && imageSourcePathFromText(block.text) !== null +} diff --git a/mobile/src/session/mobile-native-chat-message-styles.ts b/mobile/src/session/mobile-native-chat-message-styles.ts index c67ffb738cc..41d77564fd2 100644 --- a/mobile/src/session/mobile-native-chat-message-styles.ts +++ b/mobile/src/session/mobile-native-chat-message-styles.ts @@ -141,6 +141,14 @@ export const styles = StyleSheet.create({ color: colors.textSecondary, fontSize: TEXT_SIZE }, + imageThumb: { + width: 200, + height: 150, + borderRadius: radii.card, + backgroundColor: colors.bgRaised, + borderWidth: StyleSheet.hairlineWidth, + borderColor: colors.borderSubtle + }, diff: { borderRadius: radii.button, backgroundColor: colors.bgPanel, diff --git a/mobile/src/session/mobile-native-chat-render-data.test.ts b/mobile/src/session/mobile-native-chat-render-data.test.ts index e933971220d..4dabff8f600 100644 --- a/mobile/src/session/mobile-native-chat-render-data.test.ts +++ b/mobile/src/session/mobile-native-chat-render-data.test.ts @@ -63,6 +63,55 @@ describe('buildMobileNativeChatData', () => { expect(last.blocks).toEqual([{ type: 'text', text: 'queued' }]) }) + it('renders a pending send with images as text followed by image-ref thumbnails', () => { + const { data } = buildMobileNativeChatData({ + messages: [], + pending: [{ id: 'p1', text: 'look', images: ['file:///a.jpg', 'file:///b.jpg'] }] + }) + const last = data[data.length - 1] + expect(last.role).toBe('user') + expect(last.blocks).toEqual([ + { type: 'text', text: 'look' }, + { type: 'image-ref', url: 'file:///a.jpg' }, + { type: 'image-ref', url: 'file:///b.jpg' } + ]) + }) + + it('renders an image-only pending send (no text) as just the thumbnail', () => { + const { data } = buildMobileNativeChatData({ + messages: [], + pending: [{ id: 'p1', text: '', images: ['file:///a.jpg'] }] + }) + expect(data[data.length - 1].blocks).toEqual([{ type: 'image-ref', url: 'file:///a.jpg' }]) + }) + + it('folds transcript image marker turns into image-ref blocks (desktop parity)', () => { + // Claude records an attached image as `[Image: source: /path]` + an + // `[Image #1] `-prefixed caption turn; the fold must merge them into one + // user turn with an image-ref block instead of showing raw marker text. + const { data } = buildMobileNativeChatData({ + messages: [ + user('u1', '[Image: source: /tmp/a.png]'), + user('u2', '[Image #1] look at this'), + assistant('a1', 'nice photo') + ], + pending: [] + }) + const merged = data.find((message) => message.role === 'user') + expect(merged?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'look at this' } + ]) + }) + + it('renders a lone image marker turn (no caption) as an image-ref block', () => { + const { data } = buildMobileNativeChatData({ + messages: [user('u1', '[Image: source: /tmp/a.png]')], + pending: [] + }) + expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }]) + }) + it('adds a synthetic streaming bubble while the partial text leads the transcript', () => { const { streaming, data } = buildMobileNativeChatData({ messages: [user('u1', 'hi')], diff --git a/mobile/src/session/mobile-native-chat-render-data.ts b/mobile/src/session/mobile-native-chat-render-data.ts index 74ca01939ba..068050fc43d 100644 --- a/mobile/src/session/mobile-native-chat-render-data.ts +++ b/mobile/src/session/mobile-native-chat-render-data.ts @@ -5,6 +5,7 @@ import { } from '../../../src/shared/native-chat-empty-state' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { foldToolMessages } from './mobile-native-chat-blocks' +import { normalizeImageTranscriptMessages } from './mobile-native-chat-image-transcript-markers' import { stripNoiseMessages } from './mobile-native-chat-noise' import type { MobileNativeChatStatus } from './use-mobile-native-chat-session' @@ -34,6 +35,14 @@ export function mobileNativeChatEmptyState( } } +/** An optimistic user echo: the text and/or the local preview URIs of any images + * ridden along on the send, shown until the transcript catches up. */ +export type MobileNativeChatPendingItem = { + id: string + text: string + images?: string[] +} + /** Derive the list data from the raw transcript: fold tool turns into the * assistant turn, optionally append a synthetic streaming bubble, then the * route-owned optimistic "queued" messages at the tail. Returns the @@ -45,14 +54,16 @@ export function buildMobileNativeChatData({ }: { messages: NativeChatMessage[] streamingText?: string - pending: Array<{ id: string; text: string }> + pending: MobileNativeChatPendingItem[] }): { folded: NativeChatMessage[]; streaming: string | null; data: NativeChatMessage[] } { const folded = foldMobileNativeChatMessages(messages) return buildMobileNativeChatTransientData({ folded, streamingText, pending }) } export function foldMobileNativeChatMessages(messages: NativeChatMessage[]): NativeChatMessage[] { - return foldToolMessages(stripNoiseMessages(messages)) + // Normalize first (desktop assembler parity): image marker turns fold into + // image-ref blocks instead of rendering as raw `[Image: …]` text. + return foldToolMessages(stripNoiseMessages(normalizeImageTranscriptMessages(messages))) } export function buildMobileNativeChatTransientData({ @@ -62,7 +73,7 @@ export function buildMobileNativeChatTransientData({ }: { folded: NativeChatMessage[] streamingText?: string - pending: Array<{ id: string; text: string }> + pending: MobileNativeChatPendingItem[] }): { folded: NativeChatMessage[]; streaming: string | null; data: NativeChatMessage[] } { // Only show the streaming bubble while its text leads the transcript — once the // real assistant turn lands with the same text, drop the synthetic one. @@ -83,7 +94,12 @@ export function buildMobileNativeChatTransientData({ ...pending.map((p) => ({ id: p.id, role: 'user' as const, - blocks: [{ type: 'text' as const, text: p.text }], + // Text first (when present), then a thumbnail per ridden-along image so the + // sent photo shows immediately, before the transcript echo lands. + blocks: [ + ...(p.text ? [{ type: 'text' as const, text: p.text }] : []), + ...(p.images ?? []).map((uri) => ({ type: 'image-ref' as const, url: uri })) + ], timestamp: null, source: 'transcript' as const })) diff --git a/mobile/src/session/mobile-native-chat-scope-key.ts b/mobile/src/session/mobile-native-chat-scope-key.ts new file mode 100644 index 00000000000..e5b3b4ef544 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-scope-key.ts @@ -0,0 +1,10 @@ +/** Identity of a native-chat composer surface: host + worktree + tab. Drafts + * and pending image chips are both keyed by it, so a tab switch cannot leak + * one tab's composer state into another tab's terminal. */ +export function mobileNativeChatScopeKey( + hostId: string, + worktreeId: string, + tabId: string | null +): string | null { + return tabId ? `${hostId}\0${worktreeId}\0${tabId}` : null +} diff --git a/mobile/src/session/mobile-session-file-doc-lifecycle.test.ts b/mobile/src/session/mobile-session-file-doc-lifecycle.test.ts new file mode 100644 index 00000000000..f29685b7bca --- /dev/null +++ b/mobile/src/session/mobile-session-file-doc-lifecycle.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest' +import { + MobileSessionFileDocLifecycle, + beginMobileFileDocLoad, + createMobileFileDocLifecycle, + finishMobileFileDocLoad, + forgetMobileFileDocTab, + reconcileMobileFileDocTabs, + removeMobileFileDoc, + resetMobileFileDocLifecycle, + retainLiveMobileFileDocs, + type MobileFileTabIdentity +} from './mobile-session-file-doc-lifecycle' +import type { MobileFileTabDoc } from '../files/mobile-file-tab-doc' + +type TestFileDoc = MobileFileTabDoc | { status: 'loading' } | { status: 'error'; message: string } + +function fileTab(id: string, relativePath = `${id}.ts`): MobileFileTabIdentity { + return { + id, + filePath: `/repo/${relativePath}`, + relativePath, + mode: 'edit' + } +} + +describe('mobile session file document lifecycle', () => { + it('retains payloads only for file tabs in the accepted live snapshot', () => { + const lifecycle = createMobileFileDocLifecycle() + reconcileMobileFileDocTabs(lifecycle, [fileTab('live-a'), fileTab('live-b')]) + const docs = new Map([ + ['closed', { payload: 'x'.repeat(1_000_000) }], + ['live-a', { payload: 'a' }], + ['live-b', { payload: 'b' }] + ]) + + const retained = retainLiveMobileFileDocs(docs, lifecycle) + + expect([...retained.keys()]).toEqual(['live-a', 'live-b']) + expect(retained.get('live-a')).toBe(docs.get('live-a')) + }) + + it('deletes a successfully closed tab immediately', () => { + const lifecycle = createMobileFileDocLifecycle() + reconcileMobileFileDocTabs(lifecycle, [fileTab('closed')]) + forgetMobileFileDocTab(lifecycle, 'closed') + + expect(removeMobileFileDoc(new Map([['closed', { payload: 'large' }]]), 'closed').size).toBe(0) + }) + + it('rejects a late read after its tab closes', () => { + const lifecycle = createMobileFileDocLifecycle() + const tab = fileTab('file') + reconcileMobileFileDocTabs(lifecycle, [tab]) + const token = beginMobileFileDocLoad(lifecycle, tab) + expect(token).not.toBeNull() + + forgetMobileFileDocTab(lifecycle, tab.id) + + expect(finishMobileFileDocLoad(lifecycle, token!)).toBe(false) + }) + + it('keeps route state empty when a read resolves after close', async () => { + const lifecycle = new MobileSessionFileDocLifecycle() + const tab = fileTab('file') + let docs = new Map() + const updateDocs = (update: (current: typeof docs) => typeof docs) => { + docs = update(docs) + } + lifecycle.reconcile([{ ...tab, type: 'file' }], updateDocs) + let resolveRead: (doc: MobileFileTabDoc) => void = () => undefined + const read = new Promise((resolve) => { + resolveRead = resolve + }) + + const pending = lifecycle.load(tab, updateDocs, () => read) + expect(docs.get(tab.id)).toEqual({ status: 'loading' }) + lifecycle.close(tab.id, updateDocs) + resolveRead({ status: 'ready', kind: 'file', content: 'late', truncated: false, byteLength: 4 }) + await pending + + expect(docs.size).toBe(0) + }) + + it('rejects a late read when the same tab id is replaced with another file', () => { + const lifecycle = createMobileFileDocLifecycle() + const oldTab = fileTab('file', 'old.ts') + reconcileMobileFileDocTabs(lifecycle, [oldTab]) + const token = beginMobileFileDocLoad(lifecycle, oldTab) + expect(token).not.toBeNull() + + const replaced = reconcileMobileFileDocTabs(lifecycle, [fileTab('file', 'replacement.ts')]) + + expect(finishMobileFileDocLoad(lifecycle, token!)).toBe(false) + expect( + retainLiveMobileFileDocs( + new Map([['file', { payload: 'old contents' }]]), + lifecycle, + replaced + ) + ).toEqual(new Map()) + }) + + it('lets only the newest read for a live tab commit', () => { + const lifecycle = createMobileFileDocLifecycle() + const tab = fileTab('file') + reconcileMobileFileDocTabs(lifecycle, [tab]) + const first = beginMobileFileDocLoad(lifecycle, tab) + const second = beginMobileFileDocLoad(lifecycle, tab) + + expect(finishMobileFileDocLoad(lifecycle, first!)).toBe(false) + expect(finishMobileFileDocLoad(lifecycle, second!)).toBe(true) + }) + + it('invalidates requests when the route scope resets even if tab ids are reused', () => { + const lifecycle = createMobileFileDocLifecycle() + const tab = fileTab('reused') + reconcileMobileFileDocTabs(lifecycle, [tab]) + const oldScope = beginMobileFileDocLoad(lifecycle, tab) + + resetMobileFileDocLifecycle(lifecycle) + reconcileMobileFileDocTabs(lifecycle, [tab]) + const newScope = beginMobileFileDocLoad(lifecycle, tab) + + expect(finishMobileFileDocLoad(lifecycle, oldScope!)).toBe(false) + expect(finishMobileFileDocLoad(lifecycle, newScope!)).toBe(true) + }) +}) diff --git a/mobile/src/session/mobile-session-file-doc-lifecycle.ts b/mobile/src/session/mobile-session-file-doc-lifecycle.ts new file mode 100644 index 00000000000..843aa21758e --- /dev/null +++ b/mobile/src/session/mobile-session-file-doc-lifecycle.ts @@ -0,0 +1,242 @@ +import type { MobileFileTabDoc } from '../files/mobile-file-tab-doc' + +export type MobileFileTabIdentity = { + id: string + filePath: string + relativePath: string + mode?: 'edit' | 'diff' + diffSource?: 'staged' | 'unstaged' | 'branch' | 'commit' +} + +export type MobileFileDocLoadToken = { + generation: number + requestId: number + tabId: string + tabIdentity: string +} + +export type MobileFileDocLifecycle = { + generation: number + nextRequestId: number + liveTabIdentityById: Map + activeRequestIdByTabId: Map +} + +type MobileFileTabCandidate = { + type: string + id: string + filePath?: unknown + relativePath?: unknown + mode?: unknown + diffSource?: unknown +} + +type MobileSessionFileDoc = + | MobileFileTabDoc + | { status: 'loading' } + | { status: 'error'; message: string } + +type MobileSessionFileDocUpdater = ( + update: (current: Map) => Map +) => void + +export class MobileSessionFileDocLifecycle { + private readonly lifecycle = createMobileFileDocLifecycle() + + reconcile( + tabs: readonly MobileFileTabCandidate[], + updateDocs: MobileSessionFileDocUpdater + ): void { + const replacedTabIds = reconcileMobileFileDocTabs( + this.lifecycle, + tabs.filter(isMobileFileTabIdentity) + ) + updateDocs((current) => retainLiveMobileFileDocs(current, this.lifecycle, replacedTabIds)) + } + + async load( + tab: MobileFileTabIdentity, + updateDocs: MobileSessionFileDocUpdater, + read: () => Promise + ): Promise { + const token = beginMobileFileDocLoad(this.lifecycle, tab) + if (!token) { + return + } + updateDocs((current) => new Map(current).set(tab.id, { status: 'loading' })) + try { + const doc = await read() + if (finishMobileFileDocLoad(this.lifecycle, token)) { + updateDocs((current) => new Map(current).set(tab.id, doc)) + } + } catch (error) { + if (finishMobileFileDocLoad(this.lifecycle, token)) { + updateDocs((current) => + new Map(current).set(tab.id, { + status: 'error', + message: getMobileFileDocLoadErrorMessage(tab, error) + }) + ) + } + } + } + + close(tabId: string, updateDocs: MobileSessionFileDocUpdater): void { + forgetMobileFileDocTab(this.lifecycle, tabId) + updateDocs((current) => removeMobileFileDoc(current, tabId)) + } + + reset(): void { + resetMobileFileDocLifecycle(this.lifecycle) + } +} + +export function createMobileFileDocLifecycle(): MobileFileDocLifecycle { + return { + generation: 0, + nextRequestId: 0, + liveTabIdentityById: new Map(), + activeRequestIdByTabId: new Map() + } +} + +export function reconcileMobileFileDocTabs( + lifecycle: MobileFileDocLifecycle, + tabs: readonly MobileFileTabIdentity[] +): ReadonlySet { + const nextIdentityById = new Map(tabs.map((tab) => [tab.id, getMobileFileTabIdentity(tab)])) + const replacedTabIds = new Set() + for (const tabId of lifecycle.activeRequestIdByTabId.keys()) { + const nextIdentity = nextIdentityById.get(tabId) + const currentIdentity = lifecycle.liveTabIdentityById.get(tabId) + if (!nextIdentity || nextIdentity !== currentIdentity) { + lifecycle.activeRequestIdByTabId.delete(tabId) + } + } + for (const [tabId, currentIdentity] of lifecycle.liveTabIdentityById) { + const nextIdentity = nextIdentityById.get(tabId) + if (nextIdentity && nextIdentity !== currentIdentity) { + replacedTabIds.add(tabId) + } + } + lifecycle.liveTabIdentityById.clear() + for (const [tabId, identity] of nextIdentityById) { + lifecycle.liveTabIdentityById.set(tabId, identity) + } + return replacedTabIds +} + +export function beginMobileFileDocLoad( + lifecycle: MobileFileDocLifecycle, + tab: MobileFileTabIdentity +): MobileFileDocLoadToken | null { + const tabIdentity = getMobileFileTabIdentity(tab) + if (lifecycle.liveTabIdentityById.get(tab.id) !== tabIdentity) { + return null + } + const requestId = ++lifecycle.nextRequestId + lifecycle.activeRequestIdByTabId.set(tab.id, requestId) + return { + generation: lifecycle.generation, + requestId, + tabId: tab.id, + tabIdentity + } +} + +export function finishMobileFileDocLoad( + lifecycle: MobileFileDocLifecycle, + token: MobileFileDocLoadToken +): boolean { + if (!isCurrentMobileFileDocLoad(lifecycle, token)) { + return false + } + lifecycle.activeRequestIdByTabId.delete(token.tabId) + return true +} + +export function forgetMobileFileDocTab(lifecycle: MobileFileDocLifecycle, tabId: string): void { + lifecycle.liveTabIdentityById.delete(tabId) + lifecycle.activeRequestIdByTabId.delete(tabId) +} + +export function resetMobileFileDocLifecycle(lifecycle: MobileFileDocLifecycle): void { + lifecycle.generation += 1 + lifecycle.liveTabIdentityById.clear() + lifecycle.activeRequestIdByTabId.clear() +} + +export function retainLiveMobileFileDocs( + docs: Map, + lifecycle: MobileFileDocLifecycle, + replacedTabIds?: ReadonlySet +): Map { + let next: Map | null = null + for (const tabId of docs.keys()) { + if (lifecycle.liveTabIdentityById.has(tabId) && !replacedTabIds?.has(tabId)) { + continue + } + next ??= new Map(docs) + next.delete(tabId) + } + return next ?? docs +} + +export function removeMobileFileDoc(docs: Map, tabId: string): Map { + if (!docs.has(tabId)) { + return docs + } + const next = new Map(docs) + next.delete(tabId) + return next +} + +function isCurrentMobileFileDocLoad( + lifecycle: MobileFileDocLifecycle, + token: MobileFileDocLoadToken +): boolean { + return ( + lifecycle.generation === token.generation && + lifecycle.liveTabIdentityById.get(token.tabId) === token.tabIdentity && + lifecycle.activeRequestIdByTabId.get(token.tabId) === token.requestId + ) +} + +function getMobileFileTabIdentity(tab: MobileFileTabIdentity): string { + return JSON.stringify([ + tab.id, + tab.filePath, + tab.relativePath, + tab.mode ?? '', + tab.diffSource ?? '' + ]) +} + +function isMobileFileTabIdentity( + tab: MobileFileTabCandidate +): tab is MobileFileTabCandidate & MobileFileTabIdentity { + return ( + tab.type === 'file' && + typeof tab.filePath === 'string' && + typeof tab.relativePath === 'string' && + (tab.mode === undefined || tab.mode === 'edit' || tab.mode === 'diff') && + (tab.diffSource === undefined || + tab.diffSource === 'staged' || + tab.diffSource === 'unstaged' || + tab.diffSource === 'branch' || + tab.diffSource === 'commit') + ) +} + +function getMobileFileDocLoadErrorMessage(tab: MobileFileTabIdentity, error: unknown): string { + const message = error instanceof Error ? error.message : '' + if (message === 'binary_file') { + return 'Binary preview unavailable' + } + if (message === 'file_too_large') { + return 'File too large for mobile preview' + } + return tab.diffSource === 'staged' || tab.diffSource === 'unstaged' + ? "Couldn't load diff preview" + : "Couldn't load file preview" +} diff --git a/mobile/src/session/mobile-session-markdown-doc-lifecycle.test.ts b/mobile/src/session/mobile-session-markdown-doc-lifecycle.test.ts new file mode 100644 index 00000000000..ce566029f3e --- /dev/null +++ b/mobile/src/session/mobile-session-markdown-doc-lifecycle.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest' +import type { + MarkdownDocState, + MobileSessionTab +} from '../../app/h/[hostId]/session/mobile-session-route-types' +import { MobileSessionMarkdownDocLifecycle } from './mobile-session-markdown-doc-lifecycle' + +type MarkdownTab = Extract +type ReadyMarkdownDoc = Extract + +function markdownTab(id: string, relativePath = `${id}.md`): MarkdownTab { + return { + type: 'markdown', + id, + title: relativePath, + filePath: `/repo/${relativePath}`, + relativePath, + isDirty: false, + isActive: false, + documentVersion: 'v1' + } +} + +function readyDoc(content: string, isDirty = false): ReadyMarkdownDoc { + return { + status: 'ready', + content, + localContent: content, + baseVersion: 'v1', + isDirty, + editable: true + } +} + +describe('mobile session markdown document lifecycle', () => { + it('retains payloads only for markdown tabs in the accepted snapshot', () => { + const lifecycle = new MobileSessionMarkdownDocLifecycle() + let docs = new Map([ + ['closed', readyDoc('x'.repeat(1_000_000))], + ['live', readyDoc('live')] + ]) + + lifecycle.reconcile([markdownTab('live')], (update) => { + docs = update(docs) + }) + + expect([...docs.keys()]).toEqual(['live']) + }) + + it('retains a dirty orphan when reconciliation keeps its draft tab', () => { + const lifecycle = new MobileSessionMarkdownDocLifecycle() + let docs = new Map([['draft', readyDoc('unsaved', true)]]) + + lifecycle.reconcile([markdownTab('draft')], (update) => { + docs = update(docs) + }) + + expect(docs.get('draft')).toEqual(readyDoc('unsaved', true)) + }) + + it('does not let a late read resurrect a closed tab', async () => { + const lifecycle = new MobileSessionMarkdownDocLifecycle() + const tab = markdownTab('late') + let docs = new Map() + const updateDocs = (update: (current: typeof docs) => typeof docs) => { + docs = update(docs) + } + lifecycle.reconcile([tab], updateDocs) + let resolveRead: (doc: ReadyMarkdownDoc) => void = () => undefined + const read = new Promise((resolve) => { + resolveRead = resolve + }) + + const pending = lifecycle.load(tab, updateDocs, () => read) + lifecycle.close(tab.id, updateDocs) + resolveRead(readyDoc('late payload')) + await pending + + expect(docs.size).toBe(0) + }) + + it('does not let an old path overwrite a replacement using the same tab id', async () => { + const lifecycle = new MobileSessionMarkdownDocLifecycle() + const oldTab = markdownTab('same', 'old.md') + let docs = new Map() + const updateDocs = (update: (current: typeof docs) => typeof docs) => { + docs = update(docs) + } + lifecycle.reconcile([oldTab], updateDocs) + let resolveRead: (doc: ReadyMarkdownDoc) => void = () => undefined + const pending = lifecycle.load( + oldTab, + updateDocs, + () => + new Promise((resolve) => { + resolveRead = resolve + }) + ) + + lifecycle.reconcile([markdownTab('same', 'new.md')], updateDocs) + resolveRead(readyDoc('old payload')) + await pending + + expect(docs.size).toBe(0) + }) +}) diff --git a/mobile/src/session/mobile-session-markdown-doc-lifecycle.ts b/mobile/src/session/mobile-session-markdown-doc-lifecycle.ts new file mode 100644 index 00000000000..29554ce3c69 --- /dev/null +++ b/mobile/src/session/mobile-session-markdown-doc-lifecycle.ts @@ -0,0 +1,147 @@ +import type { + MarkdownDocState, + MobileSessionTab +} from '../../app/h/[hostId]/session/mobile-session-route-types' + +type MobileMarkdownTab = Extract + +type MobileMarkdownDocUpdater = ( + update: (current: Map) => Map +) => void + +type MobileMarkdownDocLoadToken = { + generation: number + requestId: number + tabId: string + tabIdentity: string +} + +export class MobileSessionMarkdownDocLifecycle { + private generation = 0 + private nextRequestId = 0 + private readonly liveTabIdentityById = new Map() + private readonly activeRequestIdByTabId = new Map() + + reconcile(tabs: readonly MobileSessionTab[], updateDocs: MobileMarkdownDocUpdater): void { + const nextIdentityById = new Map( + tabs + .filter((tab): tab is MobileMarkdownTab => tab.type === 'markdown') + .map((tab) => [tab.id, markdownTabIdentity(tab)]) + ) + const replacedTabIds = new Set() + for (const [tabId, currentIdentity] of this.liveTabIdentityById) { + const nextIdentity = nextIdentityById.get(tabId) + if (nextIdentity && nextIdentity !== currentIdentity) { + replacedTabIds.add(tabId) + } + } + for (const tabId of this.activeRequestIdByTabId.keys()) { + if (nextIdentityById.get(tabId) !== this.liveTabIdentityById.get(tabId)) { + this.activeRequestIdByTabId.delete(tabId) + } + } + this.liveTabIdentityById.clear() + for (const [tabId, identity] of nextIdentityById) { + this.liveTabIdentityById.set(tabId, identity) + } + updateDocs((current) => retainLiveMarkdownDocs(current, nextIdentityById, replacedTabIds)) + } + + async load( + tab: MobileMarkdownTab, + updateDocs: MobileMarkdownDocUpdater, + read: () => Promise> + ): Promise { + const token = this.beginLoad(tab) + if (!token) { + return + } + updateDocs((current) => new Map(current).set(tab.id, { status: 'loading' })) + try { + const doc = await read() + if (this.finishLoad(token)) { + updateDocs((current) => new Map(current).set(tab.id, doc)) + } + } catch { + if (this.finishLoad(token)) { + updateDocs((current) => + new Map(current).set(tab.id, { + status: 'error', + message: "Couldn't load markdown" + }) + ) + } + } + } + + close(tabId: string, updateDocs: MobileMarkdownDocUpdater): void { + this.liveTabIdentityById.delete(tabId) + this.activeRequestIdByTabId.delete(tabId) + updateDocs((current) => removeMarkdownDoc(current, tabId)) + } + + reset(): void { + this.generation += 1 + this.liveTabIdentityById.clear() + this.activeRequestIdByTabId.clear() + } + + private beginLoad(tab: MobileMarkdownTab): MobileMarkdownDocLoadToken | null { + const tabIdentity = markdownTabIdentity(tab) + if (this.liveTabIdentityById.get(tab.id) !== tabIdentity) { + return null + } + const requestId = ++this.nextRequestId + this.activeRequestIdByTabId.set(tab.id, requestId) + return { + generation: this.generation, + requestId, + tabId: tab.id, + tabIdentity + } + } + + private finishLoad(token: MobileMarkdownDocLoadToken): boolean { + if ( + token.generation !== this.generation || + this.liveTabIdentityById.get(token.tabId) !== token.tabIdentity || + this.activeRequestIdByTabId.get(token.tabId) !== token.requestId + ) { + return false + } + this.activeRequestIdByTabId.delete(token.tabId) + return true + } +} + +function retainLiveMarkdownDocs( + docs: Map, + liveIdentityById: ReadonlyMap, + replacedTabIds: ReadonlySet +): Map { + let next: Map | null = null + for (const tabId of docs.keys()) { + if (liveIdentityById.has(tabId) && !replacedTabIds.has(tabId)) { + continue + } + next ??= new Map(docs) + next.delete(tabId) + } + return next ?? docs +} + +function removeMarkdownDoc( + docs: Map, + tabId: string +): Map { + if (!docs.has(tabId)) { + return docs + } + const next = new Map(docs) + next.delete(tabId) + return next +} + +function markdownTabIdentity(tab: MobileMarkdownTab): string { + return JSON.stringify([tab.id, tab.filePath, tab.relativePath]) +} diff --git a/mobile/src/session/mobile-terminal-diagnostics.test.ts b/mobile/src/session/mobile-terminal-diagnostics.test.ts index 831ad4bfccc..d1c49794664 100644 --- a/mobile/src/session/mobile-terminal-diagnostics.test.ts +++ b/mobile/src/session/mobile-terminal-diagnostics.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { getMobileTerminalDiagnosticErrorName, logMobileTerminalDiagnostic, + MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES, MobileTerminalDiagnostics, shortenMobileTerminalDiagnosticId } from './mobile-terminal-diagnostics' @@ -43,4 +44,28 @@ describe('mobile terminal diagnostics', () => { expect(log).toHaveBeenCalledTimes(2) log.mockRestore() }) + + it('accepts the exact diagnostic handle cap and evicts oldest at one over', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + const diagnostics = new MobileTerminalDiagnostics() + for (let index = 0; index < MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES; index += 1) { + diagnostics.streamSkipped(`terminal-${index}`, 'inactive', false) + diagnostics.firstStreamEvent(`terminal-${index}`, 1, 'subscribed') + } + expect(diagnostics.retainedHandleCountsForTests()).toEqual({ + streamGates: MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES, + firstEvents: MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES + }) + + diagnostics.streamSkipped('one-over', 'inactive', false) + diagnostics.firstStreamEvent('one-over', 1, 'subscribed') + expect(diagnostics.retainedHandleCountsForTests()).toEqual({ + streamGates: MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES, + firstEvents: MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES + }) + diagnostics.streamSkipped('terminal-0', 'inactive', false) + diagnostics.firstStreamEvent('terminal-0', 1, 'subscribed') + expect(log).toHaveBeenCalledTimes(MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES * 2 + 4) + log.mockRestore() + }) }) diff --git a/mobile/src/session/mobile-terminal-diagnostics.ts b/mobile/src/session/mobile-terminal-diagnostics.ts index 8695928ed03..4005d564ad3 100644 --- a/mobile/src/session/mobile-terminal-diagnostics.ts +++ b/mobile/src/session/mobile-terminal-diagnostics.ts @@ -1,4 +1,5 @@ const MOBILE_TERMINAL_DIAGNOSTIC_TAG = '[terminal-diagnostic]' +export const MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES = 1_024 type MobileTerminalDiagnosticValue = string | number | boolean | null | undefined @@ -90,7 +91,7 @@ export class MobileTerminalDiagnostics { if (this.streamGateByHandle.get(handle) === reason) { return } - this.streamGateByHandle.set(handle, reason) + setBoundedHandleState(this.streamGateByHandle, handle, reason) logMobileTerminalDiagnostic('stream-skipped', { handle: shortenMobileTerminalDiagnosticId(handle), reason, @@ -113,7 +114,7 @@ export class MobileTerminalDiagnostics { if (this.firstStreamEventSeqByHandle.get(handle) === seq) { return } - this.firstStreamEventSeqByHandle.set(handle, seq) + setBoundedHandleState(this.firstStreamEventSeqByHandle, handle, seq) logMobileTerminalDiagnostic('stream-first-event', { handle: shortenMobileTerminalDiagnosticId(handle), seq, @@ -267,6 +268,14 @@ export class MobileTerminalDiagnostics { }) } + /** Test-only evidence for process-retained diagnostic dedupe state. */ + retainedHandleCountsForTests(): { streamGates: number; firstEvents: number } { + return { + streamGates: this.streamGateByHandle.size, + firstEvents: this.firstStreamEventSeqByHandle.size + } + } + private logTabs( event: 'tabs-applied' | 'tabs-fetch-success', snapshot: DiagnosticTabsSnapshot, @@ -289,3 +298,15 @@ export class MobileTerminalDiagnostics { }) } } + +function setBoundedHandleState(map: Map, handle: string, value: T): void { + map.delete(handle) + map.set(handle, value) + while (map.size > MOBILE_TERMINAL_DIAGNOSTIC_MAX_TRACKED_HANDLES) { + const oldestHandle = map.keys().next().value + if (typeof oldestHandle !== 'string') { + return + } + map.delete(oldestHandle) + } +} diff --git a/mobile/src/session/pr-actions-engine.test.ts b/mobile/src/session/pr-actions-engine.test.ts index 3656b2732f6..17b22fa87f1 100644 --- a/mobile/src/session/pr-actions-engine.test.ts +++ b/mobile/src/session/pr-actions-engine.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from 'vitest' -import { PrActionsEngine, type PrActionMutations } from './pr-actions-engine' +import { + MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS, + PrActionsEngine, + type PrActionMutations +} from './pr-actions-engine' import type { GitHubPrMutationOutcome } from './github-pr-mutations' function deferred() { @@ -238,3 +242,32 @@ describe('PrActionsEngine — PR identity changes', () => { expect(refetch).not.toHaveBeenCalled() }) }) + +describe('PrActionsEngine — reviewer retention', () => { + it('releases settled reviewer fields so sequential reviewer actions do not accumulate', async () => { + const engine = makeEngine({}) + + for (let index = 0; index < MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS + 1; index += 1) { + await engine.requestReviewer(`reviewer-${index}`) + } + + expect(engine.retainedReviewerFieldCountForTests()).toBe(0) + }) + + it('accepts the exact concurrent reviewer cap and rejects one over', async () => { + const pending = deferred() + const engine = makeEngine({ requestReviewers: () => pending.promise }) + const actions = Array.from({ length: MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS }, (_, index) => + engine.requestReviewer(`reviewer-${index}`) + ) + expect(engine.retainedReviewerFieldCountForTests()).toBe(MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS) + + await expect(engine.requestReviewer('one-over')).rejects.toThrow( + 'Too many reviewer actions are pending' + ) + + pending.resolve({ ok: true }) + await Promise.all(actions) + expect(engine.retainedReviewerFieldCountForTests()).toBe(0) + }) +}) diff --git a/mobile/src/session/pr-actions-engine.ts b/mobile/src/session/pr-actions-engine.ts index ce19c9bbbf9..04bc1ed0afe 100644 --- a/mobile/src/session/pr-actions-engine.ts +++ b/mobile/src/session/pr-actions-engine.ts @@ -8,6 +8,9 @@ import type { PrActionMutations } from './pr-action-mutation-contract' export type { PrActionMutations } from './pr-action-mutation-contract' +export const MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS = 64 +const MOBILE_PR_ACTIONS_MAX_REVIEWER_LOGIN_CHARACTERS = 256 + // Pure (React-free) engine for the PR mutation actions: owns optimistic fields, // busy/error/blocked state, and the success/transient/permanent routing. The hook // is a thin adapter that subscribes to `onChange` and exposes these methods. Kept @@ -96,12 +99,25 @@ export class PrActionsEngine { private reviewerField(login: string): OptimisticField { let f = this.reviewerFields.get(login) if (!f) { + if ( + login.length === 0 || + login.length > MOBILE_PR_ACTIONS_MAX_REVIEWER_LOGIN_CHARACTERS || + this.reviewerFields.size >= MOBILE_PR_ACTIONS_MAX_REVIEWER_FIELDS + ) { + throw new Error('Too many reviewer actions are pending') + } f = createOptimisticField(this.cfg.onChange) this.reviewerFields.set(login, f) } return f } + private releaseReviewerFieldIfIdle(login: string, field: OptimisticField): void { + if (field.peek() === undefined && this.reviewerFields.get(login) === field) { + this.reviewerFields.delete(login) + } + } + // Why: action start pairs setBusy + setError(null); skip notify when unchanged // so we don't force a full PR panel re-render for free. private setBusy(key: PrActionBusyKey | null): void { @@ -264,6 +280,7 @@ export class PrActionsEngine { }) } finally { this.clearBusyIfOwned(identity, { kind: 'reviewer', login }) + this.releaseReviewerFieldIfIdle(login, field) } } @@ -286,6 +303,7 @@ export class PrActionsEngine { }) } finally { this.clearBusyIfOwned(identity, { kind: 'reviewer', login }) + this.releaseReviewerFieldIfIdle(login, field) } } @@ -319,4 +337,9 @@ export class PrActionsEngine { const f = this.reviewerFields.get(login) return f ? f.resolve(authoritative) : authoritative } + + /** Test-only evidence for retained optimistic reviewer state. */ + retainedReviewerFieldCountForTests(): number { + return this.reviewerFields.size + } } diff --git a/mobile/src/session/use-mobile-native-chat-controller.test.ts b/mobile/src/session/use-mobile-native-chat-controller.test.ts new file mode 100644 index 00000000000..487e0a4b2bb --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-controller.test.ts @@ -0,0 +1,152 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' + +const acceptSend = vi.fn() +const captureSendOrigin = vi.fn() +const clearDraftForSend = vi.fn() +const restoreRejectedDraft = vi.fn() +const holdUnconfirmedSend = vi.fn() + +// The controller composes many session hooks; each is mocked to a minimal shape +// so this test isolates the send seam (outcome -> drafts accounting). +vi.mock('./use-mobile-session-view-mode', () => ({ + useMobileSessionViewMode: () => ({ isTabChatView: () => true, toggleTabChatView: vi.fn() }) +})) +vi.mock('./use-mobile-native-chat-session', () => ({ + useMobileNativeChatSession: () => ({ messages: [] }) +})) +vi.mock('./use-mobile-native-chat-drafts', () => ({ + useMobileNativeChatDrafts: () => ({ + composerText: '', + setComposerText: vi.fn(), + pending: [], + captureSendOrigin, + clearDraftForSend, + restoreRejectedDraft, + acceptSend, + holdUnconfirmedSend + }) +})) +vi.mock('./use-mobile-native-chat-prompts', () => ({ + useMobileNativeChatPrompts: () => ({ permission: null, question: null, ask: null }) +})) +vi.mock('./use-mobile-native-chat-answer-send', () => ({ + useMobileNativeChatAnswerSend: () => ({ answerAsk: vi.fn(), cancelPending: vi.fn() }) +})) +vi.mock('./mobile-native-chat-permission-send', () => ({ + useMobileNativeChatPermissionSend: () => vi.fn() +})) +vi.mock('./use-mobile-native-chat-stop', () => ({ + useMobileNativeChatStop: () => vi.fn() +})) +vi.mock('./use-mobile-native-chat-file-search', () => ({ + useMobileNativeChatFileSearch: () => ({ nativeChatFilePaths: [], loadNativeChatFiles: vi.fn() }) +})) +vi.mock('./mobile-native-chat-send', () => ({ + sendMobileNativeChatMessageWithOutcome: vi.fn() +})) + +import { sendMobileNativeChatMessageWithOutcome } from './mobile-native-chat-send' +import { + useMobileNativeChatController, + type MobileNativeChatController +} from './use-mobile-native-chat-controller' + +const sendWithOutcome = vi.mocked(sendMobileNativeChatMessageWithOutcome) + +const ORIGIN = { + draftKey: 'h\0w\0tab-1', + pendingKey: 'h\0w\0tab-1\0session-1', + normalizedText: 'look', + baselineOccurrences: 0, + baselineTailMessageId: null +} + +describe('useMobileNativeChatController handleNativeChatSend', () => { + let renderer: ReactTestRenderer | null = null + let controller: MobileNativeChatController | null = null + const onSendError = vi.fn() + + function Harness(): null { + controller = useMobileNativeChatController({ + client: {} as RpcClient, + hostId: 'h', + worktreeId: 'w', + activeSessionTab: null, + activeSessionTabId: 'tab-1', + activeHandleRef: { current: 'term-1' }, + deviceTokenRef: { current: null }, + nativeChatTranscriptIsLocalReadable: true, + nativeChatInputLeaseReady: true, + onSendError + }) + return null + } + + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + vi.clearAllMocks() + captureSendOrigin.mockReturnValue(ORIGIN) + const original = console.error + const spy = vi.spyOn(console, 'error').mockImplementation((...a) => { + if (typeof a[0] === 'string' && a[0].includes('react-test-renderer is deprecated')) { + return + } + original(...a) + }) + try { + act(() => { + renderer = create(createElement(Harness)) + }) + } finally { + spy.mockRestore() + } + }) + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + controller = null + }) + + it('threads the optimistic-echo image URIs into acceptSend on an accepted send', async () => { + sendWithOutcome.mockResolvedValue('accepted') + let accepted = false + await act(async () => { + accepted = await controller!.handleNativeChatSend('look', ['file:///a.jpg']) + }) + expect(accepted).toBe(true) + expect(acceptSend).toHaveBeenCalledWith(ORIGIN, 'look', ['file:///a.jpg']) + // Optimistic clear happens at send time, never a restore on success. + expect(clearDraftForSend).toHaveBeenCalledWith(ORIGIN, 'look') + expect(restoreRejectedDraft).not.toHaveBeenCalled() + }) + + it('holds an unknown-outcome send without posting the optimistic echo', async () => { + sendWithOutcome.mockResolvedValue('unknown') + let accepted = false + await act(async () => { + accepted = await controller!.handleNativeChatSend('look', ['file:///a.jpg']) + }) + expect(accepted).toBe(true) + expect(acceptSend).not.toHaveBeenCalled() + expect(holdUnconfirmedSend).toHaveBeenCalledWith(ORIGIN, 'look', expect.any(Function)) + // Delivery-unknown usually means delivered — keep the composer clear. + expect(clearDraftForSend).toHaveBeenCalledWith(ORIGIN, 'look') + expect(restoreRejectedDraft).not.toHaveBeenCalled() + }) + + it('reports a rejected send and posts no echo', async () => { + sendWithOutcome.mockResolvedValue('rejected') + let accepted = true + await act(async () => { + accepted = await controller!.handleNativeChatSend('look', ['file:///a.jpg']) + }) + expect(accepted).toBe(false) + expect(acceptSend).not.toHaveBeenCalled() + expect(onSendError).toHaveBeenCalledWith('Message not sent') + // A definite rejection puts the optimistically-cleared text back. + expect(restoreRejectedDraft).toHaveBeenCalledWith(ORIGIN, 'look') + }) +}) diff --git a/mobile/src/session/use-mobile-native-chat-controller.ts b/mobile/src/session/use-mobile-native-chat-controller.ts index 6b1ae3f1033..f3a963bb3da 100644 --- a/mobile/src/session/use-mobile-native-chat-controller.ts +++ b/mobile/src/session/use-mobile-native-chat-controller.ts @@ -19,7 +19,10 @@ import { openMobileNativeChatFile } from './mobile-native-chat-open-file' import { useMobileNativeChatPermissionSend } from './mobile-native-chat-permission-send' import { sendMobileNativeChatMessageWithOutcome } from './mobile-native-chat-send' import { useMobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send' -import { useMobileNativeChatDrafts } from './use-mobile-native-chat-drafts' +import { + useMobileNativeChatDrafts, + type MobileNativeChatPendingMessage +} from './use-mobile-native-chat-drafts' import { useMobileNativeChatFileSearch } from './use-mobile-native-chat-file-search' import { useMobileNativeChatSession } from './use-mobile-native-chat-session' import { useMobileNativeChatPrompts } from './use-mobile-native-chat-prompts' @@ -38,7 +41,7 @@ export type MobileNativeChatController = { nativeChatAgent: string | null chatComposerText: string setChatComposerText: Dispatch> - chatPending: Array<{ id: string; text: string }> + chatPending: MobileNativeChatPendingMessage[] nativeChatSession: ReturnType nativeChatAgentWorking: boolean nativeChatStreamingText?: string @@ -55,7 +58,7 @@ export type MobileNativeChatController = { handleNativeChatStop: () => void nativeChatFilePaths: string[] loadNativeChatFiles: (query: string) => void - handleNativeChatSend: (text: string) => Promise + handleNativeChatSend: (text: string, images?: string[]) => Promise } /** Owns mobile native-chat state and teardown outside the already dense session @@ -224,7 +227,7 @@ export function useMobileNativeChatController(args: { }) const handleNativeChatSend = useCallback( - async (text: string): Promise => { + async (text: string, images?: string[]): Promise => { const handle = activeHandleRef.current const origin = captureSendOrigin(text) if (!client || !handle || !origin || !nativeChatInputLeaseReady) { @@ -256,7 +259,9 @@ export function useMobileNativeChatController(args: { onSendError('Message not sent') return false } - acceptSend(origin, text) + // `images` are local preview URIs for the optimistic echo only — the actual + // image bytes already rode along as a bracketed paste before this text send. + acceptSend(origin, text, images) return true }, [ diff --git a/mobile/src/session/use-mobile-native-chat-drafts.test.ts b/mobile/src/session/use-mobile-native-chat-drafts.test.ts index 3d7042d8dda..233ae34eb7c 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.test.ts @@ -218,6 +218,126 @@ describe('useMobileNativeChatDrafts', () => { expect(state?.pending.map((pending) => pending.text)).toEqual(['ping']) }) + it('keeps an image-only echo through an agent reply, clearing only when the user turn lands', async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const origin = state?.captureSendOrigin('') + act(() => { + if (origin) { + state?.acceptSend(origin, '', ['file:///a.jpg']) + } + }) + // The echo carries the preview thumbnail and has no text to match against. + expect(state?.pending.map((pending) => pending.images)).toEqual([['file:///a.jpg']]) + + // An agent reply grows the transcript but must NOT clear the photo echo early. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [assistantTextMessage('a1', 'hi'), assistantTextMessage('a2', 'nice photo')] + }) + ) + ) + expect(state?.pending.map((pending) => pending.images)).toEqual([['file:///a.jpg']]) + + // The user's own image echo landing (Claude records it as an + // `[Image: source: …]` turn) clears it. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + assistantTextMessage('a2', 'nice photo'), + userTextMessage('u1', '[Image: source: /tmp/a.png]') + ] + }) + ) + ) + expect(state?.pending).toEqual([]) + }) + + it("keeps an image-only echo when an unrelated text send's echo lands", async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const textOrigin = state?.captureSendOrigin('ping') + const imageOrigin = state?.captureSendOrigin('') + act(() => { + if (textOrigin && imageOrigin) { + state?.acceptSend(textOrigin, 'ping') + state?.acceptSend(imageOrigin, '', ['file:///a.jpg']) + } + }) + expect(state?.pending).toHaveLength(2) + + // The text echo lands first: it must clear only the text pending — a user + // turn that is not an image echo cannot reconcile the photo. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [assistantTextMessage('a1', 'hi'), userTextMessage('u1', 'ping')] + }) + ) + ) + expect(state?.pending.map((pending) => pending.images)).toEqual([['file:///a.jpg']]) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + userTextMessage('u1', 'ping'), + userTextMessage('u2', '[Image: source: /tmp/a.png]') + ] + }) + ) + ) + expect(state?.pending).toEqual([]) + }) + + it('reconciles a captioned image echo that carries the [Image #N] marker', async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const origin = state?.captureSendOrigin('look at this') + act(() => { + if (origin) { + state?.acceptSend(origin, 'look at this', ['file:///a.jpg']) + } + }) + expect(state?.pending).toHaveLength(1) + + // Claude echoes a captioned image send as two turns: the source marker and + // the caption prefixed with `[Image #1] ` — the pending must still match. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + userTextMessage('u1', '[Image: source: /tmp/a.png]'), + userTextMessage('u2', '[Image #1] look at this') + ] + }) + ) + ) + expect(state?.pending).toEqual([]) + }) + it('does not reconcile a repeated send against an older identical turn', async () => { await mount('a') await act(async () => @@ -296,6 +416,84 @@ describe('useMobileNativeChatDrafts', () => { } }) + it('reconciles an image-only unconfirmed send against the next user turn (no false warning)', async () => { + vi.useFakeTimers() + try { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + // Image-only send: empty text, so it can only reconcile against a new user turn. + const origin = state?.captureSendOrigin('') + const onUnconfirmed = vi.fn() + act(() => { + if (origin) { + state?.holdUnconfirmedSend(origin, '', onUnconfirmed) + } + }) + + // An agent reply must not confirm it... + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [assistantTextMessage('a1', 'hi'), assistantTextMessage('a2', 'ok')] + }) + ) + ) + // ...but the user's own turn landing does, so the deadline never warns. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + assistantTextMessage('a2', 'ok'), + userTextMessage('u1', '') + ] + }) + ) + ) + act(() => vi.advanceTimersByTime(30_000)) + expect(onUnconfirmed).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('clears image-only echoes one per landed user turn, not all at once', async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const origin = state?.captureSendOrigin('') + act(() => { + if (origin) { + state?.acceptSend(origin, '', ['file:///a.jpg']) + state?.acceptSend(origin, '', ['file:///b.jpg']) + } + }) + expect(state?.pending).toHaveLength(2) + + // Only one image echo has landed — exactly one photo reconciles. + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + userTextMessage('u1', '[Image: source: /tmp/a.png]') + ] + }) + ) + ) + expect(state?.pending.map((pending) => pending.images)).toEqual([['file:///b.jpg']]) + }) + it('registers no deadline when the transcript echo beat the ambiguous RPC rejection', async () => { vi.useFakeTimers() try { diff --git a/mobile/src/session/use-mobile-native-chat-drafts.ts b/mobile/src/session/use-mobile-native-chat-drafts.ts index b0adbf07dd5..a72a08f68ad 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.ts @@ -1,10 +1,25 @@ import { useCallback, useEffect, useRef, useState, type Dispatch, type SetStateAction } from 'react' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { + countImageSourceTurnsAfter, + countUserTextOccurrences, + findLandedUnconfirmedSends, + normalizedUserText, + type UnconfirmedSend +} from './mobile-native-chat-draft-reconcile' +import { mobileNativeChatScopeKey } from './mobile-native-chat-scope-key' export type MobileNativeChatPendingMessage = { id: string text: string expectedOccurrence: number + /** Local preview URIs of images ridden along on the send, rendered as thumbnails + * on the echo bubble so the sent photo shows before the transcript catches up. */ + images?: string[] + /** Transcript tail when sent — an image-only echo (no text to match) reconciles + * against new `[Image: source: …]` echo turns after this id, so pagination, + * agent replies, and unrelated text echoes can't clear it early. */ + baselineTailMessageId: string | null } export type MobileNativeChatSendOrigin = { draftKey: string @@ -20,74 +35,6 @@ const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = [] // that delivery remains unconfirmed. const UNCONFIRMED_SEND_DEADLINE_MS = 20_000 -type UnconfirmedSend = { - draftKey: string - pendingKey: string | null - text: string - normalizedText: string - baselineTailMessageId: string | null - deadline: ReturnType | null -} - -function normalizedUserText(message: NativeChatMessage): string | null { - if (message.role !== 'user') { - return null - } - const text = message.blocks - .filter((block) => block.type === 'text') - .map((block) => (block.type === 'text' ? block.text : '')) - .join('') - .trim() - return text || null -} - -function countUserTextOccurrences(messages: readonly NativeChatMessage[], text: string): number { - let count = 0 - for (const message of messages) { - if (normalizedUserText(message) === text) { - count++ - } - } - return count -} - -function findLandedUnconfirmedSends( - messages: readonly NativeChatMessage[], - entries: readonly UnconfirmedSend[] -): UnconfirmedSend[] { - // Why: pagination prepends old equal text; only unclaimed matches after each captured tail prove new echoes. - const messageIndexById = new Map() - const userMessagesByText = new Map>() - for (const [index, message] of messages.entries()) { - messageIndexById.set(message.id, index) - const text = normalizedUserText(message) - if (text) { - const current = userMessagesByText.get(text) ?? [] - current.push({ id: message.id, index }) - userMessagesByText.set(text, current) - } - } - - const claimedMessageIds = new Set() - const landed: UnconfirmedSend[] = [] - for (const entry of entries) { - const tailIndex = entry.baselineTailMessageId - ? messageIndexById.get(entry.baselineTailMessageId) - : -1 - if (tailIndex === undefined) { - continue - } - const echo = userMessagesByText - .get(entry.normalizedText) - ?.find((message) => message.index > tailIndex && !claimedMessageIds.has(message.id)) - if (echo) { - claimedMessageIds.add(echo.id) - landed.push(entry) - } - } - return landed -} - export function useMobileNativeChatDrafts(args: { hostId: string worktreeId: string @@ -103,7 +50,7 @@ export function useMobileNativeChatDrafts(args: { clearDraftForSend: (origin: MobileNativeChatSendOrigin, text: string) => void /** Put the text back after a definite rejection, unless newer edits exist. */ restoreRejectedDraft: (origin: MobileNativeChatSendOrigin, text: string) => void - acceptSend: (origin: MobileNativeChatSendOrigin, text: string) => void + acceptSend: (origin: MobileNativeChatSendOrigin, text: string, images?: string[]) => void holdUnconfirmedSend: ( origin: MobileNativeChatSendOrigin, text: string, @@ -111,7 +58,7 @@ export function useMobileNativeChatDrafts(args: { ) => void } { const { hostId, worktreeId, tabId, sessionId, messages } = args - const draftKey = tabId ? `${hostId}\0${worktreeId}\0${tabId}` : null + const draftKey = mobileNativeChatScopeKey(hostId, worktreeId, tabId) const pendingKey = draftKey && sessionId ? `${draftKey}\0${sessionId}` : null const [drafts, setDrafts] = useState>({}) const [pendingBySession, setPendingBySession] = useState< @@ -176,29 +123,46 @@ export function useMobileNativeChatDrafts(args: { ) }, []) - const acceptSend = useCallback((origin: MobileNativeChatSendOrigin, text: string) => { - // Why: the first prompt can be sent before the provider reports a session - // id; wait for an id before keying an optimistic echo. - if (!origin.pendingKey) { - return - } - const pendingKey = origin.pendingKey - pendingCounterRef.current += 1 - setPendingBySession((previous) => { - const current = previous[pendingKey] ?? NO_PENDING_MESSAGES - const earlierOutstanding = current.filter( - (pending) => - pending.text.trim() === origin.normalizedText && - pending.expectedOccurrence > origin.baselineOccurrences - ).length - const pending = { - id: `pending-${pendingCounterRef.current}`, - text, - expectedOccurrence: origin.baselineOccurrences + earlierOutstanding + 1 + const acceptSend = useCallback( + (origin: MobileNativeChatSendOrigin, text: string, images?: string[]) => { + // Why: the first prompt can be sent before the provider reports a session + // id; wait for an id before keying an optimistic echo. + if (!origin.pendingKey) { + return } - return { ...previous, [pendingKey]: [...current, pending] } - }) - }, []) + const pendingKey = origin.pendingKey + pendingCounterRef.current += 1 + setPendingBySession((previous) => { + const current = previous[pendingKey] ?? NO_PENDING_MESSAGES + const earlierOutstanding = current.filter( + (pending) => + pending.text.trim() === origin.normalizedText && + pending.expectedOccurrence > origin.baselineOccurrences + ).length + // An empty-text send reconciles by image-echo ordinal: every outstanding + // send's ridden-along images echo as `[Image: source: …]` turns after + // this send's baseline tail, ahead of this send's own echo. + const expectedImageEchoOrdinal = + current.reduce( + (sum, pending) => + sum + (pending.images?.length ?? (pending.text.trim() === '' ? 1 : 0)), + 0 + ) + 1 + const pending: MobileNativeChatPendingMessage = { + id: `pending-${pendingCounterRef.current}`, + text, + expectedOccurrence: + origin.normalizedText === '' + ? expectedImageEchoOrdinal + : origin.baselineOccurrences + earlierOutstanding + 1, + baselineTailMessageId: origin.baselineTailMessageId, + ...(images && images.length > 0 ? { images } : {}) + } + return { ...previous, [pendingKey]: [...current, pending] } + }) + }, + [] + ) // Why: a relay drop mid-send loses only the ack in the common case — the // desktop already delivered the message. Hold the send instead of claiming @@ -291,8 +255,16 @@ export function useMobileNativeChatDrafts(args: { } // Why: compare against the count captured before send; historical equal // turns cannot clear a new echo, while duplicates land one occurrence each. - const next = current.filter( - (item) => (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence + // An image-only echo has no text to match, so it reconciles by ORDINAL + // against the count of new `[Image: source: …]` echo turns after its + // baseline tail — text echoes are excluded so an unrelated outstanding + // text send cannot clear it. Ordinal-vs-count stays stable when the effect + // re-runs on the shrunken list, and ignores paginated-in history. + const next = current.filter((item) => + item.text.trim() === '' + ? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) < + item.expectedOccurrence + : (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence ) if (next.length === current.length) { return previous diff --git a/mobile/src/session/use-mobile-native-chat-file-search.test.ts b/mobile/src/session/use-mobile-native-chat-file-search.test.ts index e0500019844..bbd4e8bb4f7 100644 --- a/mobile/src/session/use-mobile-native-chat-file-search.test.ts +++ b/mobile/src/session/use-mobile-native-chat-file-search.test.ts @@ -2,7 +2,10 @@ import { createElement } from 'react' import { act, create, type ReactTestRenderer } from 'react-test-renderer' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { RpcClient } from '../transport/rpc-client' -import { useMobileNativeChatFileSearch } from './use-mobile-native-chat-file-search' +import { + retainMobileNativeChatFilePaths, + useMobileNativeChatFileSearch +} from './use-mobile-native-chat-file-search' type SearchState = ReturnType @@ -163,3 +166,15 @@ describe('useMobileNativeChatFileSearch', () => { expect(state?.nativeChatFilePaths).toEqual(['docs/readme.md']) }) }) + +describe('retainMobileNativeChatFilePaths', () => { + it('accepts exact count and retained-byte limits and truncates at one over', () => { + const result = { + files: [{ relativePath: 'a' }, { relativePath: 'bb' }, { relativePath: 'ignored' }] + } + const exactBytes = 'a'.length * 2 + 64 + ('bb'.length * 2 + 64) + + expect(retainMobileNativeChatFilePaths(result, 2, exactBytes)).toEqual(['a', 'bb']) + expect(retainMobileNativeChatFilePaths(result, 3, exactBytes - 1)).toEqual(['a']) + }) +}) diff --git a/mobile/src/session/use-mobile-native-chat-file-search.ts b/mobile/src/session/use-mobile-native-chat-file-search.ts index 53df98e3ee0..53296947f32 100644 --- a/mobile/src/session/use-mobile-native-chat-file-search.ts +++ b/mobile/src/session/use-mobile-native-chat-file-search.ts @@ -5,12 +5,31 @@ import { rankSuggestions } from './mobile-native-chat-autocomplete' const FILE_SEARCH_DEBOUNCE_MS = 120 const FILE_SEARCH_RESULT_LIMIT = 16 const FILE_SEARCH_QUERY_CACHE_LIMIT = 20 +export const MOBILE_NATIVE_CHAT_LEGACY_FILE_MAX_PATHS = 50_000 +export const MOBILE_NATIVE_CHAT_LEGACY_FILE_MAX_RETAINED_BYTES = 8 * 1024 * 1024 +const FILE_SEARCH_RESULT_MAX_RETAINED_BYTES = 256 * 1024 -function extractPaths(result: unknown): string[] { +export function retainMobileNativeChatFilePaths( + result: unknown, + maxPaths: number, + maxRetainedBytes: number +): string[] { const files = (result as { files?: Array<{ relativePath?: string }> }).files ?? [] - return files - .map((file) => file.relativePath ?? '') - .filter((path): path is string => path.length > 0) + const paths: string[] = [] + let retainedBytes = 0 + for (const file of files) { + const path = file.relativePath + if (typeof path !== 'string' || path.length === 0) { + continue + } + const nextBytes = retainedBytes + path.length * 2 + 64 + if (paths.length >= maxPaths || nextBytes > maxRetainedBytes) { + break + } + paths.push(path) + retainedBytes = nextBytes + } + return paths } /** Debounces current-host path searches, bounds the mobile result/cache, and @@ -94,7 +113,11 @@ export function useMobileNativeChatFileSearch(args: { if (!response.ok || generationRef.current !== generation) { return null } - const paths = extractPaths(response.result) + const paths = retainMobileNativeChatFilePaths( + response.result, + MOBILE_NATIVE_CHAT_LEGACY_FILE_MAX_PATHS, + MOBILE_NATIVE_CHAT_LEGACY_FILE_MAX_RETAINED_BYTES + ) legacyPathsRef.current = paths return paths }) @@ -129,7 +152,13 @@ export function useMobileNativeChatFileSearch(args: { }) if (response.ok) { searchSupportedRef.current = true - applyPaths(extractPaths(response.result)) + applyPaths( + retainMobileNativeChatFilePaths( + response.result, + FILE_SEARCH_RESULT_LIMIT, + FILE_SEARCH_RESULT_MAX_RETAINED_BYTES + ) + ) return } if (response.error.code === 'method_not_found') { diff --git a/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts b/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts new file mode 100644 index 00000000000..7c91bc4cb78 --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts @@ -0,0 +1,532 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcResponse, RpcSuccess } from '../transport/types' +import { useMobileNativeChatImageAttachments } from './use-mobile-native-chat-image-attachments' + +// Fully stub the picker so the real expo/react-native chain never loads under +// the vitest transform (react-native ships Flow syntax rolldown can't parse). +vi.mock('./mobile-image-source-picker', () => ({ + pickMobileImage: vi.fn(), + ImageLibraryPermissionError: class ImageLibraryPermissionError extends Error {} +})) + +import { pickMobileImage } from './mobile-image-source-picker' + +const pick = vi.mocked(pickMobileImage) + +function ok(id: string, result: unknown): RpcSuccess { + return { id, ok: true, result, _meta: { runtimeId: 'r' } } +} +function methodNotFound(id: string): RpcResponse { + return { + id, + ok: false, + error: { code: 'method_not_found', message: 'no' }, + _meta: { runtimeId: 'r' } + } +} +function sendResult(accepted: boolean): RpcSuccess { + return { id: 'send', ok: true, result: { send: { accepted } }, _meta: { runtimeId: 'r' } } +} + +function makeClient(responses: RpcResponse[]): Pick & { + calls: { method: string; params: Record }[] +} { + const calls: { method: string; params: Record }[] = [] + return { + calls, + sendRequest: vi.fn(async (method: string, params?: unknown) => { + calls.push({ method, params: params as Record }) + const response = responses.shift() + if (!response) { + throw new Error(`unexpected request: ${method}`) + } + return response + }) + } +} + +type HookArgs = Parameters[0] +type Hook = ReturnType + +const SCOPE_A = 'h\0w\0tab-a' + +function baseArgs(overrides: Partial & Pick): HookArgs { + return { + activeHandleRef: { current: 'term-1' }, + deviceTokenRef: { current: null }, + getActiveWorktreeConnectionId: async () => null, + connState: 'connected', + scopeKey: SCOPE_A, + enabled: true, + showToast: vi.fn(), + baseSend: vi.fn().mockResolvedValue(true), + sleep: async () => {}, + ...overrides + } +} + +describe('useMobileNativeChatImageAttachments', () => { + let renderer: ReactTestRenderer | null = null + let hook: Hook | null = null + + function Harness({ args }: { args: HookArgs }): null { + hook = useMobileNativeChatImageAttachments(args) + return null + } + + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + pick.mockReset() + }) + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + hook = null + }) + + function mount(args: HookArgs): void { + const original = console.error + const spy = vi.spyOn(console, 'error').mockImplementation((...a) => { + if (typeof a[0] === 'string' && a[0].includes('react-test-renderer is deprecated')) { + return + } + original(...a) + }) + try { + act(() => { + renderer = create(createElement(Harness, { args })) + }) + } finally { + spy.mockRestore() + } + } + + function update(args: HookArgs): void { + act(() => { + renderer!.update(createElement(Harness, { args })) + }) + } + + it('adds an uploaded image as a chip without pasting to the terminal', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) + mount( + baseArgs({ + client: client as unknown as RpcClient, + deviceTokenRef: { current: 'device-1' }, + getActiveWorktreeConnectionId: async () => 'conn-1' + }) + ) + + await act(async () => { + await hook!.attachImage('library') + }) + + expect(hook!.attachments).toEqual([ + { id: 'img-1', path: '/tmp/a.png', previewUri: 'file:///a.jpg' } + ]) + expect(client.calls.some((c) => c.method === 'terminal.send')).toBe(false) + }) + + it('rides pending images along on send: pastes the path, settles, then delegates the text', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), + sendResult(true), // Ctrl+U clear + sendResult(true) // the image paste (enter:false) + ]) + const order: string[] = [] + const sleep = vi.fn(async () => { + order.push('settle') + }) + const baseSend = vi.fn(async (t: string) => { + order.push(`text:${t}`) + return true + }) + // Record each terminal write so the paste-before-settle order is asserted, + // not just implied by the call counts. + const trackedClient: Pick = { + sendRequest: (method, params) => { + if (method === 'terminal.send') { + order.push((params as { text?: string }).text === '\x15' ? 'clear' : 'paste') + } + return client.sendRequest(method, params) + } + } + mount( + baseArgs({ + client: trackedClient as RpcClient, + deviceTokenRef: { current: 'device-1' }, + baseSend, + sleep + }) + ) + + await act(async () => { + await hook!.attachImage('library') + }) + + let accepted = false + await act(async () => { + accepted = await hook!.sendNativeChat('look at this') + }) + + expect(accepted).toBe(true) + const sendCalls = client.calls.filter((c) => c.method === 'terminal.send') + // Ctrl+U clear, then the bracketed image paste. + expect(sendCalls).toHaveLength(2) + expect(sendCalls[0]?.params).toMatchObject({ text: '\x15', enter: false }) + expect(sendCalls[1]?.params).toMatchObject({ + text: '\x1b[200~/tmp/a.png\x1b[201~', + enter: false + }) + // Clear, then paste, then settle, then the text send — in that order. + expect(order).toEqual(['clear', 'paste', 'settle', 'text:look at this']) + // The local preview URI rides along so the sent bubble shows the photo. + expect(baseSend).toHaveBeenCalledWith('look at this', ['file:///a.jpg']) + // Chips clear once the send is accepted. + expect(hook!.attachments).toEqual([]) + }) + + it('routes an attachments-only send through baseSend with empty text so the echo still shows the photo', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), + sendResult(true), // Ctrl+U clear + sendResult(true) // image paste + ]) + const baseSend = vi.fn().mockResolvedValue(true) + mount(baseArgs({ client: client as unknown as RpcClient, baseSend })) + + await act(async () => { + await hook!.attachImage('library') + }) + let accepted = false + await act(async () => { + accepted = await hook!.sendNativeChat('') + }) + + expect(accepted).toBe(true) + // Empty text still goes through baseSend (which submits the bare Enter) so the + // optimistic echo carries the preview URI. + expect(baseSend).toHaveBeenCalledWith('', ['file:///a.jpg']) + const sendCalls = client.calls.filter((c) => c.method === 'terminal.send') + // Only the clear + image paste hit the wire here; baseSend owns the submit. + expect(sendCalls).toHaveLength(2) + expect(hook!.attachments).toEqual([]) + }) + + it('delegates straight to baseSend when there are no attachments', async () => { + const client = makeClient([]) + const baseSend = vi.fn().mockResolvedValue(true) + mount(baseArgs({ client: client as unknown as RpcClient, baseSend })) + + await act(async () => { + await hook!.sendNativeChat('just text') + }) + expect(baseSend).toHaveBeenCalledWith('just text') + expect(client.calls).toHaveLength(0) + }) + + it('keeps the chips and does not submit when the image paste is rejected', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), + sendResult(true), // Ctrl+U clear + sendResult(false) // image paste rejected + ]) + const baseSend = vi.fn().mockResolvedValue(true) + const showToast = vi.fn() + mount(baseArgs({ client: client as unknown as RpcClient, baseSend, showToast })) + await act(async () => { + await hook!.attachImage('library') + }) + let accepted = true + await act(async () => { + accepted = await hook!.sendNativeChat('hi') + }) + expect(accepted).toBe(false) + expect(baseSend).not.toHaveBeenCalled() + expect(showToast).toHaveBeenCalledWith('Message not sent', 1500) + expect(hook!.attachments).toHaveLength(1) + }) + + it('keeps the chips and reports failure when the paste transport throws', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + // No terminal.send responses queued: the clear write throws (dropped transport). + const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) + const baseSend = vi.fn().mockResolvedValue(true) + const showToast = vi.fn() + mount(baseArgs({ client: client as unknown as RpcClient, baseSend, showToast })) + await act(async () => { + await hook!.attachImage('library') + }) + let accepted = true + await act(async () => { + accepted = await hook!.sendNativeChat('hi') + }) + expect(accepted).toBe(false) + expect(baseSend).not.toHaveBeenCalled() + expect(showToast).toHaveBeenCalledWith('Message not sent', 1500) + expect(hook!.attachments).toHaveLength(1) + }) + + it('surfaces a toast instead of a silent no-op when the input lease gate is closed', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) + const baseSend = vi.fn().mockResolvedValue(true) + const showToast = vi.fn() + // Attaching is allowed without the lease; only the send is gated on it. + mount(baseArgs({ client: client as unknown as RpcClient, enabled: false, baseSend, showToast })) + await act(async () => { + await hook!.attachImage('library') + }) + let accepted = true + await act(async () => { + accepted = await hook!.sendNativeChat('hi') + }) + expect(accepted).toBe(false) + expect(baseSend).not.toHaveBeenCalled() + expect(showToast).toHaveBeenCalledWith('Message not sent (disconnected)', 1500) + expect(hook!.attachments).toHaveLength(1) + }) + + it('scopes chips to the tab that attached them', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) + const baseSend = vi.fn().mockResolvedValue(true) + const args = baseArgs({ client: client as unknown as RpcClient, baseSend }) + mount(args) + await act(async () => { + await hook!.attachImage('library') + }) + expect(hook!.attachments).toHaveLength(1) + + // Another tab sees no chip, and a send there is plain text — no image paste. + update({ ...args, scopeKey: 'h\0w\0tab-b' }) + expect(hook!.attachments).toEqual([]) + await act(async () => { + await hook!.sendNativeChat('hi') + }) + expect(baseSend).toHaveBeenCalledWith('hi') + expect(client.calls.some((c) => c.method === 'terminal.send')).toBe(false) + + // Back on the original tab the chip is still pending. + update(args) + expect(hook!.attachments).toHaveLength(1) + }) + + it('keeps isAttaching true when a cancelled pick overlaps a genuine in-flight upload', async () => { + // Park a real upload right after onUploadStart (count -> 1, isAttaching true) + // by holding its getConnectionId, then fire a cancelled pick. The cancelled + // call never incremented, so its finally must not drop the shared counter. + let releaseConnection: ((id: string | null) => void) | null = null + const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) + const args = baseArgs({ + client: client as unknown as RpcClient, + getActiveWorktreeConnectionId: () => + new Promise((resolve) => { + releaseConnection = resolve + }) + }) + mount(args) + + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + let firstAttach: Promise | null = null + await act(async () => { + firstAttach = hook!.attachImage('library') + for (let i = 0; i < 50 && !releaseConnection; i++) { + await Promise.resolve() + } + }) + expect(releaseConnection).not.toBeNull() + expect(hook!.isAttaching).toBe(true) + + // A concurrent cancelled pick — its finally must leave the counter alone. + pick.mockResolvedValue(null) + await act(async () => { + await hook!.attachImage('library') + }) + expect(hook!.isAttaching).toBe(true) + + // The real upload finishes and clears the flag on its own. + await act(async () => { + releaseConnection!('conn-1') + await firstAttach + }) + expect(hook!.isAttaching).toBe(false) + expect(hook!.attachments).toHaveLength(1) + }) + + it('clears only the chips that were sent, keeping one attached mid-send', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), // first attach + sendResult(true), // Ctrl+U clear + sendResult(true), // first image paste + methodNotFound('start'), + ok('save', '/tmp/b.png') // second attach, while the send is parked on settle + ]) + const baseSend = vi.fn().mockResolvedValue(true) + let releaseSettle: (() => void) | null = null + const args = baseArgs({ + client: client as unknown as RpcClient, + baseSend, + sleep: () => + new Promise((resolve) => { + releaseSettle = resolve + }) + }) + mount(args) + await act(async () => { + await hook!.attachImage('library') + }) + + let sendPromise: Promise | null = null + await act(async () => { + sendPromise = hook!.sendNativeChat('hi') + // Drain microtasks until the send parks on the settle sleep. + for (let i = 0; i < 50 && !releaseSettle; i++) { + await Promise.resolve() + } + }) + expect(releaseSettle).not.toBeNull() + + pick.mockResolvedValue({ base64: 'BBBB', uri: 'file:///b.jpg' }) + await act(async () => { + await hook!.attachImage('library') + }) + await act(async () => { + releaseSettle!() + await sendPromise + }) + + // Only the first (sent) image rode along; the mid-send chip survives. + expect(baseSend).toHaveBeenCalledWith('hi', ['file:///a.jpg']) + expect(hook!.attachments.map((a) => a.previewUri)).toEqual(['file:///b.jpg']) + }) + + it('aborts the send when the active terminal changes during the settle window', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), + sendResult(true), // Ctrl+U clear + sendResult(true) // image paste — into term-1 + ]) + const baseSend = vi.fn().mockResolvedValue(true) + const showToast = vi.fn() + const activeHandleRef = { current: 'term-1' } + let releaseSettle: (() => void) | null = null + const args = baseArgs({ + client: client as unknown as RpcClient, + activeHandleRef, + baseSend, + showToast, + sleep: () => + new Promise((resolve) => { + releaseSettle = resolve + }) + }) + mount(args) + await act(async () => { + await hook!.attachImage('library') + }) + + let sendPromise: Promise | null = null + await act(async () => { + sendPromise = hook!.sendNativeChat('hi') + for (let i = 0; i < 50 && !releaseSettle; i++) { + await Promise.resolve() + } + }) + expect(releaseSettle).not.toBeNull() + // The user switches tabs while the paste settles: the text + Enter must not + // land in term-2 when the images went to term-1. + activeHandleRef.current = 'term-2' + let accepted = true + await act(async () => { + releaseSettle!() + accepted = await sendPromise! + }) + expect(accepted).toBe(false) + expect(baseSend).not.toHaveBeenCalled() + expect(showToast).toHaveBeenCalledWith('Message not sent', 1500) + expect(hook!.attachments).toHaveLength(1) + }) + + it('leads the next text-only send with Ctrl+U after a failed paste, even with the chip removed', async () => { + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + const client = makeClient([ + methodNotFound('start'), + ok('save', '/tmp/a.png'), + sendResult(true), // Ctrl+U clear + sendResult(false), // image paste rejected — stale input left in term-1 + sendResult(true) // healing Ctrl+U before the text-only send + ]) + const baseSend = vi.fn().mockResolvedValue(true) + mount(baseArgs({ client: client as unknown as RpcClient, baseSend })) + await act(async () => { + await hook!.attachImage('library') + }) + await act(async () => { + await hook!.sendNativeChat('hi') + }) + expect(baseSend).not.toHaveBeenCalled() + + // The user gives up on the image and removes its chip, then sends plain text. + await act(async () => { + hook!.removeAttachment('img-1') + }) + expect(hook!.attachments).toEqual([]) + let accepted = false + await act(async () => { + accepted = await hook!.sendNativeChat('hi again') + }) + expect(accepted).toBe(true) + const sendCalls = client.calls.filter((c) => c.method === 'terminal.send') + // Failed attempt's clear + rejected paste, then the healing clear. + expect(sendCalls).toHaveLength(3) + expect(sendCalls[2]?.params).toMatchObject({ text: '\x15', enter: false }) + expect(baseSend).toHaveBeenCalledWith('hi again') + }) + + it('reports a disconnected attach failure via the live connection state', async () => { + const client = makeClient([]) + const showToast = vi.fn() + let failUpload: ((error: Error) => void) | null = null + const args = baseArgs({ + client: client as unknown as RpcClient, + showToast, + getActiveWorktreeConnectionId: () => + new Promise((_resolve, reject) => { + failUpload = reject + }) + }) + mount(args) + pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + let attach: Promise | null = null + await act(async () => { + attach = hook!.attachImage('library') + for (let i = 0; i < 50 && !failUpload; i++) { + await Promise.resolve() + } + }) + expect(failUpload).not.toBeNull() + // The connection drops mid-upload, then the in-flight RPC fails. The closure + // captured 'connected' at call time — only a live read can toast accurately. + update({ ...args, connState: 'connecting' }) + await act(async () => { + failUpload!(new Error('socket closed')) + await attach + }) + expect(showToast).toHaveBeenCalledWith('Attach failed (disconnected)', 1500) + }) +}) diff --git a/mobile/src/session/use-mobile-native-chat-image-attachments.ts b/mobile/src/session/use-mobile-native-chat-image-attachments.ts new file mode 100644 index 00000000000..fded5e2f547 --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-image-attachments.ts @@ -0,0 +1,298 @@ +import { useCallback, useRef, useState } from 'react' +import { CLIPBOARD_IMAGE_TOO_LARGE_ERROR } from '../../../src/shared/clipboard-image' +import type { RpcClient } from '../transport/rpc-client' +import type { ConnectionState } from '../transport/types' +import { + ImageLibraryPermissionError, + pickMobileImage, + type MobileImageSource +} from './mobile-image-source-picker' +import { + uploadMobileNativeChatImage, + type PendingNativeChatImage +} from './mobile-native-chat-image-attachment' +import { + MOBILE_NATIVE_CHAT_IMAGE_SETTLE_MS, + pasteMobileNativeChatImagePaths +} from './mobile-native-chat-image-send' + +type CurrentRef = { readonly current: T } +type ShowToast = (message: string, durationMs?: number) => void + +type Args = { + readonly client: RpcClient | null + readonly activeHandleRef: CurrentRef + readonly deviceTokenRef: CurrentRef + readonly getActiveWorktreeConnectionId: () => Promise + readonly connState: ConnectionState + /** Identity of the active composer surface (same key shape as the drafts hook): + * chips are scoped to the tab that picked them, so a tab switch cannot ride + * one tab's image into another tab's terminal. Null disables attaching. */ + readonly scopeKey: string | null + /** The native-chat input lease is ready — same gate `handleNativeChatSend` uses. */ + readonly enabled: boolean + readonly showToast: ShowToast + /** The plain text send (controller.handleNativeChatSend); wrapped so images ride + * along. The optional URIs drive the optimistic echo's thumbnails. */ + readonly baseSend: (text: string, imagePreviewUris?: string[]) => Promise + readonly onAttachSuccess?: () => void + readonly onError?: () => void + // Injected so the settle between image paste and submit is instant in tests. + readonly sleep?: (ms: number) => Promise +} + +export type MobileNativeChatImageAttachments = { + /** Pending chips for the active scope (tab) only. */ + readonly attachments: PendingNativeChatImage[] + readonly isAttaching: boolean + readonly attachImage: (source: MobileImageSource) => Promise + readonly removeAttachment: (id: string) => void + /** Ride any pending images along with `text`, then submit; clears the sent + * chips (and only those) once the send is accepted. */ + readonly sendNativeChat: (text: string) => Promise +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +const NO_ATTACHMENTS: PendingNativeChatImage[] = [] + +function withScopeAttachments( + byScope: Record, + scope: string, + next: PendingNativeChatImage[] +): Record { + if (next.length > 0) { + return { ...byScope, [scope]: next } + } + const remaining = { ...byScope } + delete remaining[scope] + return remaining +} + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)) + +export function useMobileNativeChatImageAttachments({ + client, + activeHandleRef, + deviceTokenRef, + getActiveWorktreeConnectionId, + connState, + scopeKey, + enabled, + showToast, + baseSend, + onAttachSuccess, + onError, + sleep = defaultSleep +}: Args): MobileNativeChatImageAttachments { + const [attachmentsByScope, setAttachmentsByScope] = useState< + Record + >({}) + const [isAttaching, setIsAttaching] = useState(false) + const idCounter = useRef(0) + // Count in-flight uploads so an overlapping attach can't clear the flag early. + const attachingCount = useRef(0) + // Live connState for attachImage's catch: the closure's value was already + // checked 'connected' at entry, so only a ref can see a mid-upload disconnect. + const connStateRef = useRef(connState) + connStateRef.current = connState + // Terminal whose input line may hold a partial paste from a failed send; the + // next send TO THAT terminal must lead with Ctrl+U even if it has no images. + const staleInputTerminalRef = useRef(null) + + const attachments = (scopeKey ? attachmentsByScope[scopeKey] : undefined) ?? NO_ATTACHMENTS + + const attachImage = useCallback( + async (source: MobileImageSource): Promise => { + // The chip lands in the scope that initiated the pick, even if the user + // switches tabs while the upload is in flight. + const scope = scopeKey + if (!client || !scope || !activeHandleRef.current || connState !== 'connected') { + return + } + // Only this call's own increment may be undone in `finally`; a cancelled + // pick or pre-upload error never ran `onUploadStart`, so decrementing the + // shared counter would clear a concurrent upload's in-flight flag early. + let started = false + try { + const uploaded = await uploadMobileNativeChatImage(source, { + client, + getConnectionId: getActiveWorktreeConnectionId, + pickImage: pickMobileImage, + onUploadStart: () => { + started = true + attachingCount.current += 1 + setIsAttaching(true) + } + }) + // Cancelled picker: no error, no toast. + if (!uploaded) { + return + } + idCounter.current += 1 + const chip = { id: `img-${idCounter.current}`, ...uploaded } + setAttachmentsByScope((prev) => ({ ...prev, [scope]: [...(prev[scope] ?? []), chip] })) + onAttachSuccess?.() + } catch (error) { + onError?.() + if (connStateRef.current !== 'connected') { + showToast('Attach failed (disconnected)', 1500) + return + } + if (error instanceof ImageLibraryPermissionError) { + showToast('Photo permission denied', 1500) + return + } + if (getErrorMessage(error) === CLIPBOARD_IMAGE_TOO_LARGE_ERROR) { + showToast('Image too large to attach', 1500) + return + } + showToast('Attach failed', 1500) + } finally { + if (started) { + attachingCount.current -= 1 + if (attachingCount.current <= 0) { + attachingCount.current = 0 + setIsAttaching(false) + } + } + } + }, + [ + activeHandleRef, + client, + connState, + getActiveWorktreeConnectionId, + onAttachSuccess, + onError, + scopeKey, + showToast + ] + ) + + const removeAttachment = useCallback( + (id: string): void => { + const scope = scopeKey + if (!scope) { + return + } + setAttachmentsByScope((prev) => + withScopeAttachments( + prev, + scope, + (prev[scope] ?? []).filter((attachment) => attachment.id !== id) + ) + ) + }, + [scopeKey] + ) + + const sendNativeChat = useCallback( + async (text: string): Promise => { + const scope = scopeKey + const pendingImages = (scope ? attachmentsByScope[scope] : undefined) ?? NO_ATTACHMENTS + if (pendingImages.length === 0 || !scope) { + // Heal a previously failed paste: a text-only send to that terminal would + // otherwise glue the stale image paste onto this message. Best-effort — + // on failure the marker stays set and the send proceeds as before. + const staleTerminal = staleInputTerminalRef.current + if (staleTerminal && staleTerminal === activeHandleRef.current && client) { + try { + await pasteMobileNativeChatImagePaths({ + client, + terminal: staleTerminal, + deviceToken: deviceTokenRef.current, + imagePaths: [] + }) + staleInputTerminalRef.current = null + } catch { + // Leave marked for the next attempt. + } + } + return baseSend(text) + } + const handle = activeHandleRef.current + if (!client || !handle || !enabled || connState !== 'connected') { + onError?.() + // Mirror the text path's failure surface (the base send is never reached). + showToast('Message not sent (disconnected)', 1500) + return false + } + try { + const pasted = await pasteMobileNativeChatImagePaths({ + client, + terminal: handle, + deviceToken: deviceTokenRef.current, + imagePaths: pendingImages.map((attachment) => attachment.path) + }) + if (!pasted) { + // Keep the chips so the user can retry; the failed paste never submitted. + staleInputTerminalRef.current = handle + onError?.() + showToast('Message not sent', 1500) + return false + } + // The paste's leading Ctrl+U cleared any earlier stale input in `handle`. + if (staleInputTerminalRef.current === handle) { + staleInputTerminalRef.current = null + } + // Let the TUI absorb the image paste before the text + Enter follow. The + // preview URIs ride along to baseSend so the sent bubble shows the photo + // immediately (empty text still submits a bare Enter through baseSend). + await sleep(MOBILE_NATIVE_CHAT_IMAGE_SETTLE_MS) + // The paste above targeted `handle`; a tab switch during the settle would + // route the text + Enter to a different terminal than the images. Abort — + // the chips keep their scope and a retry's Ctrl+U clears the stale paste. + if (activeHandleRef.current !== handle) { + staleInputTerminalRef.current = handle + onError?.() + showToast('Message not sent', 1500) + return false + } + const accepted = await baseSend( + text, + pendingImages.map((attachment) => attachment.previewUri) + ) + if (accepted) { + // Drop only what rode along — a chip attached while this send was in + // flight keeps waiting for its own send. + const sentIds = new Set(pendingImages.map((attachment) => attachment.id)) + setAttachmentsByScope((prev) => + withScopeAttachments( + prev, + scope, + (prev[scope] ?? []).filter((attachment) => !sentIds.has(attachment.id)) + ) + ) + } + return accepted + } catch { + // A thrown paste/send (network/RPC) keeps the chips and honors the + // Promise contract instead of rejecting. Retry-safe: the next + // attempt's leading Ctrl+U clears whatever fraction of the paste landed. + staleInputTerminalRef.current = handle + onError?.() + showToast('Message not sent', 1500) + return false + } + }, + [ + activeHandleRef, + attachmentsByScope, + baseSend, + client, + connState, + deviceTokenRef, + enabled, + onError, + scopeKey, + showToast, + sleep + ] + ) + + return { attachments, isAttaching, attachImage, removeAttachment, sendNativeChat } +} diff --git a/mobile/src/session/use-mobile-session-image-attachments.ts b/mobile/src/session/use-mobile-session-image-attachments.ts new file mode 100644 index 00000000000..47698b031e2 --- /dev/null +++ b/mobile/src/session/use-mobile-session-image-attachments.ts @@ -0,0 +1,81 @@ +import type { RpcClient } from '../transport/rpc-client' +import type { ConnectionState } from '../transport/types' +import type { MobileImageSource } from './mobile-image-source-picker' +import { useMobileImageAttachment } from './use-mobile-image-attachment' +import { + useMobileNativeChatImageAttachments, + type MobileNativeChatImageAttachments +} from './use-mobile-native-chat-image-attachments' + +type CurrentRef = { readonly current: T } + +type Args = { + readonly client: RpcClient | null + readonly activeHandle: string | null + readonly activeHandleRef: CurrentRef + readonly canSend: boolean + readonly connState: ConnectionState + readonly deviceTokenRef: CurrentRef + /** Active-tab identity (same key shape as the drafts hook) — native-chat chips + * are scoped per tab so a switch can't ride an image into another terminal. */ + readonly nativeChatScopeKey: string | null + readonly nativeChatInputLeaseReady: boolean + readonly getActiveWorktreeConnectionId: () => Promise + readonly beforeTerminalSend: (terminal: string) => Promise + readonly nativeChatBaseSend: (text: string, images?: string[]) => Promise + readonly showToast: (message: string, durationMs?: number) => void + readonly onSuccess: () => void + readonly onError: () => void +} + +/** A session exposes image attachment on two surfaces that share one upload + * pipeline and host wiring: the visible terminal input (immediate bracketed + * paste) and the native-chat composer (chips deferred to submit). Owning both + * here keeps the already-dense session route to a single wiring point. */ +export function useMobileSessionImageAttachments({ + client, + activeHandle, + activeHandleRef, + canSend, + connState, + deviceTokenRef, + nativeChatScopeKey, + nativeChatInputLeaseReady, + getActiveWorktreeConnectionId, + beforeTerminalSend, + nativeChatBaseSend, + showToast, + onSuccess, + onError +}: Args): { + attachImage: (source: MobileImageSource) => Promise + isAttaching: boolean + nativeChatImages: MobileNativeChatImageAttachments +} { + const { attachImage, isAttaching } = useMobileImageAttachment({ + client, + activeHandle, + canSend, + connState, + deviceTokenRef, + beforeTerminalSend, + getActiveWorktreeConnectionId, + showToast, + onSuccess, + onError + }) + const nativeChatImages = useMobileNativeChatImageAttachments({ + client, + activeHandleRef, + deviceTokenRef, + getActiveWorktreeConnectionId, + connState, + scopeKey: nativeChatScopeKey, + enabled: nativeChatInputLeaseReady, + showToast, + baseSend: nativeChatBaseSend, + onAttachSuccess: onSuccess, + onError + }) + return { attachImage, isAttaching, nativeChatImages } +} diff --git a/mobile/src/storage/preferences.test.ts b/mobile/src/storage/preferences.test.ts index b636ea12d3e..3090be4a130 100644 --- a/mobile/src/storage/preferences.test.ts +++ b/mobile/src/storage/preferences.test.ts @@ -1,5 +1,5 @@ import AsyncStorage from '@react-native-async-storage/async-storage' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { HOST_DOCK_MAX_WIDTH, HOST_DOCK_MIN_WIDTH, @@ -10,22 +10,29 @@ import { clampHostSidebarWidth, loadDisabledTerminalLiveInputHandles, loadHostSidebarWidth, + loadPinnedIds, loadPushNotificationsEnabled, loadTerminalAutocompleteEnabled, loadTerminalLinkOpenMode, readPushNotificationsPreference, readDisabledTerminalLiveInputHandlesPreference, + MOBILE_STORED_ID_SET_MAX_ENTRIES, + MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS, saveDisabledTerminalLiveInputHandles, saveHostSidebarWidth, + savePinnedIds, savePushNotificationsEnabled, saveTerminalAutocompleteEnabled, saveTerminalLinkOpenMode } from './preferences' import { + SESSION_VIEW_OVERRIDE_MAX_ACTIVE_BARRIERS, + SESSION_VIEW_OVERRIDE_MAX_ENTRIES, loadDefaultSessionView, loadSessionViewOverrides, readDefaultSessionViewPreference, readSessionViewOverridesPreference, + resetSessionViewPreferencesForTests, saveDefaultSessionView, updateSessionViewOverride } from './session-view-preferences' @@ -47,10 +54,15 @@ function deferred(): { promise: Promise; resolve: (value: T) => void } { describe('session view preference', () => { beforeEach(() => { + resetSessionViewPreferencesForTests() vi.mocked(AsyncStorage.getItem).mockReset() vi.mocked(AsyncStorage.setItem).mockReset() }) + afterEach(() => { + resetSessionViewPreferencesForTests() + }) + it('defaults to terminal and persists the chat default', async () => { vi.mocked(AsyncStorage.getItem).mockResolvedValue(null) await expect(loadDefaultSessionView()).resolves.toBe('terminal') @@ -270,6 +282,72 @@ describe('session view preference', () => { JSON.stringify({ tab: 'chat' }) ) }) + + it('loads the exact override count and rejects one over without parsing it into state', async () => { + const exact = Object.fromEntries( + Array.from({ length: SESSION_VIEW_OVERRIDE_MAX_ENTRIES }, (_, index) => [ + `tab-${index}`, + 'chat' + ]) + ) + vi.mocked(AsyncStorage.getItem).mockResolvedValue(JSON.stringify(exact)) + + await expect(readSessionViewOverridesPreference('host', 'worktree')).resolves.toEqual({ + overrides: new Map(Object.entries(exact)) as Map, + loaded: true + }) + + vi.mocked(AsyncStorage.getItem).mockResolvedValue( + JSON.stringify({ ...exact, 'one-over': 'terminal' }) + ) + await expect(readSessionViewOverridesPreference('host', 'worktree')).resolves.toEqual({ + overrides: new Map(), + loaded: false + }) + }) + + it('evicts the oldest override when a new user choice exceeds the count cap', async () => { + const exact = Object.fromEntries( + Array.from({ length: SESSION_VIEW_OVERRIDE_MAX_ENTRIES }, (_, index) => [ + `tab-${index}`, + 'chat' + ]) + ) + vi.mocked(AsyncStorage.getItem).mockResolvedValue(JSON.stringify(exact)) + + await updateSessionViewOverride('host', 'worktree', 'new-tab', 'terminal') + + const stored = JSON.parse(vi.mocked(AsyncStorage.setItem).mock.calls[0]![1]) as Record< + string, + string + > + expect(Object.keys(stored)).toHaveLength(SESSION_VIEW_OVERRIDE_MAX_ENTRIES) + expect(stored['tab-0']).toBeUndefined() + expect(stored['new-tab']).toBe('terminal') + }) + + it('caps unresolved scoped write barriers and accepts another after one settles', async () => { + const blocked = deferred() + vi.mocked(AsyncStorage.getItem).mockResolvedValue(null) + vi.mocked(AsyncStorage.setItem).mockImplementation(() => blocked.promise) + const writes = Array.from({ length: SESSION_VIEW_OVERRIDE_MAX_ACTIVE_BARRIERS }, (_, index) => + updateSessionViewOverride(`host-${index}`, 'worktree', 'tab', 'chat') + ) + await vi.waitFor(() => + expect(AsyncStorage.setItem).toHaveBeenCalledTimes(SESSION_VIEW_OVERRIDE_MAX_ACTIVE_BARRIERS) + ) + + await expect(updateSessionViewOverride('one-over', 'worktree', 'tab', 'chat')).rejects.toThrow( + 'Too many session view override writes are pending' + ) + + blocked.resolve() + await Promise.all(writes) + vi.mocked(AsyncStorage.setItem).mockResolvedValue(undefined) + await expect( + updateSessionViewOverride('recovered', 'worktree', 'tab', 'chat') + ).resolves.toBeUndefined() + }) }) describe('push notification preference', () => { @@ -406,6 +484,57 @@ describe('terminal live input disabled handles preference', () => { JSON.stringify(['pty-2', 'pty-1']) ) }) + + it('accepts the exact handle cap and persists the newest handles at one over', async () => { + const exact = new Set( + Array.from({ length: MOBILE_STORED_ID_SET_MAX_ENTRIES }, (_, index) => `pty-${index}`) + ) + await saveDisabledTerminalLiveInputHandles('host', 'worktree', exact) + expect(JSON.parse(vi.mocked(AsyncStorage.setItem).mock.calls[0]![1])).toHaveLength( + MOBILE_STORED_ID_SET_MAX_ENTRIES + ) + + exact.add('one-over') + await saveDisabledTerminalLiveInputHandles('host', 'worktree', exact) + const retained = JSON.parse(vi.mocked(AsyncStorage.setItem).mock.calls[1]![1]) as string[] + expect(retained).toHaveLength(MOBILE_STORED_ID_SET_MAX_ENTRIES) + expect(retained).not.toContain('pty-0') + expect(retained).toContain('one-over') + }) + + it('rejects an oversized handle payload before parsing', async () => { + vi.mocked(AsyncStorage.getItem).mockResolvedValue( + 'x'.repeat(MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS + 1) + ) + + await expect( + readDisabledTerminalLiveInputHandlesPreference('host', 'worktree') + ).resolves.toEqual({ handles: new Set(), loaded: false }) + }) +}) + +describe('pinned worktree ids preference', () => { + beforeEach(() => { + vi.mocked(AsyncStorage.getItem).mockReset() + vi.mocked(AsyncStorage.setItem).mockReset() + }) + + it('round-trips normal pins unchanged', async () => { + const pins = new Set(['worktree-2', 'worktree-1']) + await savePinnedIds('host', pins) + expect(AsyncStorage.setItem).toHaveBeenCalledWith('orca:pins:host', JSON.stringify([...pins])) + + vi.mocked(AsyncStorage.getItem).mockResolvedValue(JSON.stringify([...pins])) + await expect(loadPinnedIds('host')).resolves.toEqual(pins) + }) + + it('bounds oversized durable pin payloads', async () => { + vi.mocked(AsyncStorage.getItem).mockResolvedValue( + 'x'.repeat(MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS + 1) + ) + + await expect(loadPinnedIds('host')).resolves.toEqual(new Set()) + }) }) describe('host sidebar width preference', () => { diff --git a/mobile/src/storage/preferences.ts b/mobile/src/storage/preferences.ts index 5173ac5bc8a..d6128d466ab 100644 --- a/mobile/src/storage/preferences.ts +++ b/mobile/src/storage/preferences.ts @@ -2,6 +2,9 @@ import AsyncStorage from '@react-native-async-storage/async-storage' const PINS_PREFIX = 'orca:pins:' const NOTIF_KEY = 'orca:pushNotificationsEnabled' +export const MOBILE_STORED_ID_SET_MAX_ENTRIES = 10_000 +export const MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS = 1024 * 1024 +export const MOBILE_STORED_ID_MAX_CHARACTERS = 4_096 export type PushNotificationsPreference = { readonly value: boolean | null @@ -101,7 +104,10 @@ export async function readDisabledTerminalLiveInputHandlesPreference( if (!raw) { return { handles: new Set(), loaded: true } } - return { handles: new Set(stringArray(JSON.parse(raw))), loaded: true } + if (raw.length > MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS) { + return { handles: new Set(), loaded: false } + } + return { handles: new Set(retainStoredPreferenceIds(JSON.parse(raw)).ids), loaded: true } } catch { return { handles: new Set(), loaded: false } } @@ -122,7 +128,7 @@ export async function saveDisabledTerminalLiveInputHandles( ): Promise { await AsyncStorage.setItem( terminalLiveInputDisabledKey(hostId, worktreeId), - JSON.stringify([...handles]) + retainStoredPreferenceIds(handles).serialized ) } @@ -210,24 +216,59 @@ export async function saveTerminalLinkOpenMode(mode: MobileTerminalLinkOpenMode) await AsyncStorage.setItem(TERMINAL_LINK_OPEN_MODE_KEY, mode) } -function stringArray(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((item): item is string => typeof item === 'string') - : [] -} - export async function loadPinnedIds(hostId: string): Promise> { try { const raw = await AsyncStorage.getItem(PINS_PREFIX + hostId) if (!raw) { return new Set() } - return new Set(stringArray(JSON.parse(raw))) + if (raw.length > MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS) { + return new Set() + } + return new Set(retainStoredPreferenceIds(JSON.parse(raw)).ids) } catch { return new Set() } } export async function savePinnedIds(hostId: string, ids: Set): Promise { - await AsyncStorage.setItem(PINS_PREFIX + hostId, JSON.stringify([...ids])) + await AsyncStorage.setItem(PINS_PREFIX + hostId, retainStoredPreferenceIds(ids).serialized) +} + +function retainStoredPreferenceIds(value: unknown): { ids: string[]; serialized: string } { + const values = Array.isArray(value) + ? value + : value && typeof value === 'object' && Symbol.iterator in value + ? (value as Iterable) + : [] + const retained = new Map() + let entryCharacters = 0 + for (const candidate of values) { + if ( + typeof candidate !== 'string' || + candidate.length > MOBILE_STORED_ID_MAX_CHARACTERS || + retained.has(candidate) + ) { + continue + } + const serialized = JSON.stringify(candidate) + retained.set(candidate, serialized) + entryCharacters += serialized.length + while ( + retained.size > MOBILE_STORED_ID_SET_MAX_ENTRIES || + 2 + entryCharacters + Math.max(0, retained.size - 1) > + MOBILE_STORED_ID_SET_MAX_STORAGE_CHARACTERS + ) { + const oldestId = retained.keys().next().value + if (typeof oldestId !== 'string') { + break + } + entryCharacters -= retained.get(oldestId)?.length ?? 0 + retained.delete(oldestId) + } + } + return { + ids: [...retained.keys()], + serialized: `[${[...retained.values()].join(',')}]` + } } diff --git a/mobile/src/storage/session-view-preferences.ts b/mobile/src/storage/session-view-preferences.ts index cd9597d6f77..bef1b3b3c92 100644 --- a/mobile/src/storage/session-view-preferences.ts +++ b/mobile/src/storage/session-view-preferences.ts @@ -5,6 +5,11 @@ export type MobileSessionView = 'terminal' | 'chat' const DEFAULT_SESSION_VIEW_KEY = 'orca:defaultSessionView' const NATIVE_CHAT_TABS_PREFIX = 'orca:nativeChatTabs:' +export const SESSION_VIEW_OVERRIDE_MAX_ENTRIES = 4_096 +export const SESSION_VIEW_OVERRIDE_MAX_STORAGE_CHARACTERS = 512 * 1024 +export const SESSION_VIEW_OVERRIDE_MAX_TAB_ID_CHARACTERS = 1_024 +export const SESSION_VIEW_OVERRIDE_MAX_ACTIVE_BARRIERS = 64 +const SESSION_VIEW_OVERRIDE_MAX_SCOPE_KEY_CHARACTERS = 4_096 // Why: default stays terminal so native chat remains strictly opt-in. export const DEFAULT_SESSION_VIEW: MobileSessionView = 'terminal' @@ -12,8 +17,11 @@ export const DEFAULT_SESSION_VIEW: MobileSessionView = 'terminal' let defaultViewWriteBarrier: Promise | null = null const overrideUpdateBarriers = new Map>() -function sessionViewOverridesKey(hostId: string, worktreeId: string): string { - return `${NATIVE_CHAT_TABS_PREFIX}${encodeURIComponent(hostId)}:${encodeURIComponent(worktreeId)}` +function sessionViewOverridesKey(hostId: string, worktreeId: string): string | null { + const key = `${NATIVE_CHAT_TABS_PREFIX}${encodeURIComponent(hostId)}:${encodeURIComponent( + worktreeId + )}` + return key.length <= SESSION_VIEW_OVERRIDE_MAX_SCOPE_KEY_CHARACTERS ? key : null } function clearDefaultViewWriteBarrier(barrier: Promise): void { @@ -77,6 +85,9 @@ async function readSessionViewOverridesStorage( if (!raw) { return { overrides: new Map(), loaded: true } } + if (raw.length > SESSION_VIEW_OVERRIDE_MAX_STORAGE_CHARACTERS) { + return { overrides: new Map(), loaded: false } + } let parsed: unknown try { parsed = JSON.parse(raw) as unknown @@ -86,21 +97,38 @@ async function readSessionViewOverridesStorage( } // Legacy format: an array of tab ids that were showing native chat. if (Array.isArray(parsed)) { - return { - overrides: new Map( - parsed - .filter((id): id is string => typeof id === 'string') - .map((id) => [id, 'chat' as const]) - ), - loaded: true + const overrides = new Map() + for (const id of parsed) { + if (typeof id !== 'string' || id.length > SESSION_VIEW_OVERRIDE_MAX_TAB_ID_CHARACTERS) { + continue + } + if (!overrides.has(id) && overrides.size >= SESSION_VIEW_OVERRIDE_MAX_ENTRIES) { + return { overrides: new Map(), loaded: false } + } + overrides.set(id, 'chat') } + return { overrides, loaded: true } } if (parsed && typeof parsed === 'object') { - const entries = Object.entries(parsed as Record).filter( - (entry): entry is [string, MobileSessionView] => - entry[1] === 'terminal' || entry[1] === 'chat' - ) - return { overrides: new Map(entries), loaded: true } + const overrides = new Map() + const record = parsed as Record + for (const id in record) { + if (!Object.prototype.hasOwnProperty.call(record, id)) { + continue + } + const view = record[id] + if ( + (view !== 'terminal' && view !== 'chat') || + id.length > SESSION_VIEW_OVERRIDE_MAX_TAB_ID_CHARACTERS + ) { + continue + } + if (overrides.size >= SESSION_VIEW_OVERRIDE_MAX_ENTRIES) { + return { overrides: new Map(), loaded: false } + } + overrides.set(id, view) + } + return { overrides, loaded: true } } return { overrides: new Map(), loaded: true } } @@ -121,6 +149,9 @@ export async function readSessionViewOverridesPreference( worktreeId: string ): Promise { const key = sessionViewOverridesKey(hostId, worktreeId) + if (!key) { + return { overrides: new Map(), loaded: false } + } await overrideUpdateBarriers.get(key) return readSessionViewOverridesStorage(key) } @@ -133,7 +164,17 @@ export async function updateSessionViewOverride( view: MobileSessionView ): Promise { const key = sessionViewOverridesKey(hostId, worktreeId) - const previous = overrideUpdateBarriers.get(key) ?? Promise.resolve() + if (!key || tabId.length === 0 || tabId.length > SESSION_VIEW_OVERRIDE_MAX_TAB_ID_CHARACTERS) { + throw new Error('Session view override identifier is too large') + } + const existingBarrier = overrideUpdateBarriers.get(key) + if ( + !existingBarrier && + overrideUpdateBarriers.size >= SESSION_VIEW_OVERRIDE_MAX_ACTIVE_BARRIERS + ) { + throw new Error('Too many session view override writes are pending') + } + const previous = existingBarrier ?? Promise.resolve() const update = previous.then(async () => { const current = await readSessionViewOverridesStorage(key) // Why: a transient read failure must not replace valid saved siblings with @@ -141,8 +182,17 @@ export async function updateSessionViewOverride( if (!current.loaded) { throw new Error('Session view overrides could not be read') } + if ( + !current.overrides.has(tabId) && + current.overrides.size >= SESSION_VIEW_OVERRIDE_MAX_ENTRIES + ) { + const oldestId = current.overrides.keys().next().value + if (typeof oldestId === 'string') { + current.overrides.delete(oldestId) + } + } current.overrides.set(tabId, view) - await AsyncStorage.setItem(key, JSON.stringify(Object.fromEntries(current.overrides))) + await AsyncStorage.setItem(key, serializeSessionViewOverrides(current.overrides, tabId)) }) const barrier = update.catch(() => undefined) overrideUpdateBarriers.set(key, barrier) @@ -154,3 +204,48 @@ export async function updateSessionViewOverride( } } } + +function serializeSessionViewOverrides( + overrides: Map, + protectedId: string +): string { + while (true) { + const serialized = trySerializeSessionViewOverrides(overrides) + if (serialized !== null) { + return serialized + } + let evictionId: string | null = null + for (const id of overrides.keys()) { + if (id !== protectedId) { + evictionId = id + break + } + } + if (evictionId === null) { + throw new Error('Session view override storage limit exceeded') + } + overrides.delete(evictionId) + } +} + +function trySerializeSessionViewOverrides( + overrides: ReadonlyMap +): string | null { + const entries: string[] = [] + let characters = 2 + for (const [id, view] of overrides) { + const entry = `${JSON.stringify(id)}:${JSON.stringify(view)}` + characters += entry.length + (entries.length > 0 ? 1 : 0) + if (characters > SESSION_VIEW_OVERRIDE_MAX_STORAGE_CHARACTERS) { + return null + } + entries.push(entry) + } + return `{${entries.join(',')}}` +} + +/** Test-only: drop pending module write barriers between cases. */ +export function resetSessionViewPreferencesForTests(): void { + defaultViewWriteBarrier = null + overrideUpdateBarriers.clear() +} diff --git a/mobile/src/tasks/github-pr-file-diff.test.ts b/mobile/src/tasks/github-pr-file-diff.test.ts index 2a59c4934a1..5174dba5b64 100644 --- a/mobile/src/tasks/github-pr-file-diff.test.ts +++ b/mobile/src/tasks/github-pr-file-diff.test.ts @@ -44,6 +44,24 @@ describe('buildGitHubPrFileDiffLines', () => { ]) }) + it('preserves a terminal lone carriage return like the previous line splitter', () => { + expect(buildGitHubPrFileDiffLines('', 'line\r')).toEqual([ + { key: '0:added:1', kind: 'added', newLineNumber: 1, text: 'line\r' } + ]) + }) + + it('treats one empty LF line and CRLF line as equal', () => { + expect(buildGitHubPrFileDiffLines('\n', '\r\n')).toEqual([ + { + key: '0:context:1:1', + kind: 'context', + oldLineNumber: 1, + newLineNumber: 1, + text: '' + } + ]) + }) + it('keeps all lines for large files without exact diff truncation', () => { const original = Array.from({ length: 500 }, (_, index) => `old-${index}`).join('\n') const modified = Array.from({ length: 500 }, (_, index) => `new-${index}`).join('\n') @@ -71,6 +89,39 @@ describe('buildGitHubPrFileDiffLines', () => { }) }) + it('counts newline-dense files without materializing discarded line arrays', () => { + const original = '\n'.repeat(200_000) + + const preview = buildGitHubPrFileDiffPreview(original, 'changed', 3) + + expect(preview.totalLineCount).toBe(200_001) + expect(preview.lines).toEqual([ + { key: '0:removed:1', kind: 'removed', oldLineNumber: 1, text: '' }, + { key: '1:removed:2', kind: 'removed', oldLineNumber: 2, text: '' }, + { key: '2:removed:3', kind: 'removed', oldLineNumber: 3, text: '' } + ]) + }) + + it('preserves common prefix and suffix rows around a streamed middle diff', () => { + const originalMiddle = Array.from({ length: 500 }, (_, index) => `old-${index}`) + const modifiedMiddle = Array.from({ length: 500 }, (_, index) => `new-${index}`) + const original = ['first', ...originalMiddle, 'last'].join('\r\n') + const modified = ['first', ...modifiedMiddle, 'last'].join('\r\n') + + const preview = buildGitHubPrFileDiffPreview(original, modified, 1_002) + + expect(preview.totalLineCount).toBe(1_002) + expect(preview.lines[0]).toMatchObject({ kind: 'context', text: 'first' }) + expect(preview.lines[1]).toMatchObject({ kind: 'removed', text: 'old-0' }) + expect(preview.lines[501]).toMatchObject({ kind: 'added', text: 'new-0' }) + expect(preview.lines.at(-1)).toMatchObject({ + kind: 'context', + oldLineNumber: 502, + newLineNumber: 502, + text: 'last' + }) + }) + it('can compute the total without retaining preview rows', () => { const modified = Array.from({ length: 20 }, (_, index) => `new-${index}`).join('\n') diff --git a/mobile/src/tasks/github-pr-file-diff.ts b/mobile/src/tasks/github-pr-file-diff.ts index 1be684197d4..8165d255437 100644 --- a/mobile/src/tasks/github-pr-file-diff.ts +++ b/mobile/src/tasks/github-pr-file-diff.ts @@ -1,3 +1,11 @@ +import { + collectGitHubPrFileLineRange, + createGitHubPrFileLineSource, + findGitHubPrFileCommonLineEdges, + visitGitHubPrFileLineRange, + type GitHubPrFileLineSource +} from './github-pr-file-line-scan' + export type GitHubPrFileDiffLine = { key: string kind: 'context' | 'added' | 'removed' @@ -18,14 +26,6 @@ type DiffOperation = const EXACT_DIFF_CELL_LIMIT = 160_000 -function splitContentLines(value: string): string[] { - if (!value) { - return [] - } - const lines = value.split(/\r?\n/) - return lines.at(-1) === '' ? lines.slice(0, -1) : lines -} - function appendExactLineDiff( original: string[], modified: string[], @@ -79,37 +79,6 @@ function appendExactLineDiff( } } -function appendMiddleDiff( - original: string[], - modified: string[], - appendOperation: (operation: DiffOperation) => void -): void { - if (original.length === 0) { - for (const newLine of modified) { - appendOperation({ kind: 'added', newLine }) - } - return - } - if (modified.length === 0) { - for (const oldLine of original) { - appendOperation({ kind: 'removed', oldLine }) - } - return - } - if (original.length * modified.length <= EXACT_DIFF_CELL_LIMIT) { - appendExactLineDiff(original, modified, appendOperation) - return - } - // Why: the Tasks diff UI renders a capped preview. Stream fallback rows so a - // generated PR file does not allocate thousands of discarded row objects. - for (const oldLine of original) { - appendOperation({ kind: 'removed', oldLine }) - } - for (const newLine of modified) { - appendOperation({ kind: 'added', newLine }) - } -} - export function buildGitHubPrFileDiffLines( originalContent: string, modifiedContent: string @@ -122,42 +91,17 @@ export function buildGitHubPrFileDiffPreview( modifiedContent: string, maxLines = Number.POSITIVE_INFINITY ): GitHubPrFileDiffPreview { - const originalLines = splitContentLines(originalContent) - const modifiedLines = splitContentLines(modifiedContent) - let prefixLength = 0 - while ( - prefixLength < originalLines.length && - prefixLength < modifiedLines.length && - originalLines[prefixLength] === modifiedLines[prefixLength] - ) { - prefixLength += 1 - } - - let suffixLength = 0 - while ( - suffixLength < originalLines.length - prefixLength && - suffixLength < modifiedLines.length - prefixLength && - originalLines[originalLines.length - suffixLength - 1] === - modifiedLines[modifiedLines.length - suffixLength - 1] - ) { - suffixLength += 1 - } - - const originalMiddle = originalLines.slice( - prefixLength, - suffixLength === 0 ? originalLines.length : originalLines.length - suffixLength - ) - const modifiedMiddle = modifiedLines.slice( - prefixLength, - suffixLength === 0 ? modifiedLines.length : modifiedLines.length - suffixLength - ) - + const original = createGitHubPrFileLineSource(originalContent) + const modified = createGitHubPrFileLineSource(modifiedContent) + const { prefixLineCount, suffixLineCount } = findGitHubPrFileCommonLineEdges(original, modified) + const originalMiddleLineCount = original.lineCount - prefixLineCount - suffixLineCount + const modifiedMiddleLineCount = modified.lineCount - prefixLineCount - suffixLineCount const result: GitHubPrFileDiffLine[] = [] let oldLineNumber = 1 let newLineNumber = 1 let operationIndex = 0 let totalLineCount = 0 - const normalizedMaxLines = Math.max(0, Math.floor(maxLines)) + const normalizedMaxLines = Number.isNaN(maxLines) ? 0 : Math.max(0, Math.floor(maxLines)) function appendOperation(operation: DiffOperation): void { const index = operationIndex operationIndex += 1 @@ -199,15 +143,55 @@ export function buildGitHubPrFileDiffPreview( newLineNumber += 1 } - for (let i = 0; i < prefixLength; i += 1) { - const line = originalLines[i] ?? '' - appendOperation({ kind: 'context', oldLine: line, newLine: line }) + function skipOperations(kind: DiffOperation['kind'], count: number): void { + operationIndex += count + totalLineCount += count + if (kind === 'context') { + oldLineNumber += count + newLineNumber += count + } else if (kind === 'removed') { + oldLineNumber += count + } else { + newLineNumber += count + } } - appendMiddleDiff(originalMiddle, modifiedMiddle, appendOperation) - for (let i = originalLines.length - suffixLength; i < originalLines.length; i += 1) { - const line = originalLines[i] ?? '' - appendOperation({ kind: 'context', oldLine: line, newLine: line }) + + function appendLineRange( + kind: DiffOperation['kind'], + source: GitHubPrFileLineSource, + startLine: number, + lineCount: number + ): void { + const retainedLineCount = Math.min(lineCount, Math.max(0, normalizedMaxLines - result.length)) + visitGitHubPrFileLineRange(source, startLine, retainedLineCount, (line) => { + if (kind === 'context') { + appendOperation({ kind, oldLine: line, newLine: line }) + } else if (kind === 'removed') { + appendOperation({ kind, oldLine: line }) + } else { + appendOperation({ kind, newLine: line }) + } + }) + skipOperations(kind, lineCount - retainedLineCount) } + appendLineRange('context', original, 0, prefixLineCount) + if (originalMiddleLineCount === 0) { + appendLineRange('added', modified, prefixLineCount, modifiedMiddleLineCount) + } else if (modifiedMiddleLineCount === 0) { + appendLineRange('removed', original, prefixLineCount, originalMiddleLineCount) + } else if (originalMiddleLineCount * modifiedMiddleLineCount <= EXACT_DIFF_CELL_LIMIT) { + appendExactLineDiff( + collectGitHubPrFileLineRange(original, prefixLineCount, originalMiddleLineCount), + collectGitHubPrFileLineRange(modified, prefixLineCount, modifiedMiddleLineCount), + appendOperation + ) + } else { + // Why: large generated files need exact counts without retaining discarded preview rows. + appendLineRange('removed', original, prefixLineCount, originalMiddleLineCount) + appendLineRange('added', modified, prefixLineCount, modifiedMiddleLineCount) + } + appendLineRange('context', original, original.lineCount - suffixLineCount, suffixLineCount) + return { lines: result, totalLineCount } } diff --git a/mobile/src/tasks/github-pr-file-line-scan.ts b/mobile/src/tasks/github-pr-file-line-scan.ts new file mode 100644 index 00000000000..6031ce4d2f0 --- /dev/null +++ b/mobile/src/tasks/github-pr-file-line-scan.ts @@ -0,0 +1,171 @@ +export type GitHubPrFileLineSource = { + content: string + lineCount: number +} + +export type GitHubPrFileCommonLineEdges = { + prefixLineCount: number + suffixLineCount: number +} + +type LineBounds = { + start: number + end: number +} + +type ForwardLineCursor = { + offset: number + remaining: number + source: GitHubPrFileLineSource +} + +type ReverseLineCursor = { + boundary: number + remaining: number + source: GitHubPrFileLineSource +} + +export function createGitHubPrFileLineSource(content: string): GitHubPrFileLineSource { + if (content.length === 0) { + return { content, lineCount: 0 } + } + let lineCount = content.endsWith('\n') ? 0 : 1 + for (let index = 0; index < content.length; index += 1) { + if (content.charCodeAt(index) === 10) { + lineCount += 1 + } + } + return { content, lineCount } +} + +export function findGitHubPrFileCommonLineEdges( + original: GitHubPrFileLineSource, + modified: GitHubPrFileLineSource +): GitHubPrFileCommonLineEdges { + const comparableLineCount = Math.min(original.lineCount, modified.lineCount) + const originalForward = createForwardCursor(original) + const modifiedForward = createForwardCursor(modified) + let prefixLineCount = 0 + while (prefixLineCount < comparableLineCount) { + const originalLine = takeNextLine(originalForward)! + const modifiedLine = takeNextLine(modifiedForward)! + if (!lineBoundsEqual(original, originalLine, modified, modifiedLine)) { + break + } + prefixLineCount += 1 + } + + const originalReverse = createReverseCursor(original) + const modifiedReverse = createReverseCursor(modified) + const maxSuffixLineCount = comparableLineCount - prefixLineCount + let suffixLineCount = 0 + while (suffixLineCount < maxSuffixLineCount) { + const originalLine = takePreviousLine(originalReverse)! + const modifiedLine = takePreviousLine(modifiedReverse)! + if (!lineBoundsEqual(original, originalLine, modified, modifiedLine)) { + break + } + suffixLineCount += 1 + } + return { prefixLineCount, suffixLineCount } +} + +export function visitGitHubPrFileLineRange( + source: GitHubPrFileLineSource, + startLine: number, + lineCount: number, + visit: (line: string) => void +): void { + if (lineCount <= 0) { + return + } + const cursor = createForwardCursor(source) + for (let index = 0; index < startLine; index += 1) { + takeNextLine(cursor) + } + for (let index = 0; index < lineCount; index += 1) { + const bounds = takeNextLine(cursor) + if (!bounds) { + return + } + visit(source.content.slice(bounds.start, bounds.end)) + } +} + +export function collectGitHubPrFileLineRange( + source: GitHubPrFileLineSource, + startLine: number, + lineCount: number +): string[] { + const lines: string[] = [] + visitGitHubPrFileLineRange(source, startLine, lineCount, (line) => lines.push(line)) + return lines +} + +function createForwardCursor(source: GitHubPrFileLineSource): ForwardLineCursor { + return { source, offset: 0, remaining: source.lineCount } +} + +function takeNextLine(cursor: ForwardLineCursor): LineBounds | null { + if (cursor.remaining === 0) { + return null + } + const { content } = cursor.source + const separator = content.indexOf('\n', cursor.offset) + const rawEnd = separator === -1 ? content.length : separator + const end = + separator !== -1 && rawEnd > cursor.offset && content.charCodeAt(rawEnd - 1) === 13 + ? rawEnd - 1 + : rawEnd + const bounds = { start: cursor.offset, end } + cursor.offset = separator === -1 ? content.length : separator + 1 + cursor.remaining -= 1 + return bounds +} + +function createReverseCursor(source: GitHubPrFileLineSource): ReverseLineCursor { + return { + source, + boundary: source.content.endsWith('\n') ? source.content.length - 1 : source.content.length, + remaining: source.lineCount + } +} + +function takePreviousLine(cursor: ReverseLineCursor): LineBounds | null { + if (cursor.remaining === 0) { + return null + } + const { content } = cursor.source + const separator = cursor.boundary === 0 ? -1 : content.lastIndexOf('\n', cursor.boundary - 1) + const start = separator + 1 + const end = + cursor.boundary < content.length && + cursor.boundary > start && + content.charCodeAt(cursor.boundary - 1) === 13 + ? cursor.boundary - 1 + : cursor.boundary + cursor.boundary = separator === -1 ? 0 : separator + cursor.remaining -= 1 + return { start, end } +} + +function lineBoundsEqual( + leftSource: GitHubPrFileLineSource, + left: LineBounds, + rightSource: GitHubPrFileLineSource, + right: LineBounds +): boolean { + const length = left.end - left.start + if (length !== right.end - right.start) { + return false + } + for (let offset = 0; offset < length; offset += 1) { + if ( + leftSource.content.charCodeAt(left.start + offset) !== + rightSource.content.charCodeAt(right.start + offset) + ) { + return false + } + } + return true +} diff --git a/mobile/src/tasks/mobile-pr-file-content-cache.test.ts b/mobile/src/tasks/mobile-pr-file-content-cache.test.ts new file mode 100644 index 00000000000..bb32226a98a --- /dev/null +++ b/mobile/src/tasks/mobile-pr-file-content-cache.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest' +import type { GitHubPRFileContents } from '../../../src/shared/types' +import { + MOBILE_PR_FILE_CONTENT_CACHE_MAX_BYTES, + MOBILE_PR_FILE_CONTENT_CACHE_MAX_ENTRIES, + MobilePrFileContentCache, + createMobilePrFileContentKey, + createMobilePrFileContentScope, + getMobilePrFileContentByteCount, + getMobilePrFileContentsForScope +} from './mobile-pr-file-content-cache' +import { + createMobileItemPrFileContentScope, + createMobileProjectPrFileContentScope +} from './use-mobile-pr-file-content-cache' + +function contents(original: string, modified = ''): GitHubPRFileContents { + return { + original, + modified, + originalIsBinary: false, + modifiedIsBinary: false + } +} + +const scopeA = createMobilePrFileContentScope({ + source: 'item', + repoId: 'repo-1', + prNumber: 10, + headSha: 'head-a', + baseSha: 'base-a' +}) +const scopeB = createMobilePrFileContentScope({ + source: 'item', + repoId: 'repo-1', + prNumber: 11, + headSha: 'head-b', + baseSha: 'base-a' +}) + +describe('MobilePrFileContentCache', () => { + it('derives route scopes only from complete GitHub PR revisions', () => { + expect( + createMobileItemPrFileContentScope( + { provider: 'github', source: { type: 'pr', repoId: 'repo-1', number: 10 } }, + { provider: 'github', headSha: 'head-a', baseSha: 'base-a' } + ) + ).toBe(scopeA) + expect( + createMobileProjectPrFileContentScope( + { itemType: 'PULL_REQUEST', content: { number: 10 } }, + { id: 'repo-1' }, + { provider: 'github', headSha: 'head-a' } + ) + ).toBeNull() + expect( + createMobileProjectPrFileContentScope( + { itemType: 'PULL_REQUEST', content: { number: 10 } }, + { id: 'repo-1' }, + { provider: 'github', headSha: 'head-a', baseSha: 'base-a' }, + { host: 'github.example', owner: 'orca', repo: 'app' } + ) + ).toBe( + createMobilePrFileContentScope({ + source: 'project', + repoId: 'repo-1', + prNumber: 10, + headSha: 'head-a', + baseSha: 'base-a', + repository: { host: 'github.example', owner: 'orca', repo: 'app' } + }) + ) + }) + + it('uses the desktop diff budget as a hard byte cap with a smaller mobile entry cap', () => { + expect(MOBILE_PR_FILE_CONTENT_CACHE_MAX_ENTRIES).toBeLessThan(64) + expect(MOBILE_PR_FILE_CONTENT_CACHE_MAX_BYTES).toBe(24_000_000) + }) + + it('evicts least-recently-used files at the entry limit', () => { + const cache = new MobilePrFileContentCache(2, 100) + const a = createMobilePrFileContentKey({ path: 'a.ts' }) + const b = createMobilePrFileContentKey({ path: 'b.ts' }) + const c = createMobilePrFileContentKey({ path: 'c.ts' }) + cache.commitRequest(cache.beginRequest(scopeA, a), contents('a')) + cache.commitRequest(cache.beginRequest(scopeA, b), contents('b')) + expect(cache.select(scopeA, a).contents).toEqual(contents('a')) + + cache.commitRequest(cache.beginRequest(scopeA, c), contents('c')) + + expect(cache.evidence()).toMatchObject({ entryCount: 2, keysOldestFirst: [a, c] }) + expect(cache.select(scopeA, b).contents).toBeUndefined() + }) + + it('evicts older payloads to stay within the byte budget', () => { + const cache = new MobilePrFileContentCache(10, 8) + const a = createMobilePrFileContentKey({ path: 'a.ts' }) + const b = createMobilePrFileContentKey({ path: 'b.ts' }) + cache.commitRequest(cache.beginRequest(scopeA, a), contents('12345')) + cache.commitRequest(cache.beginRequest(scopeA, b), contents('67890')) + + expect(cache.evidence()).toMatchObject({ + entryCount: 1, + retainedBytes: 5, + keysOldestFirst: [b] + }) + }) + + it('measures retained UTF-8 bytes rather than JavaScript code units', () => { + expect(getMobilePrFileContentByteCount(contents('a😀', 'é'))).toBe(7) + }) + + it('counts the retained side when the other side is an oversized sentinel', () => { + expect( + getMobilePrFileContentByteCount({ + ...contents('', 'retained'), + originalTooLarge: true + }) + ).toBe(8) + }) + + it('rejects a response from a prior PR context after scope replacement', () => { + const cache = new MobilePrFileContentCache(2, 100) + const key = createMobilePrFileContentKey({ path: 'file.ts' }) + const oldRequest = cache.beginRequest(scopeA, key) + cache.activateScope(scopeB) + + expect(cache.commitRequest(oldRequest, contents('stale'))).toBe('stale') + expect(cache.evidence()).toMatchObject({ scope: scopeB, entryCount: 0, retainedBytes: 0 }) + }) + + it('rejects an older request after the active file selection changes', () => { + const cache = new MobilePrFileContentCache(2, 100) + const a = createMobilePrFileContentKey({ path: 'a.ts' }) + const b = createMobilePrFileContentKey({ path: 'b.ts' }) + const oldRequest = cache.beginRequest(scopeA, a) + cache.select(scopeA, b) + const currentRequest = cache.beginRequest(scopeA, b) + + expect(cache.commitRequest(oldRequest, contents('stale'))).toBe('stale') + expect(cache.commitRequest(currentRequest, contents('current'))).toBe('stored') + }) + + it('publishes only the active scope and refuses a single over-budget payload', () => { + const cache = new MobilePrFileContentCache(2, 4) + const key = createMobilePrFileContentKey({ path: 'file.ts' }) + expect(cache.commitRequest(cache.beginRequest(scopeA, key), contents('12345'))).toBe( + 'too-large' + ) + expect(getMobilePrFileContentsForScope(cache.snapshot(), scopeA)).toEqual({}) + expect(getMobilePrFileContentsForScope(cache.snapshot(), scopeB)).toEqual({}) + }) +}) diff --git a/mobile/src/tasks/mobile-pr-file-content-cache.ts b/mobile/src/tasks/mobile-pr-file-content-cache.ts new file mode 100644 index 00000000000..5dcb17a3912 --- /dev/null +++ b/mobile/src/tasks/mobile-pr-file-content-cache.ts @@ -0,0 +1,226 @@ +import { MAX_RENDERED_DIFF_COMBINED_CHARACTERS } from '../../../src/shared/large-diff-render-limit' +import type { GitHubPRFileContents, GitHubRepositoryIdentity } from '../../../src/shared/types' + +export const MOBILE_PR_FILE_CONTENT_CACHE_MAX_ENTRIES = 8 +export const MOBILE_PR_FILE_CONTENT_CACHE_MAX_BYTES = MAX_RENDERED_DIFF_COMBINED_CHARACTERS * 4 + +export type MobilePrFileContentScopeInput = { + source: 'item' | 'project' + repoId: string + prNumber: number + headSha: string + baseSha: string + repository?: GitHubRepositoryIdentity | null +} + +export type MobilePrFileContentKeyInput = { + path: string + oldPath?: string + status?: string +} + +export type MobilePrFileContentRequestToken = { + scope: string + key: string + requestId: number +} + +export type MobilePrFileContentCacheSnapshot = { + scope: string | null + contentsByKey: Readonly> +} + +export type MobilePrFileContentCacheEvidence = { + scope: string | null + entryCount: number + retainedBytes: number + keysOldestFirst: string[] +} + +type CacheEntry = { + contents: GitHubPRFileContents + byteCount: number +} + +type CacheSelection = { + contents: GitHubPRFileContents | undefined + scopeChanged: boolean +} + +export class MobilePrFileContentCache { + private readonly entries = new Map() + private scope: string | null = null + private retainedBytes = 0 + private requestSequence = 0 + private activeRequest: MobilePrFileContentRequestToken | null = null + + constructor( + private readonly maxEntries = MOBILE_PR_FILE_CONTENT_CACHE_MAX_ENTRIES, + private readonly maxBytes = MOBILE_PR_FILE_CONTENT_CACHE_MAX_BYTES + ) { + if ( + !Number.isInteger(maxEntries) || + maxEntries < 1 || + !Number.isFinite(maxBytes) || + maxBytes < 1 + ) { + throw new Error('Mobile PR file-content cache limits must be positive') + } + } + + activateScope(scope: string): boolean { + if (this.scope === scope) { + return false + } + this.scope = scope + this.entries.clear() + this.retainedBytes = 0 + this.activeRequest = null + return true + } + + clear(): void { + this.scope = null + this.entries.clear() + this.retainedBytes = 0 + this.activeRequest = null + } + + select(scope: string, key: string): CacheSelection { + const scopeChanged = this.activateScope(scope) + this.activeRequest = null + const entry = this.entries.get(key) + if (!entry) { + return { contents: undefined, scopeChanged } + } + this.entries.delete(key) + this.entries.set(key, entry) + return { contents: entry.contents, scopeChanged } + } + + beginRequest(scope: string, key: string): MobilePrFileContentRequestToken { + this.activateScope(scope) + const token = { scope, key, requestId: ++this.requestSequence } + this.activeRequest = token + return token + } + + commitRequest( + token: MobilePrFileContentRequestToken, + contents: GitHubPRFileContents + ): 'stored' | 'stale' | 'too-large' { + if (!this.isCurrentRequest(token)) { + return 'stale' + } + this.activeRequest = null + const byteCount = getMobilePrFileContentByteCount(contents) + if (byteCount > this.maxBytes) { + return 'too-large' + } + const previous = this.entries.get(token.key) + this.retainedBytes -= previous?.byteCount ?? 0 + this.entries.delete(token.key) + this.entries.set(token.key, { contents, byteCount }) + this.retainedBytes += byteCount + this.evictOverflow() + return 'stored' + } + + rejectRequest(token: MobilePrFileContentRequestToken): boolean { + if (!this.isCurrentRequest(token)) { + return false + } + this.activeRequest = null + return true + } + + snapshot(): MobilePrFileContentCacheSnapshot { + return { + scope: this.scope, + contentsByKey: Object.fromEntries( + [...this.entries].map(([key, entry]) => [key, entry.contents]) + ) + } + } + + evidence(): MobilePrFileContentCacheEvidence { + return { + scope: this.scope, + entryCount: this.entries.size, + retainedBytes: this.retainedBytes, + keysOldestFirst: [...this.entries.keys()] + } + } + + private isCurrentRequest(token: MobilePrFileContentRequestToken): boolean { + return ( + this.scope === token.scope && + this.activeRequest?.requestId === token.requestId && + this.activeRequest.scope === token.scope && + this.activeRequest.key === token.key + ) + } + + private evictOverflow(): void { + while (this.entries.size > this.maxEntries || this.retainedBytes > this.maxBytes) { + const oldestKey = this.entries.keys().next().value + if (typeof oldestKey !== 'string') { + return + } + const oldest = this.entries.get(oldestKey) + this.retainedBytes -= oldest?.byteCount ?? 0 + this.entries.delete(oldestKey) + } + } +} + +export function createMobilePrFileContentScope(input: MobilePrFileContentScopeInput): string { + return JSON.stringify([ + input.source, + input.repoId, + input.prNumber, + input.repository?.host?.toLowerCase() ?? '', + input.repository?.owner.toLowerCase() ?? '', + input.repository?.repo.toLowerCase() ?? '', + input.headSha, + input.baseSha + ]) +} + +export function createMobilePrFileContentKey(input: MobilePrFileContentKeyInput): string { + return input.path +} + +export function getMobilePrFileContentsForScope( + snapshot: MobilePrFileContentCacheSnapshot, + scope: string | null +): Readonly> { + return scope !== null && snapshot.scope === scope ? snapshot.contentsByKey : {} +} + +export function getMobilePrFileContentByteCount(contents: GitHubPRFileContents): number { + return getUtf8ByteCount(contents.original) + getUtf8ByteCount(contents.modified) +} + +function getUtf8ByteCount(value: string): number { + let byteCount = 0 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if (code < 0x80) { + byteCount += 1 + } else if (code < 0x800) { + byteCount += 2 + } else if (code >= 0xd800 && code <= 0xdbff && index + 1 < value.length) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + byteCount += 4 + index += 1 + } else { + byteCount += 3 + } + } else { + byteCount += 3 + } + } + return byteCount +} diff --git a/mobile/src/tasks/use-mobile-pr-file-content-cache.ts b/mobile/src/tasks/use-mobile-pr-file-content-cache.ts new file mode 100644 index 00000000000..d364324336b --- /dev/null +++ b/mobile/src/tasks/use-mobile-pr-file-content-cache.ts @@ -0,0 +1,183 @@ +import { useCallback, useEffect, useState } from 'react' +import type { GitHubPRFileContents, GitHubRepositoryIdentity } from '../../../src/shared/types' +import { + MobilePrFileContentCache, + createMobilePrFileContentKey, + createMobilePrFileContentScope, + getMobilePrFileContentsForScope, + type MobilePrFileContentKeyInput +} from './mobile-pr-file-content-cache' + +type MobilePrScopeTaskItem = { + provider: string + source: unknown +} | null + +type MobilePrScopeDetail = { + provider: string + headSha?: unknown + baseSha?: unknown +} | null + +type MobileProjectPrScopeItem = { + itemType: string + content: { number?: unknown } +} | null + +type MobileProjectPrScopeRepo = { id?: unknown } | null + +type MobilePrFileContentLoad = () => Promise +type MobilePrFileContentErrorSetter = (message: string) => void + +export function createMobileItemPrFileContentScope( + item: MobilePrScopeTaskItem, + detail: MobilePrScopeDetail +): string | null { + const source = + item?.source && typeof item.source === 'object' + ? (item.source as { type?: unknown; repoId?: unknown; number?: unknown }) + : null + if ( + item?.provider !== 'github' || + source?.type !== 'pr' || + typeof source.repoId !== 'string' || + typeof source.number !== 'number' || + detail?.provider !== 'github' || + typeof detail.headSha !== 'string' || + !detail.headSha || + typeof detail.baseSha !== 'string' || + !detail.baseSha + ) { + return null + } + return createMobilePrFileContentScope({ + source: 'item', + repoId: source.repoId, + prNumber: source.number, + headSha: detail.headSha, + baseSha: detail.baseSha + }) +} + +export function createMobileProjectPrFileContentScope( + item: MobileProjectPrScopeItem, + repo: MobileProjectPrScopeRepo, + detail: MobilePrScopeDetail, + repository?: GitHubRepositoryIdentity | null +): string | null { + if ( + item?.itemType !== 'PULL_REQUEST' || + typeof item.content.number !== 'number' || + typeof repo?.id !== 'string' || + detail?.provider !== 'github' || + typeof detail.headSha !== 'string' || + !detail.headSha || + typeof detail.baseSha !== 'string' || + !detail.baseSha + ) { + return null + } + return createMobilePrFileContentScope({ + source: 'project', + repoId: repo.id, + prNumber: item.content.number, + repository, + headSha: detail.headSha, + baseSha: detail.baseSha + }) +} + +export function useMobilePrFileContentCache(activeScope: string | null): { + clear: () => void + contents: Readonly> + load: ( + scope: string, + file: MobilePrFileContentKeyInput, + loadContents: MobilePrFileContentLoad, + setError: MobilePrFileContentErrorSetter + ) => Promise + loadingPath: string | null +} { + const [cache] = useState(() => new MobilePrFileContentCache()) + const [snapshot, setSnapshot] = useState(() => cache.snapshot()) + const [loadingPath, setLoadingPath] = useState(null) + const clear = useCallback(() => { + cache.clear() + setSnapshot(cache.snapshot()) + setLoadingPath(null) + }, [cache]) + + useEffect(() => { + if (activeScope === null) { + clear() + } else if (cache.activateScope(activeScope)) { + setSnapshot(cache.snapshot()) + setLoadingPath(null) + } + }, [activeScope, cache, clear]) + + const load = useCallback( + async ( + scope: string, + file: MobilePrFileContentKeyInput, + loadContents: MobilePrFileContentLoad, + setError: MobilePrFileContentErrorSetter + ): Promise => { + const key = createMobilePrFileContentKey(file) + const selection = cache.select(scope, key) + if (selection.scopeChanged) { + setSnapshot(cache.snapshot()) + } + if (selection.contents) { + setLoadingPath(null) + return + } + const token = cache.beginRequest(scope, key) + setLoadingPath(file.path) + setError('') + try { + const result = await loadContents() + if (!isGitHubPrFileContents(result)) { + throw new Error('Invalid file contents response') + } + const commit = cache.commitRequest(token, result) + if (commit === 'stale') { + return + } + if (commit === 'too-large') { + setError('File too large for mobile preview.') + } else { + setSnapshot(cache.snapshot()) + } + setLoadingPath((current) => (current === file.path ? null : current)) + } catch (error) { + if (!cache.rejectRequest(token)) { + return + } + setError(error instanceof Error ? error.message : 'Failed to load file contents') + setLoadingPath((current) => (current === file.path ? null : current)) + } + }, + [cache] + ) + + return { + clear, + contents: getMobilePrFileContentsForScope(snapshot, activeScope), + load, + loadingPath + } +} + +function isGitHubPrFileContents(value: unknown): value is GitHubPRFileContents { + if (!value || typeof value !== 'object') { + return false + } + const contents = value as Partial + return ( + typeof contents.original === 'string' && + typeof contents.modified === 'string' && + typeof contents.originalIsBinary === 'boolean' && + typeof contents.modifiedIsBinary === 'boolean' + ) +} diff --git a/mobile/src/terminal/custom-accessory-key-store.test.ts b/mobile/src/terminal/custom-accessory-key-store.test.ts new file mode 100644 index 00000000000..f479f366d47 --- /dev/null +++ b/mobile/src/terminal/custom-accessory-key-store.test.ts @@ -0,0 +1,68 @@ +import AsyncStorage from '@react-native-async-storage/async-storage' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS, + CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES, + CUSTOM_ACCESSORY_KEYS_MAX_STORAGE_CHARACTERS, + loadCustomKeys, + saveCustomKeys, + type CustomKey +} from './custom-accessory-key-store' + +vi.mock('@react-native-async-storage/async-storage', () => ({ + default: { + getItem: vi.fn(), + setItem: vi.fn() + } +})) + +function customKey(index: number, bytes = 'echo ok'): CustomKey { + return { id: `key-${index}`, label: `Key ${index}`, bytes, enter: false } +} + +describe('custom accessory key store', () => { + beforeEach(() => { + vi.mocked(AsyncStorage.getItem).mockReset() + vi.mocked(AsyncStorage.setItem).mockReset().mockResolvedValue(undefined) + }) + + it('round-trips normal custom keys unchanged', async () => { + const keys = [customKey(1), customKey(2)] + vi.mocked(AsyncStorage.getItem).mockResolvedValue(JSON.stringify(keys)) + + await expect(loadCustomKeys()).resolves.toEqual(keys) + await expect(saveCustomKeys(keys)).resolves.toEqual(keys) + expect(AsyncStorage.setItem).toHaveBeenCalledWith( + 'orca:custom-accessory-keys', + JSON.stringify(keys) + ) + }) + + it('accepts the exact key count and evicts oldest at one over', async () => { + const exact = Array.from({ length: CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES }, (_, index) => + customKey(index) + ) + await expect(saveCustomKeys(exact)).resolves.toEqual(exact) + + const oneOver = [...exact, customKey(CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES)] + const retained = await saveCustomKeys(oneOver) + expect(retained).toHaveLength(CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES) + expect(retained[0]?.id).toBe('key-1') + expect(retained.at(-1)?.id).toBe(`key-${CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES}`) + }) + + it('accepts the exact macro length and drops one character over', async () => { + const exact = customKey(1, 'x'.repeat(CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS)) + const oversized = customKey(2, 'x'.repeat(CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS + 1)) + + await expect(saveCustomKeys([exact, oversized])).resolves.toEqual([exact]) + }) + + it('rejects durable JSON over the storage character budget before parsing', async () => { + vi.mocked(AsyncStorage.getItem).mockResolvedValue( + 'x'.repeat(CUSTOM_ACCESSORY_KEYS_MAX_STORAGE_CHARACTERS + 1) + ) + + await expect(loadCustomKeys()).resolves.toEqual([]) + }) +}) diff --git a/mobile/src/terminal/custom-accessory-key-store.ts b/mobile/src/terminal/custom-accessory-key-store.ts new file mode 100644 index 00000000000..e7bde17f950 --- /dev/null +++ b/mobile/src/terminal/custom-accessory-key-store.ts @@ -0,0 +1,81 @@ +import AsyncStorage from '@react-native-async-storage/async-storage' + +export const CUSTOM_ACCESSORY_KEYS_STORAGE_KEY = 'orca:custom-accessory-keys' +export const CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES = 128 +export const CUSTOM_ACCESSORY_KEYS_MAX_STORAGE_CHARACTERS = 512 * 1024 +export const CUSTOM_ACCESSORY_KEY_MAX_ID_CHARACTERS = 256 +export const CUSTOM_ACCESSORY_KEY_MAX_LABEL_CHARACTERS = 128 +export const CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS = 16 * 1024 + +export type CustomKey = { + id: string + label: string + bytes: string + enter: boolean +} + +export async function loadCustomKeys(): Promise { + try { + const raw = await AsyncStorage.getItem(CUSTOM_ACCESSORY_KEYS_STORAGE_KEY) + if (!raw || raw.length > CUSTOM_ACCESSORY_KEYS_MAX_STORAGE_CHARACTERS) { + return [] + } + return retainCustomKeys(JSON.parse(raw) as unknown).keys + } catch { + return [] + } +} + +export async function saveCustomKeys(keys: CustomKey[]): Promise { + const retained = retainCustomKeys(keys) + await AsyncStorage.setItem(CUSTOM_ACCESSORY_KEYS_STORAGE_KEY, retained.serialized) + return retained.keys +} + +function retainCustomKeys(value: unknown): { keys: CustomKey[]; serialized: string } { + if (!Array.isArray(value)) { + return { keys: [], serialized: '[]' } + } + const retained: Array<{ key: CustomKey; serialized: string }> = [] + let storageCharacters = 2 + for (let index = value.length - 1; index >= 0; index -= 1) { + if (retained.length >= CUSTOM_ACCESSORY_KEYS_MAX_ENTRIES) { + break + } + const key = normalizeCustomKey(value[index]) + if (!key) { + continue + } + const serialized = JSON.stringify(key) + const nextCharacters = storageCharacters + serialized.length + (retained.length > 0 ? 1 : 0) + if (nextCharacters > CUSTOM_ACCESSORY_KEYS_MAX_STORAGE_CHARACTERS) { + continue + } + retained.push({ key, serialized }) + storageCharacters = nextCharacters + } + retained.reverse() + return { + keys: retained.map(({ key }) => key), + serialized: `[${retained.map(({ serialized }) => serialized).join(',')}]` + } +} + +function normalizeCustomKey(value: unknown): CustomKey | null { + if (!value || typeof value !== 'object') { + return null + } + const key = value as Partial + if ( + typeof key.id !== 'string' || + key.id.length > CUSTOM_ACCESSORY_KEY_MAX_ID_CHARACTERS || + typeof key.label !== 'string' || + key.label.length > CUSTOM_ACCESSORY_KEY_MAX_LABEL_CHARACTERS || + typeof key.bytes !== 'string' || + key.bytes.length > CUSTOM_ACCESSORY_KEY_MAX_BYTES_CHARACTERS || + typeof key.enter !== 'boolean' + ) { + return null + } + return { id: key.id, label: key.label, bytes: key.bytes, enter: key.enter } +} diff --git a/mobile/src/terminal/terminal-accessory-layout.test.ts b/mobile/src/terminal/terminal-accessory-layout.test.ts index 1a4135494ea..692267214be 100644 --- a/mobile/src/terminal/terminal-accessory-layout.test.ts +++ b/mobile/src/terminal/terminal-accessory-layout.test.ts @@ -1,6 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { + TERMINAL_ACCESSORY_LAYOUT_MAX_IDS, + TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS, TERMINAL_ACCESSORY_LAYOUT_STORAGE_KEY, createTerminalAccessoryLayoutPreference, getDefaultTerminalAccessoryBuiltInIds, @@ -83,6 +85,61 @@ describe('terminal accessory layout', () => { ) }) + it('accepts the exact storage limit and skips parsing one character more', async () => { + const preference = { + version: 2, + orderedBuiltInIds: getDefaultTerminalAccessoryBuiltInIds(), + visibleBuiltInIds: ['escape'] + } + const serialized = JSON.stringify(preference) + asyncStorageMock.getItem.mockResolvedValueOnce( + serialized + ' '.repeat(TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS - serialized.length) + ) + await expect(loadTerminalAccessoryLayout()).resolves.toMatchObject({ + visibleBuiltInIds: ['escape'] + }) + + const parse = vi.spyOn(JSON, 'parse') + asyncStorageMock.getItem.mockResolvedValueOnce( + 'x'.repeat(TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS + 1) + ) + await expect(loadTerminalAccessoryLayout()).resolves.toEqual( + createTerminalAccessoryLayoutPreference(getDefaultTerminalAccessoryLayout()) + ) + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + }) + + it('accepts the exact stored id count and falls back on one more', () => { + const exact = Array.from({ length: TERMINAL_ACCESSORY_LAYOUT_MAX_IDS }, (_, index) => + index === 0 ? 'tab' : 'escape' + ) + expect( + normalizeTerminalAccessoryLayoutPreference( + { version: 2, orderedBuiltInIds: exact, visibleBuiltInIds: ['escape'] }, + ['escape', 'tab'] + ) + ).toEqual({ + version: 2, + orderedBuiltInIds: ['tab', 'escape'], + visibleBuiltInIds: ['escape'] + }) + expect( + normalizeTerminalAccessoryLayoutPreference( + { + version: 2, + orderedBuiltInIds: [...exact, 'tab'], + visibleBuiltInIds: ['escape'] + }, + ['escape', 'tab'] + ) + ).toEqual({ + version: 2, + orderedBuiltInIds: ['escape', 'tab'], + visibleBuiltInIds: ['escape', 'tab'] + }) + }) + it('preserves a custom v2 order and its visible subset', () => { const reversed = [...getDefaultTerminalAccessoryBuiltInIds()].toReversed() diff --git a/mobile/src/terminal/terminal-accessory-layout.ts b/mobile/src/terminal/terminal-accessory-layout.ts index cee7a74173e..7dc3dc70e00 100644 --- a/mobile/src/terminal/terminal-accessory-layout.ts +++ b/mobile/src/terminal/terminal-accessory-layout.ts @@ -3,6 +3,9 @@ import AsyncStorage from '@react-native-async-storage/async-storage' import { TERMINAL_ACCESSORY_KEYS, type TerminalAccessoryKey } from './terminal-accessory-keys' export const TERMINAL_ACCESSORY_LAYOUT_STORAGE_KEY = 'orca:terminal-accessory-layout' +export const TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS = 64 * 1024 +export const TERMINAL_ACCESSORY_LAYOUT_MAX_IDS = 256 +export const TERMINAL_ACCESSORY_LAYOUT_MAX_ID_CHARACTERS = 256 export type TerminalAccessoryLayout = { orderedBuiltInIds: string[] @@ -26,17 +29,28 @@ function defaultPreference(ids = builtInIds()): TerminalAccessoryLayoutPreferenc } function stringArray(value: unknown): string[] | null { - if (!Array.isArray(value)) { + if (!Array.isArray(value) || value.length > TERMINAL_ACCESSORY_LAYOUT_MAX_IDS) { return null } - return value.every((item): item is string => typeof item === 'string') ? value : null + return value.every( + (item): item is string => + typeof item === 'string' && item.length <= TERMINAL_ACCESSORY_LAYOUT_MAX_ID_CHARACTERS + ) + ? value + : null } function dedupeKnownIds(ids: string[], builtInSet: Set): string[] { const seen = new Set() const out: string[] = [] - for (const id of ids) { - if (!builtInSet.has(id) || seen.has(id)) { + const count = Math.min(ids.length, TERMINAL_ACCESSORY_LAYOUT_MAX_IDS) + for (let index = 0; index < count; index++) { + const id = ids[index]! + if ( + id.length > TERMINAL_ACCESSORY_LAYOUT_MAX_ID_CHARACTERS || + !builtInSet.has(id) || + seen.has(id) + ) { continue } seen.add(id) @@ -218,6 +232,9 @@ export async function loadTerminalAccessoryLayout(): Promise TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS) { + return defaultPreference() + } return normalizeTerminalAccessoryLayoutPreference(JSON.parse(raw)) } catch { return defaultPreference() @@ -226,5 +243,9 @@ export async function loadTerminalAccessoryLayout(): Promise { const preference = createTerminalAccessoryLayoutPreference(layout) - await AsyncStorage.setItem(TERMINAL_ACCESSORY_LAYOUT_STORAGE_KEY, JSON.stringify(preference)) + const serialized = JSON.stringify(preference) + if (serialized.length > TERMINAL_ACCESSORY_LAYOUT_MAX_STORAGE_CHARACTERS) { + throw new Error('terminal accessory layout exceeds storage limit') + } + await AsyncStorage.setItem(TERMINAL_ACCESSORY_LAYOUT_STORAGE_KEY, serialized) } diff --git a/mobile/src/terminal/terminal-webview-default-theme.ts b/mobile/src/terminal/terminal-webview-default-theme.ts new file mode 100644 index 00000000000..0769f4283ed --- /dev/null +++ b/mobile/src/terminal/terminal-webview-default-theme.ts @@ -0,0 +1,27 @@ +import type { RuntimeMobileTerminalTheme } from '../../../src/shared/runtime-types' +import { colors } from '../theme/mobile-theme' + +export const DEFAULT_TERMINAL_WEBVIEW_THEME: RuntimeMobileTerminalTheme['theme'] = { + background: colors.terminalBg, + foreground: '#c0caf5', + cursor: '#c0caf5', + cursorAccent: colors.terminalBg, + selectionBackground: '#33467c', + selectionForeground: '#c0caf5', + black: '#15161e', + red: '#f7768e', + green: '#9ece6a', + yellow: '#e0af68', + blue: '#7aa2f7', + magenta: '#bb9af7', + cyan: '#7dcfff', + white: '#a9b1d6', + brightBlack: '#414868', + brightRed: '#f7768e', + brightGreen: '#9ece6a', + brightYellow: '#e0af68', + brightBlue: '#7aa2f7', + brightMagenta: '#bb9af7', + brightCyan: '#7dcfff', + brightWhite: '#c0caf5' +} diff --git a/mobile/src/terminal/terminal-webview-html.ts b/mobile/src/terminal/terminal-webview-html.ts index a4d9c350c47..9d66610dd0e 100644 --- a/mobile/src/terminal/terminal-webview-html.ts +++ b/mobile/src/terminal/terminal-webview-html.ts @@ -1,7 +1,7 @@ -// xterm.js WebView document + default Tokyonight theme; extracted from TerminalWebView.tsx for the max-lines budget. -import type { RuntimeMobileTerminalTheme } from '../../../src/shared/runtime-types' +// xterm.js WebView document; extracted from TerminalWebView.tsx for the max-lines budget. import { colors } from '../theme/mobile-theme' import { TERMINAL_TEXT_SCALES } from '../storage/preferences' +import { DEFAULT_TERMINAL_WEBVIEW_THEME } from './terminal-webview-default-theme' import { TERMINAL_PATH_TAP_JS } from './terminal-path-tap-injected' import { XTERM_ENGINE_CSS, XTERM_ENGINE_JS } from './terminal-webview-engine.generated' import { TERMINAL_REFLOW_JS } from './terminal-webview-reflow-injected' @@ -12,30 +12,9 @@ import { TERMINAL_QUERY_REPLY_JS } from './terminal-webview-query-reply-injected import { URL_TAP_WEBVIEW_JS } from './terminal-webview-url-tap' import { TERMINAL_WEBGL_RECOVERY_JS } from './terminal-webview-webgl-recovery-injected' -const DEFAULT_TERMINAL_THEME: RuntimeMobileTerminalTheme['theme'] = { - background: colors.terminalBg, - foreground: '#c0caf5', - cursor: '#c0caf5', - cursorAccent: colors.terminalBg, - selectionBackground: '#33467c', - selectionForeground: '#c0caf5', - black: '#15161e', - red: '#f7768e', - green: '#9ece6a', - yellow: '#e0af68', - blue: '#7aa2f7', - magenta: '#bb9af7', - cyan: '#7dcfff', - white: '#a9b1d6', - brightBlack: '#414868', - brightRed: '#f7768e', - brightGreen: '#9ece6a', - brightYellow: '#e0af68', - brightBlue: '#7aa2f7', - brightMagenta: '#bb9af7', - brightCyan: '#7dcfff', - brightWhite: '#c0caf5' -} +export const TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_UNITS = 1_000_000 +export const TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_ENTRIES = 4_096 +export const TERMINAL_WEBVIEW_AFTER_DRAIN_MAX_CALLBACKS = 256 // Why: TUI escape codes assume the desktop's cols/rows, so init xterm at those dims and fit the phone via a measured CSS scale() instead of resizing. export const XTERM_HTML = ` @@ -213,10 +192,11 @@ window.onerror = function(msg) { var scrollIndicator = document.getElementById('scroll-indicator'); var scrollThumb = document.getElementById('scroll-thumb'); var scrollIndicatorHideTimer = null; - var writeQueue = []; + var writeQueue = [], writeQueueUnits = 0; var writeQueueHead = 0; - var writesDraining = false; - var afterDrainCallbacks = []; + var writesDraining = false, writeBacklogFailed = false, afterDrainCallbacks = []; + var WRITE_QUEUE_MAX_UNITS = ${TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_UNITS}, WRITE_QUEUE_MAX_ENTRIES = ${TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_ENTRIES}; + var AFTER_DRAIN_MAX_CALLBACKS = ${TERMINAL_WEBVIEW_AFTER_DRAIN_MAX_CALLBACKS}; var termObserverDisposables = []; var ready = false; // Why: init() flips ready false on every re-init (live width reflow included) @@ -287,9 +267,10 @@ window.onerror = function(msg) { var normalScrollFrameId = null; var initRows = 24; var terminalGeneration = 0; - var defaultTheme = ${JSON.stringify(DEFAULT_TERMINAL_THEME)}; + var defaultTheme = ${JSON.stringify(DEFAULT_TERMINAL_WEBVIEW_THEME)}; var terminalThemeInput = null; var terminalTheme = defaultTheme; + var terminalMinimumContrastRatio = 3; var webglAddon = null; var webglRecoveryTimer = null; var activeAltScreenSnapshot = false; @@ -556,9 +537,22 @@ ${TERMINAL_WEBVIEW_THEME_JS} } } - function resetWriteQueue() { - writeQueue = []; - writeQueueHead = 0; + function resetWriteQueue() { writeQueue = []; writeQueueHead = 0; writeQueueUnits = 0; } + + function failWriteBacklog() { + if (writeBacklogFailed) return; + writeBacklogFailed = true; terminalGeneration++; ready = false; writesDraining = false; + resetWriteQueue(); afterDrainCallbacks = []; + reportEngineError('terminal write backlog exceeded safe limit', null, true); + } + + function reserveWriteQueueEntry(units) { + if (writeBacklogFailed) return false; + var pendingEntries = writeQueue.length - writeQueueHead; + if (pendingEntries >= WRITE_QUEUE_MAX_ENTRIES || writeQueueUnits + units > WRITE_QUEUE_MAX_UNITS) { + failWriteBacklog(); return false; + } + writeQueueUnits += units; return true; } function isStatusDotPresentationSelector(value) { @@ -589,13 +583,12 @@ ${TERMINAL_WEBVIEW_THEME_JS} return normalized; } - function enqueueWrite(data) { - writeQueue.push(normalizeStatusDotPresentation(data)); + function enqueueWrite(data) { var normalized = normalizeStatusDotPresentation(data); + if (!reserveWriteQueueEntry(normalized.length)) return false; + writeQueue.push(normalized); return true; } - function enqueueWriteBoundary(callback) { - writeQueue.push(callback); - } + function enqueueWriteBoundary(callback) { if (!reserveWriteQueueEntry(0)) return false; writeQueue.push(callback); return true; } function nextQueuedWrite() { if (writeQueueHead >= writeQueue.length) { @@ -604,6 +597,7 @@ ${TERMINAL_WEBVIEW_THEME_JS} } var next = writeQueue[writeQueueHead]; writeQueueHead++; + if (typeof next === 'string') writeQueueUnits = Math.max(0, writeQueueUnits - next.length); // Why: high-throughput terminals can enqueue faster than xterm parses; // compact consumed slots so drain work stays O(1) without retaining old chunks. if (writeQueueHead > 128 && writeQueueHead * 2 > writeQueue.length) { @@ -659,8 +653,8 @@ ${TERMINAL_WEBVIEW_THEME_JS} } function afterWritesDrained(callback) { - afterDrainCallbacks.push(callback); - pumpWrites(terminalGeneration); + if (afterDrainCallbacks.length >= AFTER_DRAIN_MAX_CALLBACKS) { failWriteBacklog(); return; } + afterDrainCallbacks.push(callback); pumpWrites(terminalGeneration); } ${TERMINAL_WEBGL_RECOVERY_JS} @@ -681,6 +675,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} webglAddon = null; ready = false; resetWriteQueue(); + writeBacklogFailed = false; statusDotPendingSelector = false; writesDraining = false; afterDrainCallbacks = []; @@ -714,6 +709,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} cols: cols || 80, rows: rows || 24, theme: terminalTheme, + minimumContrastRatio: terminalMinimumContrastRatio, fontFamily: terminalFontFamily, fontSize: fontPxForScale(currentTextScale), fontWeight: '300', @@ -766,7 +762,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} function write(data) { updateMouseModeFromData(data); - enqueueWrite(data); + if (!enqueueWrite(data)) return; pumpWrites(terminalGeneration); // Why: first live data chunk after init may widen the buffer past // what the post-replay applyFitScale measured. Re-fit once after this @@ -930,6 +926,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} } else if (msg.type === 'clear') { terminalGeneration++; resetWriteQueue(); resumeTerminalDataReplyAuthority(); // Why: clear drops the replay boundary. + writeBacklogFailed = false; statusDotPendingSelector = false; afterDrainCallbacks = []; writesDraining = false; diff --git a/mobile/src/terminal/terminal-webview-pending-messages.test.ts b/mobile/src/terminal/terminal-webview-pending-messages.test.ts new file mode 100644 index 00000000000..d7bcc5b6c29 --- /dev/null +++ b/mobile/src/terminal/terminal-webview-pending-messages.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import type { TerminalWebViewCommand } from './terminal-webview-messages' +import { + createTerminalWebViewPendingMessages, + MAX_PENDING_WEB_MESSAGES +} from './terminal-webview-pending-messages' + +function flushPending(queue: ReturnType) { + const delivered: TerminalWebViewCommand[] = [] + queue.flush((message) => delivered.push(message)) + return delivered +} + +describe('terminal WebView pending messages', () => { + it('retains only the latest snapshot when init is superseded before readiness', () => { + const queue = createTerminalWebViewPendingMessages() + queue.queue({ type: 'write', data: 'current-document-tail' }) + queue.queue({ type: 'init', cols: 80, rows: 24, initialData: 'old snapshot' }) + queue.queue({ type: 'write', data: 'covered by the replacement snapshot' }) + queue.queue({ type: 'init', cols: 100, rows: 30, initialData: 'new snapshot' }) + + expect(flushPending(queue)).toEqual([ + { type: 'write', data: 'current-document-tail' }, + { type: 'init', cols: 100, rows: 30, initialData: 'new snapshot' } + ]) + }) + + it('caps tiny control-message floods while preserving the newest state', () => { + const queue = createTerminalWebViewPendingMessages() + for (let index = 0; index < MAX_PENDING_WEB_MESSAGES + 100; index += 1) { + queue.queue({ type: 'resize', cols: index + 1, rows: 24 }) + } + + const delivered = flushPending(queue) + expect(delivered).toHaveLength(MAX_PENDING_WEB_MESSAGES) + expect(delivered.at(-1)).toEqual({ + type: 'resize', + cols: MAX_PENDING_WEB_MESSAGES + 100, + rows: 24 + }) + }) +}) diff --git a/mobile/src/terminal/terminal-webview-pending-messages.ts b/mobile/src/terminal/terminal-webview-pending-messages.ts index 5aa49e2e94f..1c3423b5cc8 100644 --- a/mobile/src/terminal/terminal-webview-pending-messages.ts +++ b/mobile/src/terminal/terminal-webview-pending-messages.ts @@ -2,6 +2,7 @@ import type { TerminalWebViewCommand } from './terminal-webview-messages' const MAX_PENDING_WEB_WRITE_BYTES = 1_000_000 const MAX_PENDING_WEB_WRITE_MESSAGES = 4096 +export const MAX_PENDING_WEB_MESSAGES = 8192 export function createTerminalWebViewPendingMessages() { let pending: TerminalWebViewCommand[] = [] @@ -18,9 +19,42 @@ export function createTerminalWebViewPendingMessages() { resetCounters() } + const removeAt = (index: number) => { + const [removed] = pending.splice(index, 1) + if (removed?.type === 'write') { + pendingWriteBytes = Math.max(0, pendingWriteBytes - removed.data.length) + pendingWriteCount = Math.max(0, pendingWriteCount - 1) + } + } + + const supersedePendingInit = (msg: Extract) => { + const existingIndex = pending.findIndex((candidate) => candidate.type === 'init') + if (existingIndex === -1) { + return false + } + while (pending.length > existingIndex) { + removeAt(pending.length - 1) + } + pending.push(msg) + return true + } + + const trimMessageCount = () => { + while (pending.length > MAX_PENDING_WEB_MESSAGES) { + const controlIndex = pending.findIndex( + (candidate) => candidate.type !== 'write' && candidate.type !== 'init' + ) + removeAt(Math.max(controlIndex, 0)) + } + } + const queue = (msg: TerminalWebViewCommand) => { + if (msg.type === 'init' && supersedePendingInit(msg)) { + return + } pending.push(msg) if (msg.type !== 'write') { + trimMessageCount() return } @@ -35,12 +69,9 @@ export function createTerminalWebViewPendingMessages() { resetCounters() return } - const [dropped] = pending.splice(dropIndex, 1) - if (dropped?.type === 'write') { - pendingWriteBytes = Math.max(0, pendingWriteBytes - dropped.data.length) - pendingWriteCount = Math.max(0, pendingWriteCount - 1) - } + removeAt(dropIndex) } + trimMessageCount() } const flush = (send: (msg: TerminalWebViewCommand) => void) => { diff --git a/mobile/src/terminal/terminal-webview-scroll-routing.test.ts b/mobile/src/terminal/terminal-webview-scroll-routing.test.ts index 391ccd3a840..ddd1f57cb8e 100644 --- a/mobile/src/terminal/terminal-webview-scroll-routing.test.ts +++ b/mobile/src/terminal/terminal-webview-scroll-routing.test.ts @@ -117,6 +117,16 @@ describe('TerminalWebView scroll routing', () => { expect(source).not.toContain('writeQueue.shift()') }) + it('fails closed when the in-WebView parser backlog reaches its exact bounds', () => { + expect(source).toContain('export const TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_UNITS = 1_000_000') + expect(source).toContain('export const TERMINAL_WEBVIEW_WRITE_QUEUE_MAX_ENTRIES = 4_096') + expect(source).toContain('export const TERMINAL_WEBVIEW_AFTER_DRAIN_MAX_CALLBACKS = 256') + expect(source).toContain('writeQueueUnits + units > WRITE_QUEUE_MAX_UNITS') + expect(source).toContain('pendingEntries >= WRITE_QUEUE_MAX_ENTRIES') + expect(source).toContain('afterDrainCallbacks.length >= AFTER_DRAIN_MAX_CALLBACKS') + expect(source).toContain("reportEngineError('terminal write backlog exceeded safe limit'") + }) + it('bounds native-side pending WebView writes while preserving control messages', () => { expect(source).toContain('const MAX_PENDING_WEB_WRITE_BYTES = 1_000_000') expect(source).toContain('const MAX_PENDING_WEB_WRITE_MESSAGES = 4096') diff --git a/mobile/src/terminal/terminal-webview-text-zoom.test.ts b/mobile/src/terminal/terminal-webview-text-zoom.test.ts index 09aeb82d3f7..1f6ba4fac01 100644 --- a/mobile/src/terminal/terminal-webview-text-zoom.test.ts +++ b/mobile/src/terminal/terminal-webview-text-zoom.test.ts @@ -91,7 +91,8 @@ describe('TerminalWebView text zoom', () => { expect(terminalHtmlSource).toContain( 'data.replace(CLAUDE_STATUS_DOT_PATTERN, CLAUDE_STATUS_DOT + TEXT_PRESENTATION_SELECTOR)' ) - expect(terminalHtmlSource).toContain('writeQueue.push(normalizeStatusDotPresentation(data))') + expect(terminalHtmlSource).toContain('var normalized = normalizeStatusDotPresentation(data)') + expect(terminalHtmlSource).toContain('writeQueue.push(normalized)') }) it('normalizes Claude status dots idempotently across write chunks', () => { diff --git a/mobile/src/terminal/terminal-webview-theme-injected.test.ts b/mobile/src/terminal/terminal-webview-theme-injected.test.ts new file mode 100644 index 00000000000..92e4127b2fc --- /dev/null +++ b/mobile/src/terminal/terminal-webview-theme-injected.test.ts @@ -0,0 +1,79 @@ +import { Script } from 'node:vm' +import { parse } from 'acorn' +import { describe, expect, it } from 'vitest' +import { TERMINAL_WEBVIEW_THEME_JS } from './terminal-webview-theme-injected' + +const DARK_FLOOR = 3 +const LIGHT_FLOOR = 4.5 + +// Eval the injected theme JS in a bare context so the declared helpers become +// callable properties on it (mirrors terminal-webview-engine.test.ts). +function loadThemeInjected(extra: Record = {}): Record { + const context: Record = { + defaultTheme: { background: '#1a1b26', foreground: '#c0caf5' }, + ...extra + } + new Script(TERMINAL_WEBVIEW_THEME_JS).runInNewContext(context) + return context +} + +describe('mobile terminal-webview contrast floor gate', () => { + it('parses at the Chrome 74 syntax floor', () => { + expect(() => parse(TERMINAL_WEBVIEW_THEME_JS, { ecmaVersion: 2019 })).not.toThrow() + }) + + it('picks the dark floor for dark composed backgrounds', () => { + const { resolveTerminalContrastFloor } = loadThemeInjected() as { + resolveTerminalContrastFloor: (bg: unknown) => number + } + for (const bg of ['#1a1b26', '#1e242a', '#282828', '#000000', 'black']) { + expect(resolveTerminalContrastFloor(bg)).toBe(DARK_FLOOR) + } + }) + + it('picks the light floor for light composed backgrounds', () => { + const { resolveTerminalContrastFloor } = loadThemeInjected() as { + resolveTerminalContrastFloor: (bg: unknown) => number + } + for (const bg of ['#ffffff', '#fbf1c7', 'white', 'rgb(240 240 240)']) { + expect(resolveTerminalContrastFloor(bg)).toBe(LIGHT_FLOOR) + } + }) + + it('composites transparency over the dark app surface before deciding', () => { + const { resolveTerminalContrastFloor } = loadThemeInjected() as { + resolveTerminalContrastFloor: (bg: unknown) => number + } + // Fully transparent → app surface (dark) → dark floor. + expect(resolveTerminalContrastFloor('transparent')).toBe(DARK_FLOOR) + // Faint white over the dark surface stays dark; opaque-enough white flips light. + expect(resolveTerminalContrastFloor('rgba(255,255,255,0.15)')).toBe(DARK_FLOOR) + expect(resolveTerminalContrastFloor('rgba(255,255,255,0.9)')).toBe(LIGHT_FLOOR) + }) + + it('defaults unparseable backgrounds to the dark floor so output never stays invisible', () => { + const { resolveTerminalContrastFloor } = loadThemeInjected() as { + resolveTerminalContrastFloor: (bg: unknown) => number + } + for (const bg of [undefined, null, '', 'not-a-color', '#12', 42]) { + expect(resolveTerminalContrastFloor(bg)).toBe(DARK_FLOOR) + } + }) + + it('writes the resolved floor onto a live terminal when the theme changes', () => { + const term = { options: { theme: undefined as unknown, minimumContrastRatio: 1 } } + const context = loadThemeInjected({ + term, + document: { + documentElement: { style: { background: '' } }, + body: { style: { background: '' } } + } + }) as Record & { applyTerminalTheme: (input: unknown) => void } + + context.applyTerminalTheme({ theme: { background: '#ffffff' } }) + expect(term.options.minimumContrastRatio).toBe(LIGHT_FLOOR) + + context.applyTerminalTheme({ theme: { background: '#1e242a' } }) + expect(term.options.minimumContrastRatio).toBe(DARK_FLOOR) + }) +}) diff --git a/mobile/src/terminal/terminal-webview-theme-injected.ts b/mobile/src/terminal/terminal-webview-theme-injected.ts index 1f798fe337e..c2d9beb4786 100644 --- a/mobile/src/terminal/terminal-webview-theme-injected.ts +++ b/mobile/src/terminal/terminal-webview-theme-injected.ts @@ -1,7 +1,85 @@ import { colors } from '../theme/mobile-theme' // Theme normalization and page-surface painting injected into the WebView IIFE. +// Mirrors the desktop minimumContrastRatio gate (src/renderer/src/lib/terminal-contrast-correction.ts, +// #7934/#10104): a dark composed background gets a mild floor of 3 to rescue near-background body text +// (e.g. Antigravity's #262b30 on #1e242a) without over-brightening vibrant ANSI colors; a light +// background keeps the WCAG-AA 4.5 floor. Gate on the composed background luminance, not app mode, +// because either theme slot can hold either kind of theme. export const TERMINAL_WEBVIEW_THEME_JS = ` + var DARK_BG_MIN_CONTRAST = 3; + var LIGHT_BG_MIN_CONTRAST = 4.5; + // Dark app surface a transparent terminal background composites over (matches desktop APP_SURFACE_COLORS.dark). + var CONTRAST_APP_SURFACE = { r: 10, g: 10, b: 10 }; + + function parseTerminalBackgroundRgba(value) { + if (typeof value !== 'string') return null; + var v = value.trim().toLowerCase(); + if (!v) return null; + if (v === 'black') return { r: 0, g: 0, b: 0, a: 1 }; + if (v === 'white') return { r: 255, g: 255, b: 255, a: 1 }; + if (v === 'transparent') return { r: 0, g: 0, b: 0, a: 0 }; + var hex = v.match(/^#([0-9a-f]{3,4}|[0-9a-f]{6}|[0-9a-f]{8})$/); + if (hex) { + var h = hex[1]; + var ch; + if (h.length === 3 || h.length === 4) { + ch = h.split('').map(function (p) { return parseInt(p + p, 16); }); + } else { + ch = []; + for (var i = 0; i < h.length; i += 2) ch.push(parseInt(h.slice(i, i + 2), 16)); + } + return { r: ch[0], g: ch[1], b: ch[2], a: ch[3] === undefined ? 1 : ch[3] / 255 }; + } + var rgb = v.match(/^rgba?\\(([^)]+)\\)$/); + if (!rgb) return null; + var parts = rgb[1].indexOf(',') >= 0 ? rgb[1].split(',') : rgb[1].split(/[\\s/]+/); + parts = parts.map(function (p) { return p.trim(); }).filter(function (p) { return p.length > 0; }); + if (parts.length < 3) return null; + var channel = function (p) { + var n = p.charAt(p.length - 1) === '%' ? (parseFloat(p) / 100) * 255 : parseFloat(p); + return isFinite(n) ? Math.min(255, Math.max(0, Math.round(n))) : null; + }; + var r = channel(parts[0]), g = channel(parts[1]), b = channel(parts[2]); + if (r === null || g === null || b === null) return null; + var a = 1; + if (parts[3] !== undefined) { + var raw = parts[3].charAt(parts[3].length - 1) === '%' ? parseFloat(parts[3]) / 100 : parseFloat(parts[3]); + a = isFinite(raw) ? Math.min(1, Math.max(0, raw)) : 1; + } + return { r: r, g: g, b: b, a: a }; + } + + function terminalRelativeLuminance(rgb) { + var lin = function (c) { + var n = c / 255; + return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4); + }; + return 0.2126 * lin(rgb.r) + 0.7152 * lin(rgb.g) + 0.0722 * lin(rgb.b); + } + + function terminalContrastRatio(a, b) { + var la = terminalRelativeLuminance(a), lb = terminalRelativeLuminance(b); + return (Math.max(la, lb) + 0.05) / (Math.min(la, lb) + 0.05); + } + + // Pick the xterm minimumContrastRatio floor from the composed terminal background. + // Unparseable input defaults to the dark floor so agent output never stays invisible. + function resolveTerminalContrastFloor(background) { + var color = parseTerminalBackgroundRgba(background); + if (!color) return DARK_BG_MIN_CONTRAST; + var composited = color.a < 1 + ? { + r: Math.round(color.r * color.a + CONTRAST_APP_SURFACE.r * (1 - color.a)), + g: Math.round(color.g * color.a + CONTRAST_APP_SURFACE.g * (1 - color.a)), + b: Math.round(color.b * color.a + CONTRAST_APP_SURFACE.b * (1 - color.a)) + } + : color; + var isLight = terminalContrastRatio({ r: 0, g: 0, b: 0 }, composited) >= + terminalContrastRatio({ r: 255, g: 255, b: 255 }, composited); + return isLight ? LIGHT_BG_MIN_CONTRAST : DARK_BG_MIN_CONTRAST; + } + function normalizeTerminalTheme(input) { var source = input && typeof input === 'object' && input.theme && typeof input.theme === 'object' ? input.theme @@ -22,6 +100,10 @@ export const TERMINAL_WEBVIEW_THEME_JS = ` var background = terminalTheme.background || '${colors.terminalBg}'; document.documentElement.style.background = background; document.body.style.background = background; - if (term) term.options.theme = terminalTheme; + terminalMinimumContrastRatio = resolveTerminalContrastFloor(background); + if (term) { + term.options.theme = terminalTheme; + term.options.minimumContrastRatio = terminalMinimumContrastRatio; + } } ` diff --git a/mobile/src/terminal/terminal-write-coalescer.test.ts b/mobile/src/terminal/terminal-write-coalescer.test.ts index 1279bbeb7d6..3562e7db0f4 100644 --- a/mobile/src/terminal/terminal-write-coalescer.test.ts +++ b/mobile/src/terminal/terminal-write-coalescer.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { createTerminalWriteCoalescer, TERMINAL_WRITE_FLUSH_WINDOW_MS, + TERMINAL_WRITE_MAX_PENDING_CHUNKS, TERMINAL_WRITE_MAX_PENDING_UNITS } from './terminal-write-coalescer' @@ -128,6 +129,20 @@ describe('terminal write coalescer', () => { expect(sink.delivered).toHaveLength(2) }) + it('flushes tiny chunks before their object count can grow unbounded', () => { + vi.useFakeTimers() + const sink = createDeliverySink() + const coalescer = createTerminalWriteCoalescer(sink.deliver) + + coalescer.write('leading') + for (let index = 0; index < TERMINAL_WRITE_MAX_PENDING_CHUNKS; index += 1) { + coalescer.write('x') + } + + expect(sink.delivered).toEqual(['leading', 'x'.repeat(TERMINAL_WRITE_MAX_PENDING_CHUNKS)]) + expect(vi.getTimerCount()).toBe(0) + }) + it('treats write("") as a no-op: no delivery, no buffer append, no timer', () => { vi.useFakeTimers() const sink = createDeliverySink() diff --git a/mobile/src/terminal/terminal-write-coalescer.ts b/mobile/src/terminal/terminal-write-coalescer.ts index b27f6c37fb2..1553253864a 100644 --- a/mobile/src/terminal/terminal-write-coalescer.ts +++ b/mobile/src/terminal/terminal-write-coalescer.ts @@ -5,6 +5,7 @@ export const TERMINAL_WRITE_FLUSH_WINDOW_MS = 48 // Why: defense-in-depth only — server ack flow control bounds inflow; this cap keeps // an upstream flow-control bug from growing the buffer unboundedly. UTF-16 code units. export const TERMINAL_WRITE_MAX_PENDING_UNITS = 512 * 1024 +export const TERMINAL_WRITE_MAX_PENDING_CHUNKS = 4_096 export function createTerminalWriteCoalescer(deliver: (data: string) => void) { let pendingChunks: string[] = [] @@ -46,7 +47,10 @@ export function createTerminalWriteCoalescer(deliver: (data: string) => void) { } pendingChunks.push(data) pendingUnits += data.length - if (pendingUnits > TERMINAL_WRITE_MAX_PENDING_UNITS) { + if ( + pendingUnits > TERMINAL_WRITE_MAX_PENDING_UNITS || + pendingChunks.length >= TERMINAL_WRITE_MAX_PENDING_CHUNKS + ) { flushNow() return } diff --git a/mobile/src/transport/bounded-process-line-reader.test.ts b/mobile/src/transport/bounded-process-line-reader.test.ts new file mode 100644 index 00000000000..3fd0083e134 --- /dev/null +++ b/mobile/src/transport/bounded-process-line-reader.test.ts @@ -0,0 +1,46 @@ +import { once } from 'node:events' +import { PassThrough } from 'node:stream' +import { describe, expect, it } from 'vitest' +import { + appendProcessOutputTail, + attachBoundedProcessLineReader, + PROCESS_LINE_MAX_BYTES +} from '../../scripts/bounded-process-line-reader.mjs' + +describe('attachBoundedProcessLineReader', () => { + it('preserves ordinary LF, CRLF, and split UTF-8 lines', () => { + const stream = new PassThrough() + const lines: string[] = [] + attachBoundedProcessLineReader(stream, (line) => lines.push(line)) + + const unicode = Buffer.from('café\n') + stream.write('first\r\nsecond\r') + stream.write('\n') + stream.write(unicode.subarray(0, -2)) + stream.end(unicode.subarray(-2)) + + expect(lines).toEqual(['first', 'second', 'café']) + }) + + it('retains only the configured prefix of an unterminated line', async () => { + const stream = new PassThrough() + const lines: string[] = [] + attachBoundedProcessLineReader(stream, (line) => lines.push(line), 4) + + const ended = once(stream, 'end') + stream.end('abcdefgh') + await ended + + expect(lines).toEqual(['abcd… [line truncated]']) + }) + + it('uses a bounded default for process output lines', () => { + expect(PROCESS_LINE_MAX_BYTES).toBe(64 * 1024) + }) + + it('keeps exact output below the tail cap and newest output above it', () => { + expect(appendProcessOutputTail('first\n', 'second', 32)).toBe('first\nsecond\n') + expect(appendProcessOutputTail('first\n', 'second', 8)).toBe('\nsecond\n') + expect(appendProcessOutputTail('first\n', 'second', 0)).toBe('') + }) +}) diff --git a/mobile/src/transport/bounded-response-body.test.ts b/mobile/src/transport/bounded-response-body.test.ts new file mode 100644 index 00000000000..183c1dedc61 --- /dev/null +++ b/mobile/src/transport/bounded-response-body.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' +import { responseBodyIncludesWithinLimit } from '../../scripts/bounded-response-body.mjs' + +function responseWithChunks(chunks: string[], contentLength?: number): Response { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue(encoder.encode(chunk)) + } + controller.close() + } + }), + { headers: contentLength === undefined ? {} : { 'content-length': String(contentLength) } } + ) +} + +describe('responseBodyIncludesWithinLimit', () => { + it('matches a marker split across admitted chunks', async () => { + const response = responseWithChunks(['packager-', 'status:running']) + await expect( + responseBodyIncludesWithinLimit(response, 'packager-status:running', 64) + ).resolves.toBe(true) + }) + + it('rejects streamed and declared bodies above the byte limit', async () => { + const streamed = responseWithChunks(['1234', '5']) + const declared = responseWithChunks(['ok'], 5) + + await expect(responseBodyIncludesWithinLimit(streamed, 'missing', 4)).resolves.toBe(false) + await expect(responseBodyIncludesWithinLimit(declared, 'ok', 4)).resolves.toBe(false) + }) +}) diff --git a/mobile/src/transport/browser-screencast-protocol.ts b/mobile/src/transport/browser-screencast-protocol.ts index 01ee4353f6c..7cde80ec547 100644 --- a/mobile/src/transport/browser-screencast-protocol.ts +++ b/mobile/src/transport/browser-screencast-protocol.ts @@ -1,3 +1,5 @@ +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' + const BROWSER_SCREENCAST_KIND = 0x62 const BROWSER_SCREENCAST_VERSION = 1 const HEADER_BYTES = 16 @@ -51,7 +53,7 @@ function byteToFormat(value: number): BrowserScreencastFormat | null { function decodeJson(bytes: Uint8Array): unknown { try { - return JSON.parse(new TextDecoder().decode(bytes)) as unknown + return parseMobileJsonTextWithinLimits(new TextDecoder().decode(bytes)) } catch { return null } diff --git a/mobile/src/transport/connection-log-buffer.test.ts b/mobile/src/transport/connection-log-buffer.test.ts index e7ce662534b..1f7d8762d4f 100644 --- a/mobile/src/transport/connection-log-buffer.test.ts +++ b/mobile/src/transport/connection-log-buffer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { createConnectionLogStore } from './connection-log-buffer' +import { CONNECTION_LOG_HOST_ID_MAX_BYTES, createConnectionLogStore } from './connection-log-buffer' import type { ConnectionLogEntry } from './types' function entry(id: number): ConnectionLogEntry { @@ -53,4 +53,81 @@ describe('connection log buffer', () => { store.append('host-a', entry(2)) expect(onA).toHaveBeenCalledTimes(1) }) + + it('evicts the least-recently-written inactive host past the host cap', () => { + const store = createConnectionLogStore(3, 2) + store.append('host-a', entry(1)) + store.append('host-b', entry(2)) + store.append('host-a', entry(3)) + store.append('host-c', entry(4)) + + expect(store.get('host-a').map((e) => e.id)).toEqual(['log-1', 'log-3']) + expect(store.get('host-b')).toEqual([]) + expect(store.get('host-c').map((e) => e.id)).toEqual(['log-4']) + }) + + it('drops removed-host entries and refreshes an active snapshot', () => { + const store = createConnectionLogStore() + const listener = vi.fn() + store.append('host-a', entry(1)) + store.subscribe('host-a', listener) + + store.delete('host-a') + + expect(store.get('host-a')).toEqual([]) + expect(listener).toHaveBeenCalledOnce() + }) + + it('accepts the exact host-id limit and rejects one more byte', () => { + const store = createConnectionLogStore() + const exactHost = 'h'.repeat(CONNECTION_LOG_HOST_ID_MAX_BYTES) + + store.append(exactHost, entry(1)) + store.append(`${exactHost}h`, entry(2)) + + expect(store.get(exactHost).map((value) => value.id)).toEqual(['log-1']) + expect(store.get(`${exactHost}h`)).toEqual([]) + }) + + it('accepts an exact entry-byte budget and rejects one more byte', () => { + const store = createConnectionLogStore(10, 10, { + maxEntryBytes: 260, + maxHostBytes: 1024, + maxStoreBytes: 2048 + }) + const exact: ConnectionLogEntry = { id: '', ts: 1, level: 'info', message: '' } + + store.append('host-a', exact) + store.append('host-a', { ...exact, message: 'x' }) + + expect(store.get('host-a')).toEqual([exact]) + }) + + it('keeps newest entries within the per-host byte budget', () => { + const store = createConnectionLogStore(10, 10, { + maxEntryBytes: 1024, + maxHostBytes: 700, + maxStoreBytes: 4096 + }) + + store.append('host-a', entry(1)) + store.append('host-a', entry(2)) + store.append('host-a', entry(3)) + + expect(store.get('host-a').map((value) => value.id)).toEqual(['log-2', 'log-3']) + }) + + it('evicts inactive hosts to stay within the aggregate byte budget', () => { + const store = createConnectionLogStore(10, 10, { + maxEntryBytes: 1024, + maxHostBytes: 1024, + maxStoreBytes: 700 + }) + + store.append('host-a', entry(1)) + store.append('host-b', entry(2)) + + expect(store.get('host-a')).toEqual([]) + expect(store.get('host-b').map((value) => value.id)).toEqual(['log-2']) + }) }) diff --git a/mobile/src/transport/connection-log-buffer.ts b/mobile/src/transport/connection-log-buffer.ts index 1b223ad8e1f..b7005efc602 100644 --- a/mobile/src/transport/connection-log-buffer.ts +++ b/mobile/src/transport/connection-log-buffer.ts @@ -1,4 +1,5 @@ import type { ConnectionLogEntry } from './types' +import { measureUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' // Why: the rpc-client's onLog entries were only wired during pairing; for // long-lived host connections everything went to console.log, invisible to @@ -8,35 +9,159 @@ import type { ConnectionLogEntry } from './types' // swaps (forceReconnect) and provider remounts (hot reload); bounded so an // all-night reconnect loop can't grow memory unbounded. const MAX_ENTRIES_PER_HOST = 200 +const MAX_RETAINED_HOSTS = 128 +export const CONNECTION_LOG_HOST_ID_MAX_BYTES = 4 * 1024 +export const CONNECTION_LOG_ENTRY_MAX_BYTES = 64 * 1024 +export const CONNECTION_LOG_HOST_MAX_RETAINED_BYTES = 256 * 1024 +export const CONNECTION_LOG_STORE_MAX_RETAINED_BYTES = 8 * 1024 * 1024 +const CONNECTION_LOG_MAX_LISTENERS_PER_HOST = 16 + +type RetainedConnectionLogEntry = { + entry: ConnectionLogEntry + bytes: number +} + +type RetainedHostLog = { + entries: RetainedConnectionLogEntry[] + bytes: number +} + +type ConnectionLogByteLimits = { + maxEntryBytes: number + maxHostBytes: number + maxStoreBytes: number +} + +function measureBoundedString(value: unknown, maxBytes: number): number | null { + if (value === undefined) { + return 0 + } + if (typeof value !== 'string') { + return null + } + const measurement = measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }) + return measurement.exceededLimit ? null : measurement.byteLength +} + +function measureConnectionLogEntry(entry: ConnectionLogEntry, maxBytes: number): number | null { + let bytes = 256 + for (const value of [entry.id, entry.level, entry.message, entry.detail]) { + const valueBytes = measureBoundedString(value, maxBytes - bytes) + if (valueBytes === null) { + return null + } + bytes += valueBytes + } + return bytes <= maxBytes ? bytes : null +} export type ConnectionLogStore = { append: (hostId: string, entry: ConnectionLogEntry) => void get: (hostId: string) => readonly ConnectionLogEntry[] subscribe: (hostId: string, listener: () => void) => () => void + delete: (hostId: string) => void } export function createConnectionLogStore( - maxEntriesPerHost: number = MAX_ENTRIES_PER_HOST + maxEntriesPerHost: number = MAX_ENTRIES_PER_HOST, + maxRetainedHosts: number = MAX_RETAINED_HOSTS, + byteLimits: Partial = {} ): ConnectionLogStore { - const entriesByHost = new Map() + const maxEntryBytes = Math.min( + byteLimits.maxEntryBytes ?? CONNECTION_LOG_ENTRY_MAX_BYTES, + CONNECTION_LOG_ENTRY_MAX_BYTES + ) + const maxHostBytes = Math.min( + byteLimits.maxHostBytes ?? CONNECTION_LOG_HOST_MAX_RETAINED_BYTES, + CONNECTION_LOG_HOST_MAX_RETAINED_BYTES + ) + const maxStoreBytes = Math.min( + byteLimits.maxStoreBytes ?? CONNECTION_LOG_STORE_MAX_RETAINED_BYTES, + CONNECTION_LOG_STORE_MAX_RETAINED_BYTES + ) + const entriesByHost = new Map() const listenersByHost = new Map void>>() // Why: useSyncExternalStore compares snapshots by reference — getSnapshot // must return the SAME array until the data actually changes, or React // loops re-rendering. Cache per host; invalidate on append. const snapshotByHost = new Map() const EMPTY: readonly ConnectionLogEntry[] = [] + let retainedStoreBytes = 0 + + const deleteHost = (hostId: string): boolean => { + const host = entriesByHost.get(hostId) + if (!host) { + return false + } + entriesByHost.delete(hostId) + retainedStoreBytes -= host.bytes + snapshotByHost.delete(hostId) + return true + } + + const evictOldestUnobservedHost = (exceptHostId?: string): boolean => { + for (const hostId of entriesByHost.keys()) { + if (hostId !== exceptHostId && !listenersByHost.has(hostId)) { + return deleteHost(hostId) + } + } + return false + } return { append(hostId, entry) { - let entries = entriesByHost.get(hostId) - if (!entries) { - entries = [] - entriesByHost.set(hostId, entries) + const hostIdBytes = measureBoundedString(hostId, CONNECTION_LOG_HOST_ID_MAX_BYTES) + const entryBytes = measureConnectionLogEntry(entry, maxEntryBytes) + if (hostIdBytes === null || entryBytes === null || entryBytes > maxHostBytes) { + return } - entries.push(entry) - if (entries.length > maxEntriesPerHost) { - entries.splice(0, entries.length - maxEntriesPerHost) + let host = entriesByHost.get(hostId) + if (!host) { + while (entriesByHost.size >= maxRetainedHosts) { + if (!evictOldestUnobservedHost()) { + return + } + } + const retainedHostKeyBytes = hostIdBytes + 128 + while (retainedStoreBytes + retainedHostKeyBytes + entryBytes > maxStoreBytes) { + if (!evictOldestUnobservedHost()) { + return + } + } + host = { entries: [], bytes: retainedHostKeyBytes } + entriesByHost.set(hostId, host) + retainedStoreBytes += retainedHostKeyBytes + } else { + entriesByHost.delete(hostId) + entriesByHost.set(hostId, host) } + while ( + host.entries.length >= maxEntriesPerHost || + host.bytes + entryBytes > maxHostBytes || + retainedStoreBytes + entryBytes > maxStoreBytes + ) { + const oldest = host.entries.shift() + if (oldest) { + host.bytes -= oldest.bytes + retainedStoreBytes -= oldest.bytes + continue + } + if (!evictOldestUnobservedHost(hostId)) { + break + } + } + if ( + host.bytes + entryBytes > maxHostBytes || + retainedStoreBytes + entryBytes > maxStoreBytes + ) { + if (host.entries.length === 0) { + deleteHost(hostId) + } + return + } + host.entries.push({ entry, bytes: entryBytes }) + host.bytes += entryBytes + retainedStoreBytes += entryBytes snapshotByHost.delete(hostId) const listeners = listenersByHost.get(hostId) if (listeners) { @@ -51,21 +176,30 @@ export function createConnectionLogStore( if (cached) { return cached } - const entries = entriesByHost.get(hostId) - if (!entries || entries.length === 0) { + const host = entriesByHost.get(hostId) + if (!host || host.entries.length === 0) { return EMPTY } - const snapshot = Object.freeze([...entries]) + const snapshot = Object.freeze(host.entries.map((retained) => retained.entry)) snapshotByHost.set(hostId, snapshot) return snapshot }, subscribe(hostId, listener) { + if (measureBoundedString(hostId, CONNECTION_LOG_HOST_ID_MAX_BYTES) === null) { + return () => {} + } let listeners = listenersByHost.get(hostId) if (!listeners) { + if (listenersByHost.size >= maxRetainedHosts) { + return () => {} + } listeners = new Set() listenersByHost.set(hostId, listeners) } + if (listeners.size >= CONNECTION_LOG_MAX_LISTENERS_PER_HOST) { + return () => {} + } listeners.add(listener) return () => { const set = listenersByHost.get(hostId) @@ -77,6 +211,16 @@ export function createConnectionLogStore( listenersByHost.delete(hostId) } } + }, + + delete(hostId) { + deleteHost(hostId) + const listeners = listenersByHost.get(hostId) + if (listeners) { + for (const listener of listeners) { + listener() + } + } } } } diff --git a/mobile/src/transport/e2ee.ts b/mobile/src/transport/e2ee.ts index 2732b3d615f..d36cb577341 100644 --- a/mobile/src/transport/e2ee.ts +++ b/mobile/src/transport/e2ee.ts @@ -4,6 +4,10 @@ // stream frames use the raw byte bundle. import nacl from 'tweetnacl' import * as ExpoCrypto from 'expo-crypto' +import { MOBILE_INBOUND_MAX_FRAME_BYTES } from './mobile-inbound-frame-queue' + +export const MOBILE_E2EE_MAX_FRAME_BASE64_CHARACTERS = + Math.ceil(MOBILE_INBOUND_MAX_FRAME_BYTES / 3) * 4 // Why: Hermes (React Native's JS engine) lacks crypto.getRandomValues, // which tweetnacl requires. expo-crypto provides a native secure RNG @@ -39,8 +43,14 @@ function uint8ToBase64(bytes: Uint8Array): string { return btoa(binary) } -function base64ToUint8(b64: string): Uint8Array { +function base64ToUint8(b64: string, maxDecodedBytes: number): Uint8Array { + if (b64.length > Math.ceil(maxDecodedBytes / 3) * 4) { + throw new Error('Base64 payload exceeds safe size') + } const binary = atob(b64) + if (binary.length > maxDecodedBytes) { + throw new Error('Base64 payload exceeds safe size') + } const bytes = new Uint8Array(binary.length) for (let i = 0; i < binary.length; i++) { bytes[i] = binary.charCodeAt(i) @@ -49,7 +59,7 @@ function base64ToUint8(b64: string): Uint8Array { } export function publicKeyFromBase64(b64: string): Uint8Array { - const key = base64ToUint8(b64) + const key = base64ToUint8(b64, 32) if (key.length !== 32) { throw new Error( `Invalid public key: expected 32 bytes, got ${key.length} from "${b64.slice(0, 20)}..."` @@ -68,7 +78,10 @@ export function encrypt(plaintext: string, sharedKey: Uint8Array): string { } export function decrypt(encrypted: string, sharedKey: Uint8Array): string | null { - const bundle = base64ToUint8(encrypted) + if (encrypted.length > MOBILE_E2EE_MAX_FRAME_BASE64_CHARACTERS) { + throw new Error('Base64 payload exceeds safe size') + } + const bundle = base64ToUint8(encrypted, MOBILE_INBOUND_MAX_FRAME_BYTES) const plaintext = decryptBytes(bundle, sharedKey) return plaintext ? new TextDecoder().decode(plaintext) : null } diff --git a/mobile/src/transport/host-client-open-registry.test.ts b/mobile/src/transport/host-client-open-registry.test.ts new file mode 100644 index 00000000000..f70cef41f95 --- /dev/null +++ b/mobile/src/transport/host-client-open-registry.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { HostClientOpenRegistry } from './host-client-open-registry' + +describe('HostClientOpenRegistry', () => { + it('cancels replaced and oldest tickets while bounding pending hosts', () => { + const registry = new HostClientOpenRegistry(2) + const first = registry.register('host-a', new Promise(() => {})) + const replacement = registry.register('host-a', new Promise(() => {})) + const second = registry.register('host-b', new Promise(() => {})) + const third = registry.register('host-c', new Promise(() => {})) + + expect(first.cancelled).toBe(true) + expect(replacement.cancelled).toBe(true) + expect(second.cancelled).toBe(false) + expect(third.cancelled).toBe(false) + expect(registry.getActivePromise('host-a')).toBeNull() + expect(registry.getActivePromise('host-b')).toBe(second.promise) + expect(registry.getActivePromise('host-c')).toBe(third.promise) + }) +}) diff --git a/mobile/src/transport/host-client-open-registry.ts b/mobile/src/transport/host-client-open-registry.ts index a47ca5a231a..ac9c6a16c96 100644 --- a/mobile/src/transport/host-client-open-registry.ts +++ b/mobile/src/transport/host-client-open-registry.ts @@ -3,15 +3,35 @@ export type HostClientOpenTicket = { promise: Promise } +export const HOST_CLIENT_OPEN_MAX_PENDING = 64 + export class HostClientOpenRegistry { private readonly pending = new Map() + constructor(private readonly maxPending = HOST_CLIENT_OPEN_MAX_PENDING) {} + getActivePromise(hostId: string): Promise | null { const ticket = this.pending.get(hostId) return ticket && !ticket.cancelled ? ticket.promise : null } register(hostId: string, promise: Promise): HostClientOpenTicket { + const prior = this.pending.get(hostId) + if (prior) { + prior.cancelled = true + this.pending.delete(hostId) + } + while (this.pending.size >= Math.max(1, this.maxPending)) { + const oldestHostId = this.pending.keys().next().value as string | undefined + if (oldestHostId === undefined) { + break + } + const oldest = this.pending.get(oldestHostId) + if (oldest) { + oldest.cancelled = true + } + this.pending.delete(oldestHostId) + } const ticket = { cancelled: false, promise } this.pending.set(hostId, ticket) return ticket diff --git a/mobile/src/transport/host-credential-cleanup.test.ts b/mobile/src/transport/host-credential-cleanup.test.ts index bd483a3280f..2ba0664522d 100644 --- a/mobile/src/transport/host-credential-cleanup.test.ts +++ b/mobile/src/transport/host-credential-cleanup.test.ts @@ -1,6 +1,8 @@ import AsyncStorage from '@react-native-async-storage/async-storage' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { + HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES, + HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS, loadPendingHostCredentialCleanup, loadPendingHostCredentialCleanupIds, resetHostCredentialCleanupForTests, @@ -270,4 +272,41 @@ describe('host credential cleanup', () => { await expect(loadPendingHostCredentialCleanupIds()).resolves.toEqual([]) }) + + it('accepts the exact pending-id cap and treats one over as unreadable', async () => { + storedPendingIds = Array.from( + { length: HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS }, + (_, index) => `host-${index}` + ) + + await expect(loadPendingHostCredentialCleanup()).resolves.toEqual({ + ids: storedPendingIds, + storageUnreadable: false + }) + + storedPendingIds = [...storedPendingIds, 'one-over'] + await expect(loadPendingHostCredentialCleanup()).resolves.toEqual({ + ids: [], + storageUnreadable: true + }) + }) + + it('caps simultaneous native deletes while preserving excess durable intents', async () => { + vi.useFakeTimers() + const deleteCredential = vi.fn(() => new Promise(() => undefined)) + const hostCount = HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES + 1 + + await Promise.all( + Array.from({ length: hostCount }, (_, index) => + scheduleHostCredentialCleanup(`host-${index}`, deleteCredential, 3_000) + ) + ) + await flushMicrotasks() + + expect(deleteCredential).toHaveBeenCalledTimes(HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES) + expect(storedPendingIds).toHaveLength(hostCount) + + await vi.advanceTimersByTimeAsync(3_000) + await expect(loadPendingHostCredentialCleanupIds()).resolves.toHaveLength(hostCount) + }) }) diff --git a/mobile/src/transport/host-credential-cleanup.ts b/mobile/src/transport/host-credential-cleanup.ts index d64e37a98dc..c50c63bca33 100644 --- a/mobile/src/transport/host-credential-cleanup.ts +++ b/mobile/src/transport/host-credential-cleanup.ts @@ -1,7 +1,12 @@ import AsyncStorage from '@react-native-async-storage/async-storage' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const PENDING_STORAGE_KEY = 'orca:pending-host-credential-cleanups' const CLEANUP_CONFIRM_TIMEOUT_MS = 3_000 +export const HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS = 256 +export const HOST_CREDENTIAL_CLEANUP_MAX_STORAGE_CHARACTERS = 256 * 1024 +export const HOST_CREDENTIAL_CLEANUP_MAX_HOST_ID_CHARACTERS = 4_096 +export const HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES = 16 type DeleteHostCredential = (hostId: string) => Promise type CleanupAttemptResult = 'cleared' | 'pending' @@ -33,7 +38,11 @@ function notifyPendingListeners(): void { } function markUnrecordedPending(hostId: string): void { - if (unrecordedPendingIds.has(hostId)) { + if ( + unrecordedPendingIds.has(hostId) || + !isTrackableHostId(hostId) || + unrecordedPendingIds.size >= HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS + ) { return } unrecordedPendingIds.add(hostId) @@ -47,17 +56,55 @@ function clearUnrecordedPending(hostId: string): void { } function parsePendingIds(raw: string): string[] | null { + if (raw.length > HOST_CREDENTIAL_CLEANUP_MAX_STORAGE_CHARACTERS) { + return null + } try { - const parsed = JSON.parse(raw) as unknown + const parsed = parseMobileJsonTextWithinLimits(raw) if (!Array.isArray(parsed)) { return null } - return [...new Set(parsed.filter((value): value is string => typeof value === 'string'))] + const ids: string[] = [] + const seen = new Set() + for (const value of parsed) { + if (typeof value !== 'string') { + continue + } + if (!isTrackableHostId(value)) { + return null + } + if (!seen.has(value)) { + if (ids.length >= HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS) { + return null + } + seen.add(value) + ids.push(value) + } + } + return ids } catch { return null } } +function isTrackableHostId(hostId: string): boolean { + return hostId.length > 0 && hostId.length <= HOST_CREDENTIAL_CLEANUP_MAX_HOST_ID_CHARACTERS +} + +function serializePendingIds(ids: string[]): string { + if ( + ids.length > HOST_CREDENTIAL_CLEANUP_MAX_PENDING_IDS || + ids.some((id) => !isTrackableHostId(id)) + ) { + throw new Error('pending host credential cleanup limit exceeded') + } + const serialized = JSON.stringify(ids) + if (serialized.length > HOST_CREDENTIAL_CLEANUP_MAX_STORAGE_CHARACTERS) { + throw new Error('pending host credential cleanup storage limit exceeded') + } + return serialized +} + function sameIdList(a: string[], b: string[]): boolean { return a.length === b.length && a.every((id, index) => id === b[index]) } @@ -103,7 +150,7 @@ async function mutatePendingIds(update: (ids: string[]) => string[]): Promise {}) @@ -142,6 +189,9 @@ function startOrJoinDelete(hostId: string, deleteCredential: DeleteHostCredentia if (existing) { return existing } + if (inflightDeletes.size >= HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES) { + return Promise.reject(new Error('host credential cleanup concurrency limit reached')) + } const cleanup = Promise.resolve() .then(() => deleteCredential(hostId)) .finally(() => { @@ -154,6 +204,9 @@ function startOrJoinDelete(hostId: string, deleteCredential: DeleteHostCredentia } async function recordCleanupIntent(hostId: string): Promise { + if (!isTrackableHostId(hostId)) { + return false + } try { await addPendingId(hostId) return true @@ -234,10 +287,10 @@ export async function retryPendingHostCredentialCleanups( deleteCredential: DeleteHostCredential ): Promise<{ clearedCount: number; remainingIds: string[]; storageUnreadable: boolean }> { const pending = await loadPendingCleanupState() - const outcomes = await Promise.all( - // Why: these ids are already durable (or a session-scoped fallback). Re-adding - // intent can race a late success and recreate a ghost row after deletion. - pending.ids.map((id) => confirmNativeCleanup(id, deleteCredential, CLEANUP_CONFIRM_TIMEOUT_MS)) + const outcomes = await retryCleanupIds( + pending.ids, + deleteCredential, + HOST_CREDENTIAL_CLEANUP_MAX_INFLIGHT_DELETES ) const remaining = await loadPendingCleanupState() return { @@ -247,6 +300,29 @@ export async function retryPendingHostCredentialCleanups( } } +async function retryCleanupIds( + ids: string[], + deleteCredential: DeleteHostCredential, + concurrency: number +): Promise { + const outcomes: CleanupAttemptResult[] = [] + let nextIndex = 0 + const worker = async (): Promise => { + while (nextIndex < ids.length) { + const index = nextIndex++ + // Why: these ids already have recovery intent; re-adding can race a late + // success and recreate a ghost row after deletion. + outcomes[index] = await confirmNativeCleanup( + ids[index]!, + deleteCredential, + CLEANUP_CONFIRM_TIMEOUT_MS + ) + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, ids.length) }, () => worker())) + return outcomes +} + /** Test-only: drop module listeners/in-flight state between cases. */ export function resetHostCredentialCleanupForTests(): void { inflightDeletes.clear() diff --git a/mobile/src/transport/host-removal-lifecycle.test.ts b/mobile/src/transport/host-removal-lifecycle.test.ts index e398062b270..d19b73673e8 100644 --- a/mobile/src/transport/host-removal-lifecycle.test.ts +++ b/mobile/src/transport/host-removal-lifecycle.test.ts @@ -1,16 +1,23 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -const removeHostMock = vi.hoisted(() => vi.fn()) +const { removeHostMock, deleteConnectionLogMock } = vi.hoisted(() => ({ + removeHostMock: vi.fn(), + deleteConnectionLogMock: vi.fn() +})) vi.mock('./host-store', () => ({ removeHost: (hostId: string) => removeHostMock(hostId) })) +vi.mock('./connection-log-buffer', () => ({ + connectionLogStore: { delete: deleteConnectionLogMock } +})) import { removeHostAndCloseClient } from './host-removal-lifecycle' describe('host removal lifecycle', () => { beforeEach(() => { removeHostMock.mockReset() + deleteConnectionLogMock.mockReset() }) it('closes the client only after metadata removal commits', async () => { @@ -28,6 +35,7 @@ describe('host removal lifecycle', () => { await removal expect(closeHostClient).toHaveBeenCalledWith('host-1') + expect(deleteConnectionLogMock).toHaveBeenCalledWith('host-1') }) it('keeps the client open when metadata removal fails', async () => { @@ -38,5 +46,18 @@ describe('host removal lifecycle', () => { 'storage unavailable' ) expect(closeHostClient).not.toHaveBeenCalled() + expect(deleteConnectionLogMock).not.toHaveBeenCalled() + }) + + it('forgets removed-host logs even when client teardown throws', async () => { + removeHostMock.mockResolvedValue(undefined) + const closeHostClient = vi.fn(() => { + throw new Error('close failed') + }) + + await expect(removeHostAndCloseClient('host-1', closeHostClient)).rejects.toThrow( + 'close failed' + ) + expect(deleteConnectionLogMock).toHaveBeenCalledWith('host-1') }) }) diff --git a/mobile/src/transport/host-removal-lifecycle.ts b/mobile/src/transport/host-removal-lifecycle.ts index ba599614804..8f3148744f9 100644 --- a/mobile/src/transport/host-removal-lifecycle.ts +++ b/mobile/src/transport/host-removal-lifecycle.ts @@ -1,4 +1,5 @@ import { removeHost } from './host-store' +import { connectionLogStore } from './connection-log-buffer' export async function removeHostAndCloseClient( hostId: string, @@ -7,5 +8,9 @@ export async function removeHostAndCloseClient( // Why: closing before the metadata commit can strand a still-paired host on // storage failure; closing immediately after success prevents socket leaks. await removeHost(hostId) - closeHostClient(hostId) + try { + closeHostClient(hostId) + } finally { + connectionLogStore.delete(hostId) + } } diff --git a/mobile/src/transport/host-store.test.ts b/mobile/src/transport/host-store.test.ts index 918e6e92f93..a9eb268d349 100644 --- a/mobile/src/transport/host-store.test.ts +++ b/mobile/src/transport/host-store.test.ts @@ -34,6 +34,8 @@ vi.mock('./host-credential-cleanup', () => ({ })) import { + HOST_TOKEN_CACHE_MAX_ENTRIES, + HOST_TOKEN_CACHE_MAX_ENTRY_CODE_UNITS, loadHosts, MobileRelayUpgradeHostRemovedError, removeHost, @@ -44,7 +46,18 @@ import { updateHostNameAndEndpoint, updateLastConnected } from './host-store' +import { + STORED_HOSTS_MAX_ENTRIES, + STORED_HOSTS_MAX_STORAGE_CHARACTERS +} from './mobile-host-list-storage' import { resetMobileRelayHostOverlayStoreForTests } from './mobile-relay-host-overlay-store' +import { + MOBILE_HOST_ID_MAX_CHARACTERS, + MOBILE_HOST_NAME_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PAIRING_ENDPOINT_MAX_CHARACTERS, + PAIRING_PUBLIC_KEY_MAX_CHARACTERS +} from './types' const HOSTS_STORAGE_KEY = 'orca:hosts' const OVERLAY_STORAGE_KEY = 'orca:mobile-relay:host-overlays:v2' @@ -222,6 +235,101 @@ describe('host-store list mutations', () => { expect(hosts.some(({ id }) => id === 'removed-by-old-build')).toBe(false) }) + it('bounds cached keychain tokens without changing loaded hosts', async () => { + const storedHosts = Array.from({ length: HOST_TOKEN_CACHE_MAX_ENTRIES + 1 }, (_, index) => ({ + ...HOST_ONE, + id: `host-${index}`, + publicKeyB64: `key-${index}` + })) + storedHostsRaw = JSON.stringify(storedHosts) + secureStoreMock.getItemAsync.mockImplementation(async (key: string) => `token-${key}`) + + await expect(loadHosts()).resolves.toHaveLength(storedHosts.length) + expect(secureStoreMock.getItemAsync).toHaveBeenCalledTimes(storedHosts.length) + + await expect(loadHosts()).resolves.toHaveLength(storedHosts.length) + expect(secureStoreMock.getItemAsync).toHaveBeenCalledTimes(storedHosts.length + 1) + }) + + it('does not retain an oversized token in the in-memory cache', async () => { + storedHostsRaw = JSON.stringify([HOST_ONE]) + secureStoreMock.getItemAsync.mockResolvedValue( + 'x'.repeat(HOST_TOKEN_CACHE_MAX_ENTRY_CODE_UNITS) + ) + + await expect(loadHosts()).resolves.toHaveLength(1) + await expect(loadHosts()).resolves.toHaveLength(1) + + expect(secureStoreMock.getItemAsync).toHaveBeenCalledTimes(2) + }) + + it('accepts the exact host count in order and rejects one more', async () => { + const exact = Array.from({ length: STORED_HOSTS_MAX_ENTRIES }, (_, index) => ({ + ...HOST_ONE, + id: `host-${index}`, + publicKeyB64: `key-${index}` + })) + storedHostsRaw = JSON.stringify(exact) + secureStoreMock.getItemAsync.mockImplementation(async (key: string) => `token-${key}`) + + await expect(loadHosts()).resolves.toMatchObject( + exact.map((host) => ({ id: host.id, publicKeyB64: host.publicKeyB64 })) + ) + + storedHostsRaw = JSON.stringify([ + ...exact, + { ...HOST_ONE, id: 'host-overflow', publicKeyB64: 'key-overflow' } + ]) + await expect(loadHosts()).resolves.toEqual([]) + }) + + it('does not parse an oversized host-list payload', async () => { + storedHostsRaw = { + length: STORED_HOSTS_MAX_STORAGE_CHARACTERS + 1 + } as unknown as string + const parse = vi.spyOn(JSON, 'parse') + + await expect(loadHosts()).resolves.toEqual([]) + expect(parse).not.toHaveBeenCalled() + expect(secureStoreMock.getItemAsync).not.toHaveBeenCalled() + parse.mockRestore() + }) + + it('accepts exact host field limits and rejects one character more', async () => { + const exact = { + id: 'i'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS), + name: 'n'.repeat(MOBILE_HOST_NAME_MAX_CHARACTERS), + endpoint: 'e'.repeat(PAIRING_ENDPOINT_MAX_CHARACTERS), + deviceToken: 't'.repeat(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS), + publicKeyB64: 'p'.repeat(PAIRING_PUBLIC_KEY_MAX_CHARACTERS), + lastConnected: 1 + } + + await expect(saveHost(exact)).resolves.toBeUndefined() + await expect(saveHost({ ...exact, name: `${exact.name}n` })).rejects.toThrow() + }) + + it('refuses a write that would exceed the stored host count', async () => { + storedHostsRaw = JSON.stringify( + Array.from({ length: STORED_HOSTS_MAX_ENTRIES }, (_, index) => ({ + ...HOST_ONE, + id: `host-${index}`, + publicKeyB64: `key-${index}` + })) + ) + + await expect( + saveHost({ + ...HOST_ONE, + id: 'host-overflow', + publicKeyB64: 'key-overflow', + deviceToken: 'token' + }) + ).rejects.toThrow(/storage limit/) + expect(asyncStorageMock.setItem).not.toHaveBeenCalledWith(HOSTS_STORAGE_KEY, expect.any(String)) + expect(secureStoreMock.setItemAsync).not.toHaveBeenCalled() + }) + it('refuses to resurrect a removed host during relay upgrade publication', async () => { storedHostsRaw = JSON.stringify([HOST_TWO]) diff --git a/mobile/src/transport/host-store.ts b/mobile/src/transport/host-store.ts index 61ed6742d12..7cffbe2d260 100644 --- a/mobile/src/transport/host-store.ts +++ b/mobile/src/transport/host-store.ts @@ -3,7 +3,7 @@ import * as SecureStore from 'expo-secure-store' import { Platform } from 'react-native' import { HostProfileSchema, - StoredHostProfileSchema, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, type HostProfile, type StoredHostProfile } from './types' @@ -21,6 +21,10 @@ import { import { deleteMobileRelayCredentialBundle } from './mobile-relay-credential-bundle' import { deleteMobileRelayDirectUpgradeJournal } from './mobile-relay-direct-upgrade-journal' import { scheduleOrphanedMobileRelayCleanup } from './mobile-relay-orphan-cleanup' +import { + parseMobileStoredHostList, + serializeMobileStoredHostList +} from './mobile-host-list-storage' const STORAGE_KEY = 'orca:hosts' // Why: SecureStore keys must match [A-Za-z0-9._-] (colons rejected), so use dots as the separator. @@ -43,10 +47,11 @@ function webTokenKey(hostId: string): string { async function readDeviceToken(hostId: string): Promise { // Why: Expo SecureStore has no working web backend; fall back to AsyncStorage only on web so native still uses the keychain. - if (Platform.OS === 'web') { - return AsyncStorage.getItem(webTokenKey(hostId)) - } - return SecureStore.getItemAsync(tokenKey(hostId), KEYCHAIN_OPTIONS) + const token = + Platform.OS === 'web' + ? await AsyncStorage.getItem(webTokenKey(hostId)) + : await SecureStore.getItemAsync(tokenKey(hostId), KEYCHAIN_OPTIONS) + return token !== null && token.length <= PAIRING_DEVICE_TOKEN_MAX_CHARACTERS ? token : null } async function writeDeviceToken(hostId: string, token: string): Promise { @@ -72,31 +77,30 @@ async function deleteHostCredentials(hostId: string): Promise { } // Why: Keychain reads are slow (50-200ms) and loadHosts() runs on every screen mount; cache per-hostId in memory, invalidate on save/remove. +export const HOST_TOKEN_CACHE_MAX_ENTRIES = 64 +export const HOST_TOKEN_CACHE_MAX_ENTRY_CODE_UNITS = 64 * 1024 const tokenCache = new Map() let inflightLoad: Promise | null = null // Why: serialize RMW of the shared hosts JSON; without a queue concurrent writers drop writes (resurrect a removed host, drop a rename). let hostListMutation: Promise = Promise.resolve() -function parseStoredHosts(raw: string | null): StoredHostProfile[] | null { - if (!raw) { - return [] +function rememberHostToken(hostId: string, token: string, allowEviction: boolean): void { + if (hostId.length + token.length > HOST_TOKEN_CACHE_MAX_ENTRY_CODE_UNITS) { + tokenCache.delete(hostId) + return } - try { - const parsed = JSON.parse(raw) as unknown - if (!Array.isArray(parsed)) { - return null + if (tokenCache.has(hostId)) { + tokenCache.delete(hostId) + } else if (tokenCache.size >= HOST_TOKEN_CACHE_MAX_ENTRIES) { + if (!allowEviction) { + return + } + const oldestHostId = tokenCache.keys().next().value + if (oldestHostId !== undefined) { + tokenCache.delete(oldestHostId) } - return parsed.flatMap((item) => { - // Why: pre-v0.0.3 records stored deviceToken in AsyncStorage; drop them (users re-pair) rather than carry a migration shim. - if (item && typeof item === 'object' && 'deviceToken' in item) { - return [] - } - const result = StoredHostProfileSchema.safeParse(item) - return result.success ? [result.data] : [] - }) - } catch { - return null } + tokenCache.set(hostId, token) } export async function loadHosts(): Promise { @@ -114,7 +118,7 @@ export async function loadHosts(): Promise { async function doLoadHosts(): Promise { const raw = await AsyncStorage.getItem(STORAGE_KEY) - const storedHosts = parseStoredHosts(raw) + const storedHosts = parseMobileStoredHostList(raw) if (!storedHosts) { return [] } @@ -143,7 +147,7 @@ async function doLoadHosts(): Promise { continue } token = fetched - tokenCache.set(stored.id, token) + rememberHostToken(stored.id, token, false) } const overlay = overlays.get(stored.id) out.push({ @@ -176,7 +180,7 @@ export async function resolvePairingHostIdentity( async function readStoredHostsForMutation(): Promise { try { - const parsed = parseStoredHosts(await AsyncStorage.getItem(STORAGE_KEY)) + const parsed = parseMobileStoredHostList(await AsyncStorage.getItem(STORAGE_KEY)) if (!parsed) { // Why: refuse to RMW over unreadable payload — treating it as [] would wipe the durable host list on the next write. throw new Error('host list storage unreadable') @@ -196,7 +200,7 @@ async function mutateStoredHosts( const mutation = hostListMutation.then(async () => { const current = await readStoredHostsForMutation() const next = update(current) - await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(next)) + await AsyncStorage.setItem(STORAGE_KEY, serializeMobileStoredHostList(next)) }) hostListMutation = mutation.catch(() => {}) return mutation @@ -249,7 +253,7 @@ async function persistHost(host: HostProfile, requireExisting: boolean): Promise }) // Why: write metadata before the keychain token so a crash leaves recoverable orphaned metadata, not an orphaned token that persists forever. await writeDeviceToken(stored.id, validated.deviceToken) - tokenCache.set(stored.id, validated.deviceToken) + rememberHostToken(stored.id, validated.deviceToken, true) if (validated.endpoints) { await saveMobileRelayHostOverlay({ v: 2, diff --git a/mobile/src/transport/mobile-direct-rpc-outbound.test.ts b/mobile/src/transport/mobile-direct-rpc-outbound.test.ts new file mode 100644 index 00000000000..ab69ca0a55c --- /dev/null +++ b/mobile/src/transport/mobile-direct-rpc-outbound.test.ts @@ -0,0 +1,177 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createMobileDirectRpcOutbound } from './mobile-direct-rpc-outbound' +import { createMobileOutboundMemoryBudget } from './mobile-outbound-memory-budget' +import { MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS } from './mobile-outbound-socket-ledger' + +vi.mock('./e2ee', () => ({ + encrypt: (plaintext: string) => `encrypted:${plaintext}` +})) + +function socket() { + return { + OPEN: 1, + bufferedAmount: 0, + readyState: 1, + send: vi.fn() + } +} + +describe('mobile direct RPC outbound queue', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('parks in FIFO order while another socket owns aggregate native buffer credit', () => { + const budget = createMobileOutboundMemoryBudget({ + maxBufferedBytes: 100, + maxQueuedBytes: 1_000 + }) + const firstSocket = socket() + const secondSocket = socket() + firstSocket.bufferedAmount = 60 + secondSocket.bufferedAmount = 41 + const first = createMobileDirectRpcOutbound({ + socket: firstSocket, + isActive: () => true, + onOverflow: vi.fn(), + memoryBudget: budget + }) + const second = createMobileDirectRpcOutbound({ + socket: secondSocket, + isActive: () => true, + onOverflow: vi.fn(), + memoryBudget: budget + }) + + expect(first.enqueue('one', new Uint8Array(32), 'rpc-1')).toBe(true) + expect(first.enqueue('two', new Uint8Array(32), 'rpc-2')).toBe(true) + expect(firstSocket.send).not.toHaveBeenCalled() + expect(budget.evidence().queuedClaimCount).toBe(2) + + secondSocket.bufferedAmount = 0 + firstSocket.bufferedAmount = 0 + vi.advanceTimersByTime(25) + + expect(firstSocket.send.mock.calls.map(([frame]) => frame)).toEqual(['encrypted:one']) + first.acknowledge('rpc-1') + vi.advanceTimersByTime(25) + expect(firstSocket.send.mock.calls.map(([frame]) => frame)).toEqual([ + 'encrypted:one', + 'encrypted:two' + ]) + first.acknowledge('rpc-2') + expect(budget.evidence()).toMatchObject({ queuedBytes: 0, queuedClaimCount: 0 }) + first.dispose() + second.dispose() + expect(budget.evidence().bufferedSourceCount).toBe(2) + first.socketClosed() + second.socketClosed() + expect(budget.evidence().bufferedSourceCount).toBe(0) + }) + + it('fails the offending connection when aggregate JavaScript admission is exhausted', () => { + const budget = createMobileOutboundMemoryBudget({ + maxBufferedBytes: 1, + maxQueuedBytes: 10 + }) + const targetSocket = socket() + targetSocket.bufferedAmount = 2 + const onOverflow = vi.fn() + const outbound = createMobileDirectRpcOutbound({ + socket: targetSocket, + isActive: () => true, + onOverflow, + memoryBudget: budget + }) + + expect(outbound.enqueue('payload', new Uint8Array(32))).toBe(false) + + expect(onOverflow).toHaveBeenCalledOnce() + expect(targetSocket.send).not.toHaveBeenCalled() + expect(budget.evidence()).toMatchObject({ + bufferedSourceCount: 1, + queuedBytes: 0, + queuedClaimCount: 0 + }) + outbound.dispose() + expect(budget.evidence().bufferedSourceCount).toBe(1) + outbound.socketClosed() + expect(budget.evidence().bufferedSourceCount).toBe(0) + }) + + it('uses matching RPC responses as backpressure when bufferedAmount is unavailable', () => { + const budget = createMobileOutboundMemoryBudget({ + maxBufferedBytes: 100, + maxQueuedBytes: 1_000 + }) + const targetSocket = socket() + targetSocket.bufferedAmount = Number.NaN + const outbound = createMobileDirectRpcOutbound({ + socket: targetSocket, + isActive: () => true, + onOverflow: vi.fn(), + memoryBudget: budget + }) + + expect(outbound.enqueue('one', new Uint8Array(32), 'rpc-1')).toBe(true) + expect(outbound.enqueue('two', new Uint8Array(32), 'rpc-2')).toBe(true) + expect(targetSocket.send.mock.calls.map(([frame]) => frame)).toEqual(['encrypted:one']) + + outbound.acknowledge('rpc-1') + vi.advanceTimersByTime(25) + expect(targetSocket.send.mock.calls.map(([frame]) => frame)).toEqual([ + 'encrypted:one', + 'encrypted:two' + ]) + expect(budget.evidence()).toMatchObject({ inFlightBytes: 60, inFlightClaimCount: 1 }) + + outbound.socketClosed() + expect(budget.evidence()).toMatchObject({ bufferedBytes: 0, bufferedSourceCount: 0 }) + }) + + it('reports synchronous native send failure instead of marking a stream as sent', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 100 }) + const targetSocket = socket() + targetSocket.send.mockImplementation(() => { + throw new Error('native send failed') + }) + const onOverflow = vi.fn() + const outbound = createMobileDirectRpcOutbound({ + socket: targetSocket, + isActive: () => true, + onOverflow, + memoryBudget: budget + }) + + expect(outbound.enqueue('one', new Uint8Array(32), 'rpc-1')).toBe(false) + expect(onOverflow).toHaveBeenCalledOnce() + expect(budget.evidence()).toMatchObject({ inFlightBytes: 0, inFlightClaimCount: 0 }) + outbound.socketClosed() + }) + + it('allows another socket after a retired socket never reports close', () => { + const budget = createMobileOutboundMemoryBudget({ + maxBufferedBytes: 100, + maxBufferedSources: 1 + }) + const first = createMobileDirectRpcOutbound({ + socket: socket(), + isActive: () => true, + onOverflow: vi.fn(), + memoryBudget: budget + }) + + first.dispose() + expect(budget.canRegisterBufferedAmount()).toBe(false) + vi.advanceTimersByTime(MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS) + expect(budget.canRegisterBufferedAmount()).toBe(true) + + const second = createMobileDirectRpcOutbound({ + socket: socket(), + isActive: () => true, + onOverflow: vi.fn(), + memoryBudget: budget + }) + second.socketClosed() + expect(budget.evidence().bufferedSourceCount).toBe(0) + }) +}) diff --git a/mobile/src/transport/mobile-direct-rpc-outbound.ts b/mobile/src/transport/mobile-direct-rpc-outbound.ts new file mode 100644 index 00000000000..8a78c30d836 --- /dev/null +++ b/mobile/src/transport/mobile-direct-rpc-outbound.ts @@ -0,0 +1,126 @@ +import { createWsOutboundBackpressureQueue } from '../../../src/shared/ws-outbound-backpressure-queue' +import { getUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' +import { encrypt } from './e2ee' +import { + processMobileOutboundMemoryBudget, + MOBILE_OUTBOUND_MAX_FRAME_BYTES, + type MobileOutboundMemoryBudget +} from './mobile-outbound-memory-budget' +import { createMobileOutboundSocketLedger } from './mobile-outbound-socket-ledger' + +type DirectRpcOutboundFrame = { + acknowledgementKey?: string + key: Uint8Array + plaintext: string +} + +type DirectRpcOutboundSocket = { + OPEN: number + bufferedAmount: number + readyState: number + send(frame: string): void +} + +export type MobileDirectRpcOutbound = { + acknowledge(key: string): void + acknowledgeAuthentication(): void + dispose(): void + enqueue(plaintext: string, key: Uint8Array, acknowledgementKey?: string): boolean + socketClosed(): void +} + +const AUTHENTICATION_ACKNOWLEDGEMENT_KEY = 'mobile-direct-authentication' + +export function createMobileDirectRpcOutbound(args: { + socket: DirectRpcOutboundSocket + isActive: () => boolean + onOverflow: () => void + memoryBudget?: MobileOutboundMemoryBudget +}): MobileDirectRpcOutbound { + const memoryBudget = args.memoryBudget ?? processMobileOutboundMemoryBudget + let stopped = false + let disposed = false + let overflowed = false + const socketLedger = createMobileOutboundSocketLedger({ + memoryBudget, + readBufferedAmount: () => args.socket.bufferedAmount + }) + const failOverflow = (): void => { + if (overflowed) { + return + } + overflowed = true + stopped = true + args.onOverflow() + } + const queue = createWsOutboundBackpressureQueue({ + send: (frame) => { + const bytes = encryptedTextFrameBytes(frame.plaintext) + const cancelClaim = socketLedger.claimSentBytes(bytes, frame.acknowledgementKey) + if (!cancelClaim) { + failOverflow() + return + } + try { + args.socket.send(encrypt(frame.plaintext, frame.key)) + } catch { + cancelClaim() + failOverflow() + } + }, + byteLengthOf: (frame) => encryptedTextFrameBytes(frame.plaintext), + getBufferedAmount: () => args.socket.bufferedAmount, + isWritable: () => args.isActive() && args.socket.readyState === args.socket.OPEN && !stopped, + canSend: (bytes) => socketLedger.canSend(bytes), + claimQueuedBytes: (bytes) => memoryBudget.claimQueuedBytes(bytes), + maxFrameBytes: MOBILE_OUTBOUND_MAX_FRAME_BYTES, + onOverflow: failOverflow + }) + + return { + acknowledge: (key) => socketLedger.acknowledge(key), + acknowledgeAuthentication: () => socketLedger.acknowledge(AUTHENTICATION_ACKNOWLEDGEMENT_KEY), + dispose(): void { + if (disposed) { + return + } + disposed = true + stopped = true + queue.dispose() + socketLedger.retire() + }, + enqueue(plaintext, key, acknowledgementKey): boolean { + if (stopped || !args.isActive() || args.socket.readyState !== args.socket.OPEN) { + return false + } + const accepted = queue.enqueue({ + plaintext, + key, + acknowledgementKey: + acknowledgementKey ?? + (isAuthenticationRequest(plaintext) ? AUTHENTICATION_ACKNOWLEDGEMENT_KEY : undefined) + }) + return accepted && !overflowed + }, + socketClosed(): void { + stopped = true + queue.dispose() + socketLedger.socketClosed() + } + } +} + +function encryptedTextFrameBytes(plaintext: string): number { + const encryptedBytes = getUtf8ByteLength(plaintext) + 40 + const wireBytes = Math.ceil(encryptedBytes / 3) * 4 + return Math.max(wireBytes, plaintext.length * 2) +} + +function isAuthenticationRequest(plaintext: string): boolean { + try { + const value = JSON.parse(plaintext) as { type?: unknown } + return value.type === 'e2ee_auth' + } catch { + return false + } +} diff --git a/mobile/src/transport/mobile-direct-rpc-sender.ts b/mobile/src/transport/mobile-direct-rpc-sender.ts new file mode 100644 index 00000000000..d2e9c4ca672 --- /dev/null +++ b/mobile/src/transport/mobile-direct-rpc-sender.ts @@ -0,0 +1,51 @@ +import type { MobileDirectRpcOutbound } from './mobile-direct-rpc-outbound' +import { stringifyMobileOutboundJson } from './mobile-outbound-json' +import type { ConnectionState } from './types' + +export function createMobileDirectRpcSender(args: { + getOutbound: () => MobileDirectRpcOutbound | null + getSharedKey: () => Uint8Array | null + getSocket: () => WebSocket | null + getState: () => ConnectionState + onSocketDesync: (socket: WebSocket) => void +}): (request: unknown) => boolean { + return (request): boolean => { + const socket = args.getSocket() + const sharedKey = args.getSharedKey() + const outbound = args.getOutbound() + if (socket && socket.readyState === WebSocket.OPEN && sharedKey && outbound) { + try { + return outbound.enqueue( + stringifyMobileOutboundJson(request), + sharedKey, + requestAcknowledgementKey(request) + ) + } catch (error) { + console.warn('[net] outbound request rejected', error) + return false + } + } + const state = args.getState() + console.log('[net] sendEncrypted FAILED — channel not ready', { + hasWs: !!socket, + readyState: socket?.readyState, + hasKey: !!sharedKey, + state + }) + if (state === 'connected' && socket && socket.readyState !== WebSocket.OPEN) { + console.log('[net] sendEncrypted detected ws desync — forcing reconnect', { + readyState: socket.readyState + }) + args.onSocketDesync(socket) + } + return false + } +} + +function requestAcknowledgementKey(request: unknown): string | undefined { + if (!request || typeof request !== 'object') { + return undefined + } + const id = (request as { id?: unknown }).id + return typeof id === 'string' ? id : undefined +} diff --git a/mobile/src/transport/mobile-e2ee-legacy-fixtures.test.ts b/mobile/src/transport/mobile-e2ee-legacy-fixtures.test.ts index 7e0628960e1..aea6f3d7a55 100644 --- a/mobile/src/transport/mobile-e2ee-legacy-fixtures.test.ts +++ b/mobile/src/transport/mobile-e2ee-legacy-fixtures.test.ts @@ -6,7 +6,13 @@ vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length).fill(9) })) -import { decrypt, decryptBytes, deriveSharedKey } from './e2ee' +import { + MOBILE_E2EE_MAX_FRAME_BASE64_CHARACTERS, + decrypt, + decryptBytes, + deriveSharedKey, + publicKeyFromBase64 +} from './e2ee' describe('mobile legacy E2EE fixtures', () => { it('matches the captured desktop key and text/binary frames', () => { @@ -19,6 +25,17 @@ describe('mobile legacy E2EE fixtures', () => { expect(decrypt(fixture.authFrameB64, shared)).toBe(fixture.authPlaintext) expect(decryptBytes(fromHex(fixture.binaryFrameHex), shared)).toEqual(fixture.binaryPlaintext) }) + + it('rejects oversized base64 before calling the decoder', () => { + const decode = vi.spyOn(globalThis, 'atob') + const oversizedFrame = { + length: MOBILE_E2EE_MAX_FRAME_BASE64_CHARACTERS + 1 + } as unknown as string + + expect(() => publicKeyFromBase64('A'.repeat(45))).toThrow(/safe size/) + expect(() => decrypt(oversizedFrame, new Uint8Array(32))).toThrow(/safe size/) + expect(decode).not.toHaveBeenCalled() + }) }) function hex(bytes: Uint8Array): string { diff --git a/mobile/src/transport/mobile-e2ee-v2-client-session.test.ts b/mobile/src/transport/mobile-e2ee-v2-client-session.test.ts index 6cff0ef3523..3cccc28260c 100644 --- a/mobile/src/transport/mobile-e2ee-v2-client-session.test.ts +++ b/mobile/src/transport/mobile-e2ee-v2-client-session.test.ts @@ -12,8 +12,12 @@ vi.mock('expo-crypto', () => ({ })) import { deriveSharedKey } from './e2ee' -import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session' +import { + MOBILE_E2EE_V2_MAX_TEXT_FRAME_BASE64_CHARACTERS, + MobileE2EEV2ClientSession +} from './mobile-e2ee-v2-client-session' import { deriveMobileE2EEV2KeySchedule } from './mobile-e2ee-v2-key-schedule' +import { MOBILE_INBOUND_MAX_FRAME_BYTES } from './mobile-inbound-frame-queue' const desktop = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(1)) const client = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(2)) @@ -82,4 +86,19 @@ describe('mobile E2EE v2 client session', () => { expect(session.openText(encoded)).toBe('authenticated') expect(session.openText(encoded)).toBeNull() }) + + it('rejects oversized text and binary frames before materialization', () => { + const { session } = setup() + const decode = vi.spyOn(globalThis, 'atob') + const oversizedText = { + length: MOBILE_E2EE_V2_MAX_TEXT_FRAME_BASE64_CHARACTERS + 1 + } as unknown as string + const oversizedBinary = { + byteLength: MOBILE_INBOUND_MAX_FRAME_BYTES + 1 + } as unknown as Uint8Array + + expect(session.openText(oversizedText)).toBeNull() + expect(session.openBinary(oversizedBinary)).toBeNull() + expect(decode).not.toHaveBeenCalled() + }) }) diff --git a/mobile/src/transport/mobile-e2ee-v2-client-session.ts b/mobile/src/transport/mobile-e2ee-v2-client-session.ts index 1cce2946860..80246607a8c 100644 --- a/mobile/src/transport/mobile-e2ee-v2-client-session.ts +++ b/mobile/src/transport/mobile-e2ee-v2-client-session.ts @@ -11,6 +11,10 @@ import { } from '../../../src/shared/mobile-e2ee-v2-framing' import { deriveSharedKey, generateKeyPair, publicKeyFromBase64, publicKeyToBase64 } from './e2ee' import { deriveMobileE2EEV2KeySchedule } from './mobile-e2ee-v2-key-schedule' +import { MOBILE_INBOUND_MAX_FRAME_BYTES } from './mobile-inbound-frame-queue' + +export const MOBILE_E2EE_V2_MAX_TEXT_FRAME_BASE64_CHARACTERS = + Math.ceil(MOBILE_INBOUND_MAX_FRAME_BYTES / 3) * 4 export class MobileE2EEV2ClientSession { readonly hello: MobileE2EEV2Hello @@ -91,6 +95,9 @@ export class MobileE2EEV2ClientSession { } openBinary(frame: Uint8Array): Uint8Array | null { + if (frame.byteLength > MOBILE_INBOUND_MAX_FRAME_BYTES) { + return null + } return this.open(frame, 'binary') } @@ -146,8 +153,14 @@ function encodeBase64(bytes: Uint8Array): string { } function decodeCanonicalBase64(value: string): Uint8Array | null { + if (value.length > MOBILE_E2EE_V2_MAX_TEXT_FRAME_BASE64_CHARACTERS) { + return null + } try { const binary = atob(value) + if (binary.length > MOBILE_INBOUND_MAX_FRAME_BYTES) { + return null + } const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0)) return encodeBase64(bytes) === value ? bytes : null } catch { diff --git a/mobile/src/transport/mobile-e2ee-v2-physical-channel.test.ts b/mobile/src/transport/mobile-e2ee-v2-physical-channel.test.ts index a7aae9e6d0f..064b117e168 100644 --- a/mobile/src/transport/mobile-e2ee-v2-physical-channel.test.ts +++ b/mobile/src/transport/mobile-e2ee-v2-physical-channel.test.ts @@ -22,11 +22,16 @@ import { MobileE2EEV2PhysicalChannel, type MobileE2EEV2Socket } from './mobile-e2ee-v2-physical-channel' +import { createMobileOutboundMemoryBudget } from './mobile-outbound-memory-budget' +import { MOBILE_OUTBOUND_MAX_JSON_BYTES } from './mobile-outbound-json' const desktop = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(1)) const client = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(2)) -function setup(decodeBinary: (raw: unknown) => Promise) { +function setup( + decodeBinary: (raw: unknown) => Promise, + deviceToken = 'valid-token' +) { const session = MobileE2EEV2ClientSession.create({ desktopPublicKeyB64: Buffer.from(desktop.publicKey).toString('base64'), transport: 'relay', @@ -44,15 +49,17 @@ function setup(decodeBinary: (raw: unknown) => Promise) { const events: string[] = [] const onAuthenticated = vi.fn(() => events.push('authenticated')) const onError = vi.fn() + const outboundMemoryBudget = createMobileOutboundMemoryBudget() const channel = new MobileE2EEV2PhysicalChannel({ session, socket, - deviceToken: 'valid-token', + deviceToken, decodeBinary, onAuthenticated, onText: (plaintext) => events.push(`text:${plaintext}`), onBinary: (plaintext) => events.push(`binary:${plaintext[0]}`), - onError + onError, + outboundMemoryBudget }) channel.start() @@ -72,7 +79,18 @@ function setup(decodeBinary: (raw: unknown) => Promise) { clientNonce: handshake.clientNonce, desktopNonce: handshake.desktopNonce }) - return { channel, session, socket, sent, events, onAuthenticated, onError, ready, schedule } + return { + channel, + session, + socket, + sent, + events, + onAuthenticated, + onError, + outboundMemoryBudget, + ready, + schedule + } } function serverFrame( @@ -122,6 +140,17 @@ describe('mobile E2EE v2 physical channel', () => { expect(ctx.onError).not.toHaveBeenCalled() }) + it('rejects ready JSON above the nesting cap before object materialization', async () => { + const ctx = setup(async () => null) + + await ctx.channel.handleMessage(`${'['.repeat(129)}0${']'.repeat(129)}`) + + expect(ctx.onError).toHaveBeenCalledWith( + expect.objectContaining({ message: 'Invalid E2EE v2 ready JSON' }) + ) + expect(ctx.sent).toHaveLength(1) + }) + it('classifies the encrypted desktop device-token rejection as global auth failure', async () => { const ctx = setup(async () => null) await ctx.channel.handleMessage(JSON.stringify(ctx.ready)) @@ -140,6 +169,18 @@ describe('mobile E2EE v2 physical channel', () => { expect(ctx.onAuthenticated).not.toHaveBeenCalled() }) + it('rejects oversized authentication JSON before sealing or sending it', async () => { + const ctx = setup(async () => null, 'x'.repeat(MOBILE_OUTBOUND_MAX_JSON_BYTES)) + + await ctx.channel.handleMessage(JSON.stringify(ctx.ready)) + + expect(ctx.sent).toHaveLength(1) + expect(ctx.onError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('Mobile outbound JSON exceeds') }) + ) + ctx.channel.socketClosed() + }) + it('serializes delayed binary conversion before a later text counter', async () => { let releaseBinary!: (bytes: Uint8Array) => void const pendingBinary = new Promise((resolve) => (releaseBinary = resolve)) @@ -160,6 +201,27 @@ describe('mobile E2EE v2 physical channel', () => { expect(ctx.onError).not.toHaveBeenCalled() }) + it('fails closed when delayed inbound conversion lets frames exceed the backlog bound', async () => { + let releaseBinary!: (bytes: Uint8Array) => void + const pendingBinary = new Promise((resolve) => (releaseBinary = resolve)) + const ctx = setup(async () => pendingBinary) + await authenticate(ctx) + const binary = serverFrame(new Uint8Array([7]), 'binary', 1n, ctx.schedule) + const pending = Array.from({ length: 65 }, () => + ctx.channel.handleMessage({ delayedBlob: true }) + ) + + await Promise.resolve() + try { + expect(ctx.onError).toHaveBeenCalledOnce() + expect(ctx.onError.mock.calls[0]![0].message).toBe('E2EE v2 inbound buffer overflow') + } finally { + ctx.channel.dispose() + releaseBinary(binary) + await Promise.all(pending) + } + }) + it('queues outbound text and binary in one counter order', async () => { const ctx = setup(async () => null) await authenticate(ctx) @@ -194,15 +256,61 @@ describe('mobile E2EE v2 physical channel', () => { ).toEqual(new Uint8Array([2])) }) + it('releases sent-frame reservations only after the matching response arrives', async () => { + const ctx = setup(async () => null) + await authenticate(ctx) + expect(ctx.outboundMemoryBudget.evidence().inFlightBytes).toBe(0) + + expect(ctx.channel.sendText(JSON.stringify({ id: 'rpc-1', method: 'status.get' }))).toBe(true) + expect(ctx.outboundMemoryBudget.evidence().inFlightClaimCount).toBe(1) + + const response = serverFrame( + new TextEncoder().encode(JSON.stringify({ id: 'rpc-1', ok: true, result: {} })), + 'text', + 1n, + ctx.schedule + ) + await ctx.channel.handleMessage(Buffer.from(response).toString('base64')) + + expect(ctx.outboundMemoryBudget.evidence()).toMatchObject({ + inFlightBytes: 0, + inFlightClaimCount: 0 + }) + ctx.channel.socketClosed() + }) + + it('reports synchronous native send failure to relay callers', async () => { + const ctx = setup(async () => null) + await authenticate(ctx) + ctx.socket.send = () => { + throw new Error('native send failed') + } + + expect(ctx.channel.sendText(JSON.stringify({ id: 'rpc-1' }))).toBe(false) + expect(ctx.onError).toHaveBeenCalledWith( + expect.objectContaining({ message: 'E2EE v2 outbound buffer overflow' }) + ) + expect(ctx.outboundMemoryBudget.evidence()).toMatchObject({ + inFlightBytes: 0, + inFlightClaimCount: 0 + }) + ctx.channel.socketClosed() + }) + it('bounds the unified outbound queue and reports a wedged link', async () => { const ctx = setup(async () => null) await authenticate(ctx) ctx.socket.bufferedAmount = 9 * 1024 * 1024 const megabyte = new Uint8Array(1024 * 1024) for (let index = 0; index < 65; index++) { - expect(ctx.channel.sendBinary(megabyte)).toBe(true) + ctx.channel.sendBinary(megabyte) } expect(ctx.onError).toHaveBeenCalledOnce() expect(ctx.onError.mock.calls[0]![0].message).toBe('E2EE v2 outbound buffer overflow') + expect(ctx.outboundMemoryBudget.evidence().bufferedSourceCount).toBe(1) + ctx.channel.dispose() + expect(ctx.outboundMemoryBudget.evidence().bufferedSourceCount).toBe(1) + ctx.channel.socketClosed() + expect(ctx.outboundMemoryBudget.evidence().bufferedSourceCount).toBe(0) }) }) diff --git a/mobile/src/transport/mobile-e2ee-v2-physical-channel.ts b/mobile/src/transport/mobile-e2ee-v2-physical-channel.ts index ec3b8088915..54ac0e398d0 100644 --- a/mobile/src/transport/mobile-e2ee-v2-physical-channel.ts +++ b/mobile/src/transport/mobile-e2ee-v2-physical-channel.ts @@ -2,10 +2,28 @@ import { createWsOutboundBackpressureQueue, type WsOutboundBackpressureQueue } from '../../../src/shared/ws-outbound-backpressure-queue' +import { getUtf8ByteLength } from '../../../src/shared/utf8-byte-limits' import type { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session' +import { + createMobileInboundFrameQueue, + MOBILE_INBOUND_MAX_FRAME_BYTES, + type MobileInboundFrameQueue +} from './mobile-inbound-frame-queue' +import { + processMobileOutboundMemoryBudget, + MOBILE_OUTBOUND_MAX_FRAME_BYTES, + type MobileOutboundMemoryBudget +} from './mobile-outbound-memory-budget' +import { stringifyMobileOutboundJson } from './mobile-outbound-json' +import { createMobileOutboundSocketLedger } from './mobile-outbound-socket-ledger' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' type ChannelState = 'awaiting-ready' | 'awaiting-authenticated' | 'ready' -type OutboundItem = { kind: 'text'; plaintext: string } | { kind: 'binary'; plaintext: Uint8Array } +type OutboundItem = + | { kind: 'text'; plaintext: string; acknowledgementKey?: string } + | { kind: 'binary'; plaintext: Uint8Array } +const E2EE_V2_FRAME_OVERHEAD_BYTES = 82 +const AUTHENTICATION_ACKNOWLEDGEMENT_KEY = 'mobile-e2ee-v2-authentication' export class MobileE2EEAuthenticationError extends Error { constructor() { @@ -23,8 +41,10 @@ export type MobileE2EEV2Socket = { export class MobileE2EEV2PhysicalChannel { private state: ChannelState = 'awaiting-ready' private generation = 0 - private inboundChain: Promise = Promise.resolve() + private readonly inboundQueue: MobileInboundFrameQueue private readonly outboundQueue: WsOutboundBackpressureQueue + private readonly outboundSocketLedger + private outboundOverflowed = false constructor( private readonly args: { @@ -36,25 +56,50 @@ export class MobileE2EEV2PhysicalChannel { onText: (plaintext: string) => void onBinary: (plaintext: Uint8Array) => void onError: (error: Error) => void + outboundMemoryBudget?: MobileOutboundMemoryBudget } ) { + this.inboundQueue = createMobileInboundFrameQueue({ + process: (raw) => this.processMessage(raw, this.generation), + onError: args.onError, + overflowMessage: 'E2EE v2 inbound buffer overflow', + frameTooLargeMessage: 'E2EE v2 inbound frame too large' + }) + const outboundMemoryBudget = args.outboundMemoryBudget ?? processMobileOutboundMemoryBudget + this.outboundSocketLedger = createMobileOutboundSocketLedger({ + memoryBudget: outboundMemoryBudget, + readBufferedAmount: () => args.socket.bufferedAmount + }) this.outboundQueue = createWsOutboundBackpressureQueue({ // Why: encryption happens only when an admitted item reaches the wire, // so a bounded-queue rejection cannot burn an ordered v2 counter. send: (item) => { - args.socket.send( - item.kind === 'text' - ? args.session.sealText(item.plaintext) - : args.session.sealBinary(item.plaintext) + const cancelClaim = this.outboundSocketLedger.claimSentBytes( + outboundItemRetainedBytes(item), + item.kind === 'text' ? item.acknowledgementKey : undefined ) + if (!cancelClaim) { + this.failOutboundOverflow() + return + } + try { + args.socket.send( + item.kind === 'text' + ? args.session.sealText(item.plaintext) + : args.session.sealBinary(item.plaintext) + ) + } catch { + cancelClaim() + this.failOutboundOverflow() + } }, - byteLengthOf: (item) => - (item.kind === 'text' - ? new TextEncoder().encode(item.plaintext).length - : item.plaintext.length) + 82, + byteLengthOf: outboundItemRetainedBytes, getBufferedAmount: () => args.socket.bufferedAmount, isWritable: () => args.socket.readyState === args.socket.OPEN, - onOverflow: () => args.onError(new Error('E2EE v2 outbound buffer overflow')) + canSend: (bytes) => this.outboundSocketLedger.canSend(bytes), + claimQueuedBytes: (bytes) => outboundMemoryBudget.claimQueuedBytes(bytes), + maxFrameBytes: MOBILE_OUTBOUND_MAX_FRAME_BYTES, + onOverflow: () => this.failOutboundOverflow() }) } @@ -63,19 +108,15 @@ export class MobileE2EEV2PhysicalChannel { } handleMessage(raw: unknown): Promise { - const generation = this.generation - this.inboundChain = this.inboundChain - .then(() => this.processMessage(raw, generation)) - .catch((error: unknown) => { - if (generation === this.generation) { - this.args.onError(error instanceof Error ? error : new Error(String(error))) - } - }) - return this.inboundChain + return this.inboundQueue.enqueue(raw) } sendText(plaintext: string): boolean { - return this.enqueueReady({ kind: 'text', plaintext }) + return this.enqueueReady({ + kind: 'text', + plaintext, + acknowledgementKey: outboundTextAcknowledgementKey(plaintext) + }) } sendBinary(plaintext: Uint8Array): boolean { @@ -84,7 +125,14 @@ export class MobileE2EEV2PhysicalChannel { dispose(): void { this.generation++ + this.inboundQueue.dispose() this.outboundQueue.dispose() + this.outboundSocketLedger.retire() + } + + socketClosed(): void { + this.dispose() + this.outboundSocketLedger.socketClosed() } private async processMessage(raw: unknown, generation: number): Promise { @@ -104,6 +152,7 @@ export class MobileE2EEV2PhysicalChannel { return } if (this.state === 'awaiting-authenticated') { + this.outboundSocketLedger.acknowledge(AUTHENTICATION_ACKNOWLEDGEMENT_KEY) if (typeof plaintext === 'string' && isAuthenticationRejection(plaintext)) { throw new MobileE2EEAuthenticationError() } @@ -113,6 +162,10 @@ export class MobileE2EEV2PhysicalChannel { this.state = 'ready' this.args.onAuthenticated() } else if (typeof plaintext === 'string') { + const acknowledgementKey = outboundTextAcknowledgementKey(plaintext) + if (acknowledgementKey) { + this.outboundSocketLedger.acknowledge(acknowledgementKey) + } this.args.onText(plaintext) } else { this.args.onBinary(plaintext) @@ -125,7 +178,7 @@ export class MobileE2EEV2PhysicalChannel { } let ready: unknown try { - ready = JSON.parse(raw) + ready = parseMobileJsonTextWithinLimits(raw) } catch { throw new Error('Invalid E2EE v2 ready JSON') } @@ -135,7 +188,8 @@ export class MobileE2EEV2PhysicalChannel { this.state = 'awaiting-authenticated' this.outboundQueue.enqueue({ kind: 'text', - plaintext: JSON.stringify({ + acknowledgementKey: AUTHENTICATION_ACKNOWLEDGEMENT_KEY, + plaintext: stringifyMobileOutboundJson({ type: 'e2ee_auth', v: 2, transcriptHashB64: this.args.session.transcriptHashB64, @@ -149,12 +203,15 @@ export class MobileE2EEV2PhysicalChannel { if (!bytes || generation !== this.generation) { return null } + if (bytes.byteLength > MOBILE_INBOUND_MAX_FRAME_BYTES) { + throw new Error('E2EE v2 inbound frame too large') + } return this.args.session.openBinary(bytes) } private isAuthenticated(plaintext: string): boolean { try { - const message = JSON.parse(plaintext) as Record + const message = parseMobileJsonTextWithinLimits>(plaintext) return ( Object.keys(message).sort().join(',') === 'transcriptHashB64,type,v' && message.type === 'e2ee_authenticated' && @@ -170,16 +227,41 @@ export class MobileE2EEV2PhysicalChannel { if (this.state !== 'ready') { return false } - this.outboundQueue.enqueue(item) - return true + return this.outboundQueue.enqueue(item) && !this.outboundOverflowed + } + + private failOutboundOverflow(): void { + if (this.outboundOverflowed) { + return + } + this.outboundOverflowed = true + this.args.onError(new Error('E2EE v2 outbound buffer overflow')) } } function isAuthenticationRejection(plaintext: string): boolean { try { - const message = JSON.parse(plaintext) as Record + const message = parseMobileJsonTextWithinLimits>(plaintext) return message.type === 'e2ee_error' } catch { return false } } + +function outboundItemRetainedBytes(item: OutboundItem): number { + if (item.kind === 'binary') { + return item.plaintext.byteLength + E2EE_V2_FRAME_OVERHEAD_BYTES + } + const encryptedBytes = getUtf8ByteLength(item.plaintext) + E2EE_V2_FRAME_OVERHEAD_BYTES + const wireBytes = Math.ceil(encryptedBytes / 3) * 4 + return Math.max(wireBytes, item.plaintext.length * 2) +} + +function outboundTextAcknowledgementKey(plaintext: string): string | undefined { + try { + const value = parseMobileJsonTextWithinLimits<{ id?: unknown }>(plaintext) + return typeof value.id === 'string' ? value.id : undefined + } catch { + return undefined + } +} diff --git a/mobile/src/transport/mobile-host-list-storage.ts b/mobile/src/transport/mobile-host-list-storage.ts new file mode 100644 index 00000000000..76ccd6fddd2 --- /dev/null +++ b/mobile/src/transport/mobile-host-list-storage.ts @@ -0,0 +1,43 @@ +import { StoredHostProfileSchema, type StoredHostProfile } from './types' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' + +export const STORED_HOSTS_MAX_ENTRIES = 256 +export const STORED_HOSTS_MAX_STORAGE_CHARACTERS = 8 * 1024 * 1024 + +export function parseMobileStoredHostList(raw: string | null): StoredHostProfile[] | null { + if (!raw) { + return [] + } + if (raw.length > STORED_HOSTS_MAX_STORAGE_CHARACTERS) { + return null + } + try { + const parsed = parseMobileJsonTextWithinLimits(raw) + if (!Array.isArray(parsed) || parsed.length > STORED_HOSTS_MAX_ENTRIES) { + return null + } + return parsed.flatMap((item) => { + // Why: pre-v0.0.3 records stored deviceToken in AsyncStorage; dropping + // them keeps bearer material out of the metadata migration path. + if (item && typeof item === 'object' && 'deviceToken' in item) { + return [] + } + const result = StoredHostProfileSchema.safeParse(item) + return result.success ? [result.data] : [] + }) + } catch { + return null + } +} + +export function serializeMobileStoredHostList(hosts: StoredHostProfile[]): string { + if (hosts.length > STORED_HOSTS_MAX_ENTRIES) { + throw new Error('host list storage limit exceeded') + } + const validated = hosts.map((host) => StoredHostProfileSchema.parse(host)) + const serialized = JSON.stringify(validated) + if (serialized.length > STORED_HOSTS_MAX_STORAGE_CHARACTERS) { + throw new Error('host list storage limit exceeded') + } + return serialized +} diff --git a/mobile/src/transport/mobile-inbound-frame-queue-memory.test.ts b/mobile/src/transport/mobile-inbound-frame-queue-memory.test.ts new file mode 100644 index 00000000000..fe722ae8359 --- /dev/null +++ b/mobile/src/transport/mobile-inbound-frame-queue-memory.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it, vi } from 'vitest' +import { createMobileInboundFrameQueue } from './mobile-inbound-frame-queue' +import { createMobileInboundMemoryLedger } from './mobile-inbound-memory-ledger' + +function expectEmpty(ledger: ReturnType): void { + expect(ledger.evidence()).toMatchObject({ claimCount: 0, retainedBytes: 0 }) +} + +describe('mobile inbound frame queue aggregate memory', () => { + it('releases a claim after synchronous processing succeeds', async () => { + const ledger = createMobileInboundMemoryLedger(10) + const queue = createMobileInboundFrameQueue({ + process: vi.fn(), + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + + await queue.enqueue(new Uint8Array(3)) + + expectEmpty(ledger) + }) + + it('releases a claim after synchronous processing throws', async () => { + const ledger = createMobileInboundMemoryLedger(10) + const onError = vi.fn() + const queue = createMobileInboundFrameQueue({ + process: () => { + throw new Error('processing failed') + }, + onError, + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + + await queue.enqueue(new Uint8Array(3)) + + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'processing failed' })) + expectEmpty(ledger) + }) + + it('rejects a frame when another physical queue owns the remaining process credit', async () => { + const ledger = createMobileInboundMemoryLedger(5) + let releaseFirst!: () => void + const first = createMobileInboundFrameQueue({ + process: () => new Promise((resolve) => (releaseFirst = resolve)), + onError: vi.fn(), + overflowMessage: 'first overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + const secondProcess = vi.fn() + const secondError = vi.fn() + const second = createMobileInboundFrameQueue({ + process: secondProcess, + onError: secondError, + overflowMessage: 'aggregate overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + + const firstPending = first.enqueue(new Uint8Array(3)) + await second.enqueue(new Uint8Array(3)) + + expect(secondProcess).not.toHaveBeenCalled() + expect(secondError).toHaveBeenCalledWith( + expect.objectContaining({ message: 'aggregate overflow' }) + ) + expect(ledger.evidence()).toMatchObject({ claimCount: 1, retainedBytes: 3 }) + releaseFirst() + await firstPending + expectEmpty(ledger) + }) + + it('releases current and queued claims after processing rejects', async () => { + const ledger = createMobileInboundMemoryLedger(10) + let rejectCurrent!: (error: Error) => void + const onError = vi.fn() + const queue = createMobileInboundFrameQueue({ + process: () => new Promise((_resolve, reject) => (rejectCurrent = reject)), + onError, + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + const first = queue.enqueue(new Uint8Array(2)) + const second = queue.enqueue(new Uint8Array(3)) + expect(ledger.evidence()).toMatchObject({ claimCount: 2, retainedBytes: 5 }) + + rejectCurrent(new Error('processing failed')) + await Promise.all([first, second]) + + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'processing failed' })) + expectEmpty(ledger) + }) + + it('releases queued claims on dispose and the active claim when processing settles', async () => { + const ledger = createMobileInboundMemoryLedger(10) + let releaseCurrent!: () => void + const queue = createMobileInboundFrameQueue({ + process: () => new Promise((resolve) => (releaseCurrent = resolve)), + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + memoryLedger: ledger + }) + const first = queue.enqueue(new Uint8Array(2)) + const second = queue.enqueue(new Uint8Array(3)) + + queue.dispose() + expect(ledger.evidence()).toMatchObject({ claimCount: 1, retainedBytes: 2 }) + releaseCurrent() + await Promise.all([first, second]) + expectEmpty(ledger) + }) + + it('releases every claim after per-queue overflow', async () => { + const ledger = createMobileInboundMemoryLedger(10) + let releaseCurrent!: () => void + const queue = createMobileInboundFrameQueue({ + process: () => new Promise((resolve) => (releaseCurrent = resolve)), + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + maxBufferedBytes: 4, + memoryLedger: ledger + }) + const first = queue.enqueue(new Uint8Array(3)) + const rejected = queue.enqueue(new Uint8Array(2)) + + expect(ledger.evidence()).toMatchObject({ claimCount: 1, retainedBytes: 3 }) + releaseCurrent() + await Promise.all([first, rejected]) + expectEmpty(ledger) + }) + + it('does not claim process memory for an individually oversized frame', async () => { + const ledger = createMobileInboundMemoryLedger(10) + const queue = createMobileInboundFrameQueue({ + process: vi.fn(), + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + maxFrameBytes: 2, + memoryLedger: ledger + }) + + await queue.enqueue(new Uint8Array(3)) + + expectEmpty(ledger) + }) +}) diff --git a/mobile/src/transport/mobile-inbound-frame-queue.test.ts b/mobile/src/transport/mobile-inbound-frame-queue.test.ts new file mode 100644 index 00000000000..1a006a9d094 --- /dev/null +++ b/mobile/src/transport/mobile-inbound-frame-queue.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from 'vitest' +import { createMobileInboundFrameQueue } from './mobile-inbound-frame-queue' + +describe('mobile inbound frame queue', () => { + it('serializes async processing without losing frame order', async () => { + let release!: () => void + const gate = new Promise((resolve) => (release = resolve)) + const processed: string[] = [] + const queue = createMobileInboundFrameQueue({ + process: async (raw) => { + if (raw === 'first') { + await gate + } + processed.push(String(raw)) + }, + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large' + }) + + const first = queue.enqueue('first') + const second = queue.enqueue('second') + expect(processed).toEqual([]) + release() + await Promise.all([first, second]) + expect(processed).toEqual(['first', 'second']) + }) + + it('does not retain processed frame slots while a steady stream keeps the queue busy', async () => { + let releaseCurrent!: () => void + const queue = createMobileInboundFrameQueue({ + process: () => new Promise((resolve) => (releaseCurrent = resolve)), + onError: vi.fn(), + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large' + }) + let current = queue.enqueue({ byteLength: 1, id: 0 }) + let next = queue.enqueue({ byteLength: 1, id: 1 }) + + for (let id = 2; id < 256; id += 1) { + releaseCurrent() + await current + current = next + next = queue.enqueue({ byteLength: 1, id }) + expect(queue.evidence().storageSlots).toBeLessThanOrEqual(64) + } + + releaseCurrent() + await current + releaseCurrent() + await next + expect(queue.evidence()).toEqual({ retainedBytes: 0, retainedFrames: 0, storageSlots: 0 }) + }) + + it('drops and settles the backlog when retained frame count reaches its cap', async () => { + let release!: () => void + const gate = new Promise((resolve) => (release = resolve)) + const onError = vi.fn() + const queue = createMobileInboundFrameQueue({ + process: async () => gate, + onError, + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + maxBufferedFrames: 2, + maxBufferedBytes: 100, + maxFrameBytes: 100 + }) + + const pending = [ + queue.enqueue(new Uint8Array(1)), + queue.enqueue(new Uint8Array(1)), + queue.enqueue(new Uint8Array(1)) + ] + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'overflow' })) + release() + await Promise.all(pending) + }) + + it('fails closed when aggregate retained bytes exceed their cap', async () => { + let release!: () => void + const gate = new Promise((resolve) => (release = resolve)) + const onError = vi.fn() + const queue = createMobileInboundFrameQueue({ + process: async () => gate, + onError, + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + maxBufferedFrames: 10, + maxBufferedBytes: 5, + maxFrameBytes: 4 + }) + + const first = queue.enqueue(new Uint8Array(3)) + const second = queue.enqueue(new Uint8Array(3)) + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'overflow' })) + release() + await Promise.all([first, second]) + }) + + it('rejects a single known oversized frame without invoking its processor', async () => { + const process = vi.fn() + const onError = vi.fn() + const queue = createMobileInboundFrameQueue({ + process, + onError, + overflowMessage: 'overflow', + frameTooLargeMessage: 'too large', + maxFrameBytes: 4 + }) + + await queue.enqueue(new Uint8Array(5)) + + expect(process).not.toHaveBeenCalled() + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'too large' })) + }) +}) diff --git a/mobile/src/transport/mobile-inbound-frame-queue.ts b/mobile/src/transport/mobile-inbound-frame-queue.ts new file mode 100644 index 00000000000..4396d0f4832 --- /dev/null +++ b/mobile/src/transport/mobile-inbound-frame-queue.ts @@ -0,0 +1,209 @@ +import { + processMobileInboundMemoryLedger, + type MobileInboundMemoryLedger +} from './mobile-inbound-memory-ledger' + +// Why: a valid 10 MiB image preview expands twice through nested base64 framing to ~36 MiB of JS text. +export const MOBILE_INBOUND_MAX_FRAME_BYTES = 64 * 1024 * 1024 +// Why: two valid large previews may overlap, but a stalled consumer cannot retain a third. +export const MOBILE_INBOUND_MAX_BUFFERED_BYTES = 96 * 1024 * 1024 +// Why: tiny-frame floods need a count bound independent of their serialized size. +export const MOBILE_INBOUND_MAX_BUFFERED_FRAMES = 64 + +type PendingFrame = { + raw: unknown + releaseMemory: () => void + retainedBytes: number + settle: () => void +} + +export type MobileInboundFrameQueueEvidence = { + retainedBytes: number + retainedFrames: number + storageSlots: number +} + +export type MobileInboundFrameQueue = { + enqueue(raw: unknown): Promise + dispose(): void + evidence(): MobileInboundFrameQueueEvidence +} + +const QUEUE_COMPACTION_HEAD_THRESHOLD = 64 + +export function createMobileInboundFrameQueue(options: { + process: (raw: unknown) => Promise | void + onError: (error: Error) => void + overflowMessage: string + frameTooLargeMessage: string + maxFrameBytes?: number + maxBufferedBytes?: number + maxBufferedFrames?: number + memoryLedger?: MobileInboundMemoryLedger +}): MobileInboundFrameQueue { + const maxFrameBytes = options.maxFrameBytes ?? MOBILE_INBOUND_MAX_FRAME_BYTES + const maxBufferedBytes = options.maxBufferedBytes ?? MOBILE_INBOUND_MAX_BUFFERED_BYTES + const maxBufferedFrames = options.maxBufferedFrames ?? MOBILE_INBOUND_MAX_BUFFERED_FRAMES + const memoryLedger = options.memoryLedger ?? processMobileInboundMemoryLedger + const queue: Array = [] + let queueHead = 0 + let retainedBytes = 0 + let retainedFrames = 0 + let draining = false + let stopped = false + + const release = (frame: PendingFrame): void => { + retainedBytes -= frame.retainedBytes + retainedFrames -= 1 + frame.releaseMemory() + frame.settle() + } + + const dropQueued = (): void => { + while (queueHead < queue.length) { + release(queue[queueHead++]!) + } + queue.length = 0 + queueHead = 0 + } + + const fail = (error: Error): void => { + if (stopped) { + return + } + stopped = true + dropQueued() + options.onError(error) + } + + const finishDrain = (): void => { + if (queueHead === queue.length) { + queue.length = 0 + queueHead = 0 + } + draining = false + } + + const continueDrain = (): void => { + while (!stopped && queueHead < queue.length) { + const frame = queue[queueHead++]! + queue[queueHead - 1] = undefined + if (queueHead >= QUEUE_COMPACTION_HEAD_THRESHOLD) { + queue.splice(0, queueHead) + queueHead = 0 + } + let processing: Promise | void + try { + processing = options.process(frame.raw) + } catch (error) { + release(frame) + fail(error instanceof Error ? error : new Error(String(error))) + finishDrain() + return + } + if (processing) { + void processing.then( + () => { + release(frame) + if (stopped) { + finishDrain() + return + } + continueDrain() + }, + (error: unknown) => { + release(frame) + fail(error instanceof Error ? error : new Error(String(error))) + finishDrain() + } + ) + return + } + release(frame) + } + finishDrain() + } + + const drain = (): void => { + if (draining || stopped) { + return + } + draining = true + continueDrain() + } + + return { + enqueue(raw): Promise { + if (stopped) { + return Promise.resolve() + } + const frameBytes = mobileInboundFrameRetainedBytes(raw, maxFrameBytes) + if (frameBytes > maxFrameBytes) { + fail(new Error(options.frameTooLargeMessage)) + return Promise.resolve() + } + if (retainedFrames >= maxBufferedFrames || retainedBytes + frameBytes > maxBufferedBytes) { + fail(new Error(options.overflowMessage)) + return Promise.resolve() + } + const releaseMemory = memoryLedger.claim(frameBytes) + if (!releaseMemory) { + fail(new Error(options.overflowMessage)) + return Promise.resolve() + } + return new Promise((settle) => { + queue.push({ raw, releaseMemory, retainedBytes: frameBytes, settle }) + retainedBytes += frameBytes + retainedFrames += 1 + drain() + }) + }, + dispose(): void { + if (stopped) { + return + } + stopped = true + dropQueued() + }, + evidence(): MobileInboundFrameQueueEvidence { + return { retainedBytes, retainedFrames, storageSlots: queue.length } + } + } +} + +export function assertMobileInboundFrameSize( + raw: unknown, + message: string, + maxFrameBytes = MOBILE_INBOUND_MAX_FRAME_BYTES +): void { + if (mobileInboundFrameRetainedBytes(raw, maxFrameBytes) > maxFrameBytes) { + throw new Error(message) + } +} + +function mobileInboundFrameRetainedBytes(raw: unknown, unknownFrameBytes: number): number { + if (typeof raw === 'string') { + return raw.length * 2 + } + if (raw instanceof ArrayBuffer || ArrayBuffer.isView(raw)) { + return raw.byteLength + } + if (raw && typeof raw === 'object') { + const size = numericProperty(raw, 'size') ?? numericProperty(raw, 'byteLength') + if (size !== null) { + return size + } + } + // Why: unknown RN payload wrappers may materialize their bytes only during async conversion. + return unknownFrameBytes +} + +function numericProperty(value: object, key: 'size' | 'byteLength'): number | null { + if (!(key in value)) { + return null + } + const candidate = (value as Record)[key] + return typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0 + ? candidate + : null +} diff --git a/mobile/src/transport/mobile-inbound-memory-ledger.test.ts b/mobile/src/transport/mobile-inbound-memory-ledger.test.ts new file mode 100644 index 00000000000..6ebb9aa535f --- /dev/null +++ b/mobile/src/transport/mobile-inbound-memory-ledger.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { + createMobileInboundMemoryLedger, + MOBILE_PROCESS_INBOUND_MAX_RETAINED_BYTES, + MOBILE_PROCESS_INBOUND_MAX_RETAINED_FRAMES +} from './mobile-inbound-memory-ledger' + +describe('mobile inbound memory ledger', () => { + it('uses a process ceiling that preserves two full ordinary queue budgets', () => { + expect(MOBILE_PROCESS_INBOUND_MAX_RETAINED_BYTES).toBe(192 * 1024 * 1024) + expect(MOBILE_PROCESS_INBOUND_MAX_RETAINED_FRAMES).toBe(256) + }) + + it('makes claims idempotently releasable', () => { + const ledger = createMobileInboundMemoryLedger(5) + const release = ledger.claim(5) + expect(release).not.toBeNull() + expect(ledger.claim(1)).toBeNull() + + release!() + release!() + + expect(ledger.evidence()).toEqual({ + claimCount: 0, + maxRetainedBytes: 5, + maxRetainedFrames: MOBILE_PROCESS_INBOUND_MAX_RETAINED_FRAMES, + retainedBytes: 0 + }) + }) + + it('bounds aggregate zero-byte claims independently of the byte ceiling', () => { + const ledger = createMobileInboundMemoryLedger(5, 2) + const releaseFirst = ledger.claim(0) + const releaseSecond = ledger.claim(0) + + expect(releaseFirst).not.toBeNull() + expect(releaseSecond).not.toBeNull() + expect(ledger.claim(0)).toBeNull() + + releaseFirst!() + releaseSecond!() + }) +}) diff --git a/mobile/src/transport/mobile-inbound-memory-ledger.ts b/mobile/src/transport/mobile-inbound-memory-ledger.ts new file mode 100644 index 00000000000..eb92fca6fe5 --- /dev/null +++ b/mobile/src/transport/mobile-inbound-memory-ledger.ts @@ -0,0 +1,58 @@ +export const MOBILE_PROCESS_INBOUND_MAX_RETAINED_BYTES = 192 * 1024 * 1024 +export const MOBILE_PROCESS_INBOUND_MAX_RETAINED_FRAMES = 256 + +export type MobileInboundMemoryLedgerEvidence = { + claimCount: number + maxRetainedFrames: number + maxRetainedBytes: number + retainedBytes: number +} + +export type MobileInboundMemoryLedger = { + claim(bytes: number): (() => void) | null + evidence(): MobileInboundMemoryLedgerEvidence +} + +export function createMobileInboundMemoryLedger( + maxRetainedBytes: number, + maxRetainedFrames = MOBILE_PROCESS_INBOUND_MAX_RETAINED_FRAMES +): MobileInboundMemoryLedger { + if ( + !Number.isFinite(maxRetainedBytes) || + maxRetainedBytes < 1 || + !Number.isInteger(maxRetainedFrames) || + maxRetainedFrames < 1 + ) { + throw new Error('Mobile inbound memory limit must be positive') + } + let retainedBytes = 0 + let claimCount = 0 + return { + claim(bytes): (() => void) | null { + if (!Number.isFinite(bytes) || bytes < 0) { + return null + } + if (claimCount >= maxRetainedFrames || retainedBytes + bytes > maxRetainedBytes) { + return null + } + retainedBytes += bytes + claimCount += 1 + let released = false + return () => { + if (released) { + return + } + released = true + retainedBytes -= bytes + claimCount -= 1 + } + }, + evidence(): MobileInboundMemoryLedgerEvidence { + return { claimCount, maxRetainedBytes, maxRetainedFrames, retainedBytes } + } + } +} + +export const processMobileInboundMemoryLedger = createMobileInboundMemoryLedger( + MOBILE_PROCESS_INBOUND_MAX_RETAINED_BYTES +) diff --git a/mobile/src/transport/mobile-json-text-admission.test.ts b/mobile/src/transport/mobile-json-text-admission.test.ts new file mode 100644 index 00000000000..2e8c12be208 --- /dev/null +++ b/mobile/src/transport/mobile-json-text-admission.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import { + isMobileJsonStructureCapacityError, + parseMobileJsonTextWithinLimits +} from './mobile-json-text-admission' + +describe('mobile JSON text admission', () => { + it('accepts the exact structural-token limit and rejects one more', () => { + const limits = { structuralTokens: 4, nestingDepth: 2 } + + expect(parseMobileJsonTextWithinLimits('[0,0,0]', limits)).toEqual([0, 0, 0]) + expect(() => parseMobileJsonTextWithinLimits('[0,0,0,0]', limits)).toThrow( + 'JSON structure exceeds 4 tokens' + ) + }) + + it('accepts the exact nesting limit and rejects one more', () => { + const limits = { structuralTokens: 20, nestingDepth: 3 } + + expect(parseMobileJsonTextWithinLimits('[[[]]]', limits)).toEqual([[[]]]) + expect(() => parseMobileJsonTextWithinLimits('[[[[]]]]', limits)).toThrow( + 'JSON nesting exceeds 3 levels' + ) + }) + + it('does not count structural characters inside strings', () => { + expect( + parseMobileJsonTextWithinLimits('{"value":"[{\\\":,}]"}', { + structuralTokens: 3, + nestingDepth: 1 + }) + ).toEqual({ value: '[{":,}]' }) + }) + + it('identifies capacity failures separately from malformed JSON', () => { + const capacityError = captureError(() => + parseMobileJsonTextWithinLimits('[0,0]', { + structuralTokens: 2, + nestingDepth: 2 + }) + ) + const syntaxError = captureError(() => + parseMobileJsonTextWithinLimits('[', { + structuralTokens: 2, + nestingDepth: 2 + }) + ) + + expect(isMobileJsonStructureCapacityError(capacityError)).toBe(true) + expect(isMobileJsonStructureCapacityError(syntaxError)).toBe(false) + }) +}) + +function captureError(run: () => void): unknown { + try { + run() + return null + } catch (error) { + return error + } +} diff --git a/mobile/src/transport/mobile-json-text-admission.ts b/mobile/src/transport/mobile-json-text-admission.ts new file mode 100644 index 00000000000..7645cca69c1 --- /dev/null +++ b/mobile/src/transport/mobile-json-text-admission.ts @@ -0,0 +1,38 @@ +import { + assertJsonTextStructureWithinLimits, + JsonTextStructureCapacityError, + type JsonTextStructureLimits +} from '../../../src/shared/json-text-structure-limit' + +export const MOBILE_JSON_TEXT_STRUCTURE_LIMITS: JsonTextStructureLimits = { + structuralTokens: 1_000_000, + nestingDepth: 128 +} + +export function parseMobileJsonTextWithinLimits( + content: string, + limits: JsonTextStructureLimits = MOBILE_JSON_TEXT_STRUCTURE_LIMITS +): T { + assertJsonTextStructureWithinLimits(content, limits) + return JSON.parse(content) as T +} + +export function isMobileJsonStructureCapacityError( + error: unknown +): error is JsonTextStructureCapacityError { + return error instanceof JsonTextStructureCapacityError +} + +export function tryParseMobileJsonTextWithinLimits( + content: string, + limits: JsonTextStructureLimits = MOBILE_JSON_TEXT_STRUCTURE_LIMITS +): T | null { + try { + return parseMobileJsonTextWithinLimits(content, limits) + } catch (error) { + if (isMobileJsonStructureCapacityError(error)) { + throw error + } + return null + } +} diff --git a/mobile/src/transport/mobile-outbound-json.test.ts b/mobile/src/transport/mobile-outbound-json.test.ts new file mode 100644 index 00000000000..04d459e4780 --- /dev/null +++ b/mobile/src/transport/mobile-outbound-json.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from 'vitest' +import { stringifyMobileOutboundJson } from './mobile-outbound-json' + +describe('bounded mobile outbound JSON serialization', () => { + it('preserves ordinary JSON values', () => { + const value = { text: 'hello', escaped: '\u0000\n', items: [1, true, null] } + + expect(stringifyMobileOutboundJson(value, 1_000)).toBe(JSON.stringify(value)) + }) + + it('stops before traversing fields after an oversized string', () => { + const readAfter = vi.fn(() => 'late') + const value = { + payload: 'x'.repeat(100), + get after() { + return readAfter() + } + } + + expect(() => stringifyMobileOutboundJson(value, 32)).toThrow( + 'Mobile outbound JSON exceeds 32 bytes' + ) + expect(readAfter).not.toHaveBeenCalled() + }) + + it('accounts for JSON escaping before the native serializer allocates output', () => { + expect(stringifyMobileOutboundJson('\u0000', 8)).toBe('"\\u0000"') + expect(() => stringifyMobileOutboundJson('\u0000', 7)).toThrow( + 'Mobile outbound JSON exceeds 7 bytes' + ) + }) + + it('accepts arrays and objects whose serialized form is exactly at the limit', () => { + expect(stringifyMobileOutboundJson([1, 2], 5)).toBe('[1,2]') + expect(stringifyMobileOutboundJson({ a: 1 }, 7)).toBe('{"a":1}') + expect(() => stringifyMobileOutboundJson([1, 2], 4)).toThrow( + 'Mobile outbound JSON exceeds 4 bytes' + ) + expect(() => stringifyMobileOutboundJson({ a: 1 }, 6)).toThrow( + 'Mobile outbound JSON exceeds 6 bytes' + ) + }) + + it('does not charge object properties omitted by JSON.stringify', () => { + const value = { + omittedUndefined: undefined, + omittedFunction: () => 'ignored', + omittedSymbol: Symbol('ignored'), + kept: 1 + } + + expect(stringifyMobileOutboundJson(value, 10)).toBe('{"kept":1}') + }) + + it('serializes unsupported array elements as null within the exact limit', () => { + const value = [undefined, () => 'ignored', Symbol('ignored')] + + expect(stringifyMobileOutboundJson(value, 16)).toBe('[null,null,null]') + expect(() => stringifyMobileOutboundJson(value, 15)).toThrow( + 'Mobile outbound JSON exceeds 15 bytes' + ) + }) + + it('accounts for boxed JSON primitives without underestimating them', () => { + expect(stringifyMobileOutboundJson(new String('\u0000'), 8)).toBe('"\\u0000"') + expect(stringifyMobileOutboundJson(new Number(12345), 5)).toBe('12345') + expect(stringifyMobileOutboundJson(new Boolean(false), 5)).toBe('false') + expect(() => stringifyMobileOutboundJson(new String('\u0000'), 7)).toThrow( + 'Mobile outbound JSON exceeds 7 bytes' + ) + }) +}) diff --git a/mobile/src/transport/mobile-outbound-json.ts b/mobile/src/transport/mobile-outbound-json.ts new file mode 100644 index 00000000000..fc367f985eb --- /dev/null +++ b/mobile/src/transport/mobile-outbound-json.ts @@ -0,0 +1,123 @@ +export const MOBILE_OUTBOUND_MAX_JSON_BYTES = 4 * 1024 * 1024 + +export function stringifyMobileOutboundJson( + value: unknown, + maxBytes = MOBILE_OUTBOUND_MAX_JSON_BYTES +): string { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new Error('Mobile outbound JSON limit must be a positive safe integer') + } + let estimatedBytes = 0 + let root = true + const emittedMembers = new WeakMap() + const serialized = JSON.stringify(value, function (key, item: unknown) { + const isRoot = root + root = false + const parent = this as object + const inArray = Array.isArray(parent) + if (!isRoot && !inArray && isOmittedObjectValue(item)) { + return item + } + if (!isRoot) { + const emitted = emittedMembers.get(parent) ?? 0 + estimatedBytes += emitted > 0 ? 1 : 0 + if (!inArray) { + estimatedBytes += escapedJsonStringBytes(key) + 1 + } + emittedMembers.set(parent, emitted + 1) + } + estimatedBytes += inArray && isOmittedObjectValue(item) ? 4 : jsonValueBytes(item) + if (estimatedBytes > maxBytes) { + throw new Error(`Mobile outbound JSON exceeds ${maxBytes} bytes`) + } + return item + }) + if (serialized === undefined) { + throw new Error('Mobile outbound value is not JSON serializable') + } + return serialized +} + +function isOmittedObjectValue(value: unknown): boolean { + return value === undefined || typeof value === 'function' || typeof value === 'symbol' +} + +function jsonValueBytes(value: unknown): number { + if (value === null) { + return 4 + } + if (typeof value === 'string') { + return escapedJsonStringBytes(value) + } + if (typeof value === 'boolean') { + return value ? 4 : 5 + } + if (typeof value === 'number') { + return Number.isFinite(value) ? String(value).length : 4 + } + if (typeof value === 'object' && value !== null) { + return boxedPrimitiveJsonBytes(value) ?? 2 + } + return 4 +} + +function boxedPrimitiveJsonBytes(value: object): number | null { + try { + return escapedJsonStringBytes(String.prototype.valueOf.call(value)) + } catch {} + try { + const number = Number.prototype.valueOf.call(value) + return Number.isFinite(number) ? String(number).length : 4 + } catch {} + try { + return Boolean.prototype.valueOf.call(value) ? 4 : 5 + } catch { + return null + } +} + +function escapedJsonStringBytes(value: string): number { + let bytes = 2 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if ( + code === 0x22 || + code === 0x5c || + code === 0x08 || + code === 0x09 || + code === 0x0a || + code === 0x0c || + code === 0x0d + ) { + bytes += 2 + } else if ( + code <= 0x1f || + (code >= 0xd800 && code <= 0xdfff && !isSurrogatePair(value, index)) + ) { + bytes += 6 + } else if (code <= 0x7f) { + bytes += 1 + } else if (code <= 0x7ff) { + bytes += 2 + } else if (code >= 0xd800 && code <= 0xdbff) { + bytes += 4 + index += 1 + } else { + bytes += 3 + } + } + return bytes +} + +function isSurrogatePair(value: string, index: number): boolean { + const code = value.charCodeAt(index) + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + return next >= 0xdc00 && next <= 0xdfff + } + if (code >= 0xdc00 && code <= 0xdfff) { + const previous = value.charCodeAt(index - 1) + return previous >= 0xd800 && previous <= 0xdbff + } + return false +} diff --git a/mobile/src/transport/mobile-outbound-memory-budget.test.ts b/mobile/src/transport/mobile-outbound-memory-budget.test.ts new file mode 100644 index 00000000000..832e4e9ecbe --- /dev/null +++ b/mobile/src/transport/mobile-outbound-memory-budget.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest' +import { + createMobileOutboundMemoryBudget, + MOBILE_OUTBOUND_MAX_FRAME_BYTES, + MOBILE_PROCESS_OUTBOUND_MAX_BUFFERED_BYTES, + MOBILE_PROCESS_OUTBOUND_MAX_IN_FLIGHT_FRAMES, + MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_BYTES, + MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_FRAMES, + MOBILE_PROCESS_OUTBOUND_MAX_SOCKET_SOURCES +} from './mobile-outbound-memory-budget' + +describe('mobile outbound memory budget', () => { + it('keeps ordinary two-socket bursts within explicit process ceilings', () => { + expect(MOBILE_PROCESS_OUTBOUND_MAX_BUFFERED_BYTES).toBe(16 * 1024 * 1024) + expect(MOBILE_PROCESS_OUTBOUND_MAX_IN_FLIGHT_FRAMES).toBe(16_384) + expect(MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_BYTES).toBe(128 * 1024 * 1024) + expect(MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_FRAMES).toBe(16_384) + expect(MOBILE_PROCESS_OUTBOUND_MAX_SOCKET_SOURCES).toBe(64) + expect(MOBILE_OUTBOUND_MAX_FRAME_BYTES).toBe(8 * 1024 * 1024) + }) + + it('sums native bufferedAmount across registered physical sockets', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + let first = 4 + const firstSource = budget.registerBufferedAmount(() => first) + const secondSource = budget.registerBufferedAmount(() => 6) + expect(firstSource.canSend(0)).toBe(true) + + first = 5 + expect(firstSource.canSend(0)).toBe(false) + secondSource.release() + firstSource.release() + expect(budget.evidence()).toMatchObject({ bufferedBytes: 0, bufferedSourceCount: 0 }) + }) + + it('reserves room for the prospective native frame before allowing a send', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const source = budget.registerBufferedAmount(() => 6) + + expect(source.canSend(4)).toBe(true) + expect(source.canSend(5)).toBe(false) + }) + + it('counts unacknowledged frames when React Native does not expose bufferedAmount', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const source = budget.registerBufferedAmount(() => Number.NaN) + const release = source.claimInFlightBytes(7) + + expect(release).not.toBeNull() + expect(source.canSend(4)).toBe(false) + expect(budget.evidence()).toMatchObject({ + bufferedBytes: 7, + inFlightBytes: 7, + inFlightClaimCount: 1 + }) + + release!() + expect(source.canSend(10)).toBe(true) + }) + + it('caps sent-frame claims and retired socket readers independently', () => { + const budget = createMobileOutboundMemoryBudget({ + maxBufferedBytes: 10, + maxBufferedSources: 1, + maxInFlightFrames: 1 + }) + const source = budget.registerBufferedAmount(() => 0) + const release = source.claimInFlightBytes(0) + + expect(release).not.toBeNull() + expect(source.claimInFlightBytes(0)).toBeNull() + expect(budget.canRegisterBufferedAmount()).toBe(false) + expect(() => budget.registerBufferedAmount(() => 0)).toThrow( + 'Mobile outbound socket tracking limit exceeded' + ) + + release!() + source.release() + expect(budget.canRegisterBufferedAmount()).toBe(true) + }) + + it('bounds aggregate JavaScript backlog and releases claims idempotently', () => { + const budget = createMobileOutboundMemoryBudget({ maxQueuedBytes: 5 }) + const release = budget.claimQueuedBytes(5) + expect(release).not.toBeNull() + expect(budget.claimQueuedBytes(1)).toBeNull() + + release!() + release!() + + expect(budget.evidence()).toMatchObject({ queuedBytes: 0, queuedClaimCount: 0 }) + }) + + it('bounds aggregate zero-byte claims independently of the byte ceiling', () => { + const budget = createMobileOutboundMemoryBudget({ maxQueuedBytes: 5, maxQueuedFrames: 2 }) + const releaseFirst = budget.claimQueuedBytes(0) + const releaseSecond = budget.claimQueuedBytes(0) + + expect(releaseFirst).not.toBeNull() + expect(releaseSecond).not.toBeNull() + expect(budget.claimQueuedBytes(0)).toBeNull() + + releaseFirst!() + releaseSecond!() + }) +}) diff --git a/mobile/src/transport/mobile-outbound-memory-budget.ts b/mobile/src/transport/mobile-outbound-memory-budget.ts new file mode 100644 index 00000000000..2b94690a0c6 --- /dev/null +++ b/mobile/src/transport/mobile-outbound-memory-budget.ts @@ -0,0 +1,188 @@ +export const MOBILE_PROCESS_OUTBOUND_MAX_BUFFERED_BYTES = 16 * 1024 * 1024 +export const MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_BYTES = 128 * 1024 * 1024 +export const MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_FRAMES = 16_384 +export const MOBILE_PROCESS_OUTBOUND_MAX_IN_FLIGHT_FRAMES = 16_384 +export const MOBILE_PROCESS_OUTBOUND_MAX_SOCKET_SOURCES = 64 +export const MOBILE_OUTBOUND_MAX_FRAME_BYTES = 8 * 1024 * 1024 + +export type MobileOutboundMemoryBudgetEvidence = { + bufferedBytes: number + bufferedSourceCount: number + inFlightBytes: number + inFlightClaimCount: number + queuedBytes: number + queuedClaimCount: number +} + +export type MobileOutboundSocketMemory = { + canSend(bytes: number): boolean + claimInFlightBytes(bytes: number): (() => void) | null + release(): void +} + +export type MobileOutboundMemoryBudget = { + canRegisterBufferedAmount(): boolean + claimQueuedBytes(bytes: number): (() => void) | null + evidence(): MobileOutboundMemoryBudgetEvidence + registerBufferedAmount(readBufferedAmount: () => number): MobileOutboundSocketMemory +} + +export function createMobileOutboundMemoryBudget(options?: { + maxBufferedBytes?: number + maxBufferedSources?: number + maxInFlightFrames?: number + maxQueuedBytes?: number + maxQueuedFrames?: number +}): MobileOutboundMemoryBudget { + const maxBufferedBytes = options?.maxBufferedBytes ?? MOBILE_PROCESS_OUTBOUND_MAX_BUFFERED_BYTES + const maxBufferedSources = + options?.maxBufferedSources ?? MOBILE_PROCESS_OUTBOUND_MAX_SOCKET_SOURCES + const maxInFlightFrames = + options?.maxInFlightFrames ?? MOBILE_PROCESS_OUTBOUND_MAX_IN_FLIGHT_FRAMES + const maxQueuedBytes = options?.maxQueuedBytes ?? MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_BYTES + const maxQueuedFrames = options?.maxQueuedFrames ?? MOBILE_PROCESS_OUTBOUND_MAX_QUEUED_FRAMES + if ( + !Number.isFinite(maxBufferedBytes) || + maxBufferedBytes < 1 || + !Number.isInteger(maxBufferedSources) || + maxBufferedSources < 1 || + !Number.isInteger(maxInFlightFrames) || + maxInFlightFrames < 1 || + !Number.isFinite(maxQueuedBytes) || + maxQueuedBytes < 1 || + !Number.isInteger(maxQueuedFrames) || + maxQueuedFrames < 1 + ) { + throw new Error('Mobile outbound memory limits must be positive') + } + type BufferedSource = { + inFlightBytes: number + inFlightClaimCount: number + readBufferedAmount: () => number + } + const bufferedSources = new Set() + let queuedBytes = 0 + let queuedClaimCount = 0 + + const bufferedBytes = (): number => { + let total = 0 + for (const source of bufferedSources) { + let observed = 0 + try { + const value = source.readBufferedAmount() + if (Number.isFinite(value) && value > 0) { + observed = value + } + } catch { + // Closed native sockets may reject late reads; their close path unregisters them. + } + // Some runtimes update bufferedAmount asynchronously, so unacknowledged + // claims remain additive instead of assuming the native value includes them. + total += observed + source.inFlightBytes + } + return total + } + + const inFlightClaimCount = (): number => { + let count = 0 + for (const source of bufferedSources) { + count += source.inFlightClaimCount + } + return count + } + + const inFlightBytes = (): number => { + let total = 0 + for (const source of bufferedSources) { + total += source.inFlightBytes + } + return total + } + + return { + canRegisterBufferedAmount: () => bufferedSources.size < maxBufferedSources, + claimQueuedBytes(bytes): (() => void) | null { + if ( + !Number.isFinite(bytes) || + bytes < 0 || + queuedClaimCount >= maxQueuedFrames || + queuedBytes + bytes > maxQueuedBytes + ) { + return null + } + queuedBytes += bytes + queuedClaimCount += 1 + let released = false + return () => { + if (released) { + return + } + released = true + queuedBytes -= bytes + queuedClaimCount -= 1 + } + }, + evidence(): MobileOutboundMemoryBudgetEvidence { + return { + bufferedBytes: bufferedBytes(), + bufferedSourceCount: bufferedSources.size, + inFlightBytes: inFlightBytes(), + inFlightClaimCount: inFlightClaimCount(), + queuedBytes, + queuedClaimCount + } + }, + registerBufferedAmount(readBufferedAmount): MobileOutboundSocketMemory { + if (bufferedSources.size >= maxBufferedSources) { + throw new Error('Mobile outbound socket tracking limit exceeded') + } + const source: BufferedSource = { + inFlightBytes: 0, + inFlightClaimCount: 0, + readBufferedAmount + } + bufferedSources.add(source) + let registered = true + return { + canSend(bytes): boolean { + return ( + registered && + Number.isFinite(bytes) && + bytes >= 0 && + inFlightClaimCount() < maxInFlightFrames && + bytes <= maxBufferedBytes - bufferedBytes() + ) + }, + claimInFlightBytes(bytes): (() => void) | null { + if (!this.canSend(bytes)) { + return null + } + source.inFlightBytes += bytes + source.inFlightClaimCount += 1 + let released = false + return () => { + if (released) { + return + } + released = true + if (registered) { + source.inFlightBytes -= bytes + source.inFlightClaimCount -= 1 + } + } + }, + release(): void { + if (!registered) { + return + } + registered = false + bufferedSources.delete(source) + source.inFlightBytes = 0 + source.inFlightClaimCount = 0 + } + } + } + } +} + +export const processMobileOutboundMemoryBudget = createMobileOutboundMemoryBudget() diff --git a/mobile/src/transport/mobile-outbound-socket-ledger.test.ts b/mobile/src/transport/mobile-outbound-socket-ledger.test.ts new file mode 100644 index 00000000000..5dd43ede6dd --- /dev/null +++ b/mobile/src/transport/mobile-outbound-socket-ledger.test.ts @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { createMobileOutboundMemoryBudget } from './mobile-outbound-memory-budget' +import { + createMobileOutboundSocketLedger, + MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS +} from './mobile-outbound-socket-ledger' + +describe('mobile outbound socket ledger', () => { + afterEach(() => vi.useRealTimers()) + + it('releases only the matching delivered frame while the socket stays open', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => Number.NaN + }) + + expect(ledger.claimSentBytes(4, 'rpc-1')).not.toBeNull() + expect(ledger.claimSentBytes(5, 'rpc-2')).not.toBeNull() + expect(ledger.canSend(2)).toBe(false) + + ledger.acknowledge('rpc-1') + + expect(budget.evidence()).toMatchObject({ inFlightBytes: 5, inFlightClaimCount: 1 }) + expect(ledger.canSend(5)).toBe(true) + }) + + it('keeps unacknowledged bytes counted across logical disposal until native close', () => { + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => Number.NaN + }) + expect(ledger.claimSentBytes(6, 'rpc-1')).not.toBeNull() + + expect(budget.evidence()).toMatchObject({ bufferedBytes: 6, bufferedSourceCount: 1 }) + + ledger.socketClosed() + ledger.socketClosed() + expect(budget.evidence()).toMatchObject({ + bufferedBytes: 0, + bufferedSourceCount: 0, + inFlightClaimCount: 0 + }) + }) + + it('releases a retired socket after a missing native close callback', () => { + vi.useFakeTimers() + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => Number.NaN + }) + expect(ledger.claimSentBytes(6, 'rpc-1')).not.toBeNull() + + ledger.retire() + vi.advanceTimersByTime(MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS - 1) + expect(budget.evidence().bufferedSourceCount).toBe(1) + + vi.advanceTimersByTime(1) + expect(budget.evidence()).toMatchObject({ + bufferedBytes: 0, + bufferedSourceCount: 0, + inFlightClaimCount: 0 + }) + }) + + it('settles unkeyed frames after a runtime reports two drained polls', () => { + vi.useFakeTimers() + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => 0 + }) + expect(ledger.claimSentBytes(6)).not.toBeNull() + + vi.advanceTimersByTime(50) + + expect(budget.evidence()).toMatchObject({ inFlightBytes: 0, inFlightClaimCount: 0 }) + ledger.socketClosed() + }) + + it('requires two fresh drained polls after another unkeyed frame is sent', () => { + vi.useFakeTimers() + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => 0 + }) + expect(ledger.claimSentBytes(4)).not.toBeNull() + + vi.advanceTimersByTime(25) + expect(ledger.claimSentBytes(2)).not.toBeNull() + vi.advanceTimersByTime(25) + + expect(budget.evidence()).toMatchObject({ inFlightBytes: 6, inFlightClaimCount: 2 }) + vi.advanceTimersByTime(25) + expect(budget.evidence()).toMatchObject({ inFlightBytes: 0, inFlightClaimCount: 0 }) + ledger.socketClosed() + }) + + it('keeps unkeyed React Native claims until close when bufferedAmount is unavailable', () => { + vi.useFakeTimers() + const budget = createMobileOutboundMemoryBudget({ maxBufferedBytes: 10 }) + const ledger = createMobileOutboundSocketLedger({ + memoryBudget: budget, + readBufferedAmount: () => Number.NaN + }) + expect(ledger.claimSentBytes(6)).not.toBeNull() + + vi.advanceTimersByTime(1_000) + expect(budget.evidence()).toMatchObject({ inFlightBytes: 6, inFlightClaimCount: 1 }) + + ledger.socketClosed() + expect(budget.evidence()).toMatchObject({ inFlightBytes: 0, inFlightClaimCount: 0 }) + }) +}) diff --git a/mobile/src/transport/mobile-outbound-socket-ledger.ts b/mobile/src/transport/mobile-outbound-socket-ledger.ts new file mode 100644 index 00000000000..7ad1e41d1f0 --- /dev/null +++ b/mobile/src/transport/mobile-outbound-socket-ledger.ts @@ -0,0 +1,151 @@ +import type { MobileOutboundMemoryBudget } from './mobile-outbound-memory-budget' + +type InFlightClaim = { + acknowledgementKey?: string + releaseBytes: () => void +} + +export const MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS = 30_000 +const NATIVE_DRAIN_POLL_MS = 25 + +export type MobileOutboundSocketLedger = { + acknowledge(key: string): void + canSend(bytes: number): boolean + claimSentBytes(bytes: number, acknowledgementKey?: string): (() => void) | null + retire(): void + socketClosed(): void +} + +export function createMobileOutboundSocketLedger(args: { + memoryBudget: MobileOutboundMemoryBudget + readBufferedAmount: () => number +}): MobileOutboundSocketLedger { + const socketMemory = args.memoryBudget.registerBufferedAmount(args.readBufferedAmount) + const activeClaims = new Set() + const anonymousClaims = new Set() + const claimsByAcknowledgement = new Map>() + let drainTimer: ReturnType | null = null + let retireTimer: ReturnType | null = null + let consecutiveZeroDrainPolls = 0 + let closed = false + + const settleClaim = (claim: InFlightClaim): void => { + if (!activeClaims.delete(claim)) { + return + } + claim.releaseBytes() + if (claim.acknowledgementKey === undefined) { + anonymousClaims.delete(claim) + return + } + const keyed = claimsByAcknowledgement.get(claim.acknowledgementKey) + keyed?.delete(claim) + if (keyed?.size === 0) { + claimsByAcknowledgement.delete(claim.acknowledgementKey) + } + } + + const clearTimer = (timer: ReturnType | null): void => { + if (timer) { + clearTimeout(timer) + } + } + + const pollNativeDrain = (): void => { + drainTimer = null + if (closed || anonymousClaims.size === 0) { + return + } + let bufferedAmount: number + try { + bufferedAmount = args.readBufferedAmount() + } catch { + return + } + // React Native leaves this undefined; keyed application ACKs remain the authoritative path there. + if (!Number.isFinite(bufferedAmount) || bufferedAmount < 0) { + return + } + consecutiveZeroDrainPolls = bufferedAmount === 0 ? consecutiveZeroDrainPolls + 1 : 0 + if (consecutiveZeroDrainPolls >= 2) { + for (const claim of anonymousClaims) { + settleClaim(claim) + } + return + } + drainTimer = setTimeout(pollNativeDrain, NATIVE_DRAIN_POLL_MS) + unrefTimer(drainTimer) + } + + const scheduleNativeDrainPoll = (): void => { + if (drainTimer || closed) { + return + } + drainTimer = setTimeout(pollNativeDrain, NATIVE_DRAIN_POLL_MS) + unrefTimer(drainTimer) + } + + const closeLedger = (): void => { + if (closed) { + return + } + closed = true + clearTimer(drainTimer) + clearTimer(retireTimer) + drainTimer = null + retireTimer = null + for (const claim of activeClaims) { + settleClaim(claim) + } + anonymousClaims.clear() + claimsByAcknowledgement.clear() + socketMemory.release() + } + + return { + acknowledge(key): void { + const claims = claimsByAcknowledgement.get(key) + if (!claims) { + return + } + for (const claim of claims) { + settleClaim(claim) + } + }, + canSend: (bytes) => socketMemory.canSend(bytes), + claimSentBytes(bytes, acknowledgementKey): (() => void) | null { + const releaseBytes = socketMemory.claimInFlightBytes(bytes) + if (!releaseBytes) { + return null + } + const claim: InFlightClaim = { acknowledgementKey, releaseBytes } + activeClaims.add(claim) + if (acknowledgementKey === undefined) { + anonymousClaims.add(claim) + consecutiveZeroDrainPolls = 0 + scheduleNativeDrainPoll() + } else { + let claims = claimsByAcknowledgement.get(acknowledgementKey) + if (!claims) { + claims = new Set() + claimsByAcknowledgement.set(acknowledgementKey, claims) + } + claims.add(claim) + } + return () => settleClaim(claim) + }, + retire(): void { + if (closed || retireTimer) { + return + } + retireTimer = setTimeout(closeLedger, MOBILE_OUTBOUND_SOCKET_RETIRE_TIMEOUT_MS) + unrefTimer(retireTimer) + }, + socketClosed: closeLedger + } +} + +function unrefTimer(timer: ReturnType): void { + const nodeTimer = timer as unknown as { unref?: () => void } + nodeTimer.unref?.() +} diff --git a/mobile/src/transport/mobile-relay-credential-bundle.test.ts b/mobile/src/transport/mobile-relay-credential-bundle.test.ts index 4df9d44430b..47203867a7e 100644 --- a/mobile/src/transport/mobile-relay-credential-bundle.test.ts +++ b/mobile/src/transport/mobile-relay-credential-bundle.test.ts @@ -14,12 +14,14 @@ vi.mock('expo-secure-store', () => ({ vi.mock('react-native', () => ({ Platform: platform })) import { + MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS, deleteMobileRelayCredentialBundle, promotePairingJournalCredential, readMobileRelayCredentialBundle, writeMobileRelayCredentialBundle } from './mobile-relay-credential-bundle' import type { MobileRelayPairingJournal } from './mobile-relay-pairing-journal' +import { MOBILE_HOST_ID_MAX_CHARACTERS, PAIRING_DEVICE_TOKEN_MAX_CHARACTERS } from './types' const journal = { metadata: { @@ -106,6 +108,51 @@ describe('mobile relay credential bundle', () => { ).toThrow(/does not match/) }) + it('round-trips exact field and raw limits and rejects one character more', async () => { + const exact = { + v: 1 as const, + hostId: 'h'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS), + deviceToken: 't'.repeat(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS), + current: { + token: 'A'.repeat(43), + hash: 'B'.repeat(43), + version: 1, + expiresAt: 10_000 + } + } + + await expect(writeMobileRelayCredentialBundle(exact)).resolves.toBeUndefined() + const serialized = stored! + stored = + serialized + + ' '.repeat(MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS - serialized.length) + await expect(readMobileRelayCredentialBundle(exact.hostId)).resolves.toEqual(exact) + await expect( + writeMobileRelayCredentialBundle({ + ...exact, + deviceToken: `${exact.deviceToken}t` + }) + ).rejects.toThrow() + }) + + it('does not parse an oversized secure-store record', async () => { + stored = { + length: MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS + 1 + } as unknown as string + const parse = vi.spyOn(JSON, 'parse') + + await expect(readMobileRelayCredentialBundle('host-1')).resolves.toBeNull() + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + }) + + it('does not construct a secure-store key for an oversized host id', async () => { + await expect( + readMobileRelayCredentialBundle('h'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS + 1)) + ).resolves.toBeNull() + expect(secureStore.getItemAsync).not.toHaveBeenCalled() + }) + it('deletes the namespaced bundle and never enables it on web', async () => { await deleteMobileRelayCredentialBundle('host-1') expect(secureStore.deleteItemAsync).toHaveBeenCalledWith( diff --git a/mobile/src/transport/mobile-relay-credential-bundle.ts b/mobile/src/transport/mobile-relay-credential-bundle.ts index 87f6f3826f4..cb3e178dc83 100644 --- a/mobile/src/transport/mobile-relay-credential-bundle.ts +++ b/mobile/src/transport/mobile-relay-credential-bundle.ts @@ -3,8 +3,15 @@ import { Platform } from 'react-native' import { z } from 'zod' import type { DeviceCredentialInstalled } from '../../../src/shared/mobile-relay-credential-contract' import type { MobileRelayPairingJournal } from './mobile-relay-pairing-journal' +import { + MOBILE_HOST_ID_MAX_CHARACTERS, + MobileHostIdSchema, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS +} from './types' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS = 128 * 1024 const ResumeCredentialSchema = z .object({ token: Base64Url32ByteSchema, @@ -17,15 +24,15 @@ const ResumeCredentialSchema = z export const MobileRelayCredentialBundleSchema = z .object({ v: z.literal(1), - hostId: z.string().min(1), - deviceToken: z.string().min(1), + hostId: MobileHostIdSchema, + deviceToken: z.string().min(1).max(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS), current: ResumeCredentialSchema, grace: ResumeCredentialSchema.optional(), pending: z .object({ token: Base64Url32ByteSchema, hash: Base64Url32ByteSchema, - reqId: z.string().min(1) + reqId: z.string().min(1).max(128) }) .strict() .optional(), @@ -74,12 +81,15 @@ export async function readMobileRelayCredentialBundle( hostId: string ): Promise { requireNativeSecretStore() + if (!isValidHostId(hostId)) { + return null + } const raw = await SecureStore.getItemAsync(credentialKey(hostId), KEYCHAIN_OPTIONS) - if (raw === null) { + if (raw === null || raw.length > MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS) { return null } try { - const result = MobileRelayCredentialBundleSchema.safeParse(JSON.parse(raw)) + const result = MobileRelayCredentialBundleSchema.safeParse(parseMobileJsonTextWithinLimits(raw)) return result.success && result.data.hostId === hostId ? result.data : null } catch { return null @@ -91,20 +101,24 @@ export async function writeMobileRelayCredentialBundle( ): Promise { requireNativeSecretStore() const validated = MobileRelayCredentialBundleSchema.parse(bundle) - await SecureStore.setItemAsync( - credentialKey(validated.hostId), - JSON.stringify(validated), - KEYCHAIN_OPTIONS - ) + const serialized = JSON.stringify(validated) + if (serialized.length > MOBILE_RELAY_CREDENTIAL_BUNDLE_MAX_STORAGE_CHARACTERS) { + throw new Error('mobile relay credential bundle exceeds storage limit') + } + await SecureStore.setItemAsync(credentialKey(validated.hostId), serialized, KEYCHAIN_OPTIONS) } export async function deleteMobileRelayCredentialBundle(hostId: string): Promise { - if (Platform.OS === 'web') { + if (Platform.OS === 'web' || !isValidHostId(hostId)) { return } await SecureStore.deleteItemAsync(credentialKey(hostId), KEYCHAIN_OPTIONS) } +function isValidHostId(hostId: string): boolean { + return hostId.length > 0 && hostId.length <= MOBILE_HOST_ID_MAX_CHARACTERS +} + function requireNativeSecretStore(): void { if (Platform.OS === 'web') { throw new Error('Orca Relay credentials require a native secret store') diff --git a/mobile/src/transport/mobile-relay-direct-upgrade-journal.test.ts b/mobile/src/transport/mobile-relay-direct-upgrade-journal.test.ts new file mode 100644 index 00000000000..6a2ce32ea57 --- /dev/null +++ b/mobile/src/transport/mobile-relay-direct-upgrade-journal.test.ts @@ -0,0 +1,78 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const secureStore = vi.hoisted(() => ({ + getItemAsync: vi.fn(), + setItemAsync: vi.fn(), + deleteItemAsync: vi.fn() +})) +const platform = vi.hoisted(() => ({ OS: 'ios' })) + +vi.mock('expo-secure-store', () => ({ + WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'WHEN_UNLOCKED_THIS_DEVICE_ONLY', + ...secureStore +})) +vi.mock('react-native', () => ({ Platform: platform })) + +import { + MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS, + createMobileRelayDirectUpgradeJournal, + readMobileRelayDirectUpgradeJournal, + writeMobileRelayDirectUpgradeJournal +} from './mobile-relay-direct-upgrade-journal' +import { MOBILE_HOST_ID_MAX_CHARACTERS } from './types' + +describe('mobile relay direct-upgrade journal storage', () => { + let stored: string | null + + beforeEach(() => { + vi.clearAllMocks() + platform.OS = 'ios' + stored = null + secureStore.getItemAsync.mockImplementation(async () => stored) + secureStore.setItemAsync.mockImplementation(async (_key: string, value: string) => { + stored = value + }) + }) + + it('round-trips the exact host-id and raw limits', async () => { + const hostId = 'h'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS) + const journal = createMobileRelayDirectUpgradeJournal(hostId, (length) => + new Uint8Array(length).fill(7) + ) + + await writeMobileRelayDirectUpgradeJournal(journal) + const serialized = stored! + stored = + serialized + + ' '.repeat(MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS - serialized.length) + + await expect(readMobileRelayDirectUpgradeJournal(hostId)).resolves.toEqual(journal) + }) + + it('rejects a host id one character beyond the limit', () => { + expect(() => + createMobileRelayDirectUpgradeJournal( + 'h'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS + 1), + (length) => new Uint8Array(length) + ) + ).toThrow() + }) + + it('does not parse an oversized secure-store record', async () => { + stored = { + length: MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS + 1 + } as unknown as string + const parse = vi.spyOn(JSON, 'parse') + + await expect(readMobileRelayDirectUpgradeJournal('host-1')).resolves.toBeNull() + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + }) + + it('does not construct a secure-store key for an oversized host id', async () => { + await expect( + readMobileRelayDirectUpgradeJournal('h'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS + 1)) + ).resolves.toBeNull() + expect(secureStore.getItemAsync).not.toHaveBeenCalled() + }) +}) diff --git a/mobile/src/transport/mobile-relay-direct-upgrade-journal.ts b/mobile/src/transport/mobile-relay-direct-upgrade-journal.ts index 19775270b45..7fd3327e7dc 100644 --- a/mobile/src/transport/mobile-relay-direct-upgrade-journal.ts +++ b/mobile/src/transport/mobile-relay-direct-upgrade-journal.ts @@ -2,13 +2,16 @@ import * as SecureStore from 'expo-secure-store' import { Platform } from 'react-native' import { z } from 'zod' import { hashMobileRelayCredential } from './mobile-relay-credential-hash' +import { MOBILE_HOST_ID_MAX_CHARACTERS, MobileHostIdSchema } from './types' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS = 32 * 1024 export const MobileRelayDirectUpgradeJournalSchema = z .object({ v: z.literal(1), - hostId: z.string().min(1), + hostId: MobileHostIdSchema, reqId: z.string().min(1).max(128), pendingResumeToken: Base64Url32ByteSchema, pendingResumeTokenHash: Base64Url32ByteSchema @@ -43,12 +46,17 @@ export async function readMobileRelayDirectUpgradeJournal( hostId: string ): Promise { requireNativeSecretStore() + if (!isValidHostId(hostId)) { + return null + } const raw = await SecureStore.getItemAsync(journalKey(hostId), KEYCHAIN_OPTIONS) - if (!raw) { + if (!raw || raw.length > MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS) { return null } try { - const parsed = MobileRelayDirectUpgradeJournalSchema.safeParse(JSON.parse(raw)) + const parsed = MobileRelayDirectUpgradeJournalSchema.safeParse( + parseMobileJsonTextWithinLimits(raw) + ) return parsed.success && parsed.data.hostId === hostId ? parsed.data : null } catch { return null @@ -60,20 +68,24 @@ export async function writeMobileRelayDirectUpgradeJournal( ): Promise { requireNativeSecretStore() const parsed = MobileRelayDirectUpgradeJournalSchema.parse(journal) - await SecureStore.setItemAsync( - journalKey(parsed.hostId), - JSON.stringify(parsed), - KEYCHAIN_OPTIONS - ) + const serialized = JSON.stringify(parsed) + if (serialized.length > MOBILE_RELAY_DIRECT_UPGRADE_MAX_STORAGE_CHARACTERS) { + throw new Error('mobile relay direct-upgrade journal exceeds storage limit') + } + await SecureStore.setItemAsync(journalKey(parsed.hostId), serialized, KEYCHAIN_OPTIONS) } export async function deleteMobileRelayDirectUpgradeJournal(hostId: string): Promise { - if (Platform.OS === 'web') { + if (Platform.OS === 'web' || !isValidHostId(hostId)) { return } await SecureStore.deleteItemAsync(journalKey(hostId), KEYCHAIN_OPTIONS) } +function isValidHostId(hostId: string): boolean { + return hostId.length > 0 && hostId.length <= MOBILE_HOST_ID_MAX_CHARACTERS +} + function encodeBase64Url(value: Uint8Array): string { let binary = '' for (const byte of value) { diff --git a/mobile/src/transport/mobile-relay-e2ee-link.ts b/mobile/src/transport/mobile-relay-e2ee-link.ts index 183a8deea6d..0856732e3cc 100644 --- a/mobile/src/transport/mobile-relay-e2ee-link.ts +++ b/mobile/src/transport/mobile-relay-e2ee-link.ts @@ -4,7 +4,9 @@ import { } from '../../../src/shared/mobile-relay-phone-protocol' import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session' import { MobileE2EEV2PhysicalChannel } from './mobile-e2ee-v2-physical-channel' +import { assertMobileInboundFrameSize } from './mobile-inbound-frame-queue' import { websocketPayloadToUint8 } from './websocket-payload-bytes' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' export class RelayOuterError extends Error { constructor(readonly code: number) { @@ -32,7 +34,6 @@ export class MobileRelayE2eeLink { private readonly channel: MobileE2EEV2PhysicalChannel private outerReady = false private closed = false - private inboundChain: Promise = Promise.resolve() constructor(options: MobileRelayE2eeLinkOptions) { this.options = options @@ -44,16 +45,21 @@ export class MobileRelayE2eeLink { transport: 'relay', relayHostId: options.endpoint.relayHostId }) - this.channel = new MobileE2EEV2PhysicalChannel({ - session, - socket: this.socket, - deviceToken: options.deviceToken, - decodeBinary: websocketPayloadToUint8, - onAuthenticated: options.onAuthenticated, - onText: options.onText, - onBinary: options.onBinary, - onError: (error) => this.fail(error) - }) + try { + this.channel = new MobileE2EEV2PhysicalChannel({ + session, + socket: this.socket, + deviceToken: options.deviceToken, + decodeBinary: websocketPayloadToUint8, + onAuthenticated: options.onAuthenticated, + onText: options.onText, + onBinary: options.onBinary, + onError: (error) => this.fail(error) + }) + } catch (error) { + this.socket.close() + throw error + } this.bindSocket() } @@ -86,21 +92,25 @@ export class MobileRelayE2eeLink { ) } this.socket.onmessage = (event) => { - this.inboundChain = this.inboundChain - .then(async () => { - if (this.closed) { - return - } - if (!this.outerReady) { - this.acceptHello(event.data) - } else { - await this.channel.handleMessage(event.data) - } - }) - .catch((error: unknown) => this.fail(asError(error))) + if (this.closed) { + return + } + try { + if (!this.outerReady) { + assertMobileInboundFrameSize(event.data, 'relay hello frame too large') + this.acceptHello(event.data) + } else { + void this.channel.handleMessage(event.data) + } + } catch (error) { + this.fail(asError(error)) + } } this.socket.onerror = () => this.fail(new Error('relay transport error')) - this.socket.onclose = (event) => this.fail(new RelayOuterError(event.code || 1006)) + this.socket.onclose = (event) => { + this.channel.socketClosed() + this.fail(new RelayOuterError(event.code || 1006)) + } } private acceptHello(raw: unknown): void { @@ -109,7 +119,7 @@ export class MobileRelayE2eeLink { } let value: unknown try { - value = JSON.parse(raw) + value = parseMobileJsonTextWithinLimits(raw) } catch { throw new Error('invalid relay hello JSON') } diff --git a/mobile/src/transport/mobile-relay-host-overlay-store.test.ts b/mobile/src/transport/mobile-relay-host-overlay-store.test.ts index 812d61ebb1d..0c078b91e1e 100644 --- a/mobile/src/transport/mobile-relay-host-overlay-store.test.ts +++ b/mobile/src/transport/mobile-relay-host-overlay-store.test.ts @@ -9,6 +9,8 @@ vi.mock('@react-native-async-storage/async-storage', () => ({ default: asyncStor import { loadMobileRelayHostOverlays, + MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES, + MOBILE_RELAY_HOST_OVERLAY_MAX_STORAGE_CHARACTERS, removeMobileRelayHostOverlays, resetMobileRelayHostOverlayStoreForTests, saveMobileRelayHostOverlay @@ -98,4 +100,27 @@ describe('mobile relay host overlay store', () => { expect(asyncStorage.getItem).toHaveBeenCalledOnce() expect(asyncStorage.setItem).not.toHaveBeenCalled() }) + + it('accepts the exact overlay count and refuses to overwrite one over', async () => { + const exact = Array.from({ length: MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES }, (_, index) => ({ + ...OVERLAY, + hostId: `host-${index}` + })) + stored = JSON.stringify(exact) + const hostIds = new Set(exact.map(({ hostId }) => hostId)) + expect((await loadMobileRelayHostOverlays(hostIds)).size).toBe( + MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES + ) + + stored = JSON.stringify([...exact, { ...OVERLAY, hostId: 'one-over' }]) + await expect(saveMobileRelayHostOverlay(OVERLAY)).rejects.toThrow(/unreadable/) + expect(asyncStorage.setItem).not.toHaveBeenCalled() + }) + + it('rejects an oversized overlay payload before parsing', async () => { + stored = 'x'.repeat(MOBILE_RELAY_HOST_OVERLAY_MAX_STORAGE_CHARACTERS + 1) + + await expect(saveMobileRelayHostOverlay(OVERLAY)).rejects.toThrow(/unreadable/) + expect(asyncStorage.setItem).not.toHaveBeenCalled() + }) }) diff --git a/mobile/src/transport/mobile-relay-host-overlay-store.ts b/mobile/src/transport/mobile-relay-host-overlay-store.ts index 7f1519eb5c9..ad6f399b3d2 100644 --- a/mobile/src/transport/mobile-relay-host-overlay-store.ts +++ b/mobile/src/transport/mobile-relay-host-overlay-store.ts @@ -3,23 +3,36 @@ import { MobileRelayHostOverlaySchema, type MobileRelayHostOverlay } from './mobile-relay-host-overlay' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const OVERLAY_STORAGE_KEY = 'orca:mobile-relay:host-overlays:v2' +export const MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES = 64 +export const MOBILE_RELAY_HOST_OVERLAY_MAX_STORAGE_CHARACTERS = 512 * 1024 let overlayMutation: Promise = Promise.resolve() function parseOverlays(raw: string | null): MobileRelayHostOverlay[] | null { if (raw === null) { return [] } + if (raw.length > MOBILE_RELAY_HOST_OVERLAY_MAX_STORAGE_CHARACTERS) { + return null + } try { - const value = JSON.parse(raw) as unknown + const value = parseMobileJsonTextWithinLimits(raw) if (!Array.isArray(value)) { return null } - return value.flatMap((item) => { + if (value.length > MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES) { + return null + } + const overlays: MobileRelayHostOverlay[] = [] + for (const item of value) { const result = MobileRelayHostOverlaySchema.safeParse(item) - return result.success ? [result.data] : [] - }) + if (result.success) { + overlays.push(result.data) + } + } + return overlays } catch { return null } @@ -44,13 +57,24 @@ async function mutateOverlays( // Why: direct-only saves commonly have no overlay to remove; avoid a full // AsyncStorage write when cleanup leaves the durable list unchanged. if (next !== current) { - await AsyncStorage.setItem(OVERLAY_STORAGE_KEY, JSON.stringify(next)) + await AsyncStorage.setItem(OVERLAY_STORAGE_KEY, serializeOverlays(next)) } }) overlayMutation = mutation.catch(() => {}) return mutation } +function serializeOverlays(overlays: MobileRelayHostOverlay[]): string { + if (overlays.length > MOBILE_RELAY_HOST_OVERLAY_MAX_ENTRIES) { + throw new Error('mobile relay host overlay count limit exceeded') + } + const serialized = JSON.stringify(overlays) + if (serialized.length > MOBILE_RELAY_HOST_OVERLAY_MAX_STORAGE_CHARACTERS) { + throw new Error('mobile relay host overlay storage limit exceeded') + } + return serialized +} + export async function loadMobileRelayHostOverlays( existingHostIds: ReadonlySet ): Promise> { diff --git a/mobile/src/transport/mobile-relay-host-overlay.ts b/mobile/src/transport/mobile-relay-host-overlay.ts index 59c407c9ee3..ae15eefdf61 100644 --- a/mobile/src/transport/mobile-relay-host-overlay.ts +++ b/mobile/src/transport/mobile-relay-host-overlay.ts @@ -1,6 +1,8 @@ import { z } from 'zod' import { MobileRelayEndpointSchema } from '../../../src/shared/mobile-relay-credential-contract' +export const MOBILE_RELAY_HOST_OVERLAY_MAX_HOST_ID_CHARACTERS = 4_096 + export const MobileAccessEndpointSchema = z .object({ id: z.string().min(1).max(128), @@ -12,7 +14,7 @@ export const MobileAccessEndpointSchema = z export const MobileRelayHostOverlaySchema = z .object({ v: z.literal(2), - hostId: z.string().min(1), + hostId: z.string().min(1).max(MOBILE_RELAY_HOST_OVERLAY_MAX_HOST_ID_CHARACTERS), endpoints: z.array(MobileAccessEndpointSchema).min(1).max(16), relayHostId: z .string() diff --git a/mobile/src/transport/mobile-relay-invite-director.test.ts b/mobile/src/transport/mobile-relay-invite-director.test.ts index 04508024d07..3b0cab393df 100644 --- a/mobile/src/transport/mobile-relay-invite-director.test.ts +++ b/mobile/src/transport/mobile-relay-invite-director.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { resolvePairingInviteThroughDirector } from './mobile-relay-invite-director' +import { + MOBILE_RELAY_DIRECTOR_MAX_FRAME_BYTES, + resolvePairingInviteThroughDirector +} from './mobile-relay-invite-director' class FakeSocket { sent: string[] = [] @@ -77,4 +80,24 @@ describe('pairing invite director resolution', () => { await expect(resolving).rejects.toThrow(/not strictly newer/) }) + + it('rejects over-deep and oversized moves before materializing them', async () => { + const deepSocket = new FakeSocket() + const deep = resolvePairingInviteThroughDirector({ + relay, + createSocket: () => deepSocket as unknown as WebSocket + }) + deepSocket.onmessage?.({ data: `${'['.repeat(129)}0${']'.repeat(129)}` }) + await expect(deep).rejects.toThrow('invalid relay director move') + + const oversizedSocket = new FakeSocket() + const oversized = resolvePairingInviteThroughDirector({ + relay, + createSocket: () => oversizedSocket as unknown as WebSocket + }) + oversizedSocket.onmessage?.({ + data: 'x'.repeat(MOBILE_RELAY_DIRECTOR_MAX_FRAME_BYTES / 2 + 1) + }) + await expect(oversized).rejects.toThrow('invalid relay director move') + }) }) diff --git a/mobile/src/transport/mobile-relay-invite-director.ts b/mobile/src/transport/mobile-relay-invite-director.ts index 5dc83f526b6..1723f0e9f9e 100644 --- a/mobile/src/transport/mobile-relay-invite-director.ts +++ b/mobile/src/transport/mobile-relay-invite-director.ts @@ -1,5 +1,9 @@ import type { PairingRelay } from '../../../src/shared/mobile-relay-pairing-offer' import { RelayMovedSchema } from '../../../src/shared/mobile-relay-phone-protocol' +import { assertMobileInboundFrameSize } from './mobile-inbound-frame-queue' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' + +export const MOBILE_RELAY_DIRECTOR_MAX_FRAME_BYTES = 64 * 1024 export function resolvePairingInviteThroughDirector(args: { relay: PairingRelay @@ -32,7 +36,12 @@ export function resolvePairingInviteThroughDirector(args: { } let value: unknown try { - value = JSON.parse(event.data) + assertMobileInboundFrameSize( + event.data, + 'relay director move frame too large', + MOBILE_RELAY_DIRECTOR_MAX_FRAME_BYTES + ) + value = parseMobileJsonTextWithinLimits(event.data) } catch { finish(new Error('invalid relay director move')) return diff --git a/mobile/src/transport/mobile-relay-pairing-journal-store.test.ts b/mobile/src/transport/mobile-relay-pairing-journal-store.test.ts index 5790eb456aa..b91ff031238 100644 --- a/mobile/src/transport/mobile-relay-pairing-journal-store.test.ts +++ b/mobile/src/transport/mobile-relay-pairing-journal-store.test.ts @@ -20,7 +20,11 @@ vi.mock('expo-secure-store', () => ({ vi.mock('expo-crypto', () => ({ getRandomBytes: vi.fn() })) vi.mock('react-native', () => ({ Platform: platform })) -import { createMobileRelayPairingJournal } from './mobile-relay-pairing-journal' +import { + MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS, + MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS, + createMobileRelayPairingJournal +} from './mobile-relay-pairing-journal' import { clearMobileRelayPairingJournal, loadMobileRelayPairingJournal, @@ -28,7 +32,14 @@ import { saveMobileRelayPairingJournal, updateMobileRelayPairingJournal } from './mobile-relay-pairing-journal-store' -import type { PairingOffer } from './types' +import { + MOBILE_HOST_ID_MAX_CHARACTERS, + MOBILE_HOST_NAME_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PAIRING_ENDPOINT_MAX_CHARACTERS, + PAIRING_PUBLIC_KEY_MAX_CHARACTERS, + type PairingOffer +} from './types' const now = Date.UTC(2026, 6, 13) const offer = { @@ -94,6 +105,75 @@ describe('mobile relay pairing journal store', () => { await expect(loadMobileRelayPairingJournal()).resolves.toEqual(journal) }) + it('round-trips exact persisted field limits and rejects one character more', async () => { + const base = createMobileRelayPairingJournal({ + offer: offer as PairingOffer & { relay: NonNullable }, + hostId: 'host-1', + hostName: 'Blue Whale', + now, + randomBytes: (length) => new Uint8Array(length).fill(length) + }) + const exact = { + metadata: { + ...base.metadata, + host: { + ...base.metadata.host, + id: 'i'.repeat(MOBILE_HOST_ID_MAX_CHARACTERS), + name: 'n'.repeat(MOBILE_HOST_NAME_MAX_CHARACTERS), + endpoint: 'e'.repeat(PAIRING_ENDPOINT_MAX_CHARACTERS), + publicKeyB64: 'p'.repeat(PAIRING_PUBLIC_KEY_MAX_CHARACTERS) + } + }, + secrets: { + ...base.secrets, + deviceToken: 't'.repeat(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS) + } + } + + await expect(saveMobileRelayPairingJournal(exact)).resolves.toBeUndefined() + await expect(loadMobileRelayPairingJournal()).resolves.toEqual(exact) + await expect( + saveMobileRelayPairingJournal({ + ...exact, + secrets: { ...exact.secrets, deviceToken: `${exact.secrets.deviceToken}t` } + }) + ).rejects.toThrow() + }) + + it('repairs oversized metadata without parsing it', async () => { + metadataRaw = { + length: MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS + 1 + } as unknown as string + secretRaw = 'orphan' + const parse = vi.spyOn(JSON, 'parse') + + await expect(loadMobileRelayPairingJournal()).resolves.toBeNull() + expect(parse).not.toHaveBeenCalled() + expect(metadataRaw).toBeNull() + expect(secretRaw).toBeNull() + parse.mockRestore() + }) + + it('repairs oversized secrets without parsing them', async () => { + const journal = createMobileRelayPairingJournal({ + offer: offer as PairingOffer & { relay: NonNullable }, + hostId: 'host-1', + hostName: 'Blue Whale', + randomBytes: (length) => new Uint8Array(length).fill(9) + }) + metadataRaw = JSON.stringify(journal.metadata) + secretRaw = { + length: MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS + 1 + } as unknown as string + const parse = vi.spyOn(JSON, 'parse') + + await expect(loadMobileRelayPairingJournal()).resolves.toBeNull() + expect(parse).toHaveBeenCalledTimes(1) + expect(metadataRaw).toBeNull() + expect(secretRaw).toBeNull() + parse.mockRestore() + }) + it('records a provisional winner only for the active journal identity', async () => { const journal = createMobileRelayPairingJournal({ offer: offer as PairingOffer & { relay: NonNullable }, diff --git a/mobile/src/transport/mobile-relay-pairing-journal-store.ts b/mobile/src/transport/mobile-relay-pairing-journal-store.ts index b7044830da5..9f5841e117a 100644 --- a/mobile/src/transport/mobile-relay-pairing-journal-store.ts +++ b/mobile/src/transport/mobile-relay-pairing-journal-store.ts @@ -2,11 +2,15 @@ import AsyncStorage from '@react-native-async-storage/async-storage' import * as SecureStore from 'expo-secure-store' import { Platform } from 'react-native' import { + MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS, + MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS, MobileRelayPairingJournalMetadataSchema, MobileRelayPairingJournalSecretsSchema, type MobileRelayPairingJournal, - type MobileRelayPairingJournalMetadata + type MobileRelayPairingJournalMetadata, + type MobileRelayPairingJournalSecrets } from './mobile-relay-pairing-journal' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const JOURNAL_STORAGE_KEY = 'orca:mobile-relay:pairing-journal:v1' const JOURNAL_SECRET_KEY = 'orca.mobile-relay.pairing-journal.v1' @@ -24,6 +28,8 @@ export async function saveMobileRelayPairingJournal( if (metadata.journalId !== secrets.journalId) { throw new Error('mobile relay pairing journal identity mismatch') } + const serializedMetadata = serializeMetadata(metadata) + const serializedSecrets = serializeSecrets(secrets) const mutation = journalMutation.then(async () => { const existingRaw = await AsyncStorage.getItem(JOURNAL_STORAGE_KEY) const existing = existingRaw ? parseMetadata(existingRaw) : null @@ -38,8 +44,8 @@ export async function saveMobileRelayPairingJournal( // a new user-initiated scan may safely supersede a pre-authorization attempt. // Why: metadata-first makes a crash before the keychain write recover as // an incomplete journal, never as an untracked bearer secret. - await AsyncStorage.setItem(JOURNAL_STORAGE_KEY, JSON.stringify(metadata)) - await SecureStore.setItemAsync(JOURNAL_SECRET_KEY, JSON.stringify(secrets), KEYCHAIN_OPTIONS) + await AsyncStorage.setItem(JOURNAL_STORAGE_KEY, serializedMetadata) + await SecureStore.setItemAsync(JOURNAL_SECRET_KEY, serializedSecrets, KEYCHAIN_OPTIONS) }) journalMutation = mutation.catch(() => {}) return mutation @@ -100,7 +106,7 @@ export async function updateMobileRelayPairingJournal( if (next.journalId !== journalId) { throw new Error('mobile relay pairing journal identity mismatch') } - await AsyncStorage.setItem(JOURNAL_STORAGE_KEY, JSON.stringify(next)) + await AsyncStorage.setItem(JOURNAL_STORAGE_KEY, serializeMetadata(next)) }) journalMutation = mutation.catch(() => {}) return mutation @@ -121,23 +127,49 @@ export async function clearMobileRelayPairingJournal(journalId: string): Promise } function parseMetadata(raw: string): MobileRelayPairingJournalMetadata | null { + if (raw.length > MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS) { + return null + } try { - const result = MobileRelayPairingJournalMetadataSchema.safeParse(JSON.parse(raw)) + const result = MobileRelayPairingJournalMetadataSchema.safeParse( + parseMobileJsonTextWithinLimits(raw) + ) return result.success ? result.data : null } catch { return null } } -function parseSecrets(raw: string) { +function parseSecrets(raw: string): MobileRelayPairingJournalSecrets | null { + if (raw.length > MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS) { + return null + } try { - const result = MobileRelayPairingJournalSecretsSchema.safeParse(JSON.parse(raw)) + const result = MobileRelayPairingJournalSecretsSchema.safeParse( + parseMobileJsonTextWithinLimits(raw) + ) return result.success ? result.data : null } catch { return null } } +function serializeMetadata(metadata: MobileRelayPairingJournalMetadata): string { + const serialized = JSON.stringify(metadata) + if (serialized.length > MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS) { + throw new Error('mobile relay pairing metadata exceeds storage limit') + } + return serialized +} + +function serializeSecrets(secrets: MobileRelayPairingJournalSecrets): string { + const serialized = JSON.stringify(secrets) + if (serialized.length > MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS) { + throw new Error('mobile relay pairing secrets exceed storage limit') + } + return serialized +} + function requireNativeSecretStore(): void { if (Platform.OS === 'web') { throw new Error('Orca Relay pairing requires a native secret store') diff --git a/mobile/src/transport/mobile-relay-pairing-journal.ts b/mobile/src/transport/mobile-relay-pairing-journal.ts index 08b27ad7009..b2b0648b0c8 100644 --- a/mobile/src/transport/mobile-relay-pairing-journal.ts +++ b/mobile/src/transport/mobile-relay-pairing-journal.ts @@ -1,11 +1,23 @@ import * as ExpoCrypto from 'expo-crypto' import { sha256 } from '@noble/hashes/sha256' import { z } from 'zod' -import type { PairingRelay } from '../../../src/shared/mobile-relay-pairing-offer' +import { + PAIRING_RELAY_URL_MAX_CHARACTERS, + type PairingRelay +} from '../../../src/shared/mobile-relay-pairing-offer' import { hashMobileRelayCredential } from './mobile-relay-credential-hash' -import type { PairingOffer } from './types' +import { + MOBILE_HOST_ID_MAX_CHARACTERS, + MOBILE_HOST_NAME_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PAIRING_ENDPOINT_MAX_CHARACTERS, + PAIRING_PUBLIC_KEY_MAX_CHARACTERS, + type PairingOffer +} from './types' const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const MOBILE_RELAY_PAIRING_METADATA_MAX_STORAGE_CHARACTERS = 256 * 1024 +export const MOBILE_RELAY_PAIRING_SECRETS_MAX_STORAGE_CHARACTERS = 128 * 1024 export const MobileRelayPairingJournalMetadataSchema = z .object({ @@ -14,18 +26,18 @@ export const MobileRelayPairingJournalMetadataSchema = z offerFingerprint: Base64Url32ByteSchema, host: z .object({ - id: z.string().min(1), - name: z.string().min(1), - endpoint: z.string().min(1), - publicKeyB64: z.string().min(1), + id: z.string().min(1).max(MOBILE_HOST_ID_MAX_CHARACTERS), + name: z.string().min(1).max(MOBILE_HOST_NAME_MAX_CHARACTERS), + endpoint: z.string().min(1).max(PAIRING_ENDPOINT_MAX_CHARACTERS), + publicKeyB64: z.string().min(1).max(PAIRING_PUBLIC_KEY_MAX_CHARACTERS), lastConnected: z.number().int().nonnegative() }) .strict(), relay: z .object({ v: z.literal(1), - directorUrl: z.string().min(1), - cellUrl: z.string().min(1), + directorUrl: z.string().min(1).max(PAIRING_RELAY_URL_MAX_CHARACTERS), + cellUrl: z.string().min(1).max(PAIRING_RELAY_URL_MAX_CHARACTERS), assignmentEpoch: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), inviteExpiresAt: z.number().int().positive(), @@ -44,7 +56,7 @@ export const MobileRelayPairingJournalSecretsSchema = z .object({ v: z.literal(1), journalId: z.string().min(1).max(128), - deviceToken: z.string().min(1), + deviceToken: z.string().min(1).max(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS), inviteToken: Base64Url32ByteSchema, pendingResumeToken: Base64Url32ByteSchema }) diff --git a/mobile/src/transport/mobile-relay-physical-client.test.ts b/mobile/src/transport/mobile-relay-physical-client.test.ts index e4aaa0fe4f8..34054a9e866 100644 --- a/mobile/src/transport/mobile-relay-physical-client.test.ts +++ b/mobile/src/transport/mobile-relay-physical-client.test.ts @@ -8,7 +8,8 @@ const fakes = vi.hoisted(() => ({ start: vi.fn(), handleMessage: vi.fn(), sendText: vi.fn(() => true), - dispose: vi.fn() + dispose: vi.fn(), + socketClosed: vi.fn() })) vi.mock('./mobile-e2ee-v2-client-session', () => ({ @@ -23,6 +24,7 @@ vi.mock('./mobile-e2ee-v2-physical-channel', () => ({ handleMessage = fakes.handleMessage sendText = fakes.sendText dispose = fakes.dispose + socketClosed = fakes.socketClosed } })) @@ -132,4 +134,21 @@ describe('mobile relay physical pairing client', () => { await expect(status).rejects.toEqual(new RelayOuterError(4404)) expect(fakes.start).not.toHaveBeenCalled() }) + + it('rejects relay hello JSON above the nesting cap', async () => { + const socket = new FakeSocket() + const client = connectMobileRelayForPairing({ + relay, + deviceToken: 'device-token', + desktopPublicKeyB64: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + createSocket: () => socket as unknown as WebSocket + }) + const status = client.sendRequest('status.get') + + socket.receive(`${'['.repeat(129)}0${']'.repeat(129)}`) + + await expect(status).rejects.toThrow('invalid relay hello JSON') + expect(fakes.start).not.toHaveBeenCalled() + expect(fakes.dispose).toHaveBeenCalledOnce() + }) }) diff --git a/mobile/src/transport/mobile-relay-physical-client.ts b/mobile/src/transport/mobile-relay-physical-client.ts index acf264b8c73..2334d79e62c 100644 --- a/mobile/src/transport/mobile-relay-physical-client.ts +++ b/mobile/src/transport/mobile-relay-physical-client.ts @@ -2,9 +2,15 @@ import type { PairingRelay } from '../../../src/shared/mobile-relay-pairing-offe import { RelayPhoneHelloSchema } from '../../../src/shared/mobile-relay-phone-protocol' import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session' import { MobileE2EEV2PhysicalChannel } from './mobile-e2ee-v2-physical-channel' +import { assertMobileInboundFrameSize } from './mobile-inbound-frame-queue' +import { stringifyMobileOutboundJson } from './mobile-outbound-json' import { isRpcResponse } from './rpc-response-shape' import type { RpcResponse } from './types' import { websocketPayloadToUint8 } from './websocket-payload-bytes' +import { + isMobileJsonStructureCapacityError, + parseMobileJsonTextWithinLimits +} from './mobile-json-text-admission' export { RelayOuterError } from './mobile-relay-e2ee-link' import { RelayOuterError } from './mobile-relay-e2ee-link' @@ -47,35 +53,44 @@ export function connectMobileRelayForPairing(args: { resolveAuthenticated = resolve rejectAuthenticated = reject }) - const channel = new MobileE2EEV2PhysicalChannel({ - session, - socket, - deviceToken: args.deviceToken, - decodeBinary: websocketPayloadToUint8, - onAuthenticated: () => { - authenticated = true - resolveAuthenticated() - }, - onText: (plaintext) => { - let value: unknown - try { - value = JSON.parse(plaintext) - } catch { - return - } - if (!isRpcResponse(value)) { - return - } - const request = pending.get(value.id) - if (request) { - clearTimeout(request.timer) - pending.delete(value.id) - request.resolve(value) - } - }, - onBinary: () => {}, - onError: fail - }) + let channel: MobileE2EEV2PhysicalChannel + try { + channel = new MobileE2EEV2PhysicalChannel({ + session, + socket, + deviceToken: args.deviceToken, + decodeBinary: websocketPayloadToUint8, + onAuthenticated: () => { + authenticated = true + resolveAuthenticated() + }, + onText: (plaintext) => { + let value: unknown + try { + value = parseMobileJsonTextWithinLimits(plaintext) + } catch (error) { + if (isMobileJsonStructureCapacityError(error)) { + throw error + } + return + } + if (!isRpcResponse(value)) { + return + } + const request = pending.get(value.id) + if (request) { + clearTimeout(request.timer) + pending.delete(value.id) + request.resolve(value) + } + }, + onBinary: () => {}, + onError: fail + }) + } catch (error) { + socket.close() + throw error + } socket.onopen = () => { socket.send( @@ -87,23 +102,26 @@ export function connectMobileRelayForPairing(args: { }) ) } - let inboundChain: Promise = Promise.resolve() socket.onmessage = (event) => { - inboundChain = inboundChain - .then(async () => { - if (closed) { - return - } - if (!outerReady) { - acceptRelayHello(event.data) - return - } - await channel.handleMessage(event.data) - }) - .catch((error: unknown) => fail(asError(error))) + if (closed) { + return + } + try { + if (!outerReady) { + assertMobileInboundFrameSize(event.data, 'relay hello frame too large') + acceptRelayHello(event.data) + return + } + void channel.handleMessage(event.data) + } catch (error) { + fail(asError(error)) + } } socket.onerror = () => fail(new Error('relay transport error')) - socket.onclose = (event) => fail(new RelayOuterError(event.code || 1006)) + socket.onclose = (event) => { + channel.socketClosed() + fail(new RelayOuterError(event.code || 1006)) + } function acceptRelayHello(raw: unknown): void { if (typeof raw !== 'string') { @@ -111,7 +129,7 @@ export function connectMobileRelayForPairing(args: { } let value: unknown try { - value = JSON.parse(raw) + value = parseMobileJsonTextWithinLimits(raw) } catch { throw new Error('invalid relay hello JSON') } @@ -157,9 +175,21 @@ export function connectMobileRelayForPairing(args: { reject(new Error(`relay pairing RPC timed out: ${method}`)) }, requestTimeoutMs) pending.set(id, { resolve, reject, timer }) - if ( - !channel.sendText(JSON.stringify({ id, deviceToken: args.deviceToken, method, params })) - ) { + let serialized: string + try { + serialized = stringifyMobileOutboundJson({ + id, + deviceToken: args.deviceToken, + method, + params + }) + } catch { + clearTimeout(timer) + pending.delete(id) + reject(new Error('relay RPC request is too large')) + return + } + if (!channel.sendText(serialized)) { clearTimeout(timer) pending.delete(id) reject(new Error('relay E2EE channel not ready')) diff --git a/mobile/src/transport/mobile-relay-resume-director.test.ts b/mobile/src/transport/mobile-relay-resume-director.test.ts index 426ff89ba07..7ea28a368e2 100644 --- a/mobile/src/transport/mobile-relay-resume-director.test.ts +++ b/mobile/src/transport/mobile-relay-resume-director.test.ts @@ -63,4 +63,27 @@ describe('mobile relay resume director', () => { resolveMobileRelayEndpoint({ relay, resumeToken: 'A'.repeat(43), fetchImpl: oversized }) ).rejects.toThrow(/too large/) }) + + it('cancels a streamed response once it crosses the mobile limit', async () => { + let cancelled = false + const streamed = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(16 * 1024)) + controller.enqueue(new Uint8Array([1])) + }, + cancel() { + cancelled = true + } + }) + ) + ) + + await expect( + resolveMobileRelayEndpoint({ relay, resumeToken: 'A'.repeat(43), fetchImpl: streamed }) + ).rejects.toThrow(/too large/) + expect(cancelled).toBe(true) + }) }) diff --git a/mobile/src/transport/mobile-relay-resume-director.ts b/mobile/src/transport/mobile-relay-resume-director.ts index 86fd283b9a5..1679de56e6f 100644 --- a/mobile/src/transport/mobile-relay-resume-director.ts +++ b/mobile/src/transport/mobile-relay-resume-director.ts @@ -1,5 +1,10 @@ import { z } from 'zod' import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract' +import { + FetchResponseBodyTooLargeError, + readFetchResponseTextWithinLimit +} from '../../../src/shared/fetch-response-body' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' const MAX_RESPONSE_BYTES = 16 * 1024 const ResolveResponseSchema = z @@ -34,15 +39,16 @@ export async function resolveMobileRelayEndpoint(args: { if (!response.ok) { throw new Error(`relay director resolve failed (${response.status})`) } - const declaredLength = Number(response.headers.get('content-length') ?? 0) - if (declaredLength > MAX_RESPONSE_BYTES) { - throw new Error('relay director resolve response too large') + let raw: string + try { + raw = await readFetchResponseTextWithinLimit(response, MAX_RESPONSE_BYTES) + } catch (error) { + if (error instanceof FetchResponseBodyTooLargeError) { + throw new Error('relay director resolve response too large') + } + throw error } - const raw = await response.text() - if (new TextEncoder().encode(raw).byteLength > MAX_RESPONSE_BYTES) { - throw new Error('relay director resolve response too large') - } - const resolved = ResolveResponseSchema.parse(JSON.parse(raw) as unknown) + const resolved = ResolveResponseSchema.parse(parseMobileJsonTextWithinLimits(raw)) return { ...args.relay, cellUrl: resolved.cellUrl, diff --git a/mobile/src/transport/mobile-relay-rpc-session.ts b/mobile/src/transport/mobile-relay-rpc-session.ts index a0e71a85c1b..e3a3529bd8f 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.ts @@ -7,9 +7,14 @@ import { MobileRelayE2eeLink } from './mobile-relay-e2ee-link' import { MobileRelayRpcStreams } from './mobile-relay-rpc-streams' import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel' import { markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' +import { stringifyMobileOutboundJson } from './mobile-outbound-json' import { isRpcResponse } from './rpc-response-shape' import type { RpcClient } from './rpc-client' import type { ConnectionState, RpcResponse } from './types' +import { + isMobileJsonStructureCapacityError, + parseMobileJsonTextWithinLimits +} from './mobile-json-text-admission' type PendingRequest = { resolve: (response: RpcResponse) => void @@ -162,14 +167,23 @@ export function connectMobileRelayRpcSession(args: { } function sendFrame(request: { id: string; method: string; params?: unknown }): boolean { - return link.sendText(JSON.stringify({ ...request, deviceToken: args.deviceToken })) + try { + return link.sendText( + stringifyMobileOutboundJson({ ...request, deviceToken: args.deviceToken }) + ) + } catch { + return false + } } function handleText(plaintext: string): void { let value: unknown try { - value = JSON.parse(plaintext) - } catch { + value = parseMobileJsonTextWithinLimits(plaintext) + } catch (error) { + if (isMobileJsonStructureCapacityError(error)) { + throw error + } return } if (!isRpcResponse(value)) { diff --git a/mobile/src/transport/mobile-rpc-binary-frame-handler.ts b/mobile/src/transport/mobile-rpc-binary-frame-handler.ts new file mode 100644 index 00000000000..15e9c24061d --- /dev/null +++ b/mobile/src/transport/mobile-rpc-binary-frame-handler.ts @@ -0,0 +1,18 @@ +import { decryptBytes } from './e2ee' +import { websocketPayloadToUint8 } from './websocket-payload-bytes' + +export async function handleMobileRpcSocketBinaryMessage(args: { + rawData: unknown + key: Uint8Array + isCurrent: () => boolean + onFrame: (plaintext: Uint8Array) => void +}): Promise { + const bytes = await websocketPayloadToUint8(args.rawData) + if (!args.isCurrent() || !bytes) { + return + } + const plaintextBytes = decryptBytes(bytes, args.key) + if (plaintextBytes) { + args.onFrame(plaintextBytes) + } +} diff --git a/mobile/src/transport/pairing.test.ts b/mobile/src/transport/pairing.test.ts index 9b7b7514369..5f14f925d6f 100644 --- a/mobile/src/transport/pairing.test.ts +++ b/mobile/src/transport/pairing.test.ts @@ -1,6 +1,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { decodePairingUrl, extractPairingCodeFromUrl, parsePairingCode } from './pairing' -import type { PairingOffer } from './types' +import { + PAIRING_CODE_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PairingOfferSchema, + type PairingOffer +} from './types' const offer: PairingOffer = { v: 2, @@ -72,4 +77,31 @@ describe('pairing deep links', () => { expect(parsePairingCode(code)).toEqual(proxiedOffer) }) + + it('rejects an oversized code before base64 decoding', () => { + const decode = vi.fn(() => { + throw new Error('must not decode') + }) + vi.stubGlobal('atob', decode) + const oversized = 'A'.repeat(PAIRING_CODE_MAX_CHARACTERS + 1) + + expect(parsePairingCode(oversized)).toBeNull() + expect(decodePairingUrl(`orca://pair?code=${oversized}`)).toBeNull() + expect(decode).not.toHaveBeenCalled() + }) + + it('accepts the device-token limit and rejects one character more', () => { + expect( + PairingOfferSchema.safeParse({ + ...offer, + deviceToken: 't'.repeat(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS) + }).success + ).toBe(true) + expect( + PairingOfferSchema.safeParse({ + ...offer, + deviceToken: 't'.repeat(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS + 1) + }).success + ).toBe(false) + }) }) diff --git a/mobile/src/transport/pairing.ts b/mobile/src/transport/pairing.ts index d330f1834fa..0322c107b4b 100644 --- a/mobile/src/transport/pairing.ts +++ b/mobile/src/transport/pairing.ts @@ -1,4 +1,10 @@ -import { PairingOfferSchema, type PairingOffer } from './types' +import { + PAIRING_CODE_MAX_CHARACTERS, + PAIRING_INPUT_MAX_CHARACTERS, + PairingOfferSchema, + type PairingOffer +} from './types' +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' // Why: this file mirrors src/shared/pairing.ts (which is covered by CI // vitest) but uses atob/btoa because Metro/Hermes don't ship Node's @@ -6,6 +12,9 @@ import { PairingOfferSchema, type PairingOffer } from './types' // the other. export function decodePairingUrl(url: string): PairingOffer | null { + if (url.length > PAIRING_INPUT_MAX_CHARACTERS) { + return null + } try { const code = extractPairingCodeFromUrl(url) if (!code) { @@ -21,6 +30,9 @@ export function decodePairingUrl(url: string): PairingOffer | null { // extraction here makes QR scan, paste, and external deep-link flows // accept the same URL shapes. export function extractPairingCodeFromUrl(url: string): string | null { + if (url.length > PAIRING_INPUT_MAX_CHARACTERS) { + return null + } const trimmed = url.trim() const match = /^orca:\/\/([^/?#]*)([^?#]*)?/i.exec(trimmed) if (!match) { @@ -53,6 +65,9 @@ export function extractPairingCodeFromUrl(url: string): string | null { // string so the paste-pair flow can take whichever the user actually // copied from desktop. export function parsePairingCode(input: string): PairingOffer | null { + if (input.length > PAIRING_INPUT_MAX_CHARACTERS) { + return null + } const trimmed = input.trim() if (!trimmed) { return null @@ -68,11 +83,18 @@ export function parsePairingCode(input: string): PairingOffer | null { } function decodePairingBase64(base64url: string): PairingOffer { + if ( + base64url.length === 0 || + base64url.length > PAIRING_CODE_MAX_CHARACTERS || + !/^[A-Za-z0-9+/_-]+={0,2}$/.test(base64url) + ) { + throw new Error('Invalid pairing code') + } // Why: desktop intentionally strips base64 padding from QR payloads. Some // mobile JS runtimes reject unpadded atob input, so restore it before decode. const base64 = padBase64(base64url.replace(/-/g, '+').replace(/_/g, '/')) const json = atob(base64) - return PairingOfferSchema.parse(JSON.parse(json)) + return PairingOfferSchema.parse(parseMobileJsonTextWithinLimits(json)) } function padBase64(base64: string): string { diff --git a/mobile/src/transport/redacted-websocket-endpoint.ts b/mobile/src/transport/redacted-websocket-endpoint.ts new file mode 100644 index 00000000000..b19431099ee --- /dev/null +++ b/mobile/src/transport/redacted-websocket-endpoint.ts @@ -0,0 +1,9 @@ +// Why: keep device tokens and full URLs out of connection logs. +export function redactedWebSocketEndpoint(endpoint: string): string { + try { + const url = new URL(endpoint) + return (url.protocol === 'ws:' || url.protocol === 'wss:') && url.host ? url.host : 'unknown' + } catch { + return 'unknown' + } +} diff --git a/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts b/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts index 216fd2aec56..8030dec8ce6 100644 --- a/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts +++ b/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts @@ -32,4 +32,92 @@ describe('handleTerminalBinaryFrame', () => { expect(recordValidatedInboundTraffic).toHaveBeenCalledTimes(1) expect(listener).toHaveBeenCalledWith({ type: 'metadata', streamId: 42, cwd: '/repo/src' }) }) + + it('assembles an ordinary snapshot without changing its payload', () => { + const listener = vi.fn() + const terminalSnapshots = new Map() + const options = { + terminalSnapshots, + getListener: () => listener, + recordValidatedInboundTraffic: vi.fn() + } + + handleTerminalBinaryFrame( + encodeFrame(TerminalStreamOpcode.SnapshotStart, 7, { kind: 'scrollback' }), + options + ) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, 'hello '), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, 'world'), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotEnd, 7, ''), options) + + expect(listener).toHaveBeenCalledWith({ + kind: 'scrollback', + serialized: 'hello world', + streamId: 7, + type: 'scrollback' + }) + expect(terminalSnapshots.size).toBe(0) + }) + + it('drops cumulative snapshot data at the configured byte bound', () => { + const listener = vi.fn() + const terminalSnapshots = new Map() + const options = { + terminalSnapshots, + getListener: () => listener, + recordValidatedInboundTraffic: vi.fn(), + maxSnapshotBytes: 5 + } + + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotStart, 7, {}), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, '12345'), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, '6'), options) + + expect(terminalSnapshots.size).toBe(0) + expect(listener).toHaveBeenLastCalledWith({ + message: 'Terminal snapshot exceeded size limit.', + streamId: 7, + type: 'error' + }) + }) + + it('bounds tiny snapshot chunks independently of byte size', () => { + const listener = vi.fn() + const terminalSnapshots = new Map() + const options = { + terminalSnapshots, + getListener: () => listener, + recordValidatedInboundTraffic: vi.fn(), + maxSnapshotChunks: 2 + } + + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotStart, 7, {}), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, ''), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, ''), options) + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.SnapshotChunk, 7, ''), options) + + expect(terminalSnapshots.size).toBe(0) + expect(listener).toHaveBeenLastCalledWith({ + message: 'Terminal snapshot exceeded size limit.', + streamId: 7, + type: 'error' + }) + }) + + it('rejects a terminal frame before decoding an oversized payload', () => { + const listener = vi.fn() + + handleTerminalBinaryFrame(encodeFrame(TerminalStreamOpcode.Metadata, 42, { value: 'large' }), { + terminalSnapshots: new Map(), + getListener: () => listener, + recordValidatedInboundTraffic: vi.fn(), + maxFramePayloadBytes: 4 + }) + + expect(listener).toHaveBeenCalledWith({ + message: 'Terminal stream frame exceeded size limit.', + streamId: 42, + type: 'error' + }) + }) }) diff --git a/mobile/src/transport/rpc-client-terminal-binary-frame.ts b/mobile/src/transport/rpc-client-terminal-binary-frame.ts index ee7da35383e..562362b7e3f 100644 --- a/mobile/src/transport/rpc-client-terminal-binary-frame.ts +++ b/mobile/src/transport/rpc-client-terminal-binary-frame.ts @@ -9,14 +9,44 @@ export type TerminalSnapshotState = { streamId: number meta: Record chunks: string[] + bytes: number } +export const MOBILE_TERMINAL_STREAM_FRAME_MAX_PAYLOAD_BYTES = 256 * 1024 +export const MOBILE_TERMINAL_SNAPSHOT_MAX_BYTES = 2 * 1024 * 1024 +export const MOBILE_TERMINAL_SNAPSHOT_MAX_AGGREGATE_BYTES = 16 * 1024 * 1024 +export const MOBILE_TERMINAL_SNAPSHOT_MAX_ACTIVE = 16 +export const MOBILE_TERMINAL_SNAPSHOT_MAX_CHUNKS = 1_024 + type StreamingListener = (result: unknown) => void type TerminalBinaryFrameOptions = { terminalSnapshots: Map getListener: (streamId: number) => StreamingListener | undefined recordValidatedInboundTraffic: () => void + maxFramePayloadBytes?: number + maxSnapshotBytes?: number + maxSnapshotAggregateBytes?: number + maxActiveSnapshots?: number + maxSnapshotChunks?: number +} + +function retainedSnapshotBytes(snapshots: Map): number { + let bytes = 0 + for (const snapshot of snapshots.values()) { + bytes += snapshot.bytes + } + return bytes +} + +function rejectSnapshot( + options: TerminalBinaryFrameOptions, + listener: StreamingListener, + streamId: number, + message: string +): void { + options.terminalSnapshots.delete(streamId) + listener({ type: 'error', streamId, message }) } export function handleTerminalBinaryFrame( @@ -32,6 +62,12 @@ export function handleTerminalBinaryFrame( options.recordValidatedInboundTraffic() return } + const maxFramePayloadBytes = + options.maxFramePayloadBytes ?? MOBILE_TERMINAL_STREAM_FRAME_MAX_PAYLOAD_BYTES + if (frame.payload.byteLength > maxFramePayloadBytes) { + rejectSnapshot(options, listener, frame.streamId, 'Terminal stream frame exceeded size limit.') + return + } if (frame.opcode === TerminalStreamOpcode.Output) { options.recordValidatedInboundTraffic() listener({ @@ -46,11 +82,20 @@ export function handleTerminalBinaryFrame( if (!meta) { return } + const maxActiveSnapshots = options.maxActiveSnapshots ?? MOBILE_TERMINAL_SNAPSHOT_MAX_ACTIVE + if ( + !options.terminalSnapshots.has(frame.streamId) && + options.terminalSnapshots.size >= maxActiveSnapshots + ) { + rejectSnapshot(options, listener, frame.streamId, 'Too many terminal snapshots are active.') + return + } options.recordValidatedInboundTraffic() options.terminalSnapshots.set(frame.streamId, { streamId: frame.streamId, meta, - chunks: [] + chunks: [], + bytes: 0 }) return } @@ -60,7 +105,21 @@ export function handleTerminalBinaryFrame( if (!snapshot) { return } + const maxSnapshotBytes = options.maxSnapshotBytes ?? MOBILE_TERMINAL_SNAPSHOT_MAX_BYTES + const maxSnapshotAggregateBytes = + options.maxSnapshotAggregateBytes ?? MOBILE_TERMINAL_SNAPSHOT_MAX_AGGREGATE_BYTES + const maxSnapshotChunks = options.maxSnapshotChunks ?? MOBILE_TERMINAL_SNAPSHOT_MAX_CHUNKS + if ( + snapshot.chunks.length >= maxSnapshotChunks || + snapshot.bytes + frame.payload.byteLength > maxSnapshotBytes || + retainedSnapshotBytes(options.terminalSnapshots) + frame.payload.byteLength > + maxSnapshotAggregateBytes + ) { + rejectSnapshot(options, listener, frame.streamId, 'Terminal snapshot exceeded size limit.') + return + } snapshot.chunks.push(decodeTerminalStreamText(frame.payload)) + snapshot.bytes += frame.payload.byteLength return } if (frame.opcode === TerminalStreamOpcode.SnapshotEnd) { diff --git a/mobile/src/transport/rpc-client.test.ts b/mobile/src/transport/rpc-client.test.ts index b2adb18c4ce..73302fb8b5d 100644 --- a/mobile/src/transport/rpc-client.test.ts +++ b/mobile/src/transport/rpc-client.test.ts @@ -153,6 +153,25 @@ describe('mobile rpc-client connection timeout', () => { client.close() }) + it('keeps WebSocket credentials and URL paths out of retained connection logs', () => { + const details: Array = [] + const client = connect( + 'wss://user:password@desktop.example:8443/relay?access_token=secret', + 'token', + 'server-key', + { + onLog: (entry) => details.push(entry.detail) + } + ) + + expect(details).toContain('desktop.example:8443') + expect(details.join('\n')).not.toContain('password') + expect(details.join('\n')).not.toContain('access_token') + expect(details.join('\n')).not.toContain('/relay') + + client.close() + }) + it('ignores stale socket opens after reconnect swaps in a new socket', () => { const client = connect('ws://desktop.invalid', 'token', 'server-key') const firstSocket = mockSockets[0]! diff --git a/mobile/src/transport/rpc-client.ts b/mobile/src/transport/rpc-client.ts index 9b0b6246f1c..54d4c4cd9e3 100644 --- a/mobile/src/transport/rpc-client.ts +++ b/mobile/src/transport/rpc-client.ts @@ -10,9 +10,7 @@ import { deriveSharedKey, publicKeyFromBase64, publicKeyToBase64, - encrypt, - decrypt, - decryptBytes + decrypt } from './e2ee' import { handleTerminalBinaryFrame, @@ -30,7 +28,13 @@ import { import { describeSocketEvent } from './socket-event-debug' import { markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' import { isRpcResponse } from './rpc-response-shape' -import { websocketPayloadToUint8 } from './websocket-payload-bytes' +import { createMobileInboundFrameQueue } from './mobile-inbound-frame-queue' +import { createMobileDirectRpcOutbound } from './mobile-direct-rpc-outbound' +import { createMobileDirectRpcSender } from './mobile-direct-rpc-sender' +import { handleMobileRpcSocketBinaryMessage } from './mobile-rpc-binary-frame-handler' +import { processMobileOutboundMemoryBudget } from './mobile-outbound-memory-budget' +import { redactedWebSocketEndpoint } from './redacted-websocket-endpoint' +import { tryParseMobileJsonTextWithinLimits } from './mobile-json-text-admission' type PendingRequest = { resolve: (response: RpcResponse) => void @@ -142,6 +146,7 @@ export function connect( }) } let ws: WebSocket | null = null + let outbound: ReturnType | null = null let state: ConnectionState = 'disconnected' let requestCounter = 0 let reconnectAttempt = 0 @@ -204,7 +209,7 @@ export function connect( to: next, dweltMs: dwelt, attempt: reconnectAttempt, - endpoint: redactedEndpoint(endpoint) + endpoint: redactedWebSocketEndpoint(endpoint) }) if (next === 'connected') { lastConnectedAt = Date.now() @@ -226,16 +231,6 @@ export function connect( } } - // Why: keep device tokens / full URLs out of log scrolls — truncate to host:port. - function redactedEndpoint(ep: string): string { - try { - const m = ep.match(/^wss?:\/\/([^/]+)/i) - return m ? m[1] : 'unknown' - } catch { - return 'unknown' - } - } - function waitForConnected(timeoutMs?: number): Promise { if (state === 'connected') { return Promise.resolve() @@ -270,6 +265,11 @@ export function connect( return `rpc-${++requestCounter}-${Date.now()}` } + function disposeActiveOutbound(): void { + outbound?.dispose() + outbound = null + } + function openConnection() { if (intentionallyClosed) { return @@ -279,7 +279,7 @@ export function connect( wsConstructionCounter++ console.log('[net] openConnection', { attempt: reconnectAttempt, - endpoint: redactedEndpoint(endpoint), + endpoint: redactedWebSocketEndpoint(endpoint), // Why: diagnostic for RN/OkHttp pool corruption — high wsCount + repeated 1006 closes means process-state stuck. wsCount: wsConstructionCounter, msSinceLastConnected: lastConnectedAt != null ? now - lastConnectedAt : null, @@ -293,11 +293,37 @@ export function connect( emitLog( 'info', reconnectAttempt > 0 ? `Reconnecting (attempt ${reconnectAttempt + 1})` : 'Opening WebSocket', - endpoint + redactedWebSocketEndpoint(endpoint) ) + if (!processMobileOutboundMemoryBudget.canRegisterBufferedAmount()) { + emitLog('error', 'WebSocket reconnect deferred', 'Retired socket buffers are still draining') + setState('reconnecting') + scheduleReconnect() + return + } + ws = new WebSocket(endpoint) const openingWs = ws + const closeForOverload = (direction: 'Inbound' | 'Outbound', detail: string): void => { + emitLog('error', `${direction} WebSocket overload`, detail) + openingWs.close() + if (ws === openingWs) { + handleSocketClosed(openingWs) + } + } + const openingOutbound = createMobileDirectRpcOutbound({ + socket: openingWs, + isActive: () => ws === openingWs, + onOverflow: () => closeForOverload('Outbound', 'Mobile RPC outbound buffer overflow') + }) + outbound = openingOutbound + const inboundQueue = createMobileInboundFrameQueue({ + process: handleSocketMessage, + onError: (error) => closeForOverload('Inbound', error.message), + overflowMessage: 'Mobile RPC inbound buffer overflow', + frameTooLargeMessage: 'Mobile RPC inbound frame too large' + }) const ignoreStaleSocketEvent = (eventName: string): boolean => { if (ws === openingWs) { return false @@ -373,10 +399,10 @@ export function connect( if (ignoreStaleSocketEvent('message')) { return } - void handleSocketMessage(event.data) + void inboundQueue.enqueue(event.data) } - async function handleSocketMessage(rawData: unknown) { + function handleSocketMessage(rawData: unknown): Promise | void { lastInboundAt = Date.now() const raw = typeof rawData === 'string' ? rawData : null @@ -385,15 +411,11 @@ export function connect( if (raw === null) { return } - try { - const msg = JSON.parse(raw) - if (msg.type === 'e2ee_ready') { - emitLog('success', 'Received e2ee_ready', 'Sending device token') - sendEncrypted({ type: 'e2ee_auth', deviceToken }) - return - } - } catch { - // Not plaintext JSON — fall through and try encrypted handshake messages. + const plaintextControl = tryParseMobileJsonTextWithinLimits>(raw) + if (plaintextControl?.type === 'e2ee_ready') { + emitLog('success', 'Received e2ee_ready', 'Sending device token') + sendEncrypted({ type: 'e2ee_auth', deviceToken }) + return } if (!sharedKey || sharedKey.length !== 32) { @@ -405,9 +427,10 @@ export function connect( return } - try { - const msg = JSON.parse(plaintext) + const msg = tryParseMobileJsonTextWithinLimits>(plaintext) + if (msg) { if (msg.type === 'e2ee_authenticated') { + openingOutbound.acknowledgeAuthentication() if (handshakeTimer) { clearTimeout(handshakeTimer) handshakeTimer = null @@ -441,7 +464,11 @@ export function connect( removeStreamListener(id) } } - } else if (msg.type === 'e2ee_error' || (!msg.ok && msg.error?.code === 'unauthorized')) { + } else if ( + msg.type === 'e2ee_error' || + (!msg.ok && (msg.error as { code?: unknown } | undefined)?.code === 'unauthorized') + ) { + openingOutbound.acknowledgeAuthentication() console.log('[net] e2ee auth FAILED', { msgType: msg.type, error: msg.error }) if (handshakeTimer) { clearTimeout(handshakeTimer) @@ -449,8 +476,6 @@ export function connect( } handleAuthRejection('Unauthorized — pairing may be revoked') } - } catch { - // Not JSON — ignore during handshake. } return } @@ -461,19 +486,12 @@ export function connect( } if (raw === null) { - const bytes = await websocketPayloadToUint8(rawData) - if (ws !== openingWs) { - return - } - if (!bytes) { - return - } - const plaintextBytes = decryptBytes(bytes, sharedKey) - if (!plaintextBytes) { - return - } - handleBinaryFrame(plaintextBytes) - return + return handleMobileRpcSocketBinaryMessage({ + rawData, + key: sharedKey, + isCurrent: () => ws === openingWs, + onFrame: handleBinaryFrame + }) } const plaintext = decrypt(raw, sharedKey) @@ -481,15 +499,14 @@ export function connect( return } - let response: unknown - try { - response = JSON.parse(plaintext) - } catch { + const response = tryParseMobileJsonTextWithinLimits(plaintext) + if (response === null) { return } if (!isRpcResponse(response)) { return } + openingOutbound.acknowledge(response.id) recordValidatedInboundTraffic() // Why: a mid-session unauthorized may be transient (issue #5200) — handleAuthRejection retries before latching auth-failed. @@ -580,6 +597,11 @@ export function connect( } ws.onclose = (event) => { + inboundQueue.dispose() + openingOutbound.socketClosed() + if (outbound === openingOutbound) { + disposeActiveOutbound() + } const e = event as { code?: number; reason?: string; wasClean?: boolean } | undefined const closeAt = Date.now() // Why: time-since-construct classifies the failure — instant close = RST/unreachable, slow = SYN timeout/packet loss. @@ -596,7 +618,7 @@ export function connect( state, attempt: reconnectAttempt, intentionallyClosed, - endpoint: redactedEndpoint(endpoint), + endpoint: redactedWebSocketEndpoint(endpoint), constructToCloseMs, aliveMs, inboundIdleMs, @@ -637,6 +659,7 @@ export function connect( return } clearConnectTimer() + disposeActiveOutbound() ws = null sharedKey = null activeBrowserScreencastRequestId = null @@ -684,7 +707,7 @@ export function connect( console.log('[net] auth rejected — retrying handshake', { attempt: authRejectionCount, budget: AUTH_RETRY_BUDGET, - endpoint: redactedEndpoint(endpoint) + endpoint: redactedWebSocketEndpoint(endpoint) }) emitLog( 'warn', @@ -693,6 +716,7 @@ export function connect( ) // Why: close without setting intentionallyClosed so handleSocketClosed routes to reconnect and retries the handshake. const closing = ws + disposeActiveOutbound() ws = null sharedKey = null // Why: close cleanup stale-bails here, so mark active streams for replay. @@ -707,9 +731,10 @@ export function connect( } console.log('[net] auth rejected — budget exhausted, latching auth-failed', { attempt: authRejectionCount, - endpoint: redactedEndpoint(endpoint) + endpoint: redactedWebSocketEndpoint(endpoint) }) intentionallyClosed = true + disposeActiveOutbound() ws?.close() ws = null setState('auth-failed') @@ -929,26 +954,13 @@ export function connect( stream.onBinaryFrame?.(frame) } - function sendEncrypted(request: unknown): boolean { - if (ws && ws.readyState === WebSocket.OPEN && sharedKey) { - ws.send(encrypt(JSON.stringify(request), sharedKey)) - return true - } - console.log('[net] sendEncrypted FAILED — channel not ready', { - hasWs: !!ws, - readyState: ws?.readyState, - hasKey: !!sharedKey, - state - }) - // Why: RN can drop onclose, leaving state 'connected' over a dead socket; force reconnect or every send silently fails forever. - if (state === 'connected' && ws && ws.readyState !== WebSocket.OPEN) { - console.log('[net] sendEncrypted detected ws desync — forcing reconnect', { - readyState: ws.readyState - }) - handleSocketClosed(ws, { timedOut: false }) - } - return false - } + const sendEncrypted = createMobileDirectRpcSender({ + getOutbound: () => outbound, + getSharedKey: () => sharedKey, + getSocket: () => ws, + getState: () => state, + onSocketDesync: (socket) => handleSocketClosed(socket, { timedOut: false }) + }) function sendBrowserScreencastUnsubscribe(subscriptionId: string): void { sendEncrypted({ @@ -1159,6 +1171,7 @@ export function connect( handshakeTimer = null } stopActivityProbe() + disposeActiveOutbound() if (ws) { ws.close() ws = null diff --git a/mobile/src/transport/terminal-stream-protocol.ts b/mobile/src/transport/terminal-stream-protocol.ts index 384d66996f1..5113f38254a 100644 --- a/mobile/src/transport/terminal-stream-protocol.ts +++ b/mobile/src/transport/terminal-stream-protocol.ts @@ -1,3 +1,5 @@ +import { parseMobileJsonTextWithinLimits } from './mobile-json-text-admission' + const TERMINAL_STREAM_KIND = 0x74 const TERMINAL_STREAM_VERSION = 1 const HEADER_BYTES = 16 @@ -52,13 +54,13 @@ export function decodeTerminalStreamFrame(bytes: Uint8Array): TerminalStreamFram opcode, streamId: view.getUint32(4, true), seq: high * 0x100000000 + low, - payload: bytes.slice(HEADER_BYTES) + payload: bytes.subarray(HEADER_BYTES) } } export function decodeTerminalStreamJson(payload: Uint8Array): T | null { try { - return JSON.parse(new TextDecoder().decode(payload)) as T + return parseMobileJsonTextWithinLimits(new TextDecoder().decode(payload)) } catch { return null } diff --git a/mobile/src/transport/types.ts b/mobile/src/transport/types.ts index d031d11ff08..ffd4afddd06 100644 --- a/mobile/src/transport/types.ts +++ b/mobile/src/transport/types.ts @@ -1,5 +1,10 @@ import { z } from 'zod' import { + PAIRING_CODE_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PAIRING_ENDPOINT_MAX_CHARACTERS, + PAIRING_INPUT_MAX_CHARACTERS, + PAIRING_PUBLIC_KEY_MAX_CHARACTERS, PairingOfferSchema, type PairingOffer } from '../../../src/shared/mobile-relay-pairing-offer' @@ -10,9 +15,20 @@ import { } from './mobile-relay-host-overlay' import { MobileRelayEndpointSchema } from '../../../src/shared/mobile-relay-credential-contract' -export { PairingOfferSchema } +export { + PAIRING_CODE_MAX_CHARACTERS, + PAIRING_DEVICE_TOKEN_MAX_CHARACTERS, + PAIRING_ENDPOINT_MAX_CHARACTERS, + PAIRING_INPUT_MAX_CHARACTERS, + PAIRING_PUBLIC_KEY_MAX_CHARACTERS, + PairingOfferSchema +} export type { PairingOffer } +export const MOBILE_HOST_ID_MAX_CHARACTERS = 4_096 +export const MOBILE_HOST_NAME_MAX_CHARACTERS = 4_096 +export const MobileHostIdSchema = z.string().min(1).max(MOBILE_HOST_ID_MAX_CHARACTERS) + export type RpcRequest = { id: string deviceToken: string @@ -72,11 +88,11 @@ export type HostProfile = { } export const HostProfileSchema = z.object({ - id: z.string().min(1), - name: z.string().min(1), - endpoint: z.string().min(1), - deviceToken: z.string().min(1), - publicKeyB64: z.string().min(1), + id: MobileHostIdSchema, + name: z.string().min(1).max(MOBILE_HOST_NAME_MAX_CHARACTERS), + endpoint: z.string().min(1).max(PAIRING_ENDPOINT_MAX_CHARACTERS), + deviceToken: z.string().min(1).max(PAIRING_DEVICE_TOKEN_MAX_CHARACTERS), + publicKeyB64: z.string().min(1).max(PAIRING_PUBLIC_KEY_MAX_CHARACTERS), lastConnected: z.number().finite(), endpoints: z.array(MobileAccessEndpointSchema).min(1).max(16).optional(), relayHostId: z @@ -90,10 +106,10 @@ export const HostProfileSchema = z.object({ // deviceToken is held in iOS Keychain via expo-secure-store and joined // in at load time; it must NOT appear in AsyncStorage anymore. export const StoredHostProfileSchema = z.object({ - id: z.string().min(1), - name: z.string().min(1), - endpoint: z.string().min(1), - publicKeyB64: z.string().min(1), + id: MobileHostIdSchema, + name: z.string().min(1).max(MOBILE_HOST_NAME_MAX_CHARACTERS), + endpoint: z.string().min(1).max(PAIRING_ENDPOINT_MAX_CHARACTERS), + publicKeyB64: z.string().min(1).max(PAIRING_PUBLIC_KEY_MAX_CHARACTERS), lastConnected: z.number().finite() }) diff --git a/mobile/src/transport/websocket-payload-bytes.test.ts b/mobile/src/transport/websocket-payload-bytes.test.ts index 4b501957ef5..dc630343b23 100644 --- a/mobile/src/transport/websocket-payload-bytes.test.ts +++ b/mobile/src/transport/websocket-payload-bytes.test.ts @@ -1,14 +1,64 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { websocketPayloadToUint8 } from './websocket-payload-bytes' describe('websocketPayloadToUint8', () => { it('returns null when a blob-like payload rejects arrayBuffer conversion', async () => { await expect( websocketPayloadToUint8({ + size: 0, arrayBuffer: async () => { throw new Error('conversion failed') } }) ).resolves.toBeNull() }) + + it('rejects a declared oversized payload before starting binary conversion', async () => { + const arrayBuffer = vi.fn(async () => new ArrayBuffer(1)) + + await expect(websocketPayloadToUint8({ size: 5, arrayBuffer }, 4)).rejects.toThrow( + 'exceeds inbound frame limit' + ) + expect(arrayBuffer).not.toHaveBeenCalled() + }) + + it('accepts an exact declared size and verifies the converted result', async () => { + const arrayBuffer = vi.fn(async () => new ArrayBuffer(4)) + + await expect(websocketPayloadToUint8({ size: 4, arrayBuffer }, 4)).resolves.toEqual( + new Uint8Array(4) + ) + expect(arrayBuffer).toHaveBeenCalledOnce() + }) + + it('rejects an unknown-size payload before starting binary conversion', async () => { + const arrayBuffer = vi.fn(async () => new ArrayBuffer(4)) + + await expect(websocketPayloadToUint8({ arrayBuffer }, 4)).rejects.toThrow( + 'exceeds inbound frame limit' + ) + expect(arrayBuffer).not.toHaveBeenCalled() + }) + + it.each([Number.NaN, -1, Number.POSITIVE_INFINITY])( + 'rejects invalid declared size %s before starting binary conversion', + async (size) => { + const arrayBuffer = vi.fn(async () => new ArrayBuffer(4)) + + await expect(websocketPayloadToUint8({ size, arrayBuffer }, 4)).rejects.toThrow( + 'exceeds inbound frame limit' + ) + expect(arrayBuffer).not.toHaveBeenCalled() + } + ) + + it('preserves direct Uint8Array and ArrayBuffer conversion', async () => { + const uint8 = new Uint8Array([1, 2, 3]) + const arrayBuffer = Uint8Array.from([4, 5, 6]).buffer + + await expect(websocketPayloadToUint8(uint8, 3)).resolves.toBe(uint8) + await expect(websocketPayloadToUint8(arrayBuffer, 3)).resolves.toEqual( + new Uint8Array([4, 5, 6]) + ) + }) }) diff --git a/mobile/src/transport/websocket-payload-bytes.ts b/mobile/src/transport/websocket-payload-bytes.ts index 1cd8ac5cdd8..af4b1685dd3 100644 --- a/mobile/src/transport/websocket-payload-bytes.ts +++ b/mobile/src/transport/websocket-payload-bytes.ts @@ -1,23 +1,48 @@ -export async function websocketPayloadToUint8(value: unknown): Promise { +import { + assertMobileInboundFrameSize, + MOBILE_INBOUND_MAX_FRAME_BYTES +} from './mobile-inbound-frame-queue' + +const OVERSIZED_BINARY_MESSAGE = 'WebSocket binary payload exceeds inbound frame limit' + +export async function websocketPayloadToUint8( + value: unknown, + maxBytes = MOBILE_INBOUND_MAX_FRAME_BYTES +): Promise { + assertMobileInboundFrameSize(value, OVERSIZED_BINARY_MESSAGE, maxBytes) if (value instanceof Uint8Array) { - return value + return enforceConvertedLimit(value, maxBytes) } if (value instanceof ArrayBuffer) { - return new Uint8Array(value) + return enforceConvertedLimit(new Uint8Array(value), maxBytes) } if (value && typeof value === 'object' && 'arrayBuffer' in value) { + const declaredBytes = declaredBinaryPayloadBytes(value) + if (declaredBytes === null || declaredBytes > maxBytes) { + throw new Error(OVERSIZED_BINARY_MESSAGE) + } const blob = value as { arrayBuffer: () => Promise } + let buffer: ArrayBuffer try { - return new Uint8Array(await blob.arrayBuffer()) + buffer = await blob.arrayBuffer() } catch { return null } + return enforceConvertedLimit(new Uint8Array(buffer), maxBytes) } if (typeof FileReader !== 'undefined' && value instanceof Blob) { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const reader = new FileReader() reader.onload = () => { - resolve(reader.result instanceof ArrayBuffer ? new Uint8Array(reader.result) : null) + if (!(reader.result instanceof ArrayBuffer)) { + resolve(null) + return + } + try { + resolve(enforceConvertedLimit(new Uint8Array(reader.result), maxBytes)) + } catch (error) { + reject(error) + } } reader.onerror = () => resolve(null) reader.readAsArrayBuffer(value) @@ -25,3 +50,23 @@ export async function websocketPayloadToUint8(value: unknown): Promise)[key] + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate + } + } + return null +} + +function enforceConvertedLimit(bytes: Uint8Array, maxBytes: number): Uint8Array { + if (bytes.byteLength > maxBytes) { + throw new Error(OVERSIZED_BINARY_MESSAGE) + } + return bytes +} diff --git a/mobile/src/worktree/last-visited-worktree-repo.test.ts b/mobile/src/worktree/last-visited-worktree-repo.test.ts index 4d484fbe715..1860871603d 100644 --- a/mobile/src/worktree/last-visited-worktree-repo.test.ts +++ b/mobile/src/worktree/last-visited-worktree-repo.test.ts @@ -1,5 +1,11 @@ -import { describe, expect, it } from 'vitest' -import { readLastVisitedWorktreeRepoId } from './last-visited-worktree-repo' +import { describe, expect, it, vi } from 'vitest' +import { + LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS, + LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS, + readLastVisitedWorktreeRecord, + readLastVisitedWorktreeRepoId, + serializeLastVisitedWorktreeRecord +} from './last-visited-worktree-repo' describe('last visited worktree repo', () => { it('extracts the repo id for the current host', () => { @@ -18,4 +24,37 @@ describe('last visited worktree repo', () => { expect(readLastVisitedWorktreeRepoId('{', 'host-1')).toBeNull() expect(readLastVisitedWorktreeRepoId(JSON.stringify({ hostId: 'host-1' }), 'host-1')).toBeNull() }) + + it('round-trips exact field limits and rejects one character more', () => { + const exact = { + hostId: 'h'.repeat(LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS), + worktreeId: 'w'.repeat(LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS) + } + const serialized = serializeLastVisitedWorktreeRecord(exact) + + expect(serialized).not.toBeNull() + expect(readLastVisitedWorktreeRecord(serialized)).toEqual(exact) + expect( + serializeLastVisitedWorktreeRecord({ + ...exact, + worktreeId: `${exact.worktreeId}w` + }) + ).toBeNull() + }) + + it('accepts the exact raw limit and does not parse one character more', () => { + const record = { hostId: 'host-1', worktreeId: 'repo::worktree' } + const serialized = JSON.stringify(record) + const exact = + serialized + ' '.repeat(LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS - serialized.length) + + expect(readLastVisitedWorktreeRecord(exact)).toEqual(record) + + const parse = vi.spyOn(JSON, 'parse') + expect( + readLastVisitedWorktreeRecord('x'.repeat(LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS + 1)) + ).toBeNull() + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + }) }) diff --git a/mobile/src/worktree/last-visited-worktree-repo.ts b/mobile/src/worktree/last-visited-worktree-repo.ts index 4ac9349c9fc..c991fe0c90b 100644 --- a/mobile/src/worktree/last-visited-worktree-repo.ts +++ b/mobile/src/worktree/last-visited-worktree-repo.ts @@ -1,8 +1,10 @@ import { getRepoIdFromMobileWorktreeId } from '../session/mobile-session-route-helpers' export const LAST_VISITED_WORKTREE_STORAGE_KEY = 'orca:last-visited-worktree' +export const LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS = 16 * 1024 +export const LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS = 4_096 -type LastVisitedWorktreeRecord = { +export type LastVisitedWorktreeRecord = { hostId: string worktreeId: string } @@ -11,8 +13,10 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null } -function readLastVisitedWorktreeRecord(raw: string | null): LastVisitedWorktreeRecord | null { - if (!raw) { +export function readLastVisitedWorktreeRecord( + raw: string | null +): LastVisitedWorktreeRecord | null { + if (!raw || raw.length > LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS) { return null } try { @@ -20,7 +24,11 @@ function readLastVisitedWorktreeRecord(raw: string | null): LastVisitedWorktreeR if ( !isRecord(parsed) || typeof parsed.hostId !== 'string' || - typeof parsed.worktreeId !== 'string' + parsed.hostId.length === 0 || + parsed.hostId.length > LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS || + typeof parsed.worktreeId !== 'string' || + parsed.worktreeId.length === 0 || + parsed.worktreeId.length > LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS ) { return null } @@ -30,6 +38,24 @@ function readLastVisitedWorktreeRecord(raw: string | null): LastVisitedWorktreeR } } +export function serializeLastVisitedWorktreeRecord( + record: LastVisitedWorktreeRecord +): string | null { + if ( + record.hostId.length === 0 || + record.hostId.length > LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS || + record.worktreeId.length === 0 || + record.worktreeId.length > LAST_VISITED_WORKTREE_MAX_ID_CHARACTERS + ) { + return null + } + const serialized = JSON.stringify(record) + if (serialized.length > LAST_VISITED_WORKTREE_MAX_STORAGE_CHARACTERS) { + return null + } + return serialized +} + export function readLastVisitedWorktreeRepoId(raw: string | null, hostId: string): string | null { const record = readLastVisitedWorktreeRecord(raw) if (!record || record.hostId !== hostId) { diff --git a/package.json b/package.json index ae3f9afa536..de4a48b0bd8 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "smoke:computer": "node config/scripts/computer-use-smoke.mjs", "verify:computer-native": "node config/scripts/verify-computer-native.mjs", "verify:cli-bin": "node config/scripts/verify-cli-bin.mjs", + "verify:cli-runtime": "node config/scripts/verify-cli-bin.mjs --run-help", "verify:localization-catalog": "node config/scripts/verify-localization-catalog.mjs", "sync:localization-catalog": "node config/scripts/verify-localization-catalog.mjs --fix", "bootstrap:locale-catalog": "node config/scripts/bootstrap-locale-catalog.mjs", @@ -65,9 +66,9 @@ "build:cli": "tsc -p config/tsconfig.cli.json --outDir out --composite false --incremental false && node config/scripts/verify-cli-bin.mjs --fix-executable --fix-package-json && node config/scripts/install-dev-cli.mjs", "build:electron-vite": "node config/scripts/run-electron-vite-build.mjs", "build:web": "node config/scripts/run-vite-web-build.mjs && node config/scripts/verify-web-build.mjs", - "build:desktop": "pnpm run typecheck && pnpm run build:relay && pnpm run build:cli && pnpm run build:electron-vite && pnpm run build:web", + "build:desktop": "pnpm run typecheck && pnpm run build:relay && pnpm run build:cli && pnpm run build:electron-vite && pnpm run verify:cli-runtime && pnpm run build:web", "build": "pnpm run build:desktop && pnpm run build:native", - "build:release": "pnpm run build:relay && pnpm run build:native && pnpm run verify:computer-native && pnpm run build:cli && pnpm run build:electron-vite && pnpm run build:web", + "build:release": "pnpm run build:relay && pnpm run build:native && pnpm run verify:computer-native && pnpm run build:cli && pnpm run build:electron-vite && pnpm run verify:cli-runtime && pnpm run build:web", "postinstall": "node config/scripts/rebuild-native-deps.mjs", "rebuild:electron": "node config/scripts/rebuild-native-deps.mjs", "rebuild:node": "pnpm rebuild node-pty", diff --git a/src/cli/computer-format.ts b/src/cli/computer-format.ts index c803074055a..b2e076edada 100644 --- a/src/cli/computer-format.ts +++ b/src/cli/computer-format.ts @@ -1,16 +1,6 @@ -import { - chmodSync, - lstatSync, - mkdirSync, - readdirSync, - rmSync, - statSync, - writeFileSync -} from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' import { formatBase64PayloadByteCount } from './base64-payload-byte-count' import { quoteCliCommandArgument } from './shell-command-quote' +import { writeComputerScreenshotTempFile } from './computer-screenshot-storage' import type { ComputerActionMetadata, ComputerActionResult, @@ -72,11 +62,11 @@ export function prepareComputerCliJsonResult( } try { const extension = screenshot.format === 'png' ? 'png' : 'img' - const outputDir = computerScreenshotTempDir() - cleanupComputerScreenshots(outputDir) - const outputPath = join(outputDir, `${safeCliFileStem(response.id)}-screenshot.${extension}`) - writeFileSync(outputPath, Buffer.from(screenshot.data, 'base64'), { mode: 0o600 }) - const expiresAt = new Date(Date.now() + COMPUTER_SCREENSHOT_TTL_MS).toISOString() + const { outputPath, expiresAt } = writeComputerScreenshotTempFile( + response.id, + screenshot.data, + extension + ) return { ...response, result: { @@ -97,63 +87,6 @@ export function prepareComputerCliJsonResult( } } -const COMPUTER_SCREENSHOT_TTL_MS = 24 * 60 * 60 * 1000 -const COMPUTER_SCREENSHOT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000 -const COMPUTER_SCREENSHOT_CLEANUP_MARKER = '.last-cleanup' - -function computerScreenshotTempDir(): string { - const outputDir = - process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR || join(tmpdir(), 'orca-computer-use') - mkdirSync(outputDir, { recursive: true, mode: 0o700 }) - const stat = lstatSync(outputDir) - if (!stat.isDirectory() || stat.isSymbolicLink()) { - throw new Error(`Unsafe computer screenshot temp path: ${outputDir}`) - } - if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { - throw new Error(`Computer screenshot temp path is not owned by the current user: ${outputDir}`) - } - chmodSync(outputDir, 0o700) - return outputDir -} - -function cleanupComputerScreenshots(outputDir: string): void { - const now = Date.now() - const markerPath = join(outputDir, COMPUTER_SCREENSHOT_CLEANUP_MARKER) - try { - // Why: agents can call computer-use CLI commands in loops; a marker keeps - // temp cleanup from becoming a synchronous directory scan per screenshot. - if (statSync(markerPath).mtimeMs > now - COMPUTER_SCREENSHOT_CLEANUP_INTERVAL_MS) { - return - } - } catch { - // Missing or unreadable marker means this process should attempt cleanup. - } - - const cutoff = now - COMPUTER_SCREENSHOT_TTL_MS - for (const entry of readdirSync(outputDir)) { - if (!entry.endsWith('-screenshot.png') && !entry.endsWith('-screenshot.img')) { - continue - } - const path = join(outputDir, entry) - try { - if (statSync(path).mtimeMs < cutoff) { - rmSync(path, { force: true }) - } - } catch { - // Best-effort cleanup only; formatting should not fail because a temp file raced. - } - } - try { - writeFileSync(markerPath, `${now}\n`, { mode: 0o600 }) - } catch { - // Best-effort marker only; stale cleanup state should not hide a screenshot. - } -} - -function safeCliFileStem(value: string): string { - return value.replaceAll(/[^a-zA-Z0-9._-]/g, '_') -} - export function formatListApps(result: ComputerListAppsResult): string { if (result.apps.length === 0) { return 'No apps found.' diff --git a/src/cli/computer-screenshot-storage.test.ts b/src/cli/computer-screenshot-storage.test.ts new file mode 100644 index 00000000000..0ee2aa968e1 --- /dev/null +++ b/src/cli/computer-screenshot-storage.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { existsSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { writeComputerScreenshotTempFile } from './computer-screenshot-storage' + +const roots: string[] = [] +const originalTempDir = process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR + +afterEach(() => { + if (originalTempDir === undefined) { + delete process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR + } else { + process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR = originalTempDir + } + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('computer screenshot storage', () => { + it('writes normal screenshot bytes and streams cleanup of expired captures', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-computer-screenshot-')) + roots.push(root) + process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR = root + const expiredPath = join(root, 'expired-screenshot.png') + const unrelatedPath = join(root, 'keep.txt') + writeFileSync(expiredPath, 'old') + writeFileSync(unrelatedPath, 'keep') + const expiredAt = new Date(Date.now() - 25 * 60 * 60 * 1000) + utimesSync(expiredPath, expiredAt, expiredAt) + + const result = writeComputerScreenshotTempFile( + 'rpc/id', + Buffer.from('png').toString('base64'), + 'png' + ) + + expect(result.outputPath).toBe(join(root, 'rpc_id-screenshot.png')) + expect(existsSync(result.outputPath)).toBe(true) + expect(existsSync(expiredPath)).toBe(false) + expect(existsSync(unrelatedPath)).toBe(true) + }) +}) diff --git a/src/cli/computer-screenshot-storage.ts b/src/cli/computer-screenshot-storage.ts new file mode 100644 index 00000000000..8c47a89d189 --- /dev/null +++ b/src/cli/computer-screenshot-storage.ts @@ -0,0 +1,96 @@ +import { + chmodSync, + lstatSync, + mkdirSync, + opendirSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const COMPUTER_SCREENSHOT_TTL_MS = 24 * 60 * 60 * 1000 +const COMPUTER_SCREENSHOT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000 +const COMPUTER_SCREENSHOT_CLEANUP_MARKER = '.last-cleanup' + +export function writeComputerScreenshotTempFile( + responseId: string, + base64Data: string, + extension: 'png' | 'img' +): { outputPath: string; expiresAt: string } { + const outputDir = computerScreenshotTempDir() + cleanupComputerScreenshots(outputDir) + const outputPath = join(outputDir, `${safeCliFileStem(responseId)}-screenshot.${extension}`) + writeFileSync(outputPath, Buffer.from(base64Data, 'base64'), { mode: 0o600 }) + return { + outputPath, + expiresAt: new Date(Date.now() + COMPUTER_SCREENSHOT_TTL_MS).toISOString() + } +} + +function computerScreenshotTempDir(): string { + const outputDir = + process.env.ORCA_COMPUTER_SCREENSHOT_TMPDIR || join(tmpdir(), 'orca-computer-use') + mkdirSync(outputDir, { recursive: true, mode: 0o700 }) + const stat = lstatSync(outputDir) + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error(`Unsafe computer screenshot temp path: ${outputDir}`) + } + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`Computer screenshot temp path is not owned by the current user: ${outputDir}`) + } + chmodSync(outputDir, 0o700) + return outputDir +} + +function cleanupComputerScreenshots(outputDir: string): void { + const now = Date.now() + const markerPath = join(outputDir, COMPUTER_SCREENSHOT_CLEANUP_MARKER) + try { + // Why: agents can call computer-use CLI commands in loops; a marker keeps + // temp cleanup from becoming a synchronous directory scan per screenshot. + if (statSync(markerPath).mtimeMs > now - COMPUTER_SCREENSHOT_CLEANUP_INTERVAL_MS) { + return + } + } catch { + // Missing or unreadable marker means this process should attempt cleanup. + } + + const cutoff = now - COMPUTER_SCREENSHOT_TTL_MS + const directory = opendirSync(outputDir) + try { + while (true) { + const entry = directory.readSync() + if (!entry) { + break + } + if (!entry.name.endsWith('-screenshot.png') && !entry.name.endsWith('-screenshot.img')) { + continue + } + const path = join(outputDir, entry.name) + try { + if (statSync(path).mtimeMs < cutoff) { + rmSync(path, { force: true }) + } + } catch { + // Best-effort cleanup only; formatting should not fail because a temp file raced. + } + } + } finally { + try { + directory.closeSync() + } catch { + // Best-effort cleanup only. + } + } + try { + writeFileSync(markerPath, `${now}\n`, { mode: 0o600 }) + } catch { + // Best-effort marker only; stale cleanup state should not hide a screenshot. + } +} + +function safeCliFileStem(value: string): string { + return value.replaceAll(/[^a-zA-Z0-9._-]/g, '_') +} diff --git a/src/cli/handlers/agent-hooks.test.ts b/src/cli/handlers/agent-hooks.test.ts index 4ae4c962d59..d1b92d44d8a 100644 --- a/src/cli/handlers/agent-hooks.test.ts +++ b/src/cli/handlers/agent-hooks.test.ts @@ -1,9 +1,10 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { getDefaultPersistedState } from '../../shared/constants' import type { PersistedState } from '../../shared/types' +import { ORCA_PERSISTED_STATE_MAX_BYTES } from '../../shared/persisted-state-file-bounds' const { applyAgentStatusHooksEnabledMock, @@ -45,9 +46,12 @@ vi.mock('../runtime-client', () => { } } + class RuntimeRpcFailureError extends Error {} + return { RuntimeClient, RuntimeClientError, + RuntimeRpcFailureError, getDefaultUserDataPath: getDefaultUserDataPathMock } }) @@ -120,4 +124,16 @@ describe('agent hooks CLI handler', () => { expect(readDataFile(userDataPath).settings.experimentalNewWorktreeCardStyle).toBe(true) }) + + it('rejects an oversized sparse offline state file without replacing it', async () => { + const dataPath = join(userDataPath, 'orca-data.json') + writeFileSync(dataPath, '') + truncateSync(dataPath, ORCA_PERSISTED_STATE_MAX_BYTES + 1) + + await runAgentHooksOff(userDataPath) + + expect(process.exitCode).toBe(1) + expect(console.log).toHaveBeenCalledWith(expect.stringContaining('File too large')) + expect(readFileSync(dataPath).byteLength).toBe(ORCA_PERSISTED_STATE_MAX_BYTES + 1) + }) }) diff --git a/src/cli/handlers/agent-hooks.ts b/src/cli/handlers/agent-hooks.ts index 2b83f5c15ea..8ef4d138b8e 100644 --- a/src/cli/handlers/agent-hooks.ts +++ b/src/cli/handlers/agent-hooks.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' import { randomUUID } from 'node:crypto' @@ -13,6 +13,11 @@ import { getManagedAgentHookStatuses } from '../../main/agent-hooks/managed-agent-hook-controls' import { getDefaultUserDataPath } from '../runtime-client' +import { + ORCA_PERSISTED_STATE_MAX_BYTES, + readPersistedStateJsonFileSync, + stringifyPrettyPersistedStateWithinLimit +} from '../../shared/persisted-state-file-bounds' type AgentHookCommandResult = { enabled: boolean @@ -34,7 +39,7 @@ function readPersistedState(dataPath: string): PersistedState { return getDefaultPersistedState(homedir()) } try { - const parsed = JSON.parse(readFileSync(dataPath, 'utf-8')) + const { value: parsed } = readPersistedStateJsonFileSync(dataPath) if (!isRecord(parsed)) { throw new Error('file does not contain a JSON object') } @@ -52,7 +57,11 @@ function writePersistedState(dataPath: string, state: PersistedState): void { const tmpPath = join(dirname(dataPath), `.${Date.now()}-${randomUUID()}.tmp`) let renamed = false try { - writeFileSync(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8') + const { serialized } = stringifyPrettyPersistedStateWithinLimit( + state, + ORCA_PERSISTED_STATE_MAX_BYTES - 1 + ) + writeFileSync(tmpPath, `${serialized}\n`, 'utf-8') renameSync(tmpPath, dataPath) renamed = true } finally { diff --git a/src/cli/handlers/computer-action-flags.ts b/src/cli/handlers/computer-action-flags.ts index 384c1dc0d80..8c771ec8f62 100644 --- a/src/cli/handlers/computer-action-flags.ts +++ b/src/cli/handlers/computer-action-flags.ts @@ -2,6 +2,11 @@ import { computerUseHotkeyValidationMessage, computerUsePressKeyValidationMessage } from '../../shared/computer-use-key-spec' +import { CLIPBOARD_TEXT_WRITE_MAX_BYTES } from '../../shared/clipboard-text' +import { + NodeReadableTextTooLargeError, + readNodeReadableTextWithinLimit +} from '../../shared/node-readable-text' import { getOptionalNonNegativeIntegerFlag, getOptionalNumberFlag, @@ -189,11 +194,17 @@ async function readStdin(): Promise { if (process.stdin.isTTY) { throw new RuntimeClientError('invalid_argument', 'stdin payload requested but stdin is a TTY') } - const chunks: Buffer[] = [] - for await (const chunk of process.stdin) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))) + try { + return await readNodeReadableTextWithinLimit(process.stdin, CLIPBOARD_TEXT_WRITE_MAX_BYTES) + } catch (error) { + if (error instanceof NodeReadableTextTooLargeError) { + throw new RuntimeClientError( + 'invalid_argument', + `stdin payload must be at most ${CLIPBOARD_TEXT_WRITE_MAX_BYTES} bytes` + ) + } + throw error } - return Buffer.concat(chunks).toString('utf8') } function getOptionalPositiveNumberFlag( diff --git a/src/cli/handlers/orchestration.ts b/src/cli/handlers/orchestration.ts index cdafcd66743..c631ecfa3db 100644 --- a/src/cli/handlers/orchestration.ts +++ b/src/cli/handlers/orchestration.ts @@ -79,6 +79,25 @@ type OrchestrationSendResult = | { message: { id: string }; lifecycle?: LifecycleSendRejection } | { messages: { id: string }[]; recipients: number } +type OrchestrationRetentionPage = { + count: number + truncated?: boolean + total?: number + remaining?: number +} + +function appendOrchestrationRetentionNotice( + body: string, + page: OrchestrationRetentionPage, + noun: string +): string { + if (!page.truncated) { + return body + } + const omitted = page.remaining ?? Math.max(0, (page.total ?? page.count) - page.count) + return `${body}\n\nTruncated: ${omitted} additional ${noun} omitted.` +} + function getOptionalStructuredMessagePayload( flags: Map ): string | undefined { @@ -399,6 +418,8 @@ export const ORCHESTRATION_HANDLERS: Record = { messages: MessageSummary[] count: number formatted?: string + truncated?: boolean + remaining?: number } let result: Awaited>> try { @@ -445,15 +466,17 @@ export const ORCHESTRATION_HANDLERS: Record = { } } printResult(result, json, (r) => { + let body: string if (r.formatted) { - return r.formatted + body = r.formatted + } else if (r.count === 0) { + body = 'No messages.' + } else { + body = r.messages + .map((m) => `${m.id} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`) + .join('\n') } - if (r.count === 0) { - return 'No messages.' - } - return r.messages - .map((m) => `${m.id} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`) - .join('\n') + return appendOrchestrationRetentionNotice(body, r, 'messages') }) }, @@ -472,31 +495,34 @@ export const ORCHESTRATION_HANDLERS: Record = { const result = await client.call<{ messages: MessageSummary[] count: number + total?: number + truncated?: boolean }>('orchestration.inbox', { limit: getOptionalPositiveIntegerFlag(flags, 'limit'), terminal: getOptionalStringFlag(flags, 'terminal') }) printResult(result, json, (r) => { - if (r.count === 0) { - return 'No messages.' - } // Why: default output omits body/payload for at-a-glance sweeps; --full prints them for auditing. - return r.messages - .map((m) => { - const head = `${m.id} ${m.from_handle} -> ${m.to_handle ?? '?'}: "${m.subject}"` - if (!full) { - return head - } - const parts = [head] - if (m.body && m.body.length > 0) { - parts.push(m.body) - } - if (m.payload) { - parts.push(`[payload] ${m.payload}`) - } - return parts.join('\n') - }) - .join(full ? '\n\n' : '\n') + const body = + r.count === 0 + ? 'No messages.' + : r.messages + .map((m) => { + const head = `${m.id} ${m.from_handle} -> ${m.to_handle ?? '?'}: "${m.subject}"` + if (!full) { + return head + } + const parts = [head] + if (m.body && m.body.length > 0) { + parts.push(m.body) + } + if (m.payload) { + parts.push(`[payload] ${m.payload}`) + } + return parts.join('\n') + }) + .join(full ? '\n\n' : '\n') + return appendOrchestrationRetentionNotice(body, r, 'messages') }) }, @@ -530,6 +556,8 @@ export const ORCHESTRATION_HANDLERS: Record = { spec_truncated?: boolean }[] count: number + total?: number + truncated?: boolean }>('orchestration.taskList', { status: getOptionalStringFlag(flags, 'status'), ready: flags.has('ready') ? true : undefined, @@ -545,19 +573,20 @@ export const ORCHESTRATION_HANDLERS: Record = { } : result printResult(output, json, (r) => { - if (r.count === 0) { - return 'No tasks.' - } - return r.tasks - .map((t) => { - const label = t.display_name ?? t.task_title ?? t.spec - const head = `${t.id} [${t.status}] ${label.slice(0, 60)}` - if (t.status === 'dispatched' && t.assignee_handle) { - return `${head} -> ${t.assignee_handle} (${t.dispatch_id ?? '?'})` - } - return head - }) - .join('\n') + const body = + r.count === 0 + ? 'No tasks.' + : r.tasks + .map((t) => { + const label = t.display_name ?? t.task_title ?? t.spec + const head = `${t.id} [${t.status}] ${label.slice(0, 60)}` + if (t.status === 'dispatched' && t.assignee_handle) { + return `${head} -> ${t.assignee_handle} (${t.dispatch_id ?? '?'})` + } + return head + }) + .join('\n') + return appendOrchestrationRetentionNotice(body, r, 'tasks') }) }, @@ -722,17 +751,18 @@ export const ORCHESTRATION_HANDLERS: Record = { const result = await client.call<{ gates: { id: string; task_id: string; question: string; status: string }[] count: number + total?: number + truncated?: boolean }>('orchestration.gateList', { task: getOptionalStringFlag(flags, 'task'), status: getOptionalStringFlag(flags, 'status') }) printResult(result, json, (r) => { - if (r.gates.length === 0) { - return 'No gates found.' - } - return r.gates - .map((g) => `${g.id} task=${g.task_id} [${g.status}] "${g.question}"`) - .join('\n') + const body = + r.gates.length === 0 + ? 'No gates found.' + : r.gates.map((g) => `${g.id} task=${g.task_id} [${g.status}] "${g.question}"`).join('\n') + return appendOrchestrationRetentionNotice(body, r, 'gates') }) }, diff --git a/src/cli/handlers/vm.ts b/src/cli/handlers/vm.ts index aaf01b08872..39c4062ce8a 100644 --- a/src/cli/handlers/vm.ts +++ b/src/cli/handlers/vm.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' import type { CommandHandler } from '../dispatch' import { RuntimeClientError } from '../runtime-client' @@ -18,6 +18,8 @@ import { runEphemeralVmRecipeStart } from '../../shared/ephemeral-vm-recipe-runner' import type { OrcaVmRecipe } from '../../shared/types' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { MAX_ORCA_YAML_BYTES } from '../../shared/orca-yaml-file-limit' export const VM_HANDLERS: Record = { 'vm recipe doctor': async ({ flags, cwd, json }) => { @@ -80,7 +82,7 @@ function doctorRecipe(repoPath: string, recipeId: string): DoctorResult { } function readTextFile(path: string): string { - return readFileSync(path, 'utf8') + return readNodeFileSyncWithinLimit(path, MAX_ORCA_YAML_BYTES).buffer.toString('utf8') } // Why: give the agent the full create/destroy output so it can self-diagnose a diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index f61b6fc09ff..f6c072e6c85 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -1,8 +1,9 @@ /* eslint-disable max-lines -- Why: CLI parser tests share one mocked runtime client and fixture queue; splitting this file would duplicate setup and make command coverage harder to audit. */ import path from 'node:path' -import { chmodSync, mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import { chmodSync, mkdtempSync, rmSync, truncateSync, writeFileSync, mkdirSync } from 'node:fs' import { tmpdir } from 'node:os' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_ORCA_YAML_BYTES } from '../shared/orca-yaml-file-limit' const { callMock, @@ -2135,6 +2136,24 @@ describe('orca cli worktree awareness', () => { } }) + it('rejects an oversized sparse orca.yaml during vm recipe doctor', async () => { + const repoPath = mkdtempSync(path.join(tmpdir(), 'orca-vm-doctor-bounds-')) + try { + const yamlPath = path.join(repoPath, 'orca.yaml') + writeFileSync(yamlPath, '') + truncateSync(yamlPath, MAX_ORCA_YAML_BYTES + 1) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['vm', 'recipe', 'doctor', 'bounded', '--repo-path', repoPath, '--json']) + + expect(process.exitCode).toBe(1) + expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('File too large')) + expect(callMock).not.toHaveBeenCalled() + } finally { + rmSync(repoPath, { recursive: true, force: true }) + } + }) + it('warns when vm recipe doctor finds no cleanup hook', async () => { const repoPath = mkdtempSync(path.join(tmpdir(), 'orca-vm-doctor-')) try { diff --git a/src/cli/linear-request-builders.ts b/src/cli/linear-request-builders.ts index 34d1ee9d473..cc8b2e5d27f 100644 --- a/src/cli/linear-request-builders.ts +++ b/src/cli/linear-request-builders.ts @@ -1,4 +1,3 @@ -import { readFile } from 'node:fs/promises' import { isAbsolute, join } from 'node:path' import type { LinearIssueInclude, @@ -21,6 +20,16 @@ import { getRequiredStringFlagAllowingEmpty } from './flags' import { RuntimeClientError } from './runtime-client' +import { + NodeReadableTextTooLargeError, + readNodeReadableTextWithinLimit +} from '../shared/node-readable-text' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../shared/node-bounded-file-reader' + +const LINEAR_WRITE_BODY_MAX_BYTES = LINEAR_WRITE_BODY_CAP * 4 const LINEAR_PRIORITY_VALUES = new Map([ ['none', 0], @@ -262,14 +271,35 @@ export async function readLinearBody( async function readLinearBodyFile(path: string, cwd: string): Promise { if (path !== '-') { - return await readFile(isAbsolute(path) ? path : join(cwd, path), 'utf8') + try { + const { buffer } = await readNodeFileWithinLimit( + isAbsolute(path) ? path : join(cwd, path), + LINEAR_WRITE_BODY_MAX_BYTES + ) + return buffer.toString('utf8') + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw linearBodyTooLargeError() + } + throw error + } } if (process.stdin.isTTY) { throw new RuntimeClientError('invalid_argument', 'stdin body requested but stdin is a TTY') } - const chunks: Buffer[] = [] - for await (const chunk of process.stdin) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))) + try { + return await readNodeReadableTextWithinLimit(process.stdin, LINEAR_WRITE_BODY_MAX_BYTES) + } catch (error) { + if (error instanceof NodeReadableTextTooLargeError) { + throw linearBodyTooLargeError() + } + throw error } - return Buffer.concat(chunks).toString('utf8') +} + +function linearBodyTooLargeError(): RuntimeClientError { + return new RuntimeClientError( + 'linear_body_too_large', + `Linear body must be at most ${LINEAR_WRITE_BODY_CAP} characters` + ) } diff --git a/src/cli/runtime/control-file-bounds.test.ts b/src/cli/runtime/control-file-bounds.test.ts new file mode 100644 index 00000000000..3649b2d0780 --- /dev/null +++ b/src/cli/runtime/control-file-bounds.test.ts @@ -0,0 +1,77 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + getRuntimeMetadataPath, + MAX_RUNTIME_METADATA_FILE_BYTES +} from '../../shared/runtime-bootstrap' +import { + getServeUpdateHandoffPath, + MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES +} from '../../shared/serve-update-handoff' +import { readMetadata, tryReadMetadata } from './metadata' +import { + readServeUpdateHandoff, + readServeUpdateHandoffSync, + recordServeUpdateHandoffFailure +} from './serve-update-supervisor' +import { RuntimeClientError } from './types' + +describe('CLI runtime control-file bounds', () => { + const paths: string[] = [] + + afterEach(() => { + for (const path of paths.splice(0)) { + rmSync(path, { recursive: true, force: true }) + } + }) + + function makeRoot(): string { + const path = mkdtempSync(join(tmpdir(), 'orca-cli-control-file-bound-')) + paths.push(path) + return path + } + + it('reports oversized runtime metadata as unavailable', () => { + const root = makeRoot() + const metadataPath = getRuntimeMetadataPath(root) + writeFileSync(metadataPath, '{"runtimeId":"runtime-1"}') + truncateSync(metadataPath, MAX_RUNTIME_METADATA_FILE_BYTES + 1) + + expect(tryReadMetadata(root)).toBeNull() + expect(() => readMetadata(root)).toThrow(RuntimeClientError) + }) + + it('ignores oversized serve-update handoff state in sync and async readers', async () => { + const handoffPath = getServeUpdateHandoffPath(makeRoot()) + writeFileSync(handoffPath, '{"schemaVersion":1}') + truncateSync(handoffPath, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES + 1) + + expect(readServeUpdateHandoffSync(handoffPath)).toBeNull() + await expect(readServeUpdateHandoff(handoffPath)).resolves.toBeNull() + }) + + it('preserves the prior handoff when bounded transactional serialization fails', async () => { + const handoffPath = getServeUpdateHandoffPath(makeRoot()) + const initial = { + schemaVersion: 1 as const, + phase: 'install-requested' as const, + fromVersion: '1.0.0', + targetVersion: '1.0.1', + servingPid: process.pid + } + writeFileSync(handoffPath, JSON.stringify(initial)) + + await expect( + recordServeUpdateHandoffFailure( + handoffPath, + initial, + 'x'.repeat(MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES + 1) + ) + ).rejects.toThrow(`JSON output exceeds ${MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES} bytes`) + + expect(JSON.parse(readFileSync(handoffPath, 'utf8'))).toEqual(initial) + expect(existsSync(`${handoffPath}.${process.pid}.tmp`)).toBe(false) + }) +}) diff --git a/src/cli/runtime/launch.test.ts b/src/cli/runtime/launch.test.ts index 235b2b2ed56..cc8c33c7cf8 100644 --- a/src/cli/runtime/launch.test.ts +++ b/src/cli/runtime/launch.test.ts @@ -519,6 +519,33 @@ describe('serveOrcaApp', () => { expect(child.unref).toHaveBeenCalledOnce() }) + it('preserves recipe JSON after 100,000 one-byte fragments', async () => { + const { child, result, stdoutSpy } = startRecipeJsonServer() + const output = Buffer.from(`${' '.repeat(100_000)}${RECIPE_JSON}\n`) + queueMicrotask(() => { + for (let index = 0; index < output.byteLength; index += 1) { + child.stdout.emit('data', output.subarray(index, index + 1)) + } + }) + + await expect(result).resolves.toBe(0) + expect(stdoutSpy).toHaveBeenCalledWith(`${RECIPE_JSON}\n`) + }) + + it('kills a server that emits an oversized recipe line without a newline', async () => { + const { child, result } = startRecipeJsonServer() + queueMicrotask(() => { + child.stdout.emit('data', Buffer.alloc(4 * 1024 * 1024 + 1, 0x78)) + }) + + await expect(result).rejects.toMatchObject({ + code: 'runtime_serve_failed', + message: 'Recipe JSON output exceeded 4194304 byte line limit.' + }) + expect(child.kill).toHaveBeenCalledWith('SIGTERM') + expect(child.stdout.listenerCount('data')).toBe(0) + }) + it('rejects when the server exits without valid recipe JSON', async () => { const { child, result, stdoutSpy, stderrSpy } = startRecipeJsonServer() const secrets = ['UPPER-SECRET', 'SLASH-SECRET', 'LEGACY-SECRET', 'PRIVATE-SECRET'] diff --git a/src/cli/runtime/launch.ts b/src/cli/runtime/launch.ts index bd7d939be5a..9bdd9f04252 100644 --- a/src/cli/runtime/launch.ts +++ b/src/cli/runtime/launch.ts @@ -1,16 +1,12 @@ import { spawn as spawnProcess, type SpawnOptions } from 'node:child_process' import { resolve } from 'node:path' -import { StringDecoder } from 'node:string_decoder' import { SERVE_UPDATE_HANDOFF_PATH_ENV, getServeUpdateHandoffPath } from '../../shared/serve-update-handoff' -import { - getEphemeralVmRecipeResultConnection, - parseEphemeralVmRecipeResult -} from '../../shared/ephemeral-vm-recipes' import { getDefaultUserDataPath } from './metadata' import { getMacAppBundlePath } from './mac-app-update-bundle' +import { waitForRecipeJson } from './recipe-json-output' import { readServeUpdateHandoffSync, resumeInterruptedServeUpdate, @@ -18,8 +14,6 @@ import { } from './serve-update-supervisor' import { RuntimeClientError } from './types' -const IGNORED_NON_RECIPE_STDOUT = '[serve] ignored non-recipe stdout' - export function launchOrcaApp(): void { const overrideCommand = process.env.ORCA_OPEN_COMMAND if (typeof overrideCommand === 'string' && overrideCommand.trim().length > 0) { @@ -165,97 +159,6 @@ export function serveOrcaApp( }) } -function waitForRecipeJson(child: ReturnType): Promise { - return new Promise((resolve, reject) => { - let output = '' - let settled = false - const timeout = setTimeout(() => { - finish(new RuntimeClientError('runtime_serve_failed', 'Timed out waiting for recipe JSON.')) - child.kill('SIGTERM') - }, 60000) - const finish = (error?: Error): void => { - if (settled) { - return - } - settled = true - clearTimeout(timeout) - child.stdout?.off('data', onData) - child.off('error', onError) - child.off('close', onClose) - if (error) { - reject(error) - return - } - child.stdout?.destroy?.() - child.unref() - resolve(0) - } - const writeIgnoredRecipeStdout = (): void => { - // Why: non-readiness child stdout is untrusted and cannot be safely - // redacted, including schema-valid results with arbitrary user data. - process.stderr.write(`${IGNORED_NON_RECIPE_STDOUT}\n`) - } - const processRecipeOutputLine = (line: string): void => { - const normalizedLine = line.endsWith('\r') ? line.slice(0, -1) : line - if (!normalizedLine.trim()) { - return - } - const parsed = parseEphemeralVmRecipeResult(normalizedLine) - if (!parsed.ok) { - writeIgnoredRecipeStdout() - return - } - if (getEphemeralVmRecipeResultConnection(parsed.result).type !== 'orca-server') { - writeIgnoredRecipeStdout() - return - } - process.stdout.write(`${normalizedLine.trim()}\n`) - finish() - } - const stdoutDecoder = new StringDecoder('utf8') - const onData = (chunk: Buffer | string): void => { - output += typeof chunk === 'string' ? chunk : stdoutDecoder.write(chunk) - while (!settled) { - const newlineIndex = output.indexOf('\n') - if (newlineIndex === -1) { - return - } - const line = output.slice(0, newlineIndex) - output = output.slice(newlineIndex + 1) - processRecipeOutputLine(line) - } - } - const onError = (error: Error): void => { - finish(error) - } - const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { - if (settled) { - return - } - output += stdoutDecoder.end() - if (output.trim()) { - processRecipeOutputLine(output) - } - if (settled) { - return - } - finish( - new RuntimeClientError( - 'runtime_serve_failed', - typeof code === 'number' - ? `Orca serve exited before printing valid recipe JSON with code ${code}.` - : `Orca serve exited before printing valid recipe JSON via ${signal}.` - ) - ) - } - child.stdout?.on('data', onData) - child.once('error', onError) - // Why: `exit` can precede the final piped stdout data. `close` waits until - // stdio closes so a last recipe chunk is not mistaken for missing output. - child.once('close', onClose) - }) -} - function getExecutableAppArgs(): string[] { return process.env.ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT === '1' ? [resolveAppRoot()] : [] } diff --git a/src/cli/runtime/mac-app-update-bundle.test.ts b/src/cli/runtime/mac-app-update-bundle.test.ts new file mode 100644 index 00000000000..3f802e3bbd6 --- /dev/null +++ b/src/cli/runtime/mac-app-update-bundle.test.ts @@ -0,0 +1,33 @@ +import { mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { MAC_BUNDLE_INFO_PLIST_MAX_BYTES, waitForMacBundleVersion } from './mac-app-update-bundle' + +const roots: string[] = [] + +afterEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +const originalPlatform = process.platform + +describe('waitForMacBundleVersion', () => { + it('rejects a sparse oversized Info.plist without reading its payload', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' }) + const root = mkdtempSync(join(tmpdir(), 'orca-mac-bundle-bounds-')) + roots.push(root) + const contentsPath = join(root, 'Orca.app', 'Contents') + const executable = join(contentsPath, 'MacOS', 'Orca') + mkdirSync(join(contentsPath, 'MacOS'), { recursive: true }) + writeFileSync(executable, '') + const plistPath = join(contentsPath, 'Info.plist') + writeFileSync(plistPath, '') + truncateSync(plistPath, MAC_BUNDLE_INFO_PLIST_MAX_BYTES + 1) + + await expect(waitForMacBundleVersion(executable, '1.2.3', 1)).resolves.toBe(false) + }) +}) diff --git a/src/cli/runtime/mac-app-update-bundle.ts b/src/cli/runtime/mac-app-update-bundle.ts index d7770707cb8..4edc066b0fa 100644 --- a/src/cli/runtime/mac-app-update-bundle.ts +++ b/src/cli/runtime/mac-app-update-bundle.ts @@ -1,8 +1,9 @@ import { watch, type FSWatcher } from 'node:fs' -import { readFile } from 'node:fs/promises' import { dirname, resolve } from 'node:path' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' const MAC_BUNDLE_UPDATE_TIMEOUT_MS = 120_000 +export const MAC_BUNDLE_INFO_PLIST_MAX_BYTES = 1024 * 1024 export function getMacAppBundlePath(executable: string): string | null { if (process.platform !== 'darwin') { @@ -78,7 +79,9 @@ export async function waitForMacBundleVersion( async function readMacBundleVersion(infoPlistPath: string): Promise { try { - const plist = await readFile(infoPlistPath, 'utf8') + const plist = ( + await readNodeFileWithinLimit(infoPlistPath, MAC_BUNDLE_INFO_PLIST_MAX_BYTES) + ).buffer.toString('utf8') const match = /CFBundleShortVersionString<\/key>\s*([^<]+)<\/string>/.exec(plist) return match?.[1]?.trim() || null } catch { diff --git a/src/cli/runtime/metadata.ts b/src/cli/runtime/metadata.ts index de1a8122fc2..2e617ca03dc 100644 --- a/src/cli/runtime/metadata.ts +++ b/src/cli/runtime/metadata.ts @@ -1,9 +1,11 @@ import { homedir } from 'node:os' import { join } from 'node:path' -import { readFileSync } from 'node:fs' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import { findTransport, getRuntimeMetadataPath, + MAX_RUNTIME_METADATA_FILE_BYTES, + parseRuntimeMetadataJson, type RuntimeMetadata } from '../../shared/runtime-bootstrap' import { RuntimeClientError } from './types' @@ -11,7 +13,11 @@ import { RuntimeClientError } from './types' export function readMetadata(userDataPath: string): RuntimeMetadata { const metadataPath = getRuntimeMetadataPath(userDataPath) try { - const metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as RuntimeMetadata | null + const metadata = parseRuntimeMetadataJson( + readNodeFileSyncWithinLimit(metadataPath, MAX_RUNTIME_METADATA_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) if (!metadata || !findTransport(metadata, 'unix', 'named-pipe') || !metadata.authToken) { throw new RuntimeClientError( 'runtime_unavailable', @@ -33,7 +39,11 @@ export function readMetadata(userDataPath: string): RuntimeMetadata { export function tryReadMetadata(userDataPath: string): RuntimeMetadata | null { const metadataPath = getRuntimeMetadataPath(userDataPath) try { - return JSON.parse(readFileSync(metadataPath, 'utf8')) as RuntimeMetadata | null + return parseRuntimeMetadataJson( + readNodeFileSyncWithinLimit(metadataPath, MAX_RUNTIME_METADATA_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) } catch { return null } diff --git a/src/cli/runtime/recipe-json-output.ts b/src/cli/runtime/recipe-json-output.ts new file mode 100644 index 00000000000..bcbe5e3fc2e --- /dev/null +++ b/src/cli/runtime/recipe-json-output.ts @@ -0,0 +1,120 @@ +import type { ChildProcess } from 'node:child_process' +import { + getEphemeralVmRecipeResultConnection, + parseEphemeralVmRecipeResult +} from '../../shared/ephemeral-vm-recipes' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' +import { RuntimeClientError } from './types' + +const IGNORED_NON_RECIPE_STDOUT = '[serve] ignored non-recipe stdout' +const MAX_RECIPE_OUTPUT_LINE_BYTES = 4 * 1024 * 1024 + +export function waitForRecipeJson(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + const output = new GrowingByteBuffer() + let settled = false + const timeout = setTimeout(() => { + finish(new RuntimeClientError('runtime_serve_failed', 'Timed out waiting for recipe JSON.')) + child.kill('SIGTERM') + }, 60000) + const finish = (error?: Error): void => { + if (settled) { + return + } + settled = true + clearTimeout(timeout) + output.clear() + child.stdout?.off('data', onData) + child.off('error', onError) + child.off('close', onClose) + if (error) { + reject(error) + return + } + child.stdout?.destroy?.() + child.unref() + resolve(0) + } + const writeIgnoredRecipeStdout = (): void => { + // Why: non-readiness child stdout can contain arbitrary user data. + process.stderr.write(`${IGNORED_NON_RECIPE_STDOUT}\n`) + } + const processRecipeOutputLine = (line: string): void => { + const normalizedLine = line.endsWith('\r') ? line.slice(0, -1) : line + if (!normalizedLine.trim()) { + return + } + const parsed = parseEphemeralVmRecipeResult(normalizedLine) + if (!parsed.ok) { + writeIgnoredRecipeStdout() + return + } + if (getEphemeralVmRecipeResultConnection(parsed.result).type !== 'orca-server') { + writeIgnoredRecipeStdout() + return + } + process.stdout.write(`${normalizedLine.trim()}\n`) + finish() + } + const oversizedLineError = (): RuntimeClientError => + new RuntimeClientError( + 'runtime_serve_failed', + `Recipe JSON output exceeded ${MAX_RECIPE_OUTPUT_LINE_BYTES} byte line limit.` + ) + const onData = (chunk: Buffer | string): void => { + if (settled) { + return + } + output.append(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) + while (!settled) { + const newlineIndex = output.indexOfByte(0x0a) + if (newlineIndex === -1) { + break + } + if (newlineIndex > MAX_RECIPE_OUTPUT_LINE_BYTES) { + finish(oversizedLineError()) + child.kill('SIGTERM') + return + } + const line = output.takePrefixString(newlineIndex) + output.discardPrefix(1) + processRecipeOutputLine(line) + } + if (!settled && output.byteLength > MAX_RECIPE_OUTPUT_LINE_BYTES) { + finish(oversizedLineError()) + child.kill('SIGTERM') + } + } + const onError = (error: Error): void => { + finish(error) + } + const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { + if (settled) { + return + } + if (output.byteLength > MAX_RECIPE_OUTPUT_LINE_BYTES) { + finish(oversizedLineError()) + return + } + const finalOutput = output.takeString() + if (finalOutput.trim()) { + processRecipeOutputLine(finalOutput) + } + if (settled) { + return + } + finish( + new RuntimeClientError( + 'runtime_serve_failed', + typeof code === 'number' + ? `Orca serve exited before printing valid recipe JSON with code ${code}.` + : `Orca serve exited before printing valid recipe JSON via ${signal}.` + ) + ) + } + child.stdout?.on('data', onData) + child.once('error', onError) + // Why: `close` waits for final piped stdout data that can arrive after `exit`. + child.once('close', onClose) + }) +} diff --git a/src/cli/runtime/serve-update-supervisor.ts b/src/cli/runtime/serve-update-supervisor.ts index 343b8b56373..61c3cb737ee 100644 --- a/src/cli/runtime/serve-update-supervisor.ts +++ b/src/cli/runtime/serve-update-supervisor.ts @@ -1,9 +1,14 @@ import type { ChildProcess, SpawnOptions, spawn } from 'node:child_process' -import { readFileSync } from 'node:fs' -import { readFile, rename, unlink, writeFile } from 'node:fs/promises' +import { rename, unlink, writeFile } from 'node:fs/promises' import { + readNodeFileSyncWithinLimit, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { + MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES, + parseServeUpdateHandoffJson, parseServeSupervisorMessage, - parseServeUpdateHandoffState, type ServeUpdateHandoffState } from '../../shared/serve-update-handoff' import { RuntimeClientError } from './types' @@ -213,7 +218,11 @@ export async function readServeUpdateHandoff( handoffPath: string ): Promise { try { - return parseServeUpdateHandoffState(JSON.parse(await readFile(handoffPath, 'utf8'))) + const { buffer } = await readNodeFileWithinLimit( + handoffPath, + MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES + ) + return parseServeUpdateHandoffJson(buffer.toString('utf8')) } catch { return null } @@ -221,7 +230,8 @@ export async function readServeUpdateHandoff( export function readServeUpdateHandoffSync(handoffPath: string): ServeUpdateHandoffState | null { try { - return parseServeUpdateHandoffState(JSON.parse(readFileSync(handoffPath, 'utf8'))) + const { buffer } = readNodeFileSyncWithinLimit(handoffPath, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES) + return parseServeUpdateHandoffJson(buffer.toString('utf8')) } catch { return null } @@ -259,6 +269,12 @@ async function writeServeUpdateHandoffState( state: ServeUpdateHandoffState ): Promise { const temporaryPath = `${handoffPath}.${process.pid}.tmp` - await writeFile(temporaryPath, JSON.stringify(state), { mode: 0o600 }) - await rename(temporaryPath, handoffPath) + try { + const { serialized } = stringifyJsonWithinByteLimit(state, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES) + await writeFile(temporaryPath, serialized, { mode: 0o600 }) + await rename(temporaryPath, handoffPath) + } catch (error) { + await unlink(temporaryPath).catch(() => undefined) + throw error + } } diff --git a/src/cli/runtime/transport.test.ts b/src/cli/runtime/transport.test.ts index d722de22065..cf084f5d80a 100644 --- a/src/cli/runtime/transport.test.ts +++ b/src/cli/runtime/transport.test.ts @@ -2,9 +2,9 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { createServer, type Socket } from 'node:net' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import type { RuntimeMetadata } from '../../shared/runtime-bootstrap' -import { sendRequest } from './transport' +import { CLI_RUNTIME_JSON_STRUCTURE_LIMITS, sendRequest } from './transport' const servers = new Set>() const sockets = new Set() @@ -114,4 +114,36 @@ describe.skipIf(process.platform === 'win32')('runtime transport', () => { }) expect(Date.now() - start).toBeLessThan(5000) }) + + it('rejects excessive response nesting before JSON.parse', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-transport-')) + const endpoint = join(userDataPath, 'runtime.sock') + const server = createServer((socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', () => { + const depth = CLI_RUNTIME_JSON_STRUCTURE_LIMITS.nestingDepth + 1 + socket.write(`${'['.repeat(depth)}0${']'.repeat(depth)}\n`) + }) + }) + servers.add(server) + await new Promise((resolve) => server.listen(endpoint, resolve)) + const parseSpy = vi.spyOn(JSON, 'parse') + const metadata: RuntimeMetadata = { + runtimeId: 'runtime-1', + pid: 123, + transports: [{ kind: 'unix', endpoint }], + authToken: 'token', + startedAt: 1 + } + + try { + await expect(sendRequest(metadata, 'status.get', undefined, 1000)).rejects.toMatchObject({ + code: 'invalid_runtime_response' + }) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) }) diff --git a/src/cli/runtime/transport.ts b/src/cli/runtime/transport.ts index 60da1e2a980..ceaac6ad5b6 100644 --- a/src/cli/runtime/transport.ts +++ b/src/cli/runtime/transport.ts @@ -1,9 +1,17 @@ import { createConnection } from 'node:net' import { randomUUID } from 'node:crypto' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' import { findTransport, type RuntimeMetadata } from '../../shared/runtime-bootstrap' import { isKeepaliveFrame, RuntimeRpcEnvelopeSchema } from './envelope-schema' import { RuntimeClientError, type RuntimeRpcResponse } from './types' +const MAX_RUNTIME_RESPONSE_LINE_BYTES = 64 * 1024 * 1024 +export const CLI_RUNTIME_JSON_STRUCTURE_LIMITS = { + structuralTokens: 1_000_000, + nestingDepth: 128 +} as const + export async function sendRequest( metadata: RuntimeMetadata, method: string, @@ -22,7 +30,7 @@ export async function sendRequest( return } const socket = createConnection(transport.endpoint) - let buffer = '' + const buffer = new GrowingByteBuffer() let settled = false const requestId = randomUUID() @@ -48,15 +56,16 @@ export async function sendRequest( } settled = true clearTimeout(timeout) - socket.end() + buffer.clear() if (result.ok === false) { + socket.destroy() reject(result.error) } else { + socket.end() resolve(result.response) } } - socket.setEncoding('utf8') socket.once('error', () => { finish({ ok: false, @@ -79,24 +88,39 @@ export async function sendRequest( ) }) }) - socket.on('data', (chunk) => { - buffer += chunk + socket.on('data', (chunk: Buffer) => { + if (settled) { + return + } + buffer.append(chunk) // Why: the server may interleave `{"_keepalive":true}\n` frames with the // final success/failure frame to keep both idle timers alive during a // long-poll (see design doc §3.1). Read frames in a loop until we see a // terminal frame. Each keepalive refreshes the client-side timer so a // 10 min wait doesn't trip the 60 s default ceiling. - let newlineIndex = buffer.indexOf('\n') + let newlineIndex = buffer.indexOfByte(0x0a) while (newlineIndex !== -1 && !settled) { - const line = buffer.slice(0, newlineIndex) - buffer = buffer.slice(newlineIndex + 1) + const lineBytes = newlineIndex + if (lineBytes > MAX_RUNTIME_RESPONSE_LINE_BYTES) { + finish({ + ok: false, + error: new RuntimeClientError( + 'invalid_runtime_response', + `The Orca runtime response exceeded the ${MAX_RUNTIME_RESPONSE_LINE_BYTES} byte frame limit.` + ) + }) + return + } + const line = buffer.takePrefixString(lineBytes) + buffer.discardPrefix(1) if (line.trim().length === 0) { - newlineIndex = buffer.indexOf('\n') + newlineIndex = buffer.indexOfByte(0x0a) continue } let raw: unknown try { + assertJsonTextStructureWithinLimits(line, CLI_RUNTIME_JSON_STRUCTURE_LIMITS) raw = JSON.parse(line) } catch { finish({ @@ -115,7 +139,7 @@ export async function sendRequest( // major). See §7 risk #9. if (isKeepaliveFrame(raw)) { timeout.refresh() - newlineIndex = buffer.indexOf('\n') + newlineIndex = buffer.indexOfByte(0x0a) continue } @@ -141,7 +165,7 @@ export async function sendRequest( const frame = parsed.data if ('_keepalive' in frame) { timeout.refresh() - newlineIndex = buffer.indexOf('\n') + newlineIndex = buffer.indexOfByte(0x0a) continue } @@ -169,6 +193,15 @@ export async function sendRequest( finish({ ok: true, response }) return } + if (buffer.byteLength > MAX_RUNTIME_RESPONSE_LINE_BYTES) { + finish({ + ok: false, + error: new RuntimeClientError( + 'invalid_runtime_response', + `The Orca runtime response exceeded the ${MAX_RUNTIME_RESPONSE_LINE_BYTES} byte frame limit.` + ) + }) + } }) socket.on('connect', () => { socket.write( diff --git a/src/main/active-view-preference.test.ts b/src/main/active-view-preference.test.ts index 972009f722c..8d264b27fa8 100644 --- a/src/main/active-view-preference.test.ts +++ b/src/main/active-view-preference.test.ts @@ -1,8 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { ActiveViewPreference, getActiveViewPreferenceFile } from './active-view-preference' +import { + ActiveViewPreference, + getActiveViewPreferenceFile, + MAX_ACTIVE_VIEW_PREFERENCE_FILE_BYTES +} from './active-view-preference' describe('ActiveViewPreference', () => { let dir: string @@ -88,4 +92,12 @@ describe('ActiveViewPreference', () => { expect(preference.set('__proto__')).toBe(false) expect(preference.get()).toBe('tasks') }) + + it('falls back without loading an oversized sparse sidecar', () => { + const preferenceFile = getActiveViewPreferenceFile(dataFile) + writeFileSync(preferenceFile, '{"activeView":"settings"}', 'utf-8') + truncateSync(preferenceFile, MAX_ACTIVE_VIEW_PREFERENCE_FILE_BYTES + 1) + + expect(new ActiveViewPreference(dataFile, 'tasks').get()).toBe('tasks') + }) }) diff --git a/src/main/active-view-preference.ts b/src/main/active-view-preference.ts index beed983aef4..d0728c448f6 100644 --- a/src/main/active-view-preference.ts +++ b/src/main/active-view-preference.ts @@ -1,11 +1,13 @@ -import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { mkdirSync, renameSync, writeFileSync } from 'node:fs' import { mkdir, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' import type { TopLevelView } from '../shared/types' import { isTopLevelView } from '../shared/top-level-view' const ACTIVE_VIEW_FILE_NAME = 'active-view.json' const SAVE_DEBOUNCE_MS = 100 +export const MAX_ACTIVE_VIEW_PREFERENCE_FILE_BYTES = 4 * 1024 type ActiveViewFile = { activeView: TopLevelView @@ -17,7 +19,11 @@ export function getActiveViewPreferenceFile(dataFile: string): string { function readActiveView(file: string): TopLevelView | null { try { - const parsed = JSON.parse(readFileSync(file, 'utf-8')) as Partial + const parsed = JSON.parse( + readNodeFileSyncWithinLimit(file, MAX_ACTIVE_VIEW_PREFERENCE_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) as Partial return isTopLevelView(parsed.activeView) ? parsed.activeView : null } catch { return null diff --git a/src/main/agent-hooks/agent-hook-file-bounds.test.ts b/src/main/agent-hooks/agent-hook-file-bounds.test.ts new file mode 100644 index 00000000000..9cb28f5a897 --- /dev/null +++ b/src/main/agent-hooks/agent-hook-file-bounds.test.ts @@ -0,0 +1,165 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + statSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { homedirMock } = vi.hoisted(() => ({ + homedirMock: vi.fn<() => string>() +})) + +vi.mock('os', async () => { + const actual = (await vi.importActual('os')) as Record + return { ...actual, homedir: homedirMock } +}) + +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { AmpHookService, _internals as ampInternals } from '../amp/hook-service' +import { DevinHookService } from '../devin/hook-service' +import { getDevinConfigPath, getDevinManagedScriptPath } from '../devin/hook-settings' +import { HermesHookService, _internals as hermesInternals } from '../hermes/hook-service' +import { KimiHookService } from '../kimi/hook-service' +import { + AGENT_HOOK_CONFIG_MAX_BYTES, + AGENT_HOOK_CONFIG_MAX_STRUCTURAL_TOKENS, + AGENT_HOOK_MANAGED_SCRIPT_MAX_BYTES, + AGENT_HOOK_PLUGIN_MAX_BYTES +} from './agent-hook-file-limits' +import { + readHooksJson, + readHooksJsonRawForGenerationCheck, + writeHooksJson, + writeManagedScript +} from './installer-utils' + +let root: string + +function writeSparseFile(path: string, bytes: number): void { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, '') + truncateSync(path, bytes) +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-agent-hook-bounds-')) + homedirMock.mockReturnValue(root) + vi.stubEnv('HERMES_HOME', join(root, '.hermes')) + vi.stubEnv('KIMI_CODE_HOME', join(root, '.kimi-code')) + vi.stubEnv('APPDATA', join(root, 'AppData', 'Roaming')) +}) + +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() + rmSync(root, { recursive: true, force: true }) +}) + +describe('agent hook file bounds', () => { + it('fails closed when hooks JSON exceeds the comparison and generation limit', () => { + const configPath = join(root, 'hooks.json') + writeSparseFile(configPath, AGENT_HOOK_CONFIG_MAX_BYTES + 1) + const originalSize = statSync(configPath).size + + expect(readHooksJson(configPath)).toBeNull() + expect(() => readHooksJsonRawForGenerationCheck(configPath)).toThrow(NodeFileReadTooLargeError) + expect(() => writeHooksJson(configPath, { hooks: {} })).toThrow(NodeFileReadTooLargeError) + expect(statSync(configPath).size).toBe(originalSize) + }) + + it('rejects structurally amplified hooks JSON before parsing', () => { + const configPath = join(root, 'hooks.json') + writeFileSync(configPath, `[${'0,'.repeat(AGENT_HOOK_CONFIG_MAX_STRUCTURAL_TOKENS)}0]`, 'utf8') + const parseSpy = vi.spyOn(JSON, 'parse') + + expect(readHooksJson(configPath)).toBeNull() + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('does not replace an oversized managed script during its equality check', () => { + const scriptPath = join(root, '.orca', 'agent-hooks', 'status-hook.sh') + writeSparseFile(scriptPath, AGENT_HOOK_MANAGED_SCRIPT_MAX_BYTES + 1) + const originalSize = statSync(scriptPath).size + + expect(() => writeManagedScript(scriptPath, '#!/bin/sh\n')).toThrow(NodeFileReadTooLargeError) + expect(statSync(scriptPath).size).toBe(originalSize) + }) + + it('does not replace an oversized Amp plugin', () => { + const pluginPath = ampInternals.getPluginPath() + writeSparseFile(pluginPath, AGENT_HOOK_PLUGIN_MAX_BYTES + 1) + const originalSize = statSync(pluginPath).size + + const status = new AmpHookService().install() + + expect(status).toMatchObject({ state: 'error', managedHooksPresent: false }) + expect(status.detail).toContain('File too large') + expect(statSync(pluginPath).size).toBe(originalSize) + }) + + it('does not create Hermes plugin files when config.yaml is oversized', () => { + const configPath = join(process.env.HERMES_HOME!, 'config.yaml') + writeSparseFile(configPath, AGENT_HOOK_CONFIG_MAX_BYTES + 1) + const originalSize = statSync(configPath).size + + const status = new HermesHookService().install() + + expect(status.state).toBe('error') + expect(status.detail).toContain('File too large') + expect(statSync(configPath).size).toBe(originalSize) + expect( + existsSync( + join(process.env.HERMES_HOME!, 'plugins', hermesInternals.HERMES_PLUGIN_NAME, 'plugin.yaml') + ) + ).toBe(false) + }) + + it('does not replace an oversized Hermes plugin manifest', () => { + const manifestPath = join( + process.env.HERMES_HOME!, + 'plugins', + hermesInternals.HERMES_PLUGIN_NAME, + 'plugin.yaml' + ) + writeSparseFile(manifestPath, AGENT_HOOK_PLUGIN_MAX_BYTES + 1) + const originalSize = statSync(manifestPath).size + + const status = new HermesHookService().install() + + expect(status.state).toBe('error') + expect(status.detail).toContain('File too large') + expect(statSync(manifestPath).size).toBe(originalSize) + expect(existsSync(join(process.env.HERMES_HOME!, 'config.yaml'))).toBe(false) + }) + + it('does not create a Kimi script or replace an oversized config.toml', () => { + const configPath = join(process.env.KIMI_CODE_HOME!, 'config.toml') + const scriptPath = join(root, '.orca', 'agent-hooks', 'kimi-hook.sh') + writeSparseFile(configPath, AGENT_HOOK_CONFIG_MAX_BYTES + 1) + const originalSize = statSync(configPath).size + + const status = new KimiHookService().install() + + expect(status.state).toBe('error') + expect(statSync(configPath).size).toBe(originalSize) + expect(existsSync(scriptPath)).toBe(false) + }) + + it('does not create a Devin script or replace an oversized config.json', () => { + const configPath = getDevinConfigPath() + writeSparseFile(configPath, AGENT_HOOK_CONFIG_MAX_BYTES + 1) + const originalSize = statSync(configPath).size + + const status = new DevinHookService().install() + + expect(status.state).toBe('error') + expect(statSync(configPath).size).toBe(originalSize) + expect(existsSync(getDevinManagedScriptPath())).toBe(false) + }) +}) diff --git a/src/main/agent-hooks/agent-hook-file-comparison.ts b/src/main/agent-hooks/agent-hook-file-comparison.ts new file mode 100644 index 00000000000..870161be708 --- /dev/null +++ b/src/main/agent-hooks/agent-hook-file-comparison.ts @@ -0,0 +1,15 @@ +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' + +export function readAgentHookFileForComparison(path: string, maxBytes: number): string | null { + try { + return readNodeFileSyncWithinLimit(path, maxBytes).buffer.toString('utf8') + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } + return null + } +} diff --git a/src/main/agent-hooks/agent-hook-file-limits.ts b/src/main/agent-hooks/agent-hook-file-limits.ts new file mode 100644 index 00000000000..f321f114c18 --- /dev/null +++ b/src/main/agent-hooks/agent-hook-file-limits.ts @@ -0,0 +1,6 @@ +// Why: leave room for the supported 30,000-hook cleanup, including longer encoded Windows commands. +export const AGENT_HOOK_CONFIG_MAX_BYTES = 64 * 1024 * 1024 +export const AGENT_HOOK_CONFIG_MAX_STRUCTURAL_TOKENS = 1_000_000 +export const AGENT_HOOK_CONFIG_MAX_NESTING_DEPTH = 128 +export const AGENT_HOOK_PLUGIN_MAX_BYTES = 1024 * 1024 +export const AGENT_HOOK_MANAGED_SCRIPT_MAX_BYTES = 1024 * 1024 diff --git a/src/main/agent-hooks/agent-hook-sftp-text-reader.ts b/src/main/agent-hooks/agent-hook-sftp-text-reader.ts new file mode 100644 index 00000000000..e5eb23fd40c --- /dev/null +++ b/src/main/agent-hooks/agent-hook-sftp-text-reader.ts @@ -0,0 +1,151 @@ +import type { SFTPWrapper } from 'ssh2' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' + +const READ_CHUNK_BYTES = 64 * 1024 + +type SftpWithBoundedAgentHookRead = SFTPWrapper & { + orcaReadFileWithinLimit?: ( + remotePath: string, + maxBytes: number, + callback: (error: unknown, data?: string | Buffer) => void + ) => void +} + +export async function readAgentHookRemoteTextFile( + sftp: SFTPWrapper, + remotePath: string, + maxBytes: number, + timeoutMs: number +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Remote file read limit must be a non-negative safe integer') + } + const boundedSftp = sftp as SftpWithBoundedAgentHookRead + if (typeof boundedSftp.orcaReadFileWithinLimit === 'function') { + return readWithCallback(remotePath, maxBytes, timeoutMs, (callback) => + boundedSftp.orcaReadFileWithinLimit!(remotePath, maxBytes, callback) + ) + } + if (typeof sftp.createReadStream !== 'function') { + return readWithCompatibilityFallback(sftp, remotePath, maxBytes, timeoutMs) + } + + const stream = sftp.createReadStream(remotePath, { + start: 0, + end: maxBytes, + highWaterMark: READ_CHUNK_BYTES + }) + return new Promise((resolve, reject) => { + let settled = false + let totalBytes = 0 + let retained = Buffer.alloc(0) + const timer = setTimeout(() => { + fail(new Error(`Timed out waiting for SFTP readFile ${remotePath}`)) + }, timeoutMs) + if (typeof timer === 'object' && 'unref' in timer) { + timer.unref() + } + + function fail(error: unknown): void { + if (settled) { + return + } + settled = true + clearTimeout(timer) + stream.destroy() + reject(error) + } + + stream.on('data', (chunk: string | Buffer | Uint8Array) => { + if (settled) { + return + } + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + totalBytes += bytes.byteLength + if (totalBytes > maxBytes) { + fail(new NodeFileReadTooLargeError(totalBytes, maxBytes)) + return + } + if (retained.length < totalBytes) { + const nextCapacity = Math.min( + maxBytes, + Math.max(READ_CHUNK_BYTES, retained.length * 2, totalBytes) + ) + const next = Buffer.allocUnsafe(nextCapacity) + retained.copy(next) + retained = next + } + bytes.copy(retained, totalBytes - bytes.byteLength) + }) + stream.once('error', fail) + stream.once('end', () => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + resolve(retained.toString('utf8', 0, totalBytes)) + }) + }) +} + +function readWithCompatibilityFallback( + sftp: SFTPWrapper, + remotePath: string, + maxBytes: number, + timeoutMs: number +): Promise { + return readWithCallback(remotePath, maxBytes, timeoutMs, (callback) => { + sftp.readFile(remotePath, 'utf8', callback) + }) +} + +function readWithCallback( + remotePath: string, + maxBytes: number, + timeoutMs: number, + start: (callback: (error: unknown, data?: string | Buffer) => void) => void +): Promise { + return new Promise((resolve, reject) => { + let settled = false + const timer = setTimeout(() => { + finish(new Error(`Timed out waiting for SFTP readFile ${remotePath}`)) + }, timeoutMs) + if (typeof timer === 'object' && 'unref' in timer) { + timer.unref() + } + + function finish(error: unknown, data?: string | Buffer): void { + if (settled) { + return + } + settled = true + clearTimeout(timer) + if (error) { + reject(error) + return + } + if (typeof data === 'string') { + const byteLength = Buffer.byteLength(data, 'utf8') + if (byteLength > maxBytes) { + reject(new NodeFileReadTooLargeError(byteLength, maxBytes)) + return + } + resolve(data) + return + } + const buffer = data ?? Buffer.alloc(0) + if (buffer.byteLength > maxBytes) { + reject(new NodeFileReadTooLargeError(buffer.byteLength, maxBytes)) + return + } + resolve(buffer.toString('utf8')) + } + + try { + start(finish) + } catch (error) { + finish(error) + } + }) +} diff --git a/src/main/agent-hooks/branch-rename-failure-output.test.ts b/src/main/agent-hooks/branch-rename-failure-output.test.ts index 6df43fa457a..69675120485 100644 --- a/src/main/agent-hooks/branch-rename-failure-output.test.ts +++ b/src/main/agent-hooks/branch-rename-failure-output.test.ts @@ -1,5 +1,8 @@ import { beforeEach, describe, expect, it } from 'vitest' import { + BRANCH_RENAME_FAILURE_KEY_MAX_BYTES, + BRANCH_RENAME_FAILURE_OUTPUT_MAX_BYTES, + __getBranchRenameFailureOutputCountForTests, __resetBranchRenameFailureOutputForTests, readBranchRenameFailureOutputForDisplay, rememberBranchRenameFailureOutput @@ -48,4 +51,27 @@ describe('branch rename failure output store', () => { expect(readBranchRenameFailureOutputForDisplay('wt-0')).not.toBeNull() expect(readBranchRenameFailureOutputForDisplay('wt-1')).toBeNull() }) + + it('admits exact byte boundaries and skips oversized retained text', () => { + const exactKey = 'k'.repeat(BRANCH_RENAME_FAILURE_KEY_MAX_BYTES) + const oversizedKey = `${exactKey}x` + rememberBranchRenameFailureOutput(exactKey, { + label: '', + exitCode: 1, + stdout: 'x'.repeat(BRANCH_RENAME_FAILURE_OUTPUT_MAX_BYTES), + stderr: '' + }) + rememberBranchRenameFailureOutput(oversizedKey, output('oversized-key')) + rememberBranchRenameFailureOutput('oversized-output', { + label: '', + exitCode: 1, + stdout: 'x'.repeat(BRANCH_RENAME_FAILURE_OUTPUT_MAX_BYTES + 1), + stderr: '' + }) + + expect(__getBranchRenameFailureOutputCountForTests()).toBe(1) + expect(readBranchRenameFailureOutputForDisplay(exactKey)).not.toBeNull() + expect(readBranchRenameFailureOutputForDisplay(oversizedKey)).toBeNull() + expect(readBranchRenameFailureOutputForDisplay('oversized-output')).toBeNull() + }) }) diff --git a/src/main/agent-hooks/branch-rename-failure-output.ts b/src/main/agent-hooks/branch-rename-failure-output.ts index 8b8660484b7..bff56937a3e 100644 --- a/src/main/agent-hooks/branch-rename-failure-output.ts +++ b/src/main/agent-hooks/branch-rename-failure-output.ts @@ -2,14 +2,29 @@ import { formatAgentGenerationFailureOutputForDisplay, type AgentGenerationFailureOutput } from '../text-generation/agent-failure-output' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' // Why: the full CLI output is a diagnostic for the local user only. Keeping it // in memory (never in worktree metadata) means nothing environment-identifying // is persisted or synced to paired clients; a restart just loses the on-demand // view while the sanitized excerpt badge survives. const MAX_ENTRIES = 32 +export const BRANCH_RENAME_FAILURE_KEY_MAX_BYTES = 4 * 1024 +export const BRANCH_RENAME_FAILURE_OUTPUT_MAX_BYTES = 520 * 1024 const entriesByWorktreeId = new Map() +function fitsByteBudget(values: readonly string[], maxBytes: number): boolean { + let remainingBytes = maxBytes + for (const value of values) { + const measured = measureUtf8ByteLength(value, { stopAfterBytes: remainingBytes }) + if (measured.exceededLimit) { + return false + } + remainingBytes -= measured.byteLength + } + return true +} + export function rememberBranchRenameFailureOutput( worktreeId: string, output: AgentGenerationFailureOutput | null | undefined @@ -20,6 +35,15 @@ export function rememberBranchRenameFailureOutput( if (!output) { return } + if ( + !fitsByteBudget([worktreeId], BRANCH_RENAME_FAILURE_KEY_MAX_BYTES) || + !fitsByteBudget( + [output.label, output.stdout, output.stderr], + BRANCH_RENAME_FAILURE_OUTPUT_MAX_BYTES + ) + ) { + return + } entriesByWorktreeId.set(worktreeId, output) while (entriesByWorktreeId.size > MAX_ENTRIES) { const oldest = entriesByWorktreeId.keys().next().value @@ -38,3 +62,7 @@ export function readBranchRenameFailureOutputForDisplay(worktreeId: string): str export function __resetBranchRenameFailureOutputForTests(): void { entriesByWorktreeId.clear() } + +export function __getBranchRenameFailureOutputCountForTests(): number { + return entriesByWorktreeId.size +} diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index e8d8cd5f373..7c9088f1d01 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -44,7 +44,9 @@ vi.mock('../ipc/worktree-logic', () => ({ })) import { + FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT, FIRST_WORK_BRANCH_RENAME_SETTLED_CACHE_LIMIT, + getFirstWorkBranchRenameStateForTests, maybeAutoRenameBranchOnFirstWork, resetFirstWorkBranchRenameState, type FirstWorkBranchRenameDeps @@ -297,6 +299,45 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { expect(gitExecFileAsyncMock).toHaveBeenCalledTimes(1) }) + it('bounds simultaneous auto-rename attempts and recovers after they settle', async () => { + let releaseGit!: () => void + const gitGate = new Promise((resolve) => { + releaseGit = resolve + }) + gitExecFileAsyncMock.mockImplementation(async () => { + await gitGate + return { stdout: 'you/custom-branch\n', stderr: '' } + }) + const { deps } = makeDeps() + const attempts = Array.from({ length: FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT }, (_, index) => + maybeAutoRenameBranchOnFirstWork( + workingEvent({ + tabId: undefined, + paneKey: '', + worktreeId: `${REPO_ID}${WORKTREE_ID_SEPARATOR}/repo/concurrent-${index}` + }), + deps + ) + ) + + expect(getFirstWorkBranchRenameStateForTests().inFlight).toBe( + FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT + ) + await maybeAutoRenameBranchOnFirstWork( + workingEvent({ + tabId: undefined, + paneKey: '', + worktreeId: `${REPO_ID}${WORKTREE_ID_SEPARATOR}/repo/overflow` + }), + deps + ) + expect(gitExecFileAsyncMock).toHaveBeenCalledTimes(FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT) + + releaseGit() + await Promise.all(attempts) + expect(getFirstWorkBranchRenameStateForTests().inFlight).toBe(0) + }) + it('retries on a later event after a transient failure (does not poison the worktree)', async () => { generateBranchNameMock.mockResolvedValueOnce({ success: false, error: 'agent not ready' }) const { deps, onRenamed, setRenameError } = makeDeps() diff --git a/src/main/agent-hooks/first-work-branch-rename.ts b/src/main/agent-hooks/first-work-branch-rename.ts index c9703c51637..ae06d5cfbf1 100644 --- a/src/main/agent-hooks/first-work-branch-rename.ts +++ b/src/main/agent-hooks/first-work-branch-rename.ts @@ -1,4 +1,5 @@ // On first agent work in a fresh workspace, replace the auto-generated creature branch (e.g. `you/Nautilus`) with a short work-derived name. +import { createHash } from 'node:crypto' import type { GlobalSettings, Repo } from '../../shared/types' import { getRepoIdFromWorktreeId, splitWorktreeIdForFilesystem } from '../../shared/worktree-id' import { parseWorkspaceKey } from '../../shared/workspace-scope' @@ -71,6 +72,7 @@ export type FirstWorkBranchRenameDeps = { const inFlightWorktreeIds = new Set() const settledWorktreeIds = new Set() export const FIRST_WORK_BRANCH_RENAME_SETTLED_CACHE_LIMIT = 500 +export const FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT = 32 /** Test seam: clear the per-process dedupe sets. */ export function resetFirstWorkBranchRenameState(): void { @@ -91,6 +93,20 @@ function rememberSettledWorktreeId(worktreeId: string): void { } } +function worktreeStateKey(worktreeId: string): string { + return createHash('sha256').update(worktreeId).digest('base64url') +} + +export function getFirstWorkBranchRenameStateForTests(): { + inFlight: number + settled: number +} { + return { + inFlight: inFlightWorktreeIds.size, + settled: settledWorktreeIds.size + } +} + export async function maybeAutoRenameBranchOnFirstWork( event: FirstWorkBranchRenameEvent, deps: FirstWorkBranchRenameDeps @@ -108,26 +124,30 @@ export async function maybeAutoRenameBranchOnFirstWork( if (!worktreeId) { return } + const stateKey = worktreeStateKey(worktreeId) // Short-circuit settled/in-flight worktrees before any logging or work. - if (settledWorktreeIds.has(worktreeId) || inFlightWorktreeIds.has(worktreeId)) { + if (settledWorktreeIds.has(stateKey) || inFlightWorktreeIds.has(stateKey)) { return } const prompt = event.prompt?.trim() if (!prompt) { return } - inFlightWorktreeIds.add(worktreeId) + if (inFlightWorktreeIds.size >= FIRST_WORK_BRANCH_RENAME_IN_FLIGHT_LIMIT) { + return + } + inFlightWorktreeIds.add(stateKey) try { // settled = definitive verdict (renamed/ineligible); false = transient bail to retry later. const settled = await runAutoRename(worktreeId, prompt, event.assistantMessage, deps) if (settled) { - rememberSettledWorktreeId(worktreeId) + rememberSettledWorktreeId(stateKey) } } catch (error) { // Why: best-effort convenience; a failure must never disrupt the user's agent, so swallow after logging. console.warn('[auto-branch-rename] rename attempt failed:', error) } finally { - inFlightWorktreeIds.delete(worktreeId) + inFlightWorktreeIds.delete(stateKey) } } diff --git a/src/main/agent-hooks/hooks-json-read.ts b/src/main/agent-hooks/hooks-json-read.ts index c8536fad0b4..a844f3950a3 100644 --- a/src/main/agent-hooks/hooks-json-read.ts +++ b/src/main/agent-hooks/hooks-json-read.ts @@ -1,5 +1,12 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' import type { HooksConfig } from './installer-utils' +import { + AGENT_HOOK_CONFIG_MAX_BYTES, + AGENT_HOOK_CONFIG_MAX_NESTING_DEPTH, + AGENT_HOOK_CONFIG_MAX_STRUCTURAL_TOKENS +} from './agent-hook-file-limits' export function isPlainObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) @@ -11,6 +18,19 @@ export type HooksJsonSnapshot = { config: HooksConfig | null } +export function parseHooksJsonText(raw: string): HooksConfig | null { + try { + assertJsonTextStructureWithinLimits(raw, { + structuralTokens: AGENT_HOOK_CONFIG_MAX_STRUCTURAL_TOKENS, + nestingDepth: AGENT_HOOK_CONFIG_MAX_NESTING_DEPTH + }) + const parsed = JSON.parse(raw) + return isPlainObject(parsed) ? parsed : null + } catch { + return null + } +} + // Why: generation guards abort a mutation when the file no longer matches the // bytes it was derived from; the raw snapshot and the parse must come from one // read or a concurrent save can slip between them unnoticed. @@ -20,18 +40,21 @@ export function readHooksJsonWithRaw(configPath: string): HooksJsonSnapshot { } let raw: string try { - raw = readFileSync(configPath, 'utf-8') + raw = readNodeFileSyncWithinLimit(configPath, AGENT_HOOK_CONFIG_MAX_BYTES).buffer.toString( + 'utf8' + ) } catch { return { raw: null, config: null } } - try { - const parsed = JSON.parse(raw) - return { raw, config: isPlainObject(parsed) ? parsed : null } - } catch { - return { raw, config: null } - } + return { raw, config: parseHooksJsonText(raw) } } export function readHooksJson(configPath: string): HooksConfig | null { return readHooksJsonWithRaw(configPath).config } + +export function readHooksJsonRawForGenerationCheck(configPath: string): string { + return readNodeFileSyncWithinLimit(configPath, AGENT_HOOK_CONFIG_MAX_BYTES).buffer.toString( + 'utf8' + ) +} diff --git a/src/main/agent-hooks/installer-utils-remote.test.ts b/src/main/agent-hooks/installer-utils-remote.test.ts index 75f9c9682fe..4b3f7606c28 100644 --- a/src/main/agent-hooks/installer-utils-remote.test.ts +++ b/src/main/agent-hooks/installer-utils-remote.test.ts @@ -1,8 +1,11 @@ +import { Readable } from 'node:stream' import { describe, expect, it, vi } from 'vitest' import type { SFTPWrapper } from 'ssh2' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import { readHooksJsonRemote, + readTextFileRemote, writeHooksJsonRemote, writeManagedScriptRemote, writeTextFileRemoteAtomic @@ -13,6 +16,7 @@ type FakeFs = { dirs: Set modes: Map openSshRenameCount: number + readdirCount: number } function createFakeSftp( @@ -30,7 +34,8 @@ function createFakeSftp( files: new Map(), dirs: new Set(['/']), modes: new Map(), - openSshRenameCount: 0 + openSshRenameCount: 0, + readdirCount: 0 } const noEntryError = (path: string): { code: number; message: string } => ({ code: 2, @@ -96,6 +101,10 @@ function createFakeSftp( cb(null) }, stat: (path: string, cb: (err: unknown, stats?: { mode: number }) => void): void => { + if (fs.dirs.has(path)) { + cb(null, fakeStats(0o040755)) + return + } if (!fs.files.has(path)) { cb(noEntryError(path)) return @@ -103,6 +112,7 @@ function createFakeSftp( cb(null, fakeStats(fs.modes.get(path) ?? 0o100644)) }, readdir: (path: string, cb: (err: unknown, list?: { filename: string }[]) => void): void => { + fs.readdirCount += 1 if (fs.dirs.has(path)) { cb(null, []) return @@ -201,6 +211,14 @@ describe('installer-utils-remote', () => { expect(fs.modes.get('/home/u/.claude/settings.json')).toBe(0o600) }) + it('probes mkdir ancestors with stat instead of materializing directory listings', async () => { + const { sftp, fs } = createFakeSftp() + + await writeHooksJsonRemote(sftp, '/home/u/.claude/settings.json', { hooks: {} }) + + expect(fs.readdirCount).toBe(0) + }) + it('preserves existing config file mode across atomic replacement', async () => { const { sftp, fs } = createFakeSftp() const path = '/home/u/.codex/config.toml' @@ -287,4 +305,44 @@ describe('installer-utils-remote', () => { await writeHooksJsonRemote(sftp, path, { hooks: {} }) expect(fs.files.get(path)).toBe(beforeKey) }) + + it('preserves ordinary UTF-8 contents through the bounded remote stream', async () => { + const { sftp } = createFakeSftp() + const content = 'stable 🐋 config' + sftp.createReadStream = (() => Readable.from([Buffer.from(content)])) as never + + await expect(readTextFileRemote(sftp, '/home/u/config.toml', 1024)).resolves.toBe(content) + }) + + it('accepts a streamed remote file at the exact byte cap', async () => { + const { sftp } = createFakeSftp() + sftp.createReadStream = (() => Readable.from([Buffer.alloc(1024, 0x61)])) as never + + await expect(readTextFileRemote(sftp, '/home/u/config.toml', 1024)).resolves.toHaveLength(1024) + }) + + it('bounds metadata when a remote emits one byte per stream event', async () => { + const { sftp } = createFakeSftp() + const byte = Buffer.from('x') + sftp.createReadStream = (() => + Readable.from(Array.from({ length: 100_000 }, () => byte))) as never + + await expect(readTextFileRemote(sftp, '/home/u/config.toml', 100_000)).resolves.toBe( + 'x'.repeat(100_000) + ) + }) + + it('stops streamed remote reads at the byte cap and leaves the existing file intact', async () => { + const { sftp, fs } = createFakeSftp() + const path = '/home/u/.config/agent/config.toml' + fs.files.set(path, 'original') + sftp.createReadStream = (() => Readable.from([Buffer.alloc(1025)])) as never + + await expect(readTextFileRemote(sftp, path, 1024)).rejects.toThrow(NodeFileReadTooLargeError) + await expect(writeTextFileRemoteAtomic(sftp, path, 'replacement', 1024)).rejects.toThrow( + NodeFileReadTooLargeError + ) + expect(fs.files.get(path)).toBe('original') + expect(Array.from(fs.files.keys()).some((key) => key.includes('.tmp'))).toBe(false) + }) }) diff --git a/src/main/agent-hooks/installer-utils-remote.ts b/src/main/agent-hooks/installer-utils-remote.ts index c8edd5e8060..b464d8135e8 100644 --- a/src/main/agent-hooks/installer-utils-remote.ts +++ b/src/main/agent-hooks/installer-utils-remote.ts @@ -12,9 +12,16 @@ // See docs/design/agent-status-over-ssh.md §8 (commit #8). import { randomUUID } from 'node:crypto' -import type { SFTPWrapper, FileEntryWithStats } from 'ssh2' +import type { SFTPWrapper } from 'ssh2' -import { isPlainObject, type HooksConfig } from './installer-utils' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { + AGENT_HOOK_CONFIG_MAX_BYTES, + AGENT_HOOK_MANAGED_SCRIPT_MAX_BYTES +} from './agent-hook-file-limits' +import { readAgentHookRemoteTextFile } from './agent-hook-sftp-text-reader' +import type { HooksConfig } from './installer-utils' +import { parseHooksJsonText } from './hooks-json-read' const DEFAULT_REMOTE_CONFIG_MODE = 0o600 const REMOTE_SFTP_OPERATION_TIMEOUT_MS = 10_000 @@ -31,19 +38,14 @@ export async function readHooksJsonRemote( ): Promise { let body: string try { - body = await readFile(sftp, remotePath) + body = await readFile(sftp, remotePath, AGENT_HOOK_CONFIG_MAX_BYTES) } catch (err) { if (isNoEntryError(err)) { return {} } throw err } - try { - const parsed = JSON.parse(body) - return isPlainObject(parsed) ? parsed : null - } catch { - return null - } + return parseHooksJsonText(body) } /** Atomically write a JSON config to the remote — write to a tmp path then @@ -62,11 +64,14 @@ export async function writeHooksJsonRemote( // Why: skip the write when on-disk content is identical so repeated // install() calls do not bump the file's mtime / inode unnecessarily. try { - const existing = await readFile(sftp, remotePath) + const existing = await readFile(sftp, remotePath, AGENT_HOOK_CONFIG_MAX_BYTES) if (existing === serialized) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // ENOENT or read error — fall through to the write below. } // Why: tmp + rename so a partial network drop mid-write does not leave a @@ -98,12 +103,15 @@ export async function writeManagedScriptRemote( const dir = dirnamePosix(remotePath) await mkdirpRemote(sftp, dir) try { - const existing = await readFile(sftp, remotePath) + const existing = await readFile(sftp, remotePath, AGENT_HOOK_MANAGED_SCRIPT_MAX_BYTES) if (existing === content) { await chmod(sftp, remotePath, 0o755) return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // ENOENT or read error — fall through to the atomic write below. } @@ -126,10 +134,11 @@ export async function writeManagedScriptRemote( export async function readTextFileRemote( sftp: SFTPWrapper, - remotePath: string + remotePath: string, + maxBytes = AGENT_HOOK_CONFIG_MAX_BYTES ): Promise { try { - return await readFile(sftp, remotePath) + return await readFile(sftp, remotePath, maxBytes) } catch (err) { if (isNoEntryError(err)) { return null @@ -141,16 +150,20 @@ export async function readTextFileRemote( export async function writeTextFileRemoteAtomic( sftp: SFTPWrapper, remotePath: string, - content: string + content: string, + maxExistingBytes = AGENT_HOOK_CONFIG_MAX_BYTES ): Promise { const dir = dirnamePosix(remotePath) await mkdirpRemote(sftp, dir) try { - const existing = await readFile(sftp, remotePath) + const existing = await readFile(sftp, remotePath, maxExistingBytes) if (existing === content) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // ENOENT or read error — fall through to the atomic write below. } @@ -211,11 +224,8 @@ function sftpOperation( }) } -async function readFile(sftp: SFTPWrapper, remotePath: string): Promise { - const data = await sftpOperation(`readFile ${remotePath}`, (callback) => { - sftp.readFile(remotePath, 'utf8', callback) - }) - return typeof data === 'string' ? data : data.toString('utf8') +async function readFile(sftp: SFTPWrapper, remotePath: string, maxBytes: number): Promise { + return readAgentHookRemoteTextFile(sftp, remotePath, maxBytes, REMOTE_SFTP_OPERATION_TIMEOUT_MS) } async function writeFile( @@ -296,12 +306,6 @@ async function chmod(sftp: SFTPWrapper, remotePath: string, mode: number): Promi }) } -async function readdir(sftp: SFTPWrapper, remotePath: string): Promise { - return await sftpOperation(`readdir ${remotePath}`, (callback) => { - sftp.readdir(remotePath, callback) - }) -} - async function mkdir(sftp: SFTPWrapper, remotePath: string): Promise { await sftpOperation(`mkdir ${remotePath}`, (callback) => { sftp.mkdir(remotePath, callback) @@ -320,7 +324,7 @@ async function mkdirpRemote(sftp: SFTPWrapper, remotePath: string): Promise { }) describe('managed-hook local filesystem', () => { + it('uses the bounded local reader instead of the legacy whole-file callback', async () => { + const home = await createTempHome() + const configPath = join(home, 'oversized-config.json') + await writeFile(configPath, '123456789', 'utf8') + const filesystem = createManagedHookLocalFilesystem() + const legacyRead = vi.spyOn(filesystem, 'readFile') + + await expect( + readAgentHookRemoteTextFile(filesystem, configPath, 8, 1_000) + ).rejects.toBeInstanceOf(NodeFileReadTooLargeError) + expect(legacyRead).not.toHaveBeenCalled() + }) + + it('probes a directory without reading or retaining any child entries', async () => { + let closeCalls = 0 + let readCalls = 0 + const filesystem = createManagedHookLocalFilesystem({ + openDirectory(_path, callback) { + callback(null, { + close(closeCallback) { + closeCalls += 1 + closeCallback(null) + }, + read() { + readCalls += 1 + } + } as never) + } + }) + + const entries = await new Promise((resolve, reject) => { + filesystem.readdir('/directory-with-millions-of-entries', (error, value) => { + if (error) { + reject(error) + return + } + resolve(value ?? []) + }) + }) + + expect(entries).toEqual([]) + expect(closeCalls).toBe(1) + expect(readCalls).toBe(0) + }) + it('supports cold and warm aggregate installs without SFTP or temp-file residue', async () => { const home = await createTempHome() const filesystem = createManagedHookLocalFilesystem() diff --git a/src/main/agent-hooks/managed-hook-local-filesystem.ts b/src/main/agent-hooks/managed-hook-local-filesystem.ts index 1a104a3a3b1..9aa736c48c1 100644 --- a/src/main/agent-hooks/managed-hook-local-filesystem.ts +++ b/src/main/agent-hooks/managed-hook-local-filesystem.ts @@ -1,8 +1,23 @@ -import { chmod, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from 'node:fs' +import { chmod, mkdir, opendir, readFile, rename, stat, unlink, writeFile } from 'node:fs' import type { SFTPWrapper } from 'ssh2' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' type Callback = (error: Error | null, value?: T) => void +type DirectoryProbeHandle = { + close(callback: (error: NodeJS.ErrnoException | null) => void): void +} + +export type ManagedHookLocalFilesystemOptions = { + openDirectory?: ( + path: string, + callback: (error: NodeJS.ErrnoException | null, directory?: DirectoryProbeHandle) => void + ) => void +} + function asSftpError(error: NodeJS.ErrnoException): Error & { code: number } { const translated = new Error(error.message, { cause: error }) as Error & { code: number } translated.code = @@ -20,11 +35,41 @@ function finish(callback: Callback, error: NodeJS.ErrnoException | null, v /** The managed installers only need this small callback-style SFTP surface. * On the remote host it turns hundreds of WAN round trips into local fs calls. */ -export function createManagedHookLocalFilesystem(): SFTPWrapper { +export function createManagedHookLocalFilesystem( + options?: ManagedHookLocalFilesystemOptions +): SFTPWrapper { + const openDirectory = + options?.openDirectory ?? + (( + path: string, + callback: (error: NodeJS.ErrnoException | null, directory?: DirectoryProbeHandle) => void + ) => { + opendir(path, callback) + }) const adapter = { readFile(path: string, _encoding: string, callback: Callback): void { readFile(path, 'utf8', (error, contents) => finish(callback, error, contents)) }, + orcaReadFileWithinLimit( + path: string, + maxBytes: number, + callback: Callback + ): void { + void readNodeFileWithinLimit(path, maxBytes).then( + ({ buffer }) => callback(null, buffer), + (error: unknown) => { + if (error instanceof NodeFileReadTooLargeError) { + callback(error) + return + } + callback( + error instanceof Error + ? asSftpError(error as NodeJS.ErrnoException) + : new Error(String(error)) + ) + } + ) + }, writeFile( path: string, contents: string, @@ -37,9 +82,15 @@ export function createManagedHookLocalFilesystem(): SFTPWrapper { stat(path, (error, stats) => finish(callback, error, stats)) }, readdir(path: string, callback: Callback<[]>): void { - // Why: installers use readdir only as an existence check; names and - // attrs would allocate work that no caller consumes. - readdir(path, (error) => finish(callback, error, [])) + // Why: installers only probe directory existence; opening and closing + // avoids materializing every child name before returning the unused []. + openDirectory(path, (error, directory) => { + if (error || !directory) { + finish(callback, error ?? new Error('Directory probe returned no handle')) + return + } + directory.close((closeError) => finish(callback, closeError, [])) + }) }, mkdir(path: string, callback: Callback): void { mkdir(path, (error) => finish(callback, error)) diff --git a/src/main/agent-hooks/managed-hook-lock-claims.ts b/src/main/agent-hooks/managed-hook-lock-claims.ts index d3299c7bfd8..4252cfde6fe 100644 --- a/src/main/agent-hooks/managed-hook-lock-claims.ts +++ b/src/main/agent-hooks/managed-hook-lock-claims.ts @@ -1,8 +1,10 @@ import { randomUUID } from 'node:crypto' -import { readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises' +import { opendir, rename, unlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import { CLAIMED_OWNER_PATTERN, + MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE, claimedOwnerFileName, claimRecordFileName, hasCode, @@ -10,6 +12,7 @@ import { ownerFileName, parseClaim, parseOwner, + readManagedHookLockRecord, removeFileIfPresent, type ManagedHookLockClaim, type ManagedHookLockOwner @@ -33,13 +36,21 @@ async function findClaimedOwner( lockParent: string, ownerToken: string ): Promise<{ claimToken: string; path: string } | null | undefined> { - const matches = (await readdir(lockParent)).flatMap((entry) => { - const match = CLAIMED_OWNER_PATTERN.exec(entry) - return match?.[1] === ownerToken && match[2] - ? [{ claimToken: match[2], path: join(lockParent, entry) }] - : [] + const directory = await opendir(lockParent, { + bufferSize: MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE }) - return matches.length === 1 ? matches[0] : matches.length === 0 ? null : undefined + let claimedOwner: { claimToken: string; path: string } | null = null + for await (const entry of directory) { + const match = CLAIMED_OWNER_PATTERN.exec(entry.name) + if (match?.[1] !== ownerToken || !match[2]) { + continue + } + if (claimedOwner) { + return undefined + } + claimedOwner = { claimToken: match[2], path: join(lockParent, entry.name) } + } + return claimedOwner } async function resolveClaimSource( @@ -53,10 +64,13 @@ async function resolveClaimSource( > { const ownerPath = join(lockParent, ownerFileName(owner.token)) try { - return parseOwner(await readFile(ownerPath, 'utf8'), owner.token) + return parseOwner(await readManagedHookLockRecord(ownerPath), owner.token) ? { kind: 'ready', sourcePath: ownerPath } : { kind: 'unverifiable' } } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + return { kind: 'unverifiable' } + } if (!hasCode(error, 'ENOENT')) { throw error } @@ -73,12 +87,12 @@ async function resolveClaimSource( let priorClaim: ManagedHookLockClaim | null try { priorClaim = parseClaim( - await readFile(priorClaimRecordPath, 'utf8'), + await readManagedHookLockRecord(priorClaimRecordPath), owner.token, claimedOwner.claimToken ) } catch (error) { - if (hasCode(error, 'ENOENT')) { + if (hasCode(error, 'ENOENT') || error instanceof NodeFileReadTooLargeError) { return { kind: 'unverifiable' } } throw error diff --git a/src/main/agent-hooks/managed-hook-lock-cleanup-bounds.test.ts b/src/main/agent-hooks/managed-hook-lock-cleanup-bounds.test.ts new file mode 100644 index 00000000000..860fe743673 --- /dev/null +++ b/src/main/agent-hooks/managed-hook-lock-cleanup-bounds.test.ts @@ -0,0 +1,203 @@ +import type { Dir, OpenDirOptions, PathLike, Stats } from 'node:fs' +import { + link, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + stat, + truncate, + writeFile +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { withManagedHookInstallLock } from './managed-hook-install-lock' +import { + cleanupManagedHookLockFiles, + MANAGED_HOOK_LOCK_CLEANUP_CONCURRENCY, + MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE, + MANAGED_HOOK_LOCK_RECORD_MAX_BYTES +} from './managed-hook-lock-records' + +const tempHomes: string[] = [] +const ioProbe = vi.hoisted(() => ({ + active: 0, + enabled: false, + opendirBufferSizes: [] as number[], + peak: 0 +})) +const identityProbe = vi.hoisted(() => ({ + processIdentity: null as string | null | undefined +})) + +type ProbedFsPromises = { + lstat: (path: PathLike) => Promise + opendir: (path: PathLike, options?: OpenDirOptions) => Promise +} + +vi.mock('node:fs/promises', async (importOriginal) => { + const original = await importOriginal() + return { + ...original, + lstat: async (...args: Parameters) => { + const tracked = ioProbe.enabled && String(args[0]).includes('managed-hook-install.owner-feed') + if (!tracked) { + return await original.lstat(...args) + } + ioProbe.active += 1 + ioProbe.peak = Math.max(ioProbe.peak, ioProbe.active) + try { + await new Promise((resolve) => setTimeout(resolve, 2)) + return await original.lstat(...args) + } finally { + ioProbe.active -= 1 + } + }, + opendir: async (...args: Parameters) => { + const options = args[1] as { bufferSize?: number } | undefined + if (options?.bufferSize !== undefined) { + ioProbe.opendirBufferSizes.push(options.bufferSize) + } + return await original.opendir(...args) + } + } +}) + +vi.mock('./managed-hook-owner-identity', () => ({ + readManagedHookProcessIdentity: vi.fn(async () => identityProbe.processIdentity) +})) + +function lockToken(prefix: string, index: number): string { + return `${prefix}${index.toString(16).padStart(4, '0')}-0000-4000-8000-${index + .toString(16) + .padStart(12, '0')}` +} + +async function createTempHome(): Promise { + const home = await mkdtemp(join(tmpdir(), 'orca-managed-hook-cleanup-bounds-')) + tempHomes.push(home) + return home +} + +async function seedUnrelatedFiles(directory: string, count: number): Promise { + for (let index = 0; index < count; index += 1) { + await writeFile(join(directory, `unrelated-${index.toString().padStart(4, '0')}.txt`), '') + } +} + +afterEach(async () => { + vi.restoreAllMocks() + ioProbe.active = 0 + ioProbe.enabled = false + ioProbe.opendirBufferSizes.length = 0 + ioProbe.peak = 0 + identityProbe.processIdentity = null + for (const home of tempHomes.splice(0)) { + await rm(home, { recursive: true, force: true }) + } +}) + +describe('managed-hook lock cleanup bounds', () => { + it('streams a large directory and cleans at most eight lock records concurrently', async () => { + const home = await createTempHome() + const lockParent = join(home, '.orca') + const hostIdentity = 'test-host' + const recordCount = 257 + const unrelatedCount = 320 + await mkdir(lockParent, { recursive: true }) + for (let index = 0; index < recordCount; index += 1) { + const token = lockToken('feed', index) + await writeFile( + join(lockParent, `managed-hook-install.owner-${token}.json`), + JSON.stringify({ + token, + pid: process.pid, + hostIdentity, + processIdentity: 'stale-process' + }) + ) + } + await seedUnrelatedFiles(lockParent, unrelatedCount) + ioProbe.enabled = true + + await cleanupManagedHookLockFiles(lockParent, hostIdentity) + + const remaining = await readdir(lockParent) + expect(remaining).toHaveLength(unrelatedCount) + expect(remaining.every((entry) => entry.startsWith('unrelated-'))).toBe(true) + expect(ioProbe.peak).toBe(MANAGED_HOOK_LOCK_CLEANUP_CONCURRENCY) + expect(ioProbe.opendirBufferSizes).toEqual([MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE]) + }) + + it('preserves an oversized lock record without allocating its contents', async () => { + const home = await createTempHome() + const lockParent = join(home, '.orca') + const token = lockToken('feed', 1) + const recordPath = join(lockParent, `managed-hook-install.owner-${token}.json`) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + await mkdir(lockParent, { recursive: true }) + await writeFile(recordPath, '') + await truncate(recordPath, MANAGED_HOOK_LOCK_RECORD_MAX_BYTES + 1) + + await cleanupManagedHookLockFiles(lockParent, 'test-host') + + expect((await stat(recordPath)).size).toBe(MANAGED_HOOK_LOCK_RECORD_MAX_BYTES + 1) + expect(warn).toHaveBeenCalledWith( + '[agent-hooks] Failed to clean managed-hook lock file', + expect.objectContaining({ name: 'NodeFileReadTooLargeError' }) + ) + }) + + it('recovers a claimed owner without collecting a large directory into an array', async () => { + const home = await createTempHome() + const lockParent = join(home, '.orca') + const lockPath = join(lockParent, 'managed-hook-install.lock') + const ownerToken = lockToken('cafe', 1) + const claimToken = lockToken('face', 2) + const ownerPath = join(lockParent, `managed-hook-install.owner-${ownerToken}.json`) + const claimedOwnerPath = join( + lockParent, + `managed-hook-install.claimed-${ownerToken}-${claimToken}.json` + ) + const hostIdentity = 'test-host' + await mkdir(lockParent, { recursive: true }) + await writeFile( + ownerPath, + JSON.stringify({ + token: ownerToken, + pid: process.pid, + hostIdentity, + processIdentity: 'stale-owner' + }) + ) + await link(ownerPath, lockPath) + await writeFile( + join(lockParent, `managed-hook-install.claim-${ownerToken}-${claimToken}.json`), + JSON.stringify({ + ownerToken, + claimToken, + pid: process.pid, + hostIdentity, + processIdentity: 'stale-claimant' + }) + ) + await rename(ownerPath, claimedOwnerPath) + await seedUnrelatedFiles(lockParent, 512) + identityProbe.processIdentity = 'current-process' + + await expect( + withManagedHookInstallLock(home, undefined, async () => 'installed', hostIdentity) + ).resolves.toBe('installed') + + expect(await readFile(join(lockParent, 'unrelated-0000.txt'), 'utf8')).toBe('') + expect(ioProbe.opendirBufferSizes.length).toBeGreaterThanOrEqual(2) + expect( + ioProbe.opendirBufferSizes.every( + (bufferSize) => bufferSize === MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE + ) + ).toBe(true) + }) +}) diff --git a/src/main/agent-hooks/managed-hook-lock-records.ts b/src/main/agent-hooks/managed-hook-lock-records.ts index 555c2a1a521..130476e22a1 100644 --- a/src/main/agent-hooks/managed-hook-lock-records.ts +++ b/src/main/agent-hooks/managed-hook-lock-records.ts @@ -1,8 +1,13 @@ import { randomUUID } from 'node:crypto' -import { link, lstat, readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises' +import { link, lstat, opendir, rename, unlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' import { readManagedHookProcessIdentity } from './managed-hook-owner-identity' +export const MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE = 32 +export const MANAGED_HOOK_LOCK_CLEANUP_CONCURRENCY = 8 +export const MANAGED_HOOK_LOCK_RECORD_MAX_BYTES = 2 * 1024 * 1024 + const UUID_PATTERN = '[\\da-f]{8}-[\\da-f]{4}-[1-5][\\da-f]{3}-[89ab][\\da-f]{3}-[\\da-f]{12}' const OWNER_FILE_PATTERN = new RegExp( `^managed-hook-install\\.owner-(${UUID_PATTERN})\\.json$`, @@ -127,10 +132,16 @@ export async function removeFileIfPresent(path: string): Promise { } } +export async function readManagedHookLockRecord(path: string): Promise { + return (await readNodeFileWithinLimit(path, MANAGED_HOOK_LOCK_RECORD_MAX_BYTES)).buffer.toString( + 'utf8' + ) +} + export async function inspectManagedHookLock(lockPath: string): Promise { let rawOwner: string try { - rawOwner = await readFile(lockPath, 'utf8') + rawOwner = await readManagedHookLockRecord(lockPath) } catch (error) { if (hasCode(error, 'ENOENT')) { return { kind: 'missing' } @@ -193,7 +204,7 @@ async function cleanOwnerEntry(path: string, token: string, hostIdentity: string if (stats.nlink !== 1) { return } - const owner = parseOwner(await readFile(path, 'utf8'), token) + const owner = parseOwner(await readManagedHookLockRecord(path), token) if (!owner || owner.hostIdentity !== hostIdentity) { return } @@ -229,7 +240,7 @@ async function cleanLockEntry( if (!claimMatch?.[1] || !claimMatch[2]) { return } - const claim = parseClaim(await readFile(path, 'utf8'), claimMatch[1], claimMatch[2]) + const claim = parseClaim(await readManagedHookLockRecord(path), claimMatch[1], claimMatch[2]) if (!claim || claim.hostIdentity !== hostIdentity) { return } @@ -250,25 +261,59 @@ async function cleanLockEntry( } } +function isManagedHookLockRecord(entry: string): boolean { + return ( + OWNER_FILE_PATTERN.test(entry) || + OWNER_DRAFT_PATTERN.test(entry) || + CLAIMED_OWNER_PATTERN.test(entry) || + CLAIM_RECORD_PATTERN.test(entry) + ) +} + +async function cleanLockEntrySafely( + entry: string, + lockParent: string, + hostIdentity: string +): Promise { + try { + await cleanLockEntry(entry, lockParent, hostIdentity) + } catch (error) { + if (!hasCode(error, 'ENOENT')) { + console.warn('[agent-hooks] Failed to clean managed-hook lock file', error) + } + } +} + export async function cleanupManagedHookLockFiles( lockParent: string, hostIdentity: string ): Promise { - let entries: string[] + let directory: Awaited> try { - entries = await readdir(lockParent) + directory = await opendir(lockParent, { + bufferSize: MANAGED_HOOK_LOCK_DIRECTORY_BUFFER_SIZE + }) } catch { return } - await Promise.all( - entries.map(async (entry) => { - try { - await cleanLockEntry(entry, lockParent, hostIdentity) - } catch (error) { - if (!hasCode(error, 'ENOENT')) { - console.warn('[agent-hooks] Failed to clean managed-hook lock file', error) - } + + const pending: string[] = [] + try { + for await (const entry of directory) { + if (!isManagedHookLockRecord(entry.name)) { + continue } - }) - ) + pending.push(entry.name) + if (pending.length === MANAGED_HOOK_LOCK_CLEANUP_CONCURRENCY) { + await Promise.all( + pending.splice(0).map((name) => cleanLockEntrySafely(name, lockParent, hostIdentity)) + ) + } + } + await Promise.all( + pending.splice(0).map((name) => cleanLockEntrySafely(name, lockParent, hostIdentity)) + ) + } catch { + // Directory enumeration is best-effort, matching the prior readdir behavior. + } } diff --git a/src/main/agent-hooks/managed-hook-owner-identity.test.ts b/src/main/agent-hooks/managed-hook-owner-identity.test.ts index ae8ac15dd12..fd500a1b8b1 100644 --- a/src/main/agent-hooks/managed-hook-owner-identity.test.ts +++ b/src/main/agent-hooks/managed-hook-owner-identity.test.ts @@ -57,14 +57,34 @@ async function loadLinuxIdentity(fixture: LinuxIdentityFixture) { uid: process.getuid?.() ?? 0 } }) + const readNodeFileWithinLimit = vi.fn(async (path: string, maxBytes: number) => { + if (String(path).endsWith('host-id') && fixture.hostToken) { + if (Buffer.byteLength(fixture.hostToken, 'utf8') > maxBytes) { + throw new Error('file too large') + } + return { + buffer: Buffer.from(fixture.hostToken, 'utf8'), + stats: { size: Buffer.byteLength(fixture.hostToken, 'utf8') } + } + } + throw Object.assign(new Error(`unavailable path: ${String(path)}`), { code: 'ENOENT' }) + }) vi.doMock('node:fs/promises', async (importOriginal) => ({ ...(await importOriginal>()), lstat, mkdir, readFile, - readlink + readlink, + writeFile: vi.fn(async () => { + throw Object.assign(new Error('host-local storage unavailable'), { code: 'EACCES' }) + }) })) - return { identity: await import('./managed-hook-owner-identity'), readFile } + vi.doMock('../../shared/node-bounded-file-reader', () => ({ readNodeFileWithinLimit })) + return { + identity: await import('./managed-hook-owner-identity'), + readFile, + readNodeFileWithinLimit + } } afterEach(() => { @@ -76,10 +96,23 @@ afterEach(() => { } vi.unstubAllEnvs() vi.doUnmock('node:fs/promises') + vi.doUnmock('../../shared/node-bounded-file-reader') vi.resetModules() }) describe('managed hook owner identity', () => { + it('rejects an oversized durable host token before materializing it', async () => { + const { identity, readNodeFileWithinLimit } = await loadLinuxIdentity({ + hostToken: 'x'.repeat(1025) + }) + + await expect(identity.readManagedHookHostIdentity()).resolves.toMatch(/^runtime:/) + expect(readNodeFileWithinLimit).toHaveBeenCalledWith( + '/var/tmp/orca-managed-hooks-1000/host-id', + 1024 + ) + }) + it('uses durable host-local identity without requiring Linux machine identity files', async () => { vi.stubEnv('SSH_CONNECTION', '198.51.100.8 53100 10.0.0.7 2222') const { identity, readFile } = await loadLinuxIdentity({ diff --git a/src/main/agent-hooks/managed-hook-owner-identity.ts b/src/main/agent-hooks/managed-hook-owner-identity.ts index e9801c3c489..8f47c5ca9ae 100644 --- a/src/main/agent-hooks/managed-hook-owner-identity.ts +++ b/src/main/agent-hooks/managed-hook-owner-identity.ts @@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto' import { link, lstat, mkdir, readFile, readlink, unlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { promisify } from 'node:util' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' const execFileAsync = promisify(execFile) const runtimeHostIdentity = `runtime:${randomUUID()}` @@ -10,6 +11,7 @@ const runtimeProcessIdentity = `runtime:${randomUUID()}` let hostIdentityPromise: Promise | undefined let bootIdentityPromise: Promise | undefined const HOST_TOKEN_PATTERN = /^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/i +const HOST_TOKEN_FILE_MAX_BYTES = 1024 function hasCode(error: unknown, code: string): boolean { return error instanceof Error && 'code' in error && error.code === code @@ -43,7 +45,8 @@ async function readPublishedHostToken(path: string, uid: number): Promise { expect(persist).toHaveBeenCalledTimes(1) expect(persist).toHaveBeenCalledWith([sibling, otherTab]) }) + + it('evicts the oldest unsupported PTY after the retention ceiling', () => { + for (let index = 0; index < MIGRATION_UNSUPPORTED_PTY_MAX_ENTRIES; index++) { + setMigrationUnsupportedPty(makeEntry(`pty-${index}`, `tab-${index}:leaf-a`)) + } + const listener = vi.fn() + const persist = vi.fn() + setMigrationUnsupportedPtyListener(listener) + setMigrationUnsupportedPtyPersistenceListener(persist) + const newest = makeEntry(`pty-${MIGRATION_UNSUPPORTED_PTY_MAX_ENTRIES}`, 'tab-newest:leaf-a') + + setMigrationUnsupportedPty(newest) + + const snapshot = getMigrationUnsupportedPtySnapshot() + expect(snapshot).toHaveLength(MIGRATION_UNSUPPORTED_PTY_MAX_ENTRIES) + expect(snapshot.some((entry) => entry.ptyId === 'pty-0')).toBe(false) + expect(snapshot.at(-1)).toEqual(newest) + expect(listener).toHaveBeenNthCalledWith(1, { type: 'clear', ptyId: 'pty-0' }) + expect(listener).toHaveBeenNthCalledWith(2, { type: 'set', entry: newest }) + expect(persist).toHaveBeenCalledTimes(1) + expect(persist).toHaveBeenCalledWith(snapshot) + }) + + it('rejects oversized entries and clears a prior value for the same PTY', () => { + const listener = vi.fn() + const persist = vi.fn() + setMigrationUnsupportedPtyListener(listener) + setMigrationUnsupportedPtyPersistenceListener(persist) + setMigrationUnsupportedPty(makeEntry('pty-1', 'tab-1:leaf-a')) + listener.mockClear() + persist.mockClear() + + setMigrationUnsupportedPty( + makeEntry('pty-1', `tab-1:${'x'.repeat(MIGRATION_UNSUPPORTED_PTY_MAX_ENTRY_BYTES)}`) + ) + + expect(getMigrationUnsupportedPtySnapshot()).toEqual([]) + expect(listener).toHaveBeenCalledOnce() + expect(listener).toHaveBeenCalledWith({ type: 'clear', ptyId: 'pty-1' }) + expect(persist).toHaveBeenCalledOnce() + expect(persist).toHaveBeenCalledWith([]) + }) }) diff --git a/src/main/agent-hooks/migration-unsupported-pty-state.ts b/src/main/agent-hooks/migration-unsupported-pty-state.ts index 36bdc838569..dc467c12727 100644 --- a/src/main/agent-hooks/migration-unsupported-pty-state.ts +++ b/src/main/agent-hooks/migration-unsupported-pty-state.ts @@ -1,10 +1,13 @@ import type { MigrationUnsupportedPtyEntry } from '../../shared/agent-status-types' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' type MigrationUnsupportedPtyEvent = | { type: 'set'; entry: MigrationUnsupportedPtyEntry } | { type: 'clear'; ptyId: string } const entriesByPtyId = new Map() +export const MIGRATION_UNSUPPORTED_PTY_MAX_ENTRIES = 500 +export const MIGRATION_UNSUPPORTED_PTY_MAX_ENTRY_BYTES = 16 * 1024 let listener: ((event: MigrationUnsupportedPtyEvent) => void) | null = null let persistenceListener: ((entries: MigrationUnsupportedPtyEntry[]) => void) | null = null @@ -25,7 +28,21 @@ export function setMigrationUnsupportedPtyPersistenceListener( } export function setMigrationUnsupportedPty(entry: MigrationUnsupportedPtyEntry): void { + try { + stringifyJsonWithinByteLimit(entry, MIGRATION_UNSUPPORTED_PTY_MAX_ENTRY_BYTES) + } catch { + clearMigrationUnsupportedPty(entry.ptyId) + return + } entriesByPtyId.set(entry.ptyId, entry) + while (entriesByPtyId.size > MIGRATION_UNSUPPORTED_PTY_MAX_ENTRIES) { + const oldestPtyId = entriesByPtyId.keys().next().value + if (oldestPtyId === undefined) { + break + } + entriesByPtyId.delete(oldestPtyId) + listener?.({ type: 'clear', ptyId: oldestPtyId }) + } listener?.({ type: 'set', entry }) persistenceListener?.(getMigrationUnsupportedPtySnapshot()) } diff --git a/src/main/agent-hooks/server-retained-metadata-bounds.test.ts b/src/main/agent-hooks/server-retained-metadata-bounds.test.ts new file mode 100644 index 00000000000..22a03c845a0 --- /dev/null +++ b/src/main/agent-hooks/server-retained-metadata-bounds.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, vi } from 'vitest' +import { + AgentHookServer, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES, + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES +} from './server' +import { makePaneKey } from '../../shared/stable-pane-id' + +vi.mock('../telemetry/client', () => ({ track: vi.fn() })) +vi.mock('../telemetry/cohort-classifier', () => ({ getCohortAtEmit: () => ({}) })) + +const PANE_KEY = makePaneKey('metadata-bounds-tab', '11111111-1111-4111-8111-111111111111') + +describe('AgentHookServer retained metadata bounds', () => { + it('preserves every metadata field at its exact UTF-8 byte limit', () => { + const server = new AgentHookServer() + const exactId = 'é'.repeat(MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES / 2) + const exactPath = 'é'.repeat(MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES / 2) + + expect(Buffer.byteLength(exactId, 'utf8')).toBe(MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES) + expect(Buffer.byteLength(exactPath, 'utf8')).toBe(MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES) + + server.ingestRemote( + { + paneKey: PANE_KEY, + tabId: 'metadata-bounds-tab', + worktreeId: exactPath, + launchToken: exactId, + promptInteractionKey: exactId, + hookEventName: exactId, + toolUseId: exactId, + toolAgentId: exactId, + toolAgentType: exactId, + providerSession: { + key: 'session_id', + id: 'session-1', + transcriptPath: exactPath + }, + payload: { state: 'working', prompt: 'ordinary prompt', agentType: 'claude' } + }, + exactId + ) + + const retained = server._getStateForTests().lastStatusByPaneKey.get(PANE_KEY) + expect(retained).toMatchObject({ + launchToken: exactId, + worktreeId: exactPath, + connectionId: exactId, + promptInteractionKey: exactId, + hookEventName: exactId, + toolUseId: exactId, + toolAgentId: exactId, + toolAgentType: exactId, + providerSession: { + key: 'session_id', + id: 'session-1', + transcriptPath: exactPath + } + }) + }) + + it('omits optional metadata one byte over its limit', () => { + const server = new AgentHookServer() + const oversizedId = `${'é'.repeat(MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES / 2)}x` + const oversizedPath = `${'é'.repeat(MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES / 2)}x` + + expect(Buffer.byteLength(oversizedId, 'utf8')).toBe(MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + 1) + expect(Buffer.byteLength(oversizedPath, 'utf8')).toBe( + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + 1 + ) + + server.ingestRemote( + { + paneKey: PANE_KEY, + tabId: 'metadata-bounds-tab', + worktreeId: oversizedPath, + launchToken: oversizedId, + promptInteractionKey: oversizedId, + hookEventName: oversizedId, + toolUseId: oversizedId, + toolAgentId: oversizedId, + toolAgentType: oversizedId, + providerSession: { + key: 'session_id', + id: 'session-1', + transcriptPath: oversizedPath + }, + payload: { state: 'working', prompt: 'ordinary prompt', agentType: 'claude' } + }, + 'connection-1' + ) + + const retained = server._getStateForTests().lastStatusByPaneKey.get(PANE_KEY) + expect(retained).toMatchObject({ + connectionId: 'connection-1', + providerSession: { key: 'session_id', id: 'session-1' } + }) + expect(retained).not.toHaveProperty('providerSession.transcriptPath') + for (const field of [ + 'launchToken', + 'worktreeId', + 'promptInteractionKey', + 'hookEventName', + 'toolUseId', + 'toolAgentId', + 'toolAgentType' + ]) { + expect(retained?.[field as keyof typeof retained]).toBeUndefined() + } + }) + + it('rejects oversized connection authority without retaining a watermark', () => { + const server = new AgentHookServer() + const listener = vi.fn() + const oversizedConnectionId = 'x'.repeat(MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + 1) + server.setPaneStatusClearListener(listener) + + server.ingestRemote( + { + paneKey: PANE_KEY, + payload: { state: 'working', prompt: 'ordinary prompt', agentType: 'claude' } + }, + oversizedConnectionId + ) + server.clearStatusEntriesForConnection(oversizedConnectionId) + + expect(server.getStatusSnapshot()).toEqual([]) + expect( + ( + server as unknown as { + connectionTimestampWatermarkById: Map + } + ).connectionTimestampWatermarkById + ).toHaveLength(0) + expect(listener).not.toHaveBeenCalled() + }) + + it('rejects a path-dependent Pi identity when its transcript path is oversized', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE_KEY, + providerSession: { + key: 'session_id', + id: 'pi-session-1', + transcriptPath: 'x'.repeat(MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + 1) + }, + providerSessionOnly: true, + payload: { state: 'done', prompt: '', agentType: 'pi' } + }, + 'connection-1' + ) + + expect(server.getStatusSnapshot()).toEqual([]) + }) +}) diff --git a/src/main/agent-hooks/server-status-cache-bound.test.ts b/src/main/agent-hooks/server-status-cache-bound.test.ts new file mode 100644 index 00000000000..8505baf7b31 --- /dev/null +++ b/src/main/agent-hooks/server-status-cache-bound.test.ts @@ -0,0 +1,154 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { + AgentHookServer, + MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS, + MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES +} from './server' +import { MAX_AGENT_HOOK_STATUS_CACHE_PANES } from '../../shared/agent-hook-status-cache' +import type { AgentHookEventPayload } from '../../shared/agent-hook-listener' +import { AGENT_STATUS_STALE_AFTER_MS } from '../../shared/agent-status-types' +import { makePaneKey } from '../../shared/stable-pane-id' + +vi.mock('../telemetry/client', () => ({ track: vi.fn() })) +vi.mock('../telemetry/cohort-classifier', () => ({ getCohortAtEmit: () => ({}) })) + +const LEAF_ID = '11111111-1111-4111-8111-111111111111' + +function paneKey(index: number): string { + return makePaneKey(`status-cache-${index}`, LEAF_ID) +} + +function cachedStatus( + index: number, + state: AgentHookEventPayload['payload']['state'], + receivedAt: number +): AgentHookEventPayload & { receivedAt: number; stateStartedAt: number } { + return { + paneKey: paneKey(index), + connectionId: null, + receivedAt, + stateStartedAt: receivedAt, + payload: { state, prompt: `prompt-${index}`, agentType: 'claude' } + } +} + +describe('AgentHookServer status cache bound', () => { + it('evicts a completed row before fresh work and clears related state', () => { + const server = new AgentHookServer() + const listener = server._getStateForTests() + const now = Date.now() + for (let index = 0; index < MAX_AGENT_HOOK_STATUS_CACHE_PANES; index += 1) { + const state = index === MAX_AGENT_HOOK_STATUS_CACHE_PANES - 1 ? 'done' : 'working' + listener.lastStatusByPaneKey.set(paneKey(index), cachedStatus(index, state, now)) + } + const completedPaneKey = paneKey(MAX_AGENT_HOOK_STATUS_CACHE_PANES - 1) + listener.lastPromptByPaneKey.set(completedPaneKey, 'cached prompt') + const onClear = vi.fn() + server.setPaneStatusClearListener(onClear) + + const currentPaneKey = paneKey(MAX_AGENT_HOOK_STATUS_CACHE_PANES) + server.ingestRemote( + { + paneKey: currentPaneKey, + payload: { state: 'working', prompt: 'current', agentType: 'claude' } + }, + 'connection-1' + ) + + expect(listener.lastStatusByPaneKey.size).toBe(MAX_AGENT_HOOK_STATUS_CACHE_PANES) + expect(listener.lastStatusByPaneKey.has(paneKey(0))).toBe(true) + expect(listener.lastStatusByPaneKey.has(completedPaneKey)).toBe(false) + expect(listener.lastStatusByPaneKey.has(currentPaneKey)).toBe(true) + expect(listener.lastPromptByPaneKey.has(completedPaneKey)).toBe(false) + expect(onClear).toHaveBeenCalledOnce() + expect(onClear).toHaveBeenCalledWith({ paneKey: completedPaneKey }) + }) + + it('evicts a stale hydrated row before older fresh work at capacity', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-agent-status-cache-')) + const endpointDir = join(userDataPath, 'agent-hooks') + mkdirSync(endpointDir, { recursive: true }) + const now = Date.now() + const entries: Record> = {} + for (let index = 0; index <= MAX_AGENT_HOOK_STATUS_CACHE_PANES; index += 1) { + const receivedAt = index === 1 ? now - AGENT_STATUS_STALE_AFTER_MS - 1 : now + entries[paneKey(index)] = cachedStatus(index, 'working', receivedAt) + } + writeFileSync(join(endpointDir, 'last-status.json'), JSON.stringify({ version: 2, entries })) + const server = new AgentHookServer() + + try { + await server.start({ env: 'production', userDataPath }) + const listener = server._getStateForTests() + + expect(listener.lastStatusByPaneKey.size).toBe(MAX_AGENT_HOOK_STATUS_CACHE_PANES) + expect(listener.lastStatusByPaneKey.has(paneKey(0))).toBe(true) + expect(listener.lastStatusByPaneKey.has(paneKey(1))).toBe(false) + expect(listener.lastStatusByPaneKey.has(paneKey(MAX_AGENT_HOOK_STATUS_CACHE_PANES))).toBe( + true + ) + } finally { + server.stop() + rmSync(userDataPath, { recursive: true, force: true }) + } + }) + + it('bounds connection watermarks and refreshes recency before overflow', () => { + const server = new AgentHookServer() + const watermarks = ( + server as unknown as { + connectionTimestampWatermarkById: Map + } + ).connectionTimestampWatermarkById + + for (let index = 0; index < MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS; index += 1) { + server.clearStatusEntriesForConnection(`connection-${index}`) + } + expect(watermarks.size).toBe(MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS) + + server.clearStatusEntriesForConnection('connection-0') + server.clearStatusEntriesForConnection( + `connection-${MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS}` + ) + + expect(watermarks.size).toBe(MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS) + expect(watermarks.has('connection-0')).toBe(true) + expect(watermarks.has('connection-1')).toBe(false) + expect(watermarks.has(`connection-${MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS}`)).toBe( + true + ) + }) + + it('persists the newest statuses without exceeding its own hydration ceiling', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-agent-status-file-bound-')) + const server = new AgentHookServer() + + try { + await server.start({ env: 'production', userDataPath }) + const listener = server._getStateForTests() + const retainedMetadata = 'x'.repeat(64 * 1024) + for (let index = 0; index < MAX_AGENT_HOOK_STATUS_CACHE_PANES; index += 1) { + listener.lastStatusByPaneKey.set(paneKey(index), { + ...cachedStatus(index, 'working', Date.now()), + worktreeId: retainedMetadata + }) + } + + server.flushStatusPersistSync() + + const path = join(userDataPath, 'agent-hooks', 'last-status.json') + const persisted = JSON.parse(readFileSync(path, 'utf8')) as { + entries: Record + } + expect(statSync(path).size).toBeLessThanOrEqual(MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES) + expect(Object.keys(persisted.entries).length).toBeLessThan(MAX_AGENT_HOOK_STATUS_CACHE_PANES) + expect(persisted.entries[paneKey(MAX_AGENT_HOOK_STATUS_CACHE_PANES - 1)]).toBeDefined() + } finally { + server.stop() + rmSync(userDataPath, { recursive: true, force: true }) + } + }) +}) diff --git a/src/main/agent-hooks/server.test.ts b/src/main/agent-hooks/server.test.ts index 6b4e12042c5..9fa87904af1 100644 --- a/src/main/agent-hooks/server.test.ts +++ b/src/main/agent-hooks/server.test.ts @@ -8,6 +8,7 @@ import { readFileSync, rmSync, statSync, + truncateSync, utimesSync, writeFileSync } from 'node:fs' @@ -17,6 +18,8 @@ import { AgentHookServer, agentHookServer, CLOSED_AGENT_STATUS_TAB_IDS_MAX, + MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES, + MAX_AGENT_HOOK_LAST_STATUS_STRUCTURAL_TOKENS, _internals } from './server' import { @@ -6617,6 +6620,46 @@ describe('Last-status persistence', () => { } }) + it('treats an oversized sparse last-status file as empty hydration', async () => { + mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) + writeFileSync(lastStatusPath(), '{"version":2,"entries":{}}', 'utf8') + truncateSync(lastStatusPath(), MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES + 1) + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const server = new AgentHookServer() + await server.start({ env: 'production', userDataPath }) + try { + expect(server.getStatusSnapshot()).toEqual([]) + expect(warnSpy).toHaveBeenCalledWith( + '[agent-hooks] failed to read last-status file:', + expect.any(Error) + ) + } finally { + server.stop() + warnSpy.mockRestore() + } + }) + + it('rejects structurally amplified last-status JSON before parsing', async () => { + mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) + const values = '0,'.repeat(MAX_AGENT_HOOK_LAST_STATUS_STRUCTURAL_TOKENS) + writeFileSync(lastStatusPath(), `[${values}0]`, 'utf8') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const parseSpy = vi.spyOn(JSON, 'parse') + const server = new AgentHookServer() + await server.start({ env: 'production', userDataPath }) + try { + expect(server.getStatusSnapshot()).toEqual([]) + expect(parseSpy).not.toHaveBeenCalled() + expect(warnSpy).toHaveBeenCalledWith( + '[agent-hooks] last-status file is invalid or too complex; ignoring' + ) + } finally { + server.stop() + warnSpy.mockRestore() + parseSpy.mockRestore() + } + }) + it('drops hydrated metadata-only entries without a resumable Pi session', async () => { mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) const receivedAt = recentTs() diff --git a/src/main/agent-hooks/server.ts b/src/main/agent-hooks/server.ts index 06dfb151560..038372491ee 100644 --- a/src/main/agent-hooks/server.ts +++ b/src/main/agent-hooks/server.ts @@ -2,13 +2,19 @@ // Why: this main-process adapter keeps listener internals in shared/ (`src/shared/agent-hook-listener.ts`) so the relay can host the same pipeline without Electron. See docs/design/agent-status-over-ssh.md §5. import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' import { createHash, randomBytes, randomUUID } from 'node:crypto' -import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { chmodSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { track } from '../telemetry/client' import { getCohortAtEmit } from '../telemetry/cohort-classifier' import { AGENT_KIND_VALUES, type AgentKind } from '../../shared/telemetry-events' import { ORCA_HOOK_PROTOCOL_VERSION } from '../../shared/agent-hook-types' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../../shared/node-bounded-json-stringify' import { clearAllListenerCaches, clearPaneCacheState, @@ -35,6 +41,7 @@ import { type HookListenerState } from '../../shared/agent-hook-listener' import type { AgentHookSource } from '../../shared/agent-hook-relay' +import { upsertBoundedAgentHookStatus } from '../../shared/agent-hook-status-cache' import { CLAUDE_STATUSLINE_PATHNAME, parseClaudeStatusLineBody, @@ -69,6 +76,7 @@ import { type AgentProviderSessionMetadata } from '../../shared/agent-session-resume' import { isCommandCodeNewTurnWhileWorking } from '../../shared/command-code-turn-boundary' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' export type { AgentHookSource } @@ -96,6 +104,9 @@ type PaneKeyAliasEntry = { // Why: co-located with the endpoint file in userData/agent-hooks/ so hook-server cross-restart artifacts stay together. const LAST_STATUS_FILE_NAME = 'last-status.json' +export const MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES = 16 * 1024 * 1024 +export const MAX_AGENT_HOOK_LAST_STATUS_STRUCTURAL_TOKENS = 1_000_000 +export const MAX_AGENT_HOOK_LAST_STATUS_NESTING_DEPTH = 128 const ASSISTANT_MESSAGE_RETRY_ATTEMPTS = 5 const ASSISTANT_MESSAGE_RETRY_MS = 50 const INTERRUPTED_DONE_LATE_WORKING_SUPPRESSION_MS = 15_000 @@ -115,6 +126,9 @@ const HYDRATE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 export const CLOSED_AGENT_STATUS_TAB_IDS_MAX = 1024 export const CLOSED_AGENT_STATUS_PANE_KEYS_MAX = 1024 export const PANE_KEY_ALIASES_MAX = 1024 +export const MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS = 1024 +export const MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES = 1024 +export const MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES = 16 * 1024 type LastStatusFile = { version: number @@ -181,6 +195,100 @@ function isValidPiProviderSessionOnly( return Boolean(providerSession && agentType === 'pi' && getAgentResumeArgv('pi', providerSession)) } +function isRetainedStringWithinByteLimit(value: string, maxBytes: number): boolean { + return !measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }).exceededLimit +} + +function retainOptionalStringWithinByteLimit(value: unknown, maxBytes: number): string | undefined { + if ( + typeof value !== 'string' || + value.length === 0 || + !isRetainedStringWithinByteLimit(value, maxBytes) + ) { + return undefined + } + return value +} + +function normalizeOptionalRetainedString(value: unknown, maxBytes: number): string | undefined { + const retained = retainOptionalStringWithinByteLimit(value, maxBytes) + if (!retained) { + return undefined + } + const trimmed = retained.trim() + return trimmed.length > 0 ? trimmed : undefined +} + +function boundProviderSessionTranscriptPath( + providerSession: AgentProviderSessionMetadata | undefined +): AgentProviderSessionMetadata | undefined { + if (!providerSession?.transcriptPath) { + return providerSession + } + const transcriptPath = retainOptionalStringWithinByteLimit( + providerSession.transcriptPath, + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + ) + return transcriptPath === providerSession.transcriptPath + ? providerSession + : { key: providerSession.key, id: providerSession.id } +} + +function boundAgentHookEventMetadata(event: T): T { + const launchToken = retainOptionalStringWithinByteLimit( + event.launchToken, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const worktreeId = retainOptionalStringWithinByteLimit( + event.worktreeId, + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + ) + const promptInteractionKey = retainOptionalStringWithinByteLimit( + event.promptInteractionKey, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const hookEventName = retainOptionalStringWithinByteLimit( + event.hookEventName, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolUseId = retainOptionalStringWithinByteLimit( + event.toolUseId, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolAgentId = retainOptionalStringWithinByteLimit( + event.toolAgentId, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolAgentType = retainOptionalStringWithinByteLimit( + event.toolAgentType, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const providerSession = boundProviderSessionTranscriptPath(event.providerSession) + if ( + launchToken === event.launchToken && + worktreeId === event.worktreeId && + promptInteractionKey === event.promptInteractionKey && + hookEventName === event.hookEventName && + toolUseId === event.toolUseId && + toolAgentId === event.toolAgentId && + toolAgentType === event.toolAgentType && + providerSession === event.providerSession + ) { + return event + } + return { + ...event, + launchToken, + worktreeId, + promptInteractionKey, + hookEventName, + toolUseId, + toolAgentId, + toolAgentType, + providerSession + } +} + function sanitizeHydratedEntry( paneKey: string, rawEntry: unknown @@ -225,7 +333,10 @@ function sanitizeHydratedEntry( let connectionId: string | null if (connectionIdRaw === null || connectionIdRaw === undefined) { connectionId = null - } else if (typeof connectionIdRaw === 'string') { + } else if ( + typeof connectionIdRaw === 'string' && + isRetainedStringWithinByteLimit(connectionIdRaw, MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES) + ) { connectionId = connectionIdRaw } else { return null @@ -234,12 +345,14 @@ function sanitizeHydratedEntry( if (!payload) { return null } - const providerSession = normalizeAgentProviderSession(record.providerSession) ?? undefined + const providerSession = boundProviderSessionTranscriptPath( + normalizeAgentProviderSession(record.providerSession) ?? undefined + ) const providerSessionOnly = record.providerSessionOnly === true if (providerSessionOnly && !isValidPiProviderSessionOnly(providerSession, payload.agentType)) { return null } - return { + return boundAgentHookEventMetadata({ paneKey, launchToken: typeof record.launchToken === 'string' ? record.launchToken : undefined, tabId: typeof tabId === 'string' ? tabId : undefined, @@ -255,7 +368,7 @@ function sanitizeHydratedEntry( payload, receivedAt, stateStartedAt - } + }) } function toAgentStatusIpcPayload(entry: EnrichedAgentHookEventPayload): AgentStatusIpcPayload { @@ -750,6 +863,48 @@ export class AgentHookServer { } } + private cacheStatusEntry(entry: EnrichedAgentHookEventPayload, now = entry.receivedAt): number { + const boundedEntry = boundAgentHookEventMetadata(entry) + const evicted = upsertBoundedAgentHookStatus(this.state, boundedEntry, { now }) + for (const { paneKey } of evicted) { + this.clearAssistantMessageRetry(paneKey) + this.runtimeObservedStatusPaneKeys.delete(paneKey) + this.promptSentDedupeByPaneKey.delete(paneKey) + this.onPaneStatusCleared?.({ paneKey }) + } + return evicted.length + } + + private rememberConnectionTimestampWatermark(connectionId: string, timestamp: number): void { + this.connectionTimestampWatermarkById.delete(connectionId) + this.connectionTimestampWatermarkById.set(connectionId, timestamp) + if ( + this.connectionTimestampWatermarkById.size <= MAX_AGENT_HOOK_CONNECTION_TIMESTAMP_WATERMARKS + ) { + return + } + const activeConnectionIds = new Set() + for (const entry of this.state.lastStatusByPaneKey.values()) { + if (entry.connectionId) { + activeConnectionIds.add(entry.connectionId) + } + } + let oldestFallback: string | undefined + for (const retainedConnectionId of this.connectionTimestampWatermarkById.keys()) { + if (retainedConnectionId === connectionId) { + continue + } + oldestFallback ??= retainedConnectionId + if (!activeConnectionIds.has(retainedConnectionId)) { + this.connectionTimestampWatermarkById.delete(retainedConnectionId) + return + } + } + if (oldestFallback) { + this.connectionTimestampWatermarkById.delete(oldestFallback) + } + } + private hashPromptForTelemetryDedupe(prompt: string): string { return createHash('sha256') .update(this.promptSentHashSalt) @@ -819,7 +974,8 @@ export class AgentHookServer { } } - private applyNormalizedStatus(payload: AgentHookEventPayload): EnrichedAgentHookEventPayload { + private applyNormalizedStatus(rawPayload: AgentHookEventPayload): EnrichedAgentHookEventPayload { + const payload = boundAgentHookEventMetadata(rawPayload) const previous = this.state.lastStatusByPaneKey.get(payload.paneKey) as | EnrichedAgentHookEventPayload | undefined @@ -829,14 +985,14 @@ export class AgentHookServer { // Why: Date.now() can repeat across reconnect; a remote replay must sort strictly after its connection's transient clear. const now = Math.max(Date.now(), (connectionClearWatermark ?? -1) + 1) if (payload.connectionId) { - this.connectionTimestampWatermarkById.set(payload.connectionId, now) + this.rememberConnectionTimestampWatermark(payload.connectionId, now) } if (payload.providerSessionOnly) { // Why: Pi session_start replaces stale turn state and survives replay, but must not emit prompt telemetry or a fabricated status. const enriched = this.attachStatusTiming(payload, now) this.clearAssistantMessageRetry(enriched.paneKey) this.runtimeObservedStatusPaneKeys.delete(enriched.paneKey) - this.state.lastStatusByPaneKey.set(enriched.paneKey, enriched) + this.cacheStatusEntry(enriched) this.scheduleStatusPersist() this.notifyStatusChangeListeners() this.onAgentStatus?.(enriched) @@ -948,7 +1104,7 @@ export class AgentHookServer { } const enriched = this.attachStatusTiming(effectivePayload, now) this.runtimeObservedStatusPaneKeys.add(enriched.paneKey) - this.state.lastStatusByPaneKey.set(enriched.paneKey, enriched) + this.cacheStatusEntry(enriched) this.scheduleStatusPersist() this.notifyStatusChangeListeners() this.onAgentStatus?.(enriched) @@ -1320,10 +1476,16 @@ export class AgentHookServer { if (this.shouldSuppressClosedTabStatus(paneKey)) { return } - const worktreeId = - event.worktreeId !== undefined && event.worktreeId.trim().length > 0 - ? event.worktreeId.trim() - : undefined + const worktreeId = normalizeOptionalRetainedString( + event.worktreeId, + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + ) + if ( + typeof event.connectionId === 'string' && + !isRetainedStringWithinByteLimit(event.connectionId, MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES) + ) { + return + } const connectionId = typeof event.connectionId === 'string' && event.connectionId.trim().length > 0 ? event.connectionId.trim() @@ -1375,6 +1537,9 @@ export class AgentHookServer { if (typeof connectionId !== 'string') { return } + if (!isRetainedStringWithinByteLimit(connectionId, MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES)) { + return + } const trimmedConnectionId = connectionId.trim() if (trimmedConnectionId.length === 0) { return @@ -1418,32 +1583,33 @@ export class AgentHookServer { if (this.shouldSuppressClosedTabStatus(paneKey)) { return } - const worktreeId = - envelope.worktreeId !== undefined && envelope.worktreeId.trim().length > 0 - ? envelope.worktreeId.trim() - : undefined - const hookEventName = - typeof envelope.hookEventName === 'string' && envelope.hookEventName.trim().length > 0 - ? envelope.hookEventName.trim() - : undefined - const promptInteractionKey = - typeof envelope.promptInteractionKey === 'string' && - envelope.promptInteractionKey.trim().length > 0 - ? envelope.promptInteractionKey.trim() - : undefined - const toolUseId = - typeof envelope.toolUseId === 'string' && envelope.toolUseId.trim().length > 0 - ? envelope.toolUseId.trim() - : undefined - const toolAgentId = - typeof envelope.toolAgentId === 'string' && envelope.toolAgentId.trim().length > 0 - ? envelope.toolAgentId.trim() - : undefined - const toolAgentType = - typeof envelope.toolAgentType === 'string' && envelope.toolAgentType.trim().length > 0 - ? envelope.toolAgentType.trim() - : undefined - const providerSession = normalizeAgentProviderSession(envelope.providerSession) ?? undefined + const worktreeId = normalizeOptionalRetainedString( + envelope.worktreeId, + MAX_AGENT_HOOK_RETAINED_PATH_UTF8_BYTES + ) + const hookEventName = normalizeOptionalRetainedString( + envelope.hookEventName, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const promptInteractionKey = normalizeOptionalRetainedString( + envelope.promptInteractionKey, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolUseId = normalizeOptionalRetainedString( + envelope.toolUseId, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolAgentId = normalizeOptionalRetainedString( + envelope.toolAgentId, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const toolAgentType = normalizeOptionalRetainedString( + envelope.toolAgentType, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ) + const providerSession = boundProviderSessionTranscriptPath( + normalizeAgentProviderSession(envelope.providerSession) ?? undefined + ) // Why: relay crosses a trust boundary — re-run the canonical normalizer to enforce caps/invariants (returns null on malformed). const normalizedPayload = normalizeAgentStatusPayload(envelope.payload) if (!normalizedPayload) { @@ -1463,7 +1629,10 @@ export class AgentHookServer { }) const event: AgentHookEventPayload = { paneKey, - launchToken: envelope.launchToken, + launchToken: normalizeOptionalRetainedString( + envelope.launchToken, + MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES + ), tabId, worktreeId, connectionId: trimmedConnectionId, @@ -1548,8 +1717,14 @@ export class AgentHookServer { trackEmptyPaneKeyHook(body) const aliasedBody = this.normalizeHookBodyPaneKeyAlias(body) const normalized = normalizeHookPayload(this.state, source, aliasedBody, this.env) - if (normalized && !this.shouldSuppressClosedTabStatus(normalized.paneKey)) { - const enriched = this.applyNormalizedStatus(normalized) + const retained = normalized ? boundAgentHookEventMetadata(normalized) : null + if ( + retained && + (!retained.providerSessionOnly || + isValidPiProviderSessionOnly(retained.providerSession, retained.payload.agentType)) && + !this.shouldSuppressClosedTabStatus(retained.paneKey) + ) { + const enriched = this.applyNormalizedStatus(retained) this.scheduleAssistantMessageRetry(source, aliasedBody, enriched) } @@ -1626,6 +1801,9 @@ export class AgentHookServer { /** Clear statuses proven to belong to one lost SSH transport. */ clearStatusEntriesForConnection(connectionId: string): void { + if (!isRetainedStringWithinByteLimit(connectionId, MAX_AGENT_HOOK_RETAINED_ID_UTF8_BYTES)) { + return + } const normalizedConnectionId = connectionId.trim() if (normalizedConnectionId.length === 0) { return @@ -1634,7 +1812,7 @@ export class AgentHookServer { Date.now(), (this.connectionTimestampWatermarkById.get(normalizedConnectionId) ?? -1) + 1 ) - this.connectionTimestampWatermarkById.set(normalizedConnectionId, clearedAt) + this.rememberConnectionTimestampWatermark(normalizedConnectionId, clearedAt) let statusChanged = false for (const [paneKey, rawEntry] of this.state.lastStatusByPaneKey) { const entry = rawEntry as EnrichedAgentHookEventPayload @@ -1829,7 +2007,10 @@ export class AgentHookServer { this.state.lastStatusByPaneKey.clear() let raw: string try { - raw = readFileSync(this.lastStatusFilePath, 'utf8') + raw = readNodeFileSyncWithinLimit( + this.lastStatusFilePath, + MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES + ).buffer.toString('utf8') } catch (err) { // Why: missing file is normal (first launch); other errors degrade to empty hydration + one warn. if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { @@ -1839,9 +2020,13 @@ export class AgentHookServer { } let parsed: unknown try { + assertJsonTextStructureWithinLimits(raw, { + structuralTokens: MAX_AGENT_HOOK_LAST_STATUS_STRUCTURAL_TOKENS, + nestingDepth: MAX_AGENT_HOOK_LAST_STATUS_NESTING_DEPTH + }) parsed = JSON.parse(raw) } catch { - console.warn('[agent-hooks] last-status file is not valid JSON; ignoring') + console.warn('[agent-hooks] last-status file is invalid or too complex; ignoring') return } if (typeof parsed !== 'object' || parsed === null) { @@ -1866,7 +2051,8 @@ export class AgentHookServer { let dropped = 0 let prunedLegacyClaudeSubagents = 0 // Why: drop entries older than HYDRATE_MAX_AGE_MS to bound disk growth (one Date.now() for a consistent cutoff). - const ttlCutoff = Date.now() - HYDRATE_MAX_AGE_MS + const hydrateNow = Date.now() + const ttlCutoff = hydrateNow - HYDRATE_MAX_AGE_MS for (const [paneKey, rawEntry] of Object.entries(entries)) { const resolvedPaneKey = this.resolvePaneKeyAlias(paneKey) const rawResolvedEntry = @@ -1881,11 +2067,11 @@ export class AgentHookServer { (entry.payload.subagents?.length ?? 0) - (hydratedPayload.subagents?.length ?? 0) entry.payload = hydratedPayload } - this.state.lastStatusByPaneKey.set(resolvedPaneKey, entry) + dropped += this.cacheStatusEntry(entry, hydrateNow) if (entry.connectionId) { // Why: a restart can see an earlier wall clock; seed ordering so new events stay after disk state. const previousWatermark = this.connectionTimestampWatermarkById.get(entry.connectionId) - this.connectionTimestampWatermarkById.set( + this.rememberConnectionTimestampWatermark( entry.connectionId, Math.max(previousWatermark ?? -1, entry.receivedAt) ) @@ -1907,7 +2093,7 @@ export class AgentHookServer { } if (dropped > 0) { console.warn( - `[agent-hooks] last-status hydrate dropped ${dropped} entries (kept ${hydrated})` + `[agent-hooks] last-status hydrate dropped ${dropped} entries (kept ${this.state.lastStatusByPaneKey.size})` ) } if (dropped > 0 || prunedLegacyClaudeSubagents > 0) { @@ -1920,17 +2106,45 @@ export class AgentHookServer { } private serializeStatusFile(): string { - const entries: Record = {} - for (const [paneKey, payload] of this.state.lastStatusByPaneKey) { + const prefix = `{"version":${LAST_STATUS_FILE_VERSION},"entries":{` + const suffix = '}}' + let remainingBytes = + MAX_AGENT_HOOK_LAST_STATUS_FILE_BYTES - + Buffer.byteLength(prefix, 'utf8') - + Buffer.byteLength(suffix, 'utf8') + const fragments: string[] = [] + const statuses = Array.from(this.state.lastStatusByPaneKey) + for (let index = statuses.length - 1; index >= 0; index -= 1) { + const [paneKey, payload] = statuses[index] // Why: never persist invalid keys (matches the hydrate-path invariant). if (!isValidPaneKey(paneKey)) { continue } const { promptInteractionKey: _promptInteractionKey, ...persistedPayload } = payload - entries[paneKey] = persistedPayload as EnrichedAgentHookEventPayload + const propertyPrefix = `${JSON.stringify(paneKey)}:` + const separatorBytes = fragments.length > 0 ? 1 : 0 + const payloadBudget = + remainingBytes - separatorBytes - Buffer.byteLength(propertyPrefix, 'utf8') + if (payloadBudget < 0) { + continue + } + let serializedPayload: string + try { + serializedPayload = stringifyJsonWithinByteLimit(persistedPayload, payloadBudget).serialized + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + continue + } + throw error + } + fragments.push(`${propertyPrefix}${serializedPayload}`) + remainingBytes -= + separatorBytes + + Buffer.byteLength(propertyPrefix, 'utf8') + + Buffer.byteLength(serializedPayload, 'utf8') } - const file: LastStatusFile = { version: LAST_STATUS_FILE_VERSION, entries } - return JSON.stringify(file) + fragments.reverse() + return `${prefix}${fragments.join(',')}${suffix}` } private scheduleStatusPersist(): void { diff --git a/src/main/agent-hooks/wsl-hook-fs-adapter.ts b/src/main/agent-hooks/wsl-hook-fs-adapter.ts index d47dd400199..826a314f2b7 100644 --- a/src/main/agent-hooks/wsl-hook-fs-adapter.ts +++ b/src/main/agent-hooks/wsl-hook-fs-adapter.ts @@ -5,10 +5,20 @@ // `installer-utils-remote.ts` touches are implemented. import type { SFTPWrapper } from 'ssh2' +import { assertFilesystemDirectoryWithinLimit } from '../../shared/filesystem-directory-listing-limit' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import type { installRemoteManagedAgentHooks } from './remote-managed-hook-installers' import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import { wslCodexRuntimeHomeForGuestHome } from '../pty/codex-home-wsl-env' -import { WSL_HOOK_FS_METHODS, type WslFsResult } from '../../shared/wsl-hook-relay-contract' +import { + WSL_HOOK_FS_METHODS, + WSL_HOOK_FS_MAX_DIRECTORY_ENTRIES, + WSL_HOOK_FS_MAX_DIRECTORY_RETAINED_BYTES, + WSL_HOOK_FS_MAX_READ_BYTES, + type WslFsFailure, + type WslFsResult, + type WslHookFsDirectoryLimits +} from '../../shared/wsl-hook-relay-contract' /** Run the shared remote hook installers against a WSL guest over the relay's * fs bridge. Codex is the one agent whose home Orca redirects for WSL @@ -44,7 +54,23 @@ const ERRNO_TO_SFTP_CODE: Record = { EEXIST: 4 } -function toSftpError(failure: { errno?: string; message?: string }): Error { +function toSftpError(failure: { + errno?: string + message?: string + fileCapacity?: { observedBytes: number; maxBytes: number } +}): Error { + if ( + failure.errno === 'EFBIG' && + Number.isSafeInteger(failure.fileCapacity?.observedBytes) && + Number.isSafeInteger(failure.fileCapacity?.maxBytes) && + failure.fileCapacity!.observedBytes >= 0 && + failure.fileCapacity!.maxBytes >= 0 + ) { + return new NodeFileReadTooLargeError( + failure.fileCapacity!.observedBytes, + failure.fileCapacity!.maxBytes + ) + } const err = new Error(failure.message ?? 'wsl fs bridge failure') as Error & { code?: number } err.code = ERRNO_TO_SFTP_CODE[failure.errno ?? ''] ?? 5 return err @@ -62,7 +88,7 @@ export function createWslHookSftpAdapter(mux: SshChannelMultiplexer): SFTPWrappe .then((raw) => { const result = raw as WslFsResult if (!result || typeof result !== 'object' || result.ok !== true) { - callback(toSftpError((result ?? {}) as { errno?: string; message?: string })) + callback(toSftpError((result ?? {}) as WslFsFailure)) return } callback(null, pick(result)) @@ -81,7 +107,15 @@ export function createWslHookSftpAdapter(mux: SshChannelMultiplexer): SFTPWrappe readFile(path: string, _encoding: unknown, callback: SftpCallback): void { call<{ content: string }, string>( WSL_HOOK_FS_METHODS.readFile, - { path }, + { path, maxBytes: WSL_HOOK_FS_MAX_READ_BYTES }, + callback, + (r) => r.content + ) + }, + orcaReadFileWithinLimit(path: string, maxBytes: number, callback: SftpCallback): void { + call<{ content: string }, string>( + WSL_HOOK_FS_METHODS.readFile, + { path, maxBytes }, callback, (r) => r.content ) @@ -122,9 +156,45 @@ export function createWslHookSftpAdapter(mux: SshChannelMultiplexer): SFTPWrappe readdir(path: string, callback: SftpCallback<{ filename: string }[]>): void { call<{ entries: { filename: string }[] }, { filename: string }[]>( WSL_HOOK_FS_METHODS.readdir, - { path }, + { + path, + maxEntries: WSL_HOOK_FS_MAX_DIRECTORY_ENTRIES, + maxRetainedBytes: WSL_HOOK_FS_MAX_DIRECTORY_RETAINED_BYTES + }, callback, - (r) => r.entries + (r) => { + const limits = { + maxEntries: WSL_HOOK_FS_MAX_DIRECTORY_ENTRIES, + maxRetainedBytes: WSL_HOOK_FS_MAX_DIRECTORY_RETAINED_BYTES + } + assertFilesystemDirectoryWithinLimit( + r.entries.map((entry) => ({ name: entry.filename })), + limits + ) + return r.entries + } + ) + }, + orcaReaddirWithinLimit( + path: string, + limits: WslHookFsDirectoryLimits, + callback: SftpCallback<{ filename: string }[]> + ): void { + call<{ entries: { filename: string }[] }, { filename: string }[]>( + WSL_HOOK_FS_METHODS.readdir, + { + path, + maxEntries: limits.maxEntries, + maxRetainedBytes: limits.maxRetainedBytes + }, + callback, + (r) => { + assertFilesystemDirectoryWithinLimit( + r.entries.map((entry) => ({ name: entry.filename })), + limits + ) + return r.entries + } ) }, mkdir(path: string, callback: SftpCallback): void { diff --git a/src/main/agent-hooks/wsl-hook-relay-deps.ts b/src/main/agent-hooks/wsl-hook-relay-deps.ts index 2c30739e25b..a40e340b593 100644 --- a/src/main/agent-hooks/wsl-hook-relay-deps.ts +++ b/src/main/agent-hooks/wsl-hook-relay-deps.ts @@ -2,12 +2,12 @@ // production wiring. Tests construct the manager with fakes for everything // that spawns wsl.exe or touches the live agentHookServer. import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' import { agentHookServer } from './server' import { installRemoteManagedAgentHooks } from './remote-managed-hook-installers' import { isWslDistroRunning, + readWslHookRelayBundle, resolveWslHookRelayBundle, runWslInstallProcess, spawnWslRelayProcess @@ -73,7 +73,7 @@ export const defaultWslHookRelayDeps: WslHookRelayManagerDeps = { return source ? createHash('sha256').update(source).digest('hex').slice(0, 12) : null }, resolveBundle: resolveWslHookRelayBundle, - readBundle: (jsPath) => readFileSync(jsPath), + readBundle: readWslHookRelayBundle, listDistros: () => listWslDistrosAsync(), isDistroRunning: isWslDistroRunning, spawnRelay: spawnWslRelayProcess, diff --git a/src/main/agent-hooks/wsl-hook-relay-file-bounds.test.ts b/src/main/agent-hooks/wsl-hook-relay-file-bounds.test.ts new file mode 100644 index 00000000000..ce2c23d12b0 --- /dev/null +++ b/src/main/agent-hooks/wsl-hook-relay-file-bounds.test.ts @@ -0,0 +1,60 @@ +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { + readWslHookRelayBundle, + readWslHookRelayBundleVersion, + WSL_HOOK_RELAY_MAX_BUNDLE_BYTES, + WSL_HOOK_RELAY_MAX_VERSION_FILE_BYTES +} from './wsl-hook-relay-launch' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function tempFile(name: string): string { + const root = mkdtempSync(join(tmpdir(), 'orca-wsl-relay-bounds-')) + roots.push(root) + return join(root, name) +} + +function createSparseFile(path: string, bytes: number): void { + const descriptor = openSync(path, 'w') + try { + ftruncateSync(descriptor, bytes) + } finally { + closeSync(descriptor) + } +} + +describe('WSL hook relay local file bounds', () => { + it('preserves ordinary version and bundle reads', () => { + const versionPath = tempFile('.version') + const bundlePath = tempFile('relay.js') + writeFileSync(versionPath, '1.2.3\n') + writeFileSync(bundlePath, 'console.log("relay")') + + expect(readWslHookRelayBundleVersion(versionPath)).toBe('1.2.3') + expect(readWslHookRelayBundle(bundlePath).toString('utf8')).toBe('console.log("relay")') + }) + + it('rejects an oversized version marker before materializing it', () => { + const versionPath = tempFile('.version') + createSparseFile(versionPath, WSL_HOOK_RELAY_MAX_VERSION_FILE_BYTES + 1) + + expect(() => readWslHookRelayBundleVersion(versionPath)).toThrow(NodeFileReadTooLargeError) + }) + + it('rejects an oversized bundle before materializing or base64-expanding it', () => { + const bundlePath = tempFile('relay.js') + createSparseFile(bundlePath, WSL_HOOK_RELAY_MAX_BUNDLE_BYTES + 1) + + expect(() => readWslHookRelayBundle(bundlePath)).toThrow(NodeFileReadTooLargeError) + }) +}) diff --git a/src/main/agent-hooks/wsl-hook-relay-launch.ts b/src/main/agent-hooks/wsl-hook-relay-launch.ts index 72ff3bb0c23..66a7f57a5f7 100644 --- a/src/main/agent-hooks/wsl-hook-relay-launch.ts +++ b/src/main/agent-hooks/wsl-hook-relay-launch.ts @@ -4,10 +4,11 @@ // MultiplexerTransport. Kept separate from the manager so the state machine // stays readable. See docs/agent-status-over-wsl.md (STA-1515). import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' import { app } from 'electron' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import type { MultiplexerTransport } from '../ssh/ssh-channel-multiplexer' import { decodeWslText, @@ -27,6 +28,8 @@ import { } from '../../shared/wsl-hook-relay-contract' const INSTALL_TIMEOUT_MS = 30_000 +export const WSL_HOOK_RELAY_MAX_VERSION_FILE_BYTES = 4 * 1024 +export const WSL_HOOK_RELAY_MAX_BUNDLE_BYTES = 8 * 1024 * 1024 export type WslHookRelayBundle = { jsPath: string; version: string } @@ -52,7 +55,12 @@ export function resolveWslHookRelayBundle(): WslHookRelayBundle | null { const jsPath = join(dir, WSL_HOOK_RELAY_BUNDLE_NAME) const versionPath = join(dir, WSL_HOOK_RELAY_VERSION_FILE) if (existsSync(jsPath) && existsSync(versionPath)) { - const version = readFileSync(versionPath, 'utf8').trim() + let version: string + try { + version = readWslHookRelayBundleVersion(versionPath) + } catch { + continue + } // Why: the version lands inside single-quoted guest shell text and in // a guest path segment — refuse anything outside the safe alphabet. if (/^[A-Za-z0-9+.-]+$/.test(version)) { @@ -63,6 +71,16 @@ export function resolveWslHookRelayBundle(): WslHookRelayBundle | null { return null } +export function readWslHookRelayBundleVersion(versionPath: string): string { + return readNodeFileSyncWithinLimit(versionPath, WSL_HOOK_RELAY_MAX_VERSION_FILE_BYTES) + .buffer.toString('utf8') + .trim() +} + +export function readWslHookRelayBundle(jsPath: string): Buffer { + return readNodeFileSyncWithinLimit(jsPath, WSL_HOOK_RELAY_MAX_BUNDLE_BYTES).buffer +} + // Why: the install dir is namespaced by bundle version so concurrent Orca // instances with different bundles (dev + prod) never reinstall over each // other; each instance launches exactly the version it shipped. diff --git a/src/main/agent-hooks/wsl-hook-relay-live.integration.test.ts b/src/main/agent-hooks/wsl-hook-relay-live.integration.test.ts index 89b0987a582..28125dd28a8 100644 --- a/src/main/agent-hooks/wsl-hook-relay-live.integration.test.ts +++ b/src/main/agent-hooks/wsl-hook-relay-live.integration.test.ts @@ -43,12 +43,11 @@ describe.skipIf(process.platform === 'win32')( let child: ChildProcessWithoutNullStreams | null beforeAll(() => { - if (!existsSync(BUNDLE_JS)) { - execFileSync(process.execPath, [join('config', 'scripts', 'build-relay.mjs')], { - cwd: process.cwd(), - stdio: 'ignore' - }) - } + // Why: a prior build can exist while its embedded fs bridge is stale relative to this checkout. + execFileSync(process.execPath, [join('config', 'scripts', 'build-relay.mjs')], { + cwd: process.cwd(), + stdio: 'ignore' + }) }, 120_000) afterEach(() => { diff --git a/src/main/agent-hooks/wsl-hook-relay-manager.test.ts b/src/main/agent-hooks/wsl-hook-relay-manager.test.ts index 7e341d408da..2568e69adb9 100644 --- a/src/main/agent-hooks/wsl-hook-relay-manager.test.ts +++ b/src/main/agent-hooks/wsl-hook-relay-manager.test.ts @@ -2,15 +2,18 @@ // bridge (including a full run of the unchanged remote hook installers), and // the per-distro relay manager state machine with fault injection. import { EventEmitter } from 'node:events' -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { ChildProcessWithoutNullStreams } from 'node:child_process' +import type { SFTPWrapper } from 'ssh2' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { RelayDispatcher } from '../../relay/dispatcher' import { registerWslHookFsHandlers } from '../../relay/wsl-hook-fs-bridge' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import { SshChannelMultiplexer, type MultiplexerTransport } from '../ssh/ssh-channel-multiplexer' +import { readTextFileRemote } from './installer-utils-remote' import { createWslHookSftpAdapter } from './wsl-hook-fs-adapter' import { installRemoteManagedAgentHooks } from './remote-managed-hook-installers' import { WslHookRelayManager } from './wsl-hook-relay-manager' @@ -113,6 +116,39 @@ describe.skipIf(process.platform === 'win32')( expect(outside).toBeInstanceOf(Error) }) + it('preserves the bounded-read error across the guest bridge and host adapter', async () => { + const adapter = createWslHookSftpAdapter(harness.mux) + const path = `${home}/oversized.json` + writeFileSync(path, 'x'.repeat(1025)) + + await expect(readTextFileRemote(adapter, path, 1024)).rejects.toBeInstanceOf( + NodeFileReadTooLargeError + ) + expect(readFileSync(path, 'utf8')).toHaveLength(1025) + }) + + it('threads requested directory limits through the host adapter', async () => { + const adapter = createWslHookSftpAdapter(harness.mux) as SFTPWrapper & { + orcaReaddirWithinLimit: ( + path: string, + limits: { maxEntries: number; maxRetainedBytes: number }, + callback: (error: Error | null, entries?: { filename: string }[]) => void + ) => void + } + writeFileSync(`${home}/one`, '') + writeFileSync(`${home}/two`, '') + + const error = await new Promise((resolve, reject) => { + adapter.orcaReaddirWithinLimit( + home, + { maxEntries: 1, maxRetainedBytes: 4096 }, + (readError) => + readError ? resolve(readError) : reject(new Error('expected limit error')) + ) + }) + expect(error).toBeInstanceOf(Error) + }) + it('runs the unchanged remote managed hook installers against a WSL guest home', async () => { const adapter = createWslHookSftpAdapter(harness.mux) const results = await installRemoteManagedAgentHooks(adapter, home) diff --git a/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts b/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts index a8085485553..bb78100c6ea 100644 --- a/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts +++ b/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts @@ -5,7 +5,11 @@ import { EventEmitter } from 'node:events' import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { afterEach, describe, expect, it, vi } from 'vitest' -import { RELAY_SENTINEL, RELAY_SENTINEL_TIMEOUT_MS } from '../ssh/relay-protocol' +import { + MAX_BUFFERED_FRAME_CHUNKS, + RELAY_SENTINEL, + RELAY_SENTINEL_TIMEOUT_MS +} from '../ssh/relay-protocol' import { MAX_STARTUP_BUFFER_BYTES, waitForWslRelaySentinel, @@ -110,6 +114,26 @@ describe('waitForWslRelaySentinel', () => { expect(received).toEqual(['DEFER']) }) + it('closes instead of retaining too many post-sentinel fragments before attachment', async () => { + const child = fakeChild() + const promise = waitForWslRelaySentinel(child) + emitStdout(child, RELAY_SENTINEL) + for (let index = 0; index <= MAX_BUFFERED_FRAME_CHUNKS; index += 1) { + emitStdout(child, Buffer.from('x')) + } + + const transport = await promise + const onClose = vi.fn() + const onData = vi.fn() + transport.onClose(onClose) + transport.onData(onData) + await Promise.resolve() + + expect(child.kill).toHaveBeenCalled() + expect(onClose).toHaveBeenCalledOnce() + expect(onData).not.toHaveBeenCalled() + }) + it('kills the child and rejects when startup output exceeds 64 KiB before the sentinel', async () => { const child = fakeChild() const settled = catchStartup(waitForWslRelaySentinel(child)) diff --git a/src/main/agent-hooks/wsl-hook-relay-sentinel.ts b/src/main/agent-hooks/wsl-hook-relay-sentinel.ts index fdabf70fe15..19a2b9a160d 100644 --- a/src/main/agent-hooks/wsl-hook-relay-sentinel.ts +++ b/src/main/agent-hooks/wsl-hook-relay-sentinel.ts @@ -4,10 +4,17 @@ // ChildProcess instead of a ClientChannel. import type { ChildProcessWithoutNullStreams } from 'node:child_process' -import { RELAY_SENTINEL, RELAY_SENTINEL_TIMEOUT_MS } from '../ssh/relay-protocol' +import { + HEADER_LENGTH, + MAX_BUFFERED_FRAME_CHUNKS, + MAX_MESSAGE_SIZE, + RELAY_SENTINEL, + RELAY_SENTINEL_TIMEOUT_MS +} from '../ssh/relay-protocol' import type { MultiplexerTransport } from '../ssh/ssh-channel-multiplexer' export const MAX_STARTUP_BUFFER_BYTES = 64 * 1024 +const MAX_PENDING_RELAY_BYTES = (MAX_MESSAGE_SIZE + HEADER_LENGTH) * 2 // Why: without WSL_UTF8, wsl.exe's own messages arrive UTF-16LE; NUL bytes // in breadcrumbs and the catastrophic-failure matcher must not depend on the @@ -45,6 +52,7 @@ export function waitForWslRelaySentinel( // decoder never sees chunks out of order. A setImmediate handoff would // NOT preserve that: it is a macrotask the next 'data' event can beat. const pendingChunks: Buffer[] = [] + let pendingChunkBytes = 0 let closedNotified = false const fail = (failure: WslRelayStartupFailure): void => { @@ -71,8 +79,22 @@ export function waitForWslRelaySentinel( } const dispatch = (chunk: Buffer): void => { + if (closedNotified) { + return + } if (dataCallbacks.length === 0) { + if ( + chunk.length > MAX_PENDING_RELAY_BYTES - pendingChunkBytes || + pendingChunks.length >= MAX_BUFFERED_FRAME_CHUNKS + ) { + pendingChunks.length = 0 + pendingChunkBytes = 0 + notifyClosed() + child.kill() + return + } pendingChunks.push(chunk) + pendingChunkBytes += chunk.length return } for (const cb of dataCallbacks) { @@ -104,23 +126,32 @@ export function waitForWslRelaySentinel( dispatch(chunk) return } - stdoutBuffer = Buffer.concat([stdoutBuffer, chunk]) - const idx = stdoutBuffer.indexOf(sentinel) + const searchableLength = Math.max( + 0, + MAX_STARTUP_BUFFER_BYTES - stdoutBuffer.length + sentinel.length + ) + const searchableChunk = chunk.subarray(0, searchableLength) + const startupOutput = + stdoutBuffer.length === 0 ? searchableChunk : Buffer.concat([stdoutBuffer, searchableChunk]) + const idx = startupOutput.indexOf(sentinel) if (idx === -1) { // Why: pre-sentinel stdout is untrusted startup noise; cap it so a // broken guest cannot grow memory until the timeout fires. - if (stdoutBuffer.length > MAX_STARTUP_BUFFER_BYTES) { + if (startupOutput.length > MAX_STARTUP_BUFFER_BYTES) { child.kill() fail({ kind: 'exit', code: null, stderr: 'startup output exceeded 64 KiB' }) + } else { + stdoutBuffer = startupOutput } return } sentinelSeen = true settled = true clearTimeout(timeout) - const trailing = stdoutBuffer.subarray(idx + sentinel.length) + const trailingOffset = idx + sentinel.length - stdoutBuffer.length + const trailing = chunk.subarray(Math.max(0, trailingOffset)) if (trailing.length > 0) { - pendingChunks.push(trailing) + dispatch(trailing) } const transport: MultiplexerTransport = { write: (data) => { @@ -135,6 +166,7 @@ export function waitForWslRelaySentinel( if (dataCallbacks.length === 1 && pendingChunks.length > 0) { queueMicrotask(() => { for (const pending of pendingChunks.splice(0)) { + pendingChunkBytes -= pending.length for (const dataCb of dataCallbacks) { dataCb(pending) } @@ -142,7 +174,13 @@ export function waitForWslRelaySentinel( }) } }, - onClose: (cb) => closeCallbacks.push(cb), + onClose: (cb) => { + if (closedNotified) { + queueMicrotask(cb) + } else { + closeCallbacks.push(cb) + } + }, close: () => child.kill() } resolve(transport) diff --git a/src/main/agent-state-file-reader.test.ts b/src/main/agent-state-file-reader.test.ts new file mode 100644 index 00000000000..34638fcbc06 --- /dev/null +++ b/src/main/agent-state-file-reader.test.ts @@ -0,0 +1,82 @@ +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { NodeFileReadTooLargeError } from '../shared/node-bounded-file-reader' +import { + MAX_AGENT_STATE_FILE_BYTES, + MAX_AGENT_STATE_JSON_NESTING_DEPTH, + MAX_AGENT_STATE_JSON_STRUCTURAL_TOKENS, + readAgentStateFileSync, + readAgentStateJsonFileSync +} from './agent-state-file-reader' + +const tempDirs: string[] = [] + +function tempFile(name: string): string { + const directory = mkdtempSync(join(tmpdir(), 'orca-agent-state-')) + tempDirs.push(directory) + return join(directory, name) +} + +afterEach(() => { + for (const directory of tempDirs.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('readAgentStateFileSync', () => { + it('preserves normal UTF-8 auth and config contents', () => { + const filePath = tempFile('auth.json') + const contents = '{"token":"你好","refresh":"abc"}\n' + writeFileSync(filePath, contents) + + expect(readAgentStateFileSync(filePath)).toBe(contents) + }) + + it('reads a sparse file at the exact 4 MiB boundary', () => { + const filePath = tempFile('snapshot.json') + writeFileSync(filePath, '') + truncateSync(filePath, MAX_AGENT_STATE_FILE_BYTES) + + const contents = readAgentStateFileSync(filePath) + + expect(contents).toHaveLength(MAX_AGENT_STATE_FILE_BYTES) + expect(contents.charCodeAt(MAX_AGENT_STATE_FILE_BYTES - 1)).toBe(0) + }) + + it('rejects a sparse file one byte over the boundary before reading its payload', () => { + const filePath = tempFile('oversized.json') + writeFileSync(filePath, '') + truncateSync(filePath, MAX_AGENT_STATE_FILE_BYTES + 1) + + expect(() => readAgentStateFileSync(filePath)).toThrow(NodeFileReadTooLargeError) + }) +}) + +describe('readAgentStateJsonFileSync', () => { + it('preserves ordinary JSON values', () => { + const filePath = tempFile('auth.json') + writeFileSync(filePath, '{"token":"你好","nested":{"enabled":true}}') + + expect(readAgentStateJsonFileSync(filePath)).toEqual({ + token: '你好', + nested: { enabled: true } + }) + }) + + it('rejects structural-token and nesting amplification before JSON.parse', () => { + const structuralPath = tempFile('structural.json') + writeFileSync(structuralPath, `[${'0,'.repeat(MAX_AGENT_STATE_JSON_STRUCTURAL_TOKENS)}0]`) + const nestedPath = tempFile('nested.json') + writeFileSync( + nestedPath, + `${'['.repeat(MAX_AGENT_STATE_JSON_NESTING_DEPTH + 1)}0${']'.repeat( + MAX_AGENT_STATE_JSON_NESTING_DEPTH + 1 + )}` + ) + + expect(() => readAgentStateJsonFileSync(structuralPath)).toThrow('JSON structure exceeds') + expect(() => readAgentStateJsonFileSync(nestedPath)).toThrow('JSON nesting exceeds') + }) +}) diff --git a/src/main/agent-state-file-reader.ts b/src/main/agent-state-file-reader.ts new file mode 100644 index 00000000000..0980fb5f935 --- /dev/null +++ b/src/main/agent-state-file-reader.ts @@ -0,0 +1,19 @@ +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { assertJsonTextStructureWithinLimits } from '../shared/json-text-structure-limit' + +export const MAX_AGENT_STATE_FILE_BYTES = 4 * 1024 * 1024 +export const MAX_AGENT_STATE_JSON_STRUCTURAL_TOKENS = 1_000_000 +export const MAX_AGENT_STATE_JSON_NESTING_DEPTH = 128 + +export function readAgentStateFileSync(filePath: string): string { + return readNodeFileSyncWithinLimit(filePath, MAX_AGENT_STATE_FILE_BYTES).buffer.toString('utf8') +} + +export function readAgentStateJsonFileSync(filePath: string): unknown { + const content = readAgentStateFileSync(filePath) + assertJsonTextStructureWithinLimits(content, { + structuralTokens: MAX_AGENT_STATE_JSON_STRUCTURAL_TOKENS, + nestingDepth: MAX_AGENT_STATE_JSON_NESTING_DEPTH + }) + return JSON.parse(content) as unknown +} diff --git a/src/main/agent-trust-presets.test.ts b/src/main/agent-trust-presets.test.ts index 2a979c60e27..e89272696a6 100644 --- a/src/main/agent-trust-presets.test.ts +++ b/src/main/agent-trust-presets.test.ts @@ -6,11 +6,14 @@ import { readFileSync, realpathSync, rmSync, + statSync, + truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_AGENT_STATE_FILE_BYTES } from './agent-state-file-reader' const testState = { fakeHomeDir: '', @@ -38,8 +41,12 @@ vi.mock('node:os', async () => { } }) -const { markCodexProjectTrusted, markCopilotFolderTrusted, markCursorWorkspaceTrusted } = - await import('./agent-trust-presets') +const { + MAX_AGENT_TRUST_GIT_POINTER_BYTES, + markCodexProjectTrusted, + markCopilotFolderTrusted, + markCursorWorkspaceTrusted +} = await import('./agent-trust-presets') beforeEach(() => { testState.fakeHomeDir = mkdtempSync(join(tmpdir(), 'orca-trust-presets-')) @@ -134,9 +141,115 @@ describe('markCopilotFolderTrusted', () => { rmSync(workspace, { recursive: true, force: true }) } }) + + it('does not overwrite a sparse config above the 4 MiB read ceiling', () => { + const workspace = mkdtempSync(join(tmpdir(), 'orca-copilot-ws-')) + const configDir = join(testState.fakeHomeDir, '.copilot') + const configPath = join(configDir, 'config.json') + try { + mkdirSync(configDir, { recursive: true }) + writeFileSync(configPath, '{"keep":true}') + truncateSync(configPath, MAX_AGENT_STATE_FILE_BYTES + 1) + + markCopilotFolderTrusted(workspace) + + expect(statSync(configPath).size).toBe(MAX_AGENT_STATE_FILE_BYTES + 1) + } finally { + rmSync(workspace, { recursive: true, force: true }) + } + }) }) describe('markCodexProjectTrusted', () => { + it('accepts both linked-worktree pointers at the exact 64 KiB boundary', () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), 'orca-codex-exact-git-pointers-')) + const repository = join(fixtureRoot, 'repo') + const workspace = join(fixtureRoot, 'worktrees', 'feature') + const worktreeGitDir = join(repository, '.git', 'worktrees', 'feature') + const padPointer = (value: string): string => + value + ' '.repeat(MAX_AGENT_TRUST_GIT_POINTER_BYTES - Buffer.byteLength(value)) + try { + mkdirSync(worktreeGitDir, { recursive: true }) + mkdirSync(workspace, { recursive: true }) + const gitPointer = padPointer(`gitdir: ${worktreeGitDir}\n`) + const backlink = padPointer(`${join(workspace, '.git')}\n`) + expect(Buffer.byteLength(gitPointer)).toBe(MAX_AGENT_TRUST_GIT_POINTER_BYTES) + expect(Buffer.byteLength(backlink)).toBe(MAX_AGENT_TRUST_GIT_POINTER_BYTES) + writeFileSync(join(workspace, '.git'), gitPointer) + writeFileSync(join(worktreeGitDir, 'gitdir'), backlink) + + markCodexProjectTrusted(workspace) + + const written = readFileSync(join(testState.fakeHomeDir, '.codex', 'config.toml'), 'utf8') + expect(written).toContain( + `[projects."${escapeTomlBasicString(realpathSync.native(repository))}"]` + ) + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }) + } + }) + + it('falls back to the workspace when its sparse .git pointer exceeds 64 KiB', () => { + const workspace = mkdtempSync(join(tmpdir(), 'orca-codex-large-git-pointer-')) + try { + const gitPointerPath = join(workspace, '.git') + writeFileSync(gitPointerPath, '') + truncateSync(gitPointerPath, MAX_AGENT_TRUST_GIT_POINTER_BYTES + 1) + + markCodexProjectTrusted(workspace) + + const written = readFileSync(join(testState.fakeHomeDir, '.codex', 'config.toml'), 'utf8') + expect(written).toContain( + `[projects."${escapeTomlBasicString(realpathSync.native(workspace))}"]` + ) + } finally { + rmSync(workspace, { recursive: true, force: true }) + } + }) + + it('falls back to the workspace when the sparse gitdir backlink exceeds 64 KiB', () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), 'orca-codex-large-gitdir-pointer-')) + const repository = join(fixtureRoot, 'repo') + const workspace = join(fixtureRoot, 'worktrees', 'feature') + const worktreeGitDir = join(repository, '.git', 'worktrees', 'feature') + try { + mkdirSync(worktreeGitDir, { recursive: true }) + mkdirSync(workspace, { recursive: true }) + writeFileSync(join(workspace, '.git'), `gitdir: ${worktreeGitDir}\n`) + const backlinkPath = join(worktreeGitDir, 'gitdir') + writeFileSync(backlinkPath, '') + truncateSync(backlinkPath, MAX_AGENT_TRUST_GIT_POINTER_BYTES + 1) + + markCodexProjectTrusted(workspace) + + const written = readFileSync(join(testState.fakeHomeDir, '.codex', 'config.toml'), 'utf8') + expect(written).toContain( + `[projects."${escapeTomlBasicString(realpathSync.native(workspace))}"]` + ) + expect(written).not.toContain( + `[projects."${escapeTomlBasicString(realpathSync.native(repository))}"]` + ) + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }) + } + }) + + it('does not overwrite a sparse config above the 4 MiB read ceiling', () => { + const workspace = mkdtempSync(join(tmpdir(), 'orca-codex-ws-')) + const configDir = join(testState.fakeHomeDir, '.codex') + const configPath = join(configDir, 'config.toml') + try { + mkdirSync(configDir, { recursive: true }) + writeFileSync(configPath, 'model = "keep"\n') + truncateSync(configPath, MAX_AGENT_STATE_FILE_BYTES + 1) + + expect(() => markCodexProjectTrusted(workspace)).toThrow('File too large') + expect(statSync(configPath).size).toBe(MAX_AGENT_STATE_FILE_BYTES + 1) + } finally { + rmSync(workspace, { recursive: true, force: true }) + } + }) + it('trusts the main repository root for a linked worktree without reading commondir', () => { const fixtureRoot = mkdtempSync(join(tmpdir(), 'orca-codex-linked-ws-')) const repository = join(fixtureRoot, 'repo') diff --git a/src/main/agent-trust-presets.ts b/src/main/agent-trust-presets.ts index 3b441d6d1fe..3d109b96acb 100644 --- a/src/main/agent-trust-presets.ts +++ b/src/main/agent-trust-presets.ts @@ -1,11 +1,14 @@ -import { existsSync, mkdirSync, readFileSync, realpathSync } from 'node:fs' +import { existsSync, mkdirSync, realpathSync } from 'node:fs' import { homedir } from 'node:os' import { basename, dirname, join, resolve } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { readAgentStateJsonFileSync } from './agent-state-file-reader' import { writeFileAtomically } from './codex-accounts/fs-utils' import { getOrcaManagedCodexHomePath } from './codex/codex-home-paths' import { upsertProjectTrustLevel } from './codex/config-toml-trust' export type AgentTrustPreset = 'cursor' | 'copilot' | 'codex' +export const MAX_AGENT_TRUST_GIT_POINTER_BYTES = 64 * 1024 /** * Pre-mark a workspace as trusted for cursor-agent, GitHub Copilot CLI, or @@ -73,8 +76,7 @@ export function markCopilotFolderTrusted(workspacePath: string): void { let config: Record = {} try { if (existsSync(configPath)) { - const raw = readFileSync(configPath, 'utf-8') - const parsed = JSON.parse(raw) + const parsed = readAgentStateJsonFileSync(configPath) if (parsed && typeof parsed === 'object') { config = parsed as Record } @@ -120,7 +122,7 @@ export function markCodexProjectTrusted(workspacePath: string): void { function resolveCodexProjectTrustRoot(workspacePath: string): string { const absPath = canonicalize(workspacePath) try { - const gitDirReference = readFileSync(join(absPath, '.git'), 'utf-8').trim() + const gitDirReference = readGitPointerFile(join(absPath, '.git')) if (!gitDirReference.startsWith('gitdir:')) { return absPath } @@ -134,7 +136,7 @@ function resolveCodexProjectTrustRoot(workspacePath: string): string { return absPath } // Why: workspace-controlled .git metadata must not broaden trust without Git's reciprocal link. - const gitDirBacklink = readFileSync(join(gitDir, 'gitdir'), 'utf-8').trim() + const gitDirBacklink = readGitPointerFile(join(gitDir, 'gitdir')) if (!gitDirBacklink) { return absPath } @@ -153,6 +155,12 @@ function resolveCodexProjectTrustRoot(workspacePath: string): string { } } +function readGitPointerFile(filePath: string): string { + return readNodeFileSyncWithinLimit(filePath, MAX_AGENT_TRUST_GIT_POINTER_BYTES) + .buffer.toString('utf8') + .trim() +} + function canonicalize(p: string): string { // Why: macOS reports `/tmp/x` and `/private/tmp/x` as the same inode, but // both Cursor and Copilot's trust comparators run realpath() before the diff --git a/src/main/ai-vault/cached-session-list.ts b/src/main/ai-vault/cached-session-list.ts index 4b39e75eb72..3943e0cb591 100644 --- a/src/main/ai-vault/cached-session-list.ts +++ b/src/main/ai-vault/cached-session-list.ts @@ -3,6 +3,7 @@ import { scanAiVaultSessions } from './session-scanner' import { getWslHomeAsync, listWslDistrosAsync } from '../wsl' import type { AiVaultListArgs, AiVaultListResult } from '../../shared/ai-vault-types' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' +import { aiVaultSessionListCacheKey, boundAiVaultListResult } from './session-list-retention' // Why: ONE module owns the scan cache so the desktop IPC handler AND the runtime // RPC method share a single cache instance — opening the desktop panel and the @@ -34,10 +35,23 @@ export function configureAiVaultSessionSources(next: AiVaultSessionSources): voi export async function listAiVaultSessions(args?: AiVaultListArgs): Promise { // Scope paths change the result set, so they must be part of the cache key. - const key = JSON.stringify({ + const key = aiVaultSessionListCacheKey({ limit: args?.limit ?? 'default', scopePaths: args?.scopePaths ?? [] }) + const scan = async (): Promise => + scanAiVaultSessions({ + limit: args?.limit, + scopePaths: args?.scopePaths, + additionalCodexSessionsDirs: + sources.getAdditionalCodexHomePaths?.().map((homePath) => join(homePath, 'sessions')) ?? [], + wslHomeDirs: await getAiVaultWslHomeDirs(), + // The shared result is restamped at the RPC edge for runtime callers. + executionHostId: LOCAL_EXECUTION_HOST_ID + }) + if (key === null) { + return boundAiVaultListResult(await scan()) + } const now = Date.now() // Why: opening this panel repeatedly should not re-parse hundreds of JSONL // transcripts; explicit refreshes bypass the cache but not an active scan. @@ -49,25 +63,15 @@ export async function listAiVaultSessions(args?: AiVaultListArgs): Promise join(homePath, 'sessions')) ?? [] - inflightList = (async () => - scanAiVaultSessions({ - limit: args?.limit, - scopePaths: args?.scopePaths, - additionalCodexSessionsDirs, - wslHomeDirs: await getAiVaultWslHomeDirs(), - // Why: this scan is always host-local; callers addressing this host by a - // runtime id get the result restamped at the RPC edge, never rescanned. - executionHostId: LOCAL_EXECUTION_HOST_ID - }))() + inflightList = scan() .then((result) => { + const bounded = boundAiVaultListResult(result) cachedList = { key, - result, + result: bounded, expiresAt: Date.now() + AI_VAULT_CACHE_TTL_MS } - return result + return bounded }) .finally(() => { // Only clear tracking if it still refers to this request: a concurrent diff --git a/src/main/ai-vault/claude-subagent-list-retention.ts b/src/main/ai-vault/claude-subagent-list-retention.ts new file mode 100644 index 00000000000..94ab1e9c691 --- /dev/null +++ b/src/main/ai-vault/claude-subagent-list-retention.ts @@ -0,0 +1,19 @@ +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { + AI_VAULT_SESSION_LIST_MAX_JSON_BYTES, + retainAiVaultSessionsWithinAggregate +} from './session-list-retention' +import { sessionSortTime } from './session-scanner-accumulator' + +export function retainClaudeSubagentSessionBatch( + retainedSessions: readonly AiVaultSession[], + batch: readonly (AiVaultSession | null)[], + maxBytes: number = AI_VAULT_SESSION_LIST_MAX_JSON_BYTES +): { sessions: AiVaultSession[]; omitted: number } { + const candidates = [ + ...retainedSessions, + ...batch.filter((session): session is AiVaultSession => session !== null) + ] + candidates.sort((left, right) => sessionSortTime(right) - sessionSortTime(left)) + return retainAiVaultSessionsWithinAggregate(candidates, maxBytes) +} diff --git a/src/main/ai-vault/remote-session-scanner-codex-index.ts b/src/main/ai-vault/remote-session-scanner-codex-index.ts index d6a305ba4cf..901d28c8681 100644 --- a/src/main/ai-vault/remote-session-scanner-codex-index.ts +++ b/src/main/ai-vault/remote-session-scanner-codex-index.ts @@ -2,6 +2,11 @@ import type { IFilesystemProvider } from '../providers/types' import type { RemoteHostPlatform } from '../ssh/ssh-remote-platform' import { joinRemotePath } from '../ssh/ssh-remote-platform' import { extractString, normalizeTitleText, parseJsonObject } from './session-scanner-values' +import { + CODEX_SESSION_INDEX_CACHE_KEY_MAX_UTF8_BYTES, + CODEX_SESSION_INDEX_TITLE_CACHE_MAX, + retainCodexSessionIndexTitle +} from './session-scanner-codex-title-index' const CODEX_SESSION_INDEX_FILE = 'session_index.jsonl' @@ -11,12 +16,25 @@ export async function remoteCodexIndexTitles(args: { hostPlatform: RemoteHostPlatform titleCaches: Map>> }): Promise> { - const cached = args.titleCaches.get(args.codexHome) + const cacheable = + Buffer.byteLength(args.codexHome, 'utf8') <= CODEX_SESSION_INDEX_CACHE_KEY_MAX_UTF8_BYTES + const cached = cacheable ? args.titleCaches.get(args.codexHome) : undefined if (cached) { + args.titleCaches.delete(args.codexHome) + args.titleCaches.set(args.codexHome, cached) return cached } const pending = readRemoteCodexIndexTitles(args.provider, args.codexHome, args.hostPlatform) - args.titleCaches.set(args.codexHome, pending) + if (cacheable) { + args.titleCaches.set(args.codexHome, pending) + while (args.titleCaches.size > CODEX_SESSION_INDEX_TITLE_CACHE_MAX) { + const oldest = args.titleCaches.keys().next().value + if (oldest === undefined) { + break + } + args.titleCaches.delete(oldest) + } + } return pending } @@ -41,7 +59,7 @@ async function readRemoteCodexIndexTitles( const sessionId = extractString(record.id) const title = normalizeTitleText(extractString(record.thread_name) ?? '') if (sessionId && title) { - titleBySessionId.set(sessionId, title) + retainCodexSessionIndexTitle(titleBySessionId, sessionId, title) } } } catch { diff --git a/src/main/ai-vault/remote-session-scanner-discovery.ts b/src/main/ai-vault/remote-session-scanner-discovery.ts index 14a47f36559..2ec43644d82 100644 --- a/src/main/ai-vault/remote-session-scanner-discovery.ts +++ b/src/main/ai-vault/remote-session-scanner-discovery.ts @@ -6,6 +6,7 @@ import { isMissingRemoteSessionPathError, statRemoteSessionFile } from './remote import { partitionSubagentTranscriptPaths } from './session-scanner-subagent-transcripts' import type { FileWithMtime } from './session-scanner-types' import { errorMessage } from './session-scanner-values' +import { AiVaultSessionDiscoveryCapacityError } from './session-discovery-budget' import type { RemoteScannerContext, RemoteSessionCandidate, @@ -13,15 +14,31 @@ import type { } from './remote-session-scanner-types' const REMOTE_DISCOVERY_CONCURRENCY = 8 +const REMOTE_DISCOVERY_ISSUE_MAX = 256 export async function discoverRemoteSourceCandidates(args: { source: RemoteSessionSource context: RemoteScannerContext issues: AiVaultScanIssue[] }): Promise { - const walked = args.source.fixedChildFileSegments - ? await listRemoteFixedChildFiles(args.source, args.context, args.issues) - : await walkRemoteSessionFiles(args.source, args.context, args.issues) + const walked: string[] = [] + try { + await (args.source.fixedChildFileSegments + ? collectRemoteFixedChildFiles(args.source, args.context, args.issues, walked) + : collectRemoteSessionFiles(args.source, args.context, args.issues, walked)) + } catch (error) { + if (error instanceof AiVaultSessionDiscoveryCapacityError) { + recordRemoteDirectoryIssue( + args.source, + args.context.executionHostId, + args.issues, + args.source.rootDir, + error + ) + } else { + throw error + } + } const partition = args.source.collectSubagentSiblingCounts ? partitionSubagentTranscriptPaths(walked) : null @@ -45,52 +62,63 @@ export async function discoverRemoteSourceCandidates(args: { })) } -async function listRemoteFixedChildFiles( +async function collectRemoteFixedChildFiles( source: RemoteSessionSource, context: RemoteScannerContext, - issues: AiVaultScanIssue[] -): Promise { + issues: AiVaultScanIssue[], + paths: string[] +): Promise { + context.discoveryBudget.enterDirectory(0) let entries try { entries = await context.provider.readDir(source.rootDir) } catch (err) { recordRemoteDirectoryIssue(source, context.executionHostId, issues, source.rootDir, err) - return [] + return } const segments = source.fixedChildFileSegments ?? [] // Why: Antigravity's transcript path is fixed. Constructing it avoids three // serialized SSH readDir round trips for every conversation directory. - return entries - .filter((entry) => entry.isDirectory && !entry.isSymlink) - .map((entry) => joinRemotePath(context.hostPlatform, source.rootDir, entry.name, ...segments)) - .filter((path) => source.filePredicate?.(path) ?? true) + for (const entry of entries) { + const entryPath = joinRemotePath(context.hostPlatform, source.rootDir, entry.name) + context.discoveryBudget.visitEntry(entryPath) + if (!entry.isDirectory || entry.isSymlink) { + continue + } + const path = joinRemotePath(context.hostPlatform, entryPath, ...segments) + if (source.filePredicate?.(path) ?? true) { + paths.push(path) + } + } } -async function walkRemoteSessionFiles( +async function collectRemoteSessionFiles( source: RemoteSessionSource, context: RemoteScannerContext, issues: AiVaultScanIssue[], + files: string[], dirPath = source.rootDir, depth = 0 -): Promise { +): Promise { + context.discoveryBudget.enterDirectory(depth) let entries try { entries = await context.provider.readDir(dirPath) } catch (err) { recordRemoteDirectoryIssue(source, context.executionHostId, issues, dirPath, err) - return [] + return } const extensions = new Set(source.extensions) - const files: string[] = [] for (const entry of entries) { const fullPath = joinRemotePath(context.hostPlatform, dirPath, entry.name) + context.discoveryBudget.visitEntry(fullPath) if ( entry.isDirectory && !entry.isSymlink && (source.directoryPredicate?.(entry.name, depth) ?? true) ) { - files.push(...(await walkRemoteSessionFiles(source, context, issues, fullPath, depth + 1))) + await collectRemoteSessionFiles(source, context, issues, files, fullPath, depth + 1) continue } if ( @@ -101,7 +129,6 @@ async function walkRemoteSessionFiles( files.push(fullPath) } } - return files } function recordRemoteDirectoryIssue( @@ -111,7 +138,7 @@ function recordRemoteDirectoryIssue( path: string, err: unknown ): void { - if (!isMissingRemoteSessionPathError(err)) { + if (!isMissingRemoteSessionPathError(err) && issues.length < REMOTE_DISCOVERY_ISSUE_MAX) { issues.push({ executionHostId, agent: source.agent, path, message: errorMessage(err) }) } } diff --git a/src/main/ai-vault/remote-session-scanner-types.ts b/src/main/ai-vault/remote-session-scanner-types.ts index 77806f38be6..4e785a7752e 100644 --- a/src/main/ai-vault/remote-session-scanner-types.ts +++ b/src/main/ai-vault/remote-session-scanner-types.ts @@ -4,6 +4,7 @@ import type { IFilesystemProvider } from '../providers/types' import type { RemoteHostPlatform } from '../ssh/ssh-remote-platform' import type { FileWithMtime } from './session-scanner-types' import type { AntigravityWorkspaceResolver } from './session-scanner-antigravity-history' +import type { AiVaultSessionDiscoveryBudget } from './session-discovery-budget' export type RemoteScannerContext = { provider: IFilesystemProvider @@ -11,6 +12,7 @@ export type RemoteScannerContext = { hostPlatform: RemoteHostPlatform titleCaches: Map>> antigravityWorkspaceResolver: AntigravityWorkspaceResolver + discoveryBudget: AiVaultSessionDiscoveryBudget } export type RemoteParserOptions = { diff --git a/src/main/ai-vault/remote-session-scanner.test.ts b/src/main/ai-vault/remote-session-scanner.test.ts index 4ba3bd4d647..49a6ff78907 100644 --- a/src/main/ai-vault/remote-session-scanner.test.ts +++ b/src/main/ai-vault/remote-session-scanner.test.ts @@ -3,6 +3,7 @@ import type { DirEntry } from '../../shared/types' import type { FileReadResult, FileStat, IFilesystemProvider } from '../providers/types' import { getRemoteHostPlatform } from '../ssh/ssh-remote-platform' import { scanRemoteAiVaultSessions } from './remote-session-scanner' +import { AI_VAULT_SESSION_ID_MAX_UTF8_BYTES } from './session-list-retention' class MemoryRemoteProvider implements IFilesystemProvider { private readonly files = new Map() @@ -668,6 +669,35 @@ describe('scanRemoteAiVaultSessions', () => { expect(result.sessions.map((session) => session.sessionId)).toEqual(['user-session']) }) + it('omits remote resume-critical overflow with a bounded issue', async () => { + const provider = new MemoryRemoteProvider() + provider.addFile( + '/home/ada/.codex/sessions/oversized.jsonl', + codexTranscript({ + sessionId: 'x'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES + 1), + title: 'Oversized remote identity', + cwd: '/home/ada/repo', + timestamp: '2026-07-04T04:00:00.000Z' + }), + 40 + ) + + const result = await scanRemoteAiVaultSessions({ + provider, + executionHostId: 'ssh:dev-box', + remoteHome: '/home/ada', + hostPlatform: getRemoteHostPlatform('linux-x64') + }) + + expect(result.sessions).toEqual([]) + expect(result.issues).toEqual([ + expect.objectContaining({ + path: '/home/ada/.codex/sessions/oversized.jsonl', + message: expect.stringContaining('session id exceeds the 65536 byte limit') + }) + ]) + }) + it('keeps scoped remote sessions even when they are older than the recency cap', async () => { const provider = new MemoryRemoteProvider() provider.addFile( @@ -706,6 +736,62 @@ describe('scanRemoteAiVaultSessions', () => { 'scoped-session' ]) }) + + it('accepts the aggregate discovery cap and reports cross-source overflow', async () => { + const exactProvider = new MemoryRemoteProvider() + exactProvider.addFile( + '/home/ada/.codex/sessions/exact.jsonl', + codexTranscript({ + sessionId: 'exact-session', + title: 'Exact capacity', + cwd: '/home/ada/repo', + timestamp: '2026-07-04T06:00:00.000Z' + }), + 60 + ) + const exact = await scanRemoteAiVaultSessions({ + provider: exactProvider, + executionHostId: 'ssh:dev-box', + remoteHome: '/home/ada', + hostPlatform: getRemoteHostPlatform('linux-x64'), + discoveryLimits: { maxEntries: 1 } + }) + + expect(exact.sessions.map((session) => session.sessionId)).toEqual(['exact-session']) + expect(exact.issues).toEqual([]) + + const overflowProvider = new MemoryRemoteProvider() + for (const [path, sessionId] of [ + ['/home/ada/.codex/sessions/default.jsonl', 'default-session'], + [ + '/home/ada/.local/share/orca/codex-runtime-home/home/sessions/managed.jsonl', + 'managed-session' + ] + ] as const) { + overflowProvider.addFile( + path, + codexTranscript({ + sessionId, + title: sessionId, + cwd: '/home/ada/repo', + timestamp: '2026-07-04T06:00:00.000Z' + }), + 60 + ) + } + const overflow = await scanRemoteAiVaultSessions({ + provider: overflowProvider, + executionHostId: 'ssh:dev-box', + remoteHome: '/home/ada', + hostPlatform: getRemoteHostPlatform('linux-x64'), + discoveryLimits: { maxEntries: 1 } + }) + + expect(overflow.sessions).toHaveLength(1) + expect(overflow.issues).toEqual([ + expect.objectContaining({ message: expect.stringContaining('safety limit') }) + ]) + }) }) function codexTranscript(args: { diff --git a/src/main/ai-vault/remote-session-scanner.ts b/src/main/ai-vault/remote-session-scanner.ts index 0ad894214e1..eb6afe3e417 100644 --- a/src/main/ai-vault/remote-session-scanner.ts +++ b/src/main/ai-vault/remote-session-scanner.ts @@ -18,6 +18,15 @@ import type { RemoteScannerContext, RemoteSessionCandidate } from './remote-sess import { sessionSortTime } from './session-scanner-accumulator' import { createAntigravityWorkspaceResolver } from './session-scanner-antigravity-history' import { errorMessage } from './session-scanner-values' +import { + AiVaultSessionDiscoveryBudget, + type AiVaultSessionDiscoveryLimits +} from './session-discovery-budget' +import { + boundAiVaultListResult, + retainAiVaultSession, + retainAiVaultSessionsWithinAggregate +} from './session-list-retention' const DEFAULT_REMOTE_SCAN_LIMIT = 1000 const REMOTE_SCAN_CONCURRENCY = 8 @@ -30,6 +39,7 @@ export async function scanRemoteAiVaultSessions(args: { hostPlatform: RemoteHostPlatform limit?: number scopePaths?: readonly string[] + discoveryLimits?: Partial }): Promise { const limit = args.limit && args.limit > 0 ? Math.floor(args.limit) : DEFAULT_REMOTE_SCAN_LIMIT const issues: AiVaultScanIssue[] = [] @@ -38,6 +48,7 @@ export async function scanRemoteAiVaultSessions(args: { executionHostId: args.executionHostId, hostPlatform: args.hostPlatform, titleCaches: new Map(), + discoveryBudget: new AiVaultSessionDiscoveryBudget(args.discoveryLimits), antigravityWorkspaceResolver: createAntigravityWorkspaceResolver(async (historyPath) => { try { const read = await args.provider.readFile(historyPath) @@ -85,11 +96,11 @@ export async function scanRemoteAiVaultSessions(args: { ...extraScopeSessions ]) - return { + return boundAiVaultListResult({ sessions: mergeRemoteSessions(cappedSessions, scopeSessions), issues, scannedAt: new Date().toISOString() - } + }) } async function parseRemoteSessionCandidates(args: { @@ -116,7 +127,12 @@ async function parseRemoteSessionCandidates(args: { ) sessions.push(...results.filter(isAiVaultSession)) const uniqueSessions = dedupeCodexSessionsBySessionId(sessions) - sessions.splice(0, sessions.length, ...uniqueSessions) + const retained = retainAiVaultSessionsWithinAggregate(uniqueSessions) + sessions.splice(0, sessions.length, ...retained.sessions) + if (retained.omitted > 0) { + addRemoteSessionCapacityIssue(args.issues, args.context.executionHostId, retained.omitted) + break + } index += batch.length } @@ -150,6 +166,12 @@ async function scanRemoteInScopeSessions(args: { isAiVaultSession(session) && isRemoteSessionInScope(session, args.scopePaths) ) ) + const retained = retainAiVaultSessionsWithinAggregate(sessions) + sessions.splice(0, sessions.length, ...retained.sessions) + if (retained.omitted > 0) { + addRemoteSessionCapacityIssue(args.issues, args.context.executionHostId, retained.omitted) + break + } } return sessions @@ -170,10 +192,9 @@ async function parseRemoteSessionCandidate( // transcript count (row badge; recoverable signal at zero turns). The // walk listing supplies it — the parser can't readdir a remote disk. const subagentTranscriptCount = candidate.subagentTranscriptCount ?? 0 - if (session && subagentTranscriptCount > 0) { - return { ...session, subagentTranscriptCount } - } - return session + const enriched = + session && subagentTranscriptCount > 0 ? { ...session, subagentTranscriptCount } : session + return enriched ? retainAiVaultSession(enriched) : null } catch (err) { issues.push({ executionHostId: context.executionHostId, @@ -233,6 +254,19 @@ function isAiVaultSession(session: AiVaultSession | null): session is AiVaultSes return Boolean(session) } +function addRemoteSessionCapacityIssue( + issues: AiVaultScanIssue[], + executionHostId: ExecutionHostId, + omitted: number +): void { + issues.push({ + executionHostId, + agent: 'codex', + path: 'AI Vault session list', + message: `AI Vault stopped after omitting ${omitted} sessions at its memory limit.` + }) +} + async function mapRemoteScanConcurrently( items: readonly T[], mapper: (item: T) => Promise diff --git a/src/main/ai-vault/session-discovery-budget.test.ts b/src/main/ai-vault/session-discovery-budget.test.ts new file mode 100644 index 00000000000..8f355588f6c --- /dev/null +++ b/src/main/ai-vault/session-discovery-budget.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'vitest' +import { + AiVaultSessionDiscoveryBudget, + AiVaultSessionDiscoveryCapacityError +} from './session-discovery-budget' + +describe('AiVaultSessionDiscoveryBudget', () => { + it('accepts the exact path-memory capacity and rejects the next entry', () => { + const budget = new AiVaultSessionDiscoveryBudget({ maxPathBytes: 130 }) + + expect(() => budget.visitEntry('a')).not.toThrow() + expect(() => budget.visitEntry('b')).toThrow(AiVaultSessionDiscoveryCapacityError) + }) +}) diff --git a/src/main/ai-vault/session-discovery-budget.ts b/src/main/ai-vault/session-discovery-budget.ts new file mode 100644 index 00000000000..ed92b15afc2 --- /dev/null +++ b/src/main/ai-vault/session-discovery-budget.ts @@ -0,0 +1,56 @@ +export const AI_VAULT_DISCOVERY_MAX_ENTRIES = 100_000 +export const AI_VAULT_DISCOVERY_MAX_DEPTH = 64 +export const AI_VAULT_DISCOVERY_MAX_PATH_BYTES = 32 * 1024 * 1024 + +const DISCOVERY_PATH_OVERHEAD_BYTES = 128 + +export type AiVaultSessionDiscoveryLimits = { + maxEntries: number + maxDepth: number + maxPathBytes: number +} + +export class AiVaultSessionDiscoveryCapacityError extends Error { + constructor() { + super( + 'AI Vault session discovery stopped at its safety limit (100,000 entries, 64 folder levels, or 32 MiB of path data).' + ) + this.name = 'AiVaultSessionDiscoveryCapacityError' + } +} + +export class AiVaultSessionDiscoveryBudget { + private entries = 0 + private pathBytes = 0 + readonly limits: AiVaultSessionDiscoveryLimits + + constructor(requested?: Partial) { + this.limits = { + maxEntries: clampLimit(requested?.maxEntries, AI_VAULT_DISCOVERY_MAX_ENTRIES), + maxDepth: clampLimit(requested?.maxDepth, AI_VAULT_DISCOVERY_MAX_DEPTH), + maxPathBytes: clampLimit(requested?.maxPathBytes, AI_VAULT_DISCOVERY_MAX_PATH_BYTES) + } + } + + enterDirectory(depth: number): void { + if (depth > this.limits.maxDepth) { + throw new AiVaultSessionDiscoveryCapacityError() + } + } + + visitEntry(path: string): void { + const nextPathBytes = this.pathBytes + path.length * 2 + DISCOVERY_PATH_OVERHEAD_BYTES + if (this.entries >= this.limits.maxEntries || nextPathBytes > this.limits.maxPathBytes) { + throw new AiVaultSessionDiscoveryCapacityError() + } + this.entries += 1 + this.pathBytes = nextPathBytes + } +} + +function clampLimit(value: number | undefined, maximum: number): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + return maximum + } + return Math.min(value, maximum) +} diff --git a/src/main/ai-vault/session-jsonl-line-reader.test.ts b/src/main/ai-vault/session-jsonl-line-reader.test.ts new file mode 100644 index 00000000000..c78449d48ab --- /dev/null +++ b/src/main/ai-vault/session-jsonl-line-reader.test.ts @@ -0,0 +1,76 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { consumeAiVaultJsonlLines, iterateAiVaultJsonlLines } from './session-jsonl-line-reader' + +const tempDirs: string[] = [] + +async function tempFile(content: string): Promise { + const directory = await mkdtemp(join(tmpdir(), 'orca-vault-jsonl-')) + tempDirs.push(directory) + const path = join(directory, 'session.jsonl') + await writeFile(path, content) + return path +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((directory) => rm(directory, { recursive: true }))) +}) + +describe('AI Vault bounded JSONL reader', () => { + it('preserves complete UTF-8 and CRLF records plus a trailing partial line', async () => { + const path = await tempFile('first🐋\r\nsecond\npartial') + const lines: string[] = [] + + const result = await consumeAiVaultJsonlLines({ + path, + onLine: (line) => lines.push(line) + }) + + expect(lines).toEqual(['first🐋', 'second']) + expect(result.trailingPartialLine).toBe('partial') + expect(result.consumedThrough).toBe(Buffer.byteLength('first🐋\r\nsecond\n')) + expect(result.bytesRead).toBe(Buffer.byteLength('first🐋\r\nsecond\npartial')) + }) + + it('drops only an oversized record and resumes at the next newline', async () => { + const path = await tempFile(`keep\n${'x'.repeat(40)}\nafter\n`) + const lines: string[] = [] + + const result = await consumeAiVaultJsonlLines({ + path, + maxRecordBytes: 8, + onLine: (line) => lines.push(line) + }) + + expect(lines).toEqual(['keep', 'after']) + expect(result.oversizedRecords).toBe(1) + expect(result.consumedThrough).toBe(Buffer.byteLength(`keep\n${'x'.repeat(40)}\nafter\n`)) + }) + + it('never materializes an oversized unterminated tail', async () => { + const path = await tempFile(`keep\n${'x'.repeat(40)}`) + const iterator = iterateAiVaultJsonlLines(path, { maxRecordBytes: 8 }) + const first = await iterator.next() + const end = await iterator.next() + + expect(first).toMatchObject({ done: false, value: 'keep' }) + expect(end.done).toBe(true) + expect(end.value).toMatchObject({ + trailingPartialLine: null, + trailingPartialOversized: true + }) + }) + + it('yields an accepted unterminated tail for one-shot parsers', async () => { + const path = await tempFile('complete\npartial') + const lines: string[] = [] + + for await (const line of iterateAiVaultJsonlLines(path)) { + lines.push(line) + } + + expect(lines).toEqual(['complete', 'partial']) + }) +}) diff --git a/src/main/ai-vault/session-jsonl-line-reader.ts b/src/main/ai-vault/session-jsonl-line-reader.ts new file mode 100644 index 00000000000..e9510c137d8 --- /dev/null +++ b/src/main/ai-vault/session-jsonl-line-reader.ts @@ -0,0 +1,116 @@ +import { createReadStream } from 'node:fs' + +const NEWLINE_BYTE = 0x0a +const CARRIAGE_RETURN_BYTE = 0x0d + +export const AI_VAULT_JSONL_MAX_RECORD_BYTES = 8 * 1024 * 1024 + +export type AiVaultJsonlReadResult = { + consumedThrough: number + trailingPartialLine: string | null + trailingPartialOversized: boolean + oversizedRecords: number + bytesRead: number +} + +function decodeLine(parts: Buffer[], byteLength: number): string { + const bytes = parts.length === 1 ? parts[0] : Buffer.concat(parts, byteLength) + const end = + byteLength > 0 && bytes[byteLength - 1] === CARRIAGE_RETURN_BYTE ? byteLength - 1 : byteLength + return bytes.toString('utf8', 0, end) +} + +export async function* iterateAiVaultJsonlLines( + path: string, + options: { + start?: number + maxRecordBytes?: number + yieldTrailingPartial?: boolean + } = {} +): AsyncGenerator { + const start = options.start ?? 0 + const maxRecordBytes = options.maxRecordBytes ?? AI_VAULT_JSONL_MAX_RECORD_BYTES + if (!Number.isSafeInteger(start) || start < 0) { + throw new RangeError('JSONL start offset must be a non-negative safe integer') + } + if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 0) { + throw new RangeError('JSONL record limit must be a non-negative safe integer') + } + + let bytesRead = 0 + let consumedThrough = start + let lineParts: Buffer[] = [] + let lineBytes = 0 + let discardingLine = false + let oversizedRecords = 0 + const stream = createReadStream(path, { start }) + + for await (const rawChunk of stream as AsyncIterable) { + const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) + const chunkOffset = start + bytesRead + bytesRead += chunk.length + let cursor = 0 + while (cursor < chunk.length) { + const newline = chunk.indexOf(NEWLINE_BYTE, cursor) + const segmentEnd = newline === -1 ? chunk.length : newline + const segmentLength = segmentEnd - cursor + + if (!discardingLine && lineBytes + segmentLength > maxRecordBytes) { + lineParts = [] + lineBytes = 0 + discardingLine = true + } + if (!discardingLine && segmentLength > 0) { + lineParts.push(chunk.subarray(cursor, segmentEnd)) + lineBytes += segmentLength + } + if (newline === -1) { + break + } + + consumedThrough = chunkOffset + newline + 1 + if (discardingLine) { + oversizedRecords += 1 + } else { + yield decodeLine(lineParts, lineBytes) + } + lineParts = [] + lineBytes = 0 + discardingLine = false + cursor = newline + 1 + } + } + + const trailingPartialLine = + !discardingLine && lineBytes > 0 ? decodeLine(lineParts, lineBytes) : null + if (options.yieldTrailingPartial !== false && trailingPartialLine !== null) { + yield trailingPartialLine + } + return { + consumedThrough, + trailingPartialLine, + trailingPartialOversized: discardingLine, + oversizedRecords, + bytesRead + } +} + +export async function consumeAiVaultJsonlLines(args: { + path: string + start?: number + maxRecordBytes?: number + onLine: (line: string) => void +}): Promise { + const iterator = iterateAiVaultJsonlLines(args.path, { + start: args.start, + maxRecordBytes: args.maxRecordBytes, + yieldTrailingPartial: false + }) + while (true) { + const next = await iterator.next() + if (next.done) { + return next.value + } + args.onLine(next.value) + } +} diff --git a/src/main/ai-vault/session-list-results.test.ts b/src/main/ai-vault/session-list-results.test.ts new file mode 100644 index 00000000000..d1cbe002046 --- /dev/null +++ b/src/main/ai-vault/session-list-results.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import type { AiVaultListResult, AiVaultSession } from '../../shared/ai-vault-types' +import { mergeAiVaultListResults } from './session-list-results' + +function session(id: string, modifiedAt: string): AiVaultSession { + return { + id, + executionHostId: 'local', + agent: 'claude', + sessionId: id, + title: id, + cwd: '/repo', + branch: 'main', + model: null, + filePath: `/${id}`, + codexHome: null, + createdAt: null, + updatedAt: null, + modifiedAt, + messageCount: 1, + totalTokens: 0, + previewMessages: [], + queuedMessageCount: 0, + subagentTranscriptCount: 0, + resumeCommand: `claude --resume ${id}`, + subagent: null + } +} + +function result(sessions: AiVaultSession[]): AiVaultListResult { + return { sessions, issues: [], scannedAt: '2026-01-01T00:00:00.000Z' } +} + +describe('AI Vault list result merging', () => { + it('applies the result limit after later hosts replace duplicate sessions', () => { + const newest = session('duplicate', '2026-04-03T00:00:00.000Z') + const fallback = session('fallback', '2026-04-02T00:00:00.000Z') + const replacement = session('duplicate', '2026-04-01T00:00:00.000Z') + const otherHost = session('other-host', '2026-04-01T12:00:00.000Z') + + const merged = mergeAiVaultListResults( + [result([newest, fallback]), result([replacement, otherHost])], + 1 + ) + + expect(merged.sessions).toEqual([fallback]) + }) +}) diff --git a/src/main/ai-vault/session-list-results.ts b/src/main/ai-vault/session-list-results.ts index 06f8df9c39a..fd6466d40ef 100644 --- a/src/main/ai-vault/session-list-results.ts +++ b/src/main/ai-vault/session-list-results.ts @@ -1,17 +1,14 @@ -import type { - AiVaultListResult, - AiVaultScanIssue, - AiVaultSession -} from '../../shared/ai-vault-types' +import type { AiVaultListResult, AiVaultSession } from '../../shared/ai-vault-types' import type { ExecutionHostId } from '../../shared/execution-host' import { sessionSortTime } from './session-scanner-accumulator' +import { boundAiVaultListResult } from './session-list-retention' export function aiVaultScanIssueResult(args: { executionHostId?: ExecutionHostId path: string message: string }): AiVaultListResult { - return { + return boundAiVaultListResult({ sessions: [], issues: [ { @@ -22,7 +19,7 @@ export function aiVaultScanIssueResult(args: { } ], scannedAt: new Date().toISOString() - } + }) } // Why: the serving-side scan is host-local and cached once for every caller @@ -33,8 +30,9 @@ export function restampAiVaultListResult( result: AiVaultListResult, executionHostId: ExecutionHostId ): AiVaultListResult { - return { - sessions: result.sessions.map((session) => + const retained = boundAiVaultListResult(result) + return boundAiVaultListResult({ + sessions: retained.sessions.map((session) => session.executionHostId === executionHostId ? session : { @@ -43,9 +41,9 @@ export function restampAiVaultListResult( id: `${executionHostId}:${session.agent}:${session.sessionId}:${session.filePath}` } ), - issues: result.issues.map((issue) => ({ ...issue, executionHostId })), - scannedAt: result.scannedAt - } + issues: retained.issues.map((issue) => ({ ...issue, executionHostId })), + scannedAt: retained.scannedAt + }) } export function mergeAiVaultListResults( @@ -53,19 +51,22 @@ export function mergeAiVaultListResults( rawLimit: number | undefined ): AiVaultListResult { const limit = rawLimit && rawLimit > 0 ? Math.floor(rawLimit) : 1000 - const byId = new Map() - const issues: AiVaultScanIssue[] = [] - for (const result of results) { - for (const session of result.sessions) { + let merged: AiVaultListResult = { sessions: [], issues: [], scannedAt: new Date().toISOString() } + for (let index = 0; index < results.length; index += 1) { + const rawResult = results[index] + const result = boundAiVaultListResult(rawResult) + const byId = new Map() + for (const session of [...merged.sessions, ...result.sessions]) { byId.set(session.id, session) } - issues.push(...result.issues) - } - return { - sessions: [...byId.values()] - .sort((left, right) => sessionSortTime(right) - sessionSortTime(left)) - .slice(0, limit), - issues, - scannedAt: new Date().toISOString() + const sessions = [...byId.values()].sort( + (left, right) => sessionSortTime(right) - sessionSortTime(left) + ) + merged = boundAiVaultListResult({ + sessions: index === results.length - 1 ? sessions.slice(0, limit) : sessions, + issues: [...merged.issues, ...result.issues], + scannedAt: new Date().toISOString() + }) } + return merged } diff --git a/src/main/ai-vault/session-list-retention.test.ts b/src/main/ai-vault/session-list-retention.test.ts new file mode 100644 index 00000000000..d15d7f96d4c --- /dev/null +++ b/src/main/ai-vault/session-list-retention.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from 'vitest' +import type { + AiVaultListResult, + AiVaultScanIssue, + AiVaultSession +} from '../../shared/ai-vault-types' +import { + AI_VAULT_SCAN_ISSUE_MAX_ENTRIES, + AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES, + AI_VAULT_SESSION_ID_MAX_UTF8_BYTES, + AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES, + AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES, + AiVaultSessionCapacityError, + aiVaultSessionListCacheKey, + boundAiVaultListResult, + retainAiVaultSession, + retainAiVaultSessionsWithinAggregate +} from './session-list-retention' + +function session(overrides: Partial = {}): AiVaultSession { + return { + id: 'local:claude:session:/transcript', + executionHostId: 'local', + agent: 'claude', + sessionId: 'session', + title: 'Title', + cwd: '/repo', + branch: 'main', + model: 'claude-test', + filePath: '/transcript', + codexHome: null, + createdAt: null, + updatedAt: null, + modifiedAt: '2026-01-01T00:00:00.000Z', + messageCount: 1, + totalTokens: 1, + previewMessages: [], + queuedMessageCount: 0, + subagentTranscriptCount: 0, + resumeCommand: 'claude --resume session', + subagent: null, + ...overrides + } +} + +function result(sessions: AiVaultSession[], issues: AiVaultScanIssue[] = []): AiVaultListResult { + return { sessions, issues, scannedAt: '2026-01-01T00:00:00.000Z' } +} + +describe('AI Vault session list retention', () => { + it('returns exact under-limit sessions and results by identity', () => { + const normalSession = session() + const normalResult = result( + [normalSession], + [{ agent: 'claude', path: '/transcript', message: 'recoverable warning' }] + ) + + expect(retainAiVaultSession(normalSession)).toBe(normalSession) + expect(boundAiVaultListResult(normalResult)).toBe(normalResult) + }) + + it('accepts exact UTF-8 identity/path limits and rejects one byte over', () => { + const exactId = 'é'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES / 2) + const exactPath = 'é'.repeat(AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES / 2) + expect(retainAiVaultSession(session({ sessionId: exactId, filePath: exactPath }))).toEqual( + expect.objectContaining({ sessionId: exactId, filePath: exactPath }) + ) + + expect(() => retainAiVaultSession(session({ sessionId: `${exactId}x` }))).toThrow( + AiVaultSessionCapacityError + ) + expect(() => retainAiVaultSession(session({ filePath: `${exactPath}x` }))).toThrow( + AiVaultSessionCapacityError + ) + }) + + it('UTF-8 truncates display-only metadata without splitting a code point', () => { + const exact = 'é'.repeat(AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES / 2) + expect(retainAiVaultSession(session({ model: exact })).model).toBe(exact) + + const retained = retainAiVaultSession(session({ model: `${exact}x` })) + expect(retained.model).toMatch(/\.\.\.$/) + expect(Buffer.byteLength(retained.model ?? '', 'utf8')).toBeLessThanOrEqual( + AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES + ) + expect(retained.model).not.toContain('\ufffd') + }) + + it('uses the exact list boundary and keeps the newest session prefix', () => { + const sessions = [session({ id: 'newest' }), session({ id: 'older' })] + const exactArrayBytes = Buffer.byteLength(JSON.stringify(sessions), 'utf8') + expect(retainAiVaultSessionsWithinAggregate(sessions, exactArrayBytes)).toEqual({ + sessions, + omitted: 0 + }) + expect(retainAiVaultSessionsWithinAggregate(sessions, exactArrayBytes - 1)).toEqual({ + sessions: [sessions[0]], + omitted: 1 + }) + + const exactResult = result(sessions) + const exactResultBytes = Buffer.byteLength(JSON.stringify(exactResult), 'utf8') + expect(boundAiVaultListResult(exactResult, exactResultBytes)).toBe(exactResult) + const overflow = boundAiVaultListResult(exactResult, exactResultBytes - 1) + expect(Buffer.byteLength(JSON.stringify(overflow), 'utf8')).toBeLessThanOrEqual( + exactResultBytes - 1 + ) + expect(overflow.issues.at(-1)?.message).toContain('AI Vault omitted') + }) + + it('turns critical overflow into one bounded issue and caps issue count', () => { + const oversized = session({ + sessionId: 'x'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES + 1) + }) + const issues = Array.from({ length: AI_VAULT_SCAN_ISSUE_MAX_ENTRIES + 1 }, (_, index) => ({ + agent: 'claude' as const, + path: `/warning/${index}`, + message: 'warning' + })) + + const bounded = boundAiVaultListResult(result([oversized], issues)) + expect(bounded.sessions).toEqual([]) + expect(bounded.issues).toHaveLength(AI_VAULT_SCAN_ISSUE_MAX_ENTRIES) + expect(bounded.issues.at(-1)?.message).toContain('1 sessions and 2 scan issues') + }) + + it('admits an exact-limit cache key and bypasses caching one byte over', () => { + const overhead = Buffer.byteLength(JSON.stringify({ scopePaths: [''] }), 'utf8') + const exact = { + scopePaths: ['x'.repeat(AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES - overhead)] + } + expect(Buffer.byteLength(JSON.stringify(exact), 'utf8')).toBe( + AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES + ) + expect(aiVaultSessionListCacheKey(exact)).toBe(JSON.stringify(exact)) + expect(aiVaultSessionListCacheKey({ scopePaths: [`${exact.scopePaths[0]}x`] })).toBeNull() + }) +}) diff --git a/src/main/ai-vault/session-list-retention.ts b/src/main/ai-vault/session-list-retention.ts new file mode 100644 index 00000000000..19e085bc79c --- /dev/null +++ b/src/main/ai-vault/session-list-retention.ts @@ -0,0 +1,321 @@ +import type { + AiVaultListResult, + AiVaultScanIssue, + AiVaultSession +} from '../../shared/ai-vault-types' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' + +export const AI_VAULT_SESSION_ID_MAX_UTF8_BYTES = 64 * 1024 +export const AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES = 128 * 1024 +export const AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES = 4 * 1024 +export const AI_VAULT_SESSION_MAX_JSON_BYTES = 1024 * 1024 +export const AI_VAULT_SESSION_LIST_MAX_JSON_BYTES = 32 * 1024 * 1024 +export const AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES = 256 * 1024 +export const AI_VAULT_SCAN_ISSUE_MAX_ENTRIES = 1024 + +const AI_VAULT_SESSION_DERIVED_FIELD_MAX_UTF8_BYTES = 512 * 1024 +const AI_VAULT_SCAN_ISSUE_FIELD_MAX_UTF8_BYTES = 16 * 1024 +const AI_VAULT_SCAN_ISSUE_MAX_JSON_BYTES = 64 * 1024 +const AI_VAULT_SESSION_PREVIEW_MAX_MESSAGES = 5 +const TRUNCATION_MARKER = '...' + +export class AiVaultSessionCapacityError extends Error { + constructor(field: string, maxBytes: number) { + super(`AI Vault omitted a session because its ${field} exceeds the ${maxBytes} byte limit.`) + this.name = 'AiVaultSessionCapacityError' + } +} + +export function retainAiVaultSession(session: AiVaultSession): AiVaultSession { + requireField(session.executionHostId, 64 * 1024, 'execution host id') + requireField(session.sessionId, AI_VAULT_SESSION_ID_MAX_UTF8_BYTES, 'session id') + requireField(session.cwd, AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES, 'working directory') + requireField(session.filePath, AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES, 'transcript path') + requireField(session.codexHome, AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES, 'Codex home') + requireField(session.id, AI_VAULT_SESSION_DERIVED_FIELD_MAX_UTF8_BYTES, 'derived id') + requireField( + session.resumeCommand, + AI_VAULT_SESSION_DERIVED_FIELD_MAX_UTF8_BYTES, + 'resume command' + ) + if (session.subagent) { + requireField( + session.subagent.parentSessionId, + AI_VAULT_SESSION_ID_MAX_UTF8_BYTES, + 'parent session id' + ) + } + + const title = truncateUtf8(session.title, AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES) + const branch = truncateNullableUtf8(session.branch, AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES) + const model = truncateNullableUtf8(session.model, AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES) + const lastUserPrompt = truncateNullableUtf8( + session.lastUserPrompt, + AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES + ) + const previewSource = + session.previewMessages.length > AI_VAULT_SESSION_PREVIEW_MAX_MESSAGES + ? session.previewMessages.slice(-AI_VAULT_SESSION_PREVIEW_MAX_MESSAGES) + : session.previewMessages + const previewMessages = previewSource.map((message) => { + const text = truncateUtf8(message.text, AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES) + return text === message.text ? message : { ...message, text } + }) + const agentType = session.subagent + ? truncateNullableUtf8( + session.subagent.agentType, + AI_VAULT_SESSION_DISPLAY_FIELD_MAX_UTF8_BYTES + ) + : undefined + const changed = + title !== session.title || + branch !== session.branch || + model !== session.model || + lastUserPrompt !== session.lastUserPrompt || + previewSource !== session.previewMessages || + previewMessages.some((message, index) => message !== session.previewMessages[index]) || + agentType !== session.subagent?.agentType + const retained = changed + ? { + ...session, + title, + branch, + model, + ...(session.lastUserPrompt === undefined ? {} : { lastUserPrompt }), + previewMessages, + ...(session.subagent + ? { subagent: { ...session.subagent, agentType: agentType ?? null } } + : {}) + } + : session + try { + stringifyJsonWithinByteLimit(retained, AI_VAULT_SESSION_MAX_JSON_BYTES) + } catch { + throw new AiVaultSessionCapacityError('serialized metadata', AI_VAULT_SESSION_MAX_JSON_BYTES) + } + return retained +} + +export function retainAiVaultSessionsWithinAggregate( + sessions: readonly AiVaultSession[], + maxBytes: number = AI_VAULT_SESSION_LIST_MAX_JSON_BYTES +): { + sessions: AiVaultSession[] + omitted: number +} { + const limit = boundedListLimit(maxBytes) + const retained: AiVaultSession[] = [] + let retainedBytes = 2 + let omitted = 0 + for (let index = 0; index < sessions.length; index += 1) { + try { + const session = retainAiVaultSession(sessions[index]) + const bytes = stringifyJsonWithinByteLimit( + session, + AI_VAULT_SESSION_MAX_JSON_BYTES + ).byteLength + const nextBytes = retainedBytes + bytes + (retained.length > 0 ? 1 : 0) + if (nextBytes > limit) { + omitted += sessions.length - index + break + } + retained.push(session) + retainedBytes = nextBytes + } catch { + omitted++ + } + } + return { sessions: retained, omitted } +} + +export function boundAiVaultListResult( + result: AiVaultListResult, + maxBytes: number = AI_VAULT_SESSION_LIST_MAX_JSON_BYTES +): AiVaultListResult { + const limit = boundedListLimit(maxBytes) + const scannedAt = truncateUtf8(result.scannedAt, 256) + const baseBytes = stringifyJsonWithinByteLimit( + { sessions: [], issues: [], scannedAt }, + limit + ).byteLength + const sessions: AiVaultSession[] = [] + const sessionBytes: number[] = [] + const issues: AiVaultScanIssue[] = [] + const issueBytes: number[] = [] + let retainedBytes = baseBytes + let omittedSessions = 0 + let omittedIssues = 0 + let firstOmittedAgent: AiVaultScanIssue['agent'] | null = null + let changed = scannedAt !== result.scannedAt + + for (let index = 0; index < result.sessions.length; index += 1) { + try { + const session = retainAiVaultSession(result.sessions[index]) + const bytes = stringifyJsonWithinByteLimit( + session, + AI_VAULT_SESSION_MAX_JSON_BYTES + ).byteLength + const nextBytes = retainedBytes + bytes + (sessions.length > 0 ? 1 : 0) + if (nextBytes > limit) { + omittedSessions += result.sessions.length - index + firstOmittedAgent ??= session.agent + changed = true + break + } + sessions.push(session) + sessionBytes.push(bytes) + retainedBytes = nextBytes + changed ||= session !== result.sessions[index] + } catch { + omittedSessions++ + firstOmittedAgent ??= result.sessions[index].agent + changed = true + } + } + + for (let index = 0; index < result.issues.length; index += 1) { + if (issues.length >= AI_VAULT_SCAN_ISSUE_MAX_ENTRIES) { + omittedIssues += result.issues.length - index + changed = true + break + } + const issue = retainScanIssue(result.issues[index]) + const bytes = stringifyJsonWithinByteLimit(issue, AI_VAULT_SCAN_ISSUE_MAX_JSON_BYTES).byteLength + const nextBytes = retainedBytes + bytes + (issues.length > 0 ? 1 : 0) + if (nextBytes > limit) { + omittedIssues += result.issues.length - index + changed = true + break + } + issues.push(issue) + issueBytes.push(bytes) + retainedBytes = nextBytes + changed ||= issue !== result.issues[index] + } + + if (omittedSessions > 0 || omittedIssues > 0) { + let summary = overflowSummary(firstOmittedAgent, omittedSessions, omittedIssues) + let summaryBytes = issueJsonBytes(summary) + while ( + issues.length >= AI_VAULT_SCAN_ISSUE_MAX_ENTRIES || + retainedBytes + summaryBytes + (issues.length > 0 ? 1 : 0) > limit + ) { + if (issues.length > 0) { + const removedBytes = issueBytes.pop() ?? 0 + issues.pop() + retainedBytes -= removedBytes + (issues.length > 0 ? 1 : 0) + omittedIssues++ + } else if (sessions.length > 0) { + const removedBytes = sessionBytes.pop() ?? 0 + sessions.pop() + retainedBytes -= removedBytes + (sessions.length > 0 ? 1 : 0) + omittedSessions++ + } else { + break + } + summary = overflowSummary(firstOmittedAgent, omittedSessions, omittedIssues) + summaryBytes = issueJsonBytes(summary) + } + issues.push(summary) + } + + return changed ? { sessions, issues, scannedAt } : result +} + +export function aiVaultSessionListCacheKey(value: unknown): string | null { + try { + return stringifyJsonWithinByteLimit(value, AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES) + .serialized + } catch { + return null + } +} + +function retainScanIssue(issue: AiVaultScanIssue): AiVaultScanIssue { + const executionHostId = + issue.executionHostId && + !exceedsUtf8Bytes(issue.executionHostId, AI_VAULT_SCAN_ISSUE_FIELD_MAX_UTF8_BYTES) + ? issue.executionHostId + : undefined + const path = truncateUtf8(issue.path, AI_VAULT_SCAN_ISSUE_FIELD_MAX_UTF8_BYTES) + const message = truncateUtf8(issue.message, AI_VAULT_SCAN_ISSUE_FIELD_MAX_UTF8_BYTES) + return executionHostId === issue.executionHostId && + path === issue.path && + message === issue.message + ? issue + : { + ...issue, + executionHostId, + path, + message + } +} + +function requireField(value: string | null, maxBytes: number, field: string): void { + if (value !== null && exceedsUtf8Bytes(value, maxBytes)) { + throw new AiVaultSessionCapacityError(field, maxBytes) + } +} + +function truncateNullableUtf8(value: string | null, maxBytes: number): string | null +function truncateNullableUtf8( + value: string | null | undefined, + maxBytes: number +): string | null | undefined +function truncateNullableUtf8( + value: string | null | undefined, + maxBytes: number +): string | null | undefined { + return value == null ? value : truncateUtf8(value, maxBytes) +} + +function truncateUtf8(value: string, maxBytes: number): string { + if (!exceedsUtf8Bytes(value, maxBytes)) { + return value + } + const prefixBytes = Math.max(0, maxBytes - TRUNCATION_MARKER.length) + const end = utf8PrefixEnd(value, prefixBytes) + return `${value.slice(0, end)}${TRUNCATION_MARKER}` +} + +function exceedsUtf8Bytes(value: string, maxBytes: number): boolean { + return utf8PrefixEnd(value, maxBytes) < value.length +} + +function utf8PrefixEnd(value: string, maxBytes: number): number { + let bytes = 0 + let index = 0 + while (index < value.length) { + const code = value.codePointAt(index) ?? 0 + const charBytes = code <= 0x7f ? 1 : code <= 0x7ff ? 2 : code <= 0xffff ? 3 : 4 + if (bytes + charBytes > maxBytes) { + break + } + bytes += charBytes + index += code > 0xffff ? 2 : 1 + } + return index +} + +function overflowSummary( + agent: AiVaultScanIssue['agent'] | null, + omittedSessions: number, + omittedIssues: number +): AiVaultScanIssue { + return { + agent: agent ?? 'codex', + path: 'AI Vault session list', + message: `AI Vault omitted ${omittedSessions} sessions and ${omittedIssues} scan issues because the result exceeded its memory limits.` + } +} + +function issueJsonBytes(issue: AiVaultScanIssue): number { + return stringifyJsonWithinByteLimit(issue, AI_VAULT_SCAN_ISSUE_MAX_JSON_BYTES).byteLength +} + +function boundedListLimit(requested: number): number { + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError('AI Vault session list limit must be a non-negative safe integer') + } + return Math.min(requested, AI_VAULT_SESSION_LIST_MAX_JSON_BYTES) +} diff --git a/src/main/ai-vault/session-parse-cache-payload.test.ts b/src/main/ai-vault/session-parse-cache-payload.test.ts new file mode 100644 index 00000000000..21fe145e1ca --- /dev/null +++ b/src/main/ai-vault/session-parse-cache-payload.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'vitest' +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { + AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES, + AI_VAULT_PARSE_CACHE_PERSIST_MAX_UTF8_BYTES, + serializeSessionParseCachePayload +} from './session-parse-cache-payload' +import type { PersistedSessionParseCacheEntry } from './session-parse-cache-retention' +import { AI_VAULT_WHOLE_JSON_MAX_BYTES } from './session-whole-json-reader' + +function entry(model: string | null = null): PersistedSessionParseCacheEntry { + return { + mtimeMs: 1, + sizeBytes: 2, + platform: process.platform, + session: session(model) + } +} + +function session(model: string | null): AiVaultSession { + return { + id: 'local:claude:session:/transcript', + executionHostId: 'local', + agent: 'claude', + sessionId: 'session', + title: 'Title', + cwd: '/repo', + branch: 'main', + model, + filePath: '/transcript', + codexHome: null, + createdAt: null, + updatedAt: null, + modifiedAt: '2026-01-01T00:00:00.000Z', + messageCount: 1, + totalTokens: 1, + previewMessages: [], + queuedMessageCount: 0, + subagentTranscriptCount: 0, + resumeCommand: 'claude --resume session', + subagent: null + } +} + +describe('session parse cache payload', () => { + it('is byte-identical to native JSON when every LRU entry fits', () => { + const entries: [string, PersistedSessionParseCacheEntry][] = [ + ['/old\n✨', entry('first')], + ['/new', entry('second')] + ] + const expected = JSON.stringify({ schemaVersion: 1, appVersion: '1.2.3', entries }) + + expect( + serializeSessionParseCachePayload({ schemaVersion: 1, appVersion: '1.2.3', entries }) + ).toBe(expected) + }) + + it('uses the exact byte boundary and prioritizes newest LRU entries', () => { + const older: [string, PersistedSessionParseCacheEntry] = ['/older', entry('x'.repeat(200))] + const middle: [string, PersistedSessionParseCacheEntry] = ['/middle', entry('middle')] + const newest: [string, PersistedSessionParseCacheEntry] = ['/newest', entry('newest')] + const expected = JSON.stringify({ + schemaVersion: 1, + appVersion: '1.2.3', + entries: [middle, newest] + }) + const exactBytes = Buffer.byteLength(expected, 'utf8') + + const exact = serializeSessionParseCachePayload({ + schemaVersion: 1, + appVersion: '1.2.3', + entries: [older, middle, newest], + maxBytes: exactBytes + }) + expect(exact).toBe(expected) + expect(Buffer.byteLength(exact, 'utf8')).toBe(exactBytes) + + const overflow = JSON.parse( + serializeSessionParseCachePayload({ + schemaVersion: 1, + appVersion: '1.2.3', + entries: [older, middle, newest], + maxBytes: exactBytes - 1 + }) + ) as { entries: [string, PersistedSessionParseCacheEntry][] } + expect(overflow.entries.map(([path]) => path)).toEqual(['/newest']) + }) + + it('caps production output below the 64 MiB reader ceiling', () => { + const payload = serializeSessionParseCachePayload({ + schemaVersion: 1, + appVersion: '1.2.3', + entries: [['/too-large', entry('x'.repeat(1024 * 1024))]], + maxBytes: 512 + }) + + expect(JSON.parse(payload).entries).toEqual([]) + expect(Buffer.byteLength(payload, 'utf8')).toBeLessThanOrEqual(512) + expect(AI_VAULT_PARSE_CACHE_PERSIST_MAX_UTF8_BYTES).toBeLessThan(AI_VAULT_WHOLE_JSON_MAX_BYTES) + }) + + it('admits an exact-limit encoded app version and rejects one byte over', () => { + const exact = 'x'.repeat(AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES - 2) + expect(Buffer.byteLength(JSON.stringify(exact), 'utf8')).toBe( + AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES + ) + expect(() => + serializeSessionParseCachePayload({ schemaVersion: 1, appVersion: exact, entries: [] }) + ).not.toThrow() + + expect(() => + serializeSessionParseCachePayload({ + schemaVersion: 1, + appVersion: `${exact}x`, + entries: [] + }) + ).toThrow(`JSON output exceeds ${AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES} bytes`) + }) +}) diff --git a/src/main/ai-vault/session-parse-cache-payload.ts b/src/main/ai-vault/session-parse-cache-payload.ts new file mode 100644 index 00000000000..c0d731473bb --- /dev/null +++ b/src/main/ai-vault/session-parse-cache-payload.ts @@ -0,0 +1,66 @@ +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../../shared/node-bounded-json-stringify' +import type { PersistedSessionParseCacheEntry } from './session-parse-cache-retention' + +// Half the loader's 64 MiB ceiling leaves headroom for the UTF-16 string and +// the writeFile encoding buffer to coexist during a save. +export const AI_VAULT_PARSE_CACHE_PERSIST_MAX_UTF8_BYTES = 32 * 1024 * 1024 +export const AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES = 4 * 1024 + +export function serializeSessionParseCachePayload(args: { + schemaVersion: number + appVersion: string + entries: readonly [string, PersistedSessionParseCacheEntry][] + maxBytes?: number +}): string { + if (!Number.isSafeInteger(args.schemaVersion)) { + throw new TypeError('Session parse cache schema version must be a safe integer') + } + const maxBytes = boundedPayloadLimit(args.maxBytes) + const appVersion = stringifyJsonWithinByteLimit( + args.appVersion, + AI_VAULT_PARSE_CACHE_APP_VERSION_MAX_JSON_BYTES + ).serialized + const prefix = `{"schemaVersion":${args.schemaVersion},"appVersion":${appVersion},"entries":[` + const suffix = ']}' + let retainedBytes = Buffer.byteLength(prefix, 'utf8') + Buffer.byteLength(suffix, 'utf8') + if (retainedBytes > maxBytes) { + throw new JsonStringifyByteLimitError(retainedBytes, maxBytes) + } + + const newestFirst: string[] = [] + for (let index = args.entries.length - 1; index >= 0; index -= 1) { + const separatorBytes = newestFirst.length === 0 ? 0 : 1 + const remaining = maxBytes - retainedBytes - separatorBytes + if (remaining < 0) { + continue + } + try { + const encoded = stringifyJsonWithinByteLimit(args.entries[index], remaining) + newestFirst.push(encoded.serialized) + retainedBytes += separatorBytes + encoded.byteLength + } catch { + // One entry that cannot fit must not prevent smaller older entries from persisting. + } + } + + const fragments = [prefix] + for (let index = newestFirst.length - 1; index >= 0; index -= 1) { + if (index < newestFirst.length - 1) { + fragments.push(',') + } + fragments.push(newestFirst[index]) + } + fragments.push(suffix) + return fragments.join('') +} + +function boundedPayloadLimit(requested: number | undefined): number { + const limit = requested ?? AI_VAULT_PARSE_CACHE_PERSIST_MAX_UTF8_BYTES + if (!Number.isSafeInteger(limit) || limit < 0) { + throw new RangeError('Session parse cache payload limit must be a non-negative safe integer') + } + return Math.min(limit, AI_VAULT_PARSE_CACHE_PERSIST_MAX_UTF8_BYTES) +} diff --git a/src/main/ai-vault/session-parse-cache-persistence.ts b/src/main/ai-vault/session-parse-cache-persistence.ts index f4a924bd82f..519e024785d 100644 --- a/src/main/ai-vault/session-parse-cache-persistence.ts +++ b/src/main/ai-vault/session-parse-cache-persistence.ts @@ -3,7 +3,7 @@ // of re-reading the whole transcript corpus (issue #9210: 6.7 GB / 109 s cold // scans). Disabled unless the composition root calls init; every failure mode // degrades to today's cold-scan behavior. -import { mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { mkdir, opendir, rename, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { seedSessionParseCache, @@ -11,6 +11,9 @@ import { type PersistedSessionParseCacheEntry, type SessionParseStats } from './session-scanner-parse-cache' +import { serializeSessionParseCachePayload } from './session-parse-cache-payload' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' +import { parseAiVaultJsonText } from './session-scanner-values' // Bump when the persisted entry layout changes; a mismatched file is discarded whole. const SCHEMA_VERSION = 1 @@ -98,11 +101,12 @@ export async function flushSessionParseCachePersistForTests(): Promise { async function loadPersistedEntries(current: SessionParseCachePersistenceOptions): Promise { await sweepOrphanedTempFiles(current.filePath) try { - const raw = await readFile(current.filePath, 'utf-8') - const entries = parsePersistedFile(JSON.parse(raw), current.appVersion) - if (entries) { - seedSessionParseCache(entries) - } + await withAiVaultWholeJsonFile(current.filePath, (content) => { + const entries = parsePersistedFile(parseAiVaultJsonText(content), current.appVersion) + if (entries) { + seedSessionParseCache(entries) + } + }) } catch { // Why: a missing/corrupt/foreign cache file must never fail the scan; // worst case is exactly today's cold scan. @@ -115,12 +119,12 @@ async function loadPersistedEntries(current: SessionParseCachePersistenceOptions async function sweepOrphanedTempFiles(filePath: string): Promise { const directory = dirname(filePath) try { - const names = await readdir(directory) - await Promise.all( - names - .filter((name) => name.startsWith('session-parse-cache-') && name.endsWith('.tmp')) - .map((name) => rm(join(directory, name), { force: true }).catch(() => {})) - ) + const entries = await opendir(directory) + for await (const entry of entries) { + if (entry.name.startsWith('session-parse-cache-') && entry.name.endsWith('.tmp')) { + await rm(join(directory, entry.name), { force: true }).catch(() => {}) + } + } } catch { // Directory missing or unreadable — nothing to sweep. } @@ -190,7 +194,7 @@ async function persistSnapshot(current: SessionParseCachePersistenceOptions): Pr const directory = dirname(current.filePath) const tempPath = join(directory, `session-parse-cache-${process.pid}-${Date.now()}.tmp`) try { - const payload = JSON.stringify({ + const payload = serializeSessionParseCachePayload({ schemaVersion: SCHEMA_VERSION, appVersion: current.appVersion, entries: snapshotSessionParseCacheForPersistence() diff --git a/src/main/ai-vault/session-parse-cache-retention.test.ts b/src/main/ai-vault/session-parse-cache-retention.test.ts new file mode 100644 index 00000000000..980e63e72bc --- /dev/null +++ b/src/main/ai-vault/session-parse-cache-retention.test.ts @@ -0,0 +1,149 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { + AI_VAULT_PARSE_CACHE_KEY_MAX_UTF8_BYTES, + AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES, + inspectSessionParseCacheRetentionForTests, + resetSessionParseCacheRetentionForTests, + seedSessionParseCache, + setSessionParseCacheMaxRetainedBytesForTests, + snapshotSessionParseCacheForPersistence, + storeSessionParseCacheEntry, + type PersistedSessionParseCacheEntry, + type SessionParseCacheEntry +} from './session-parse-cache-retention' +import type { ResumableSessionParseState } from './session-scanner-types' + +beforeEach(() => resetSessionParseCacheRetentionForTests()) +afterEach(() => resetSessionParseCacheRetentionForTests()) + +function persistedEntry(model: string | null = null): PersistedSessionParseCacheEntry { + return { + mtimeMs: 1, + sizeBytes: 2, + platform: process.platform, + session: session(model) + } +} + +function cacheEntry(model: string | null = null): SessionParseCacheEntry { + return { ...persistedEntry(model), resume: null } +} + +function cacheEntryWithResumeBytes(bytes: number): SessionParseCacheEntry { + let state: ResumableSessionParseState + state = { + consumeLine: () => {}, + clone: () => state, + retainedUtf8Bytes: () => bytes, + touchFile: () => {}, + finalize: () => null + } + return { ...persistedEntry(), resume: { state, byteOffset: 0 } } +} + +function session(model: string | null): AiVaultSession { + return { + id: 'local:claude:session:/transcript', + executionHostId: 'local', + agent: 'claude', + sessionId: 'session', + title: 'Title', + cwd: '/repo', + branch: 'main', + model, + filePath: '/transcript', + codexHome: null, + createdAt: null, + updatedAt: null, + modifiedAt: '2026-01-01T00:00:00.000Z', + messageCount: 1, + totalTokens: 1, + previewMessages: [], + queuedMessageCount: 0, + subagentTranscriptCount: 0, + resumeCommand: 'claude --resume session', + subagent: null + } +} + +function entryWithSerializedBytes(targetBytes: number): PersistedSessionParseCacheEntry { + const entry = persistedEntry('') + const baseBytes = Buffer.byteLength(JSON.stringify(entry), 'utf8') + expect(baseBytes).toBeLessThan(targetBytes) + entry.session!.model = 'x'.repeat(targetBytes - baseBytes) + expect(Buffer.byteLength(JSON.stringify(entry), 'utf8')).toBe(targetBytes) + return entry +} + +function retainedBytes(path: string, entry: PersistedSessionParseCacheEntry): number { + return Buffer.byteLength(path, 'utf8') + Buffer.byteLength(JSON.stringify(entry), 'utf8') +} + +describe('session parse cache retention', () => { + it('admits an exact-limit UTF-8 key and rejects one byte over', () => { + const exact = 'é'.repeat(AI_VAULT_PARSE_CACHE_KEY_MAX_UTF8_BYTES / 2) + const overflow = `${exact}x` + expect(Buffer.byteLength(exact, 'utf8')).toBe(AI_VAULT_PARSE_CACHE_KEY_MAX_UTF8_BYTES) + + seedSessionParseCache([ + [exact, persistedEntry()], + [overflow, persistedEntry()] + ]) + + expect(snapshotSessionParseCacheForPersistence().map(([path]) => path)).toEqual([exact]) + }) + + it('admits an exact-limit serialized value and rejects one byte over', () => { + const exact = entryWithSerializedBytes(AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES) + const overflow = entryWithSerializedBytes(AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES + 1) + + seedSessionParseCache([ + ['/exact', exact], + ['/overflow', overflow] + ]) + + expect(snapshotSessionParseCacheForPersistence()).toEqual([['/exact', exact]]) + }) + + it('includes the non-persisted resume fold in the exact value limit', () => { + const persistedBytes = Buffer.byteLength(JSON.stringify(persistedEntry()), 'utf8') + const exactResumeBytes = AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES - persistedBytes + + storeSessionParseCacheEntry('/exact-resume', cacheEntryWithResumeBytes(exactResumeBytes)) + storeSessionParseCacheEntry('/overflow-resume', cacheEntryWithResumeBytes(exactResumeBytes + 1)) + + expect(inspectSessionParseCacheRetentionForTests().paths).toEqual(['/exact-resume']) + }) + + it('fills the aggregate UTF-8 budget exactly and keeps the newest entries on overflow', () => { + const first: [string, PersistedSessionParseCacheEntry] = ['/a', persistedEntry()] + const second: [string, PersistedSessionParseCacheEntry] = ['/b', persistedEntry()] + const exactBytes = retainedBytes(...first) + retainedBytes(...second) + setSessionParseCacheMaxRetainedBytesForTests(exactBytes) + + seedSessionParseCache([first, second]) + expect(inspectSessionParseCacheRetentionForTests()).toEqual({ + paths: ['/a', '/b'], + retainedUtf8Bytes: exactBytes + }) + + setSessionParseCacheMaxRetainedBytesForTests(exactBytes - 1) + seedSessionParseCache([first, second]) + expect(inspectSessionParseCacheRetentionForTests().paths).toEqual(['/b']) + }) + + it('refreshes LRU recency before aggregate eviction', () => { + const persisted = persistedEntry() + const entry = cacheEntry() + const twoEntryBytes = retainedBytes('/a', persisted) + retainedBytes('/b', persisted) + setSessionParseCacheMaxRetainedBytesForTests(twoEntryBytes) + + storeSessionParseCacheEntry('/a', entry) + storeSessionParseCacheEntry('/b', entry) + storeSessionParseCacheEntry('/a', entry) + storeSessionParseCacheEntry('/c', entry) + + expect(inspectSessionParseCacheRetentionForTests().paths).toEqual(['/a', '/c']) + }) +}) diff --git a/src/main/ai-vault/session-parse-cache-retention.ts b/src/main/ai-vault/session-parse-cache-retention.ts new file mode 100644 index 00000000000..a6bc5e1e696 --- /dev/null +++ b/src/main/ai-vault/session-parse-cache-retention.ts @@ -0,0 +1,180 @@ +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import type { ResumableSessionParseState } from './session-scanner-types' + +// Covers the 1,000 recent plus 2,000 in-scope result caps with refresh headroom. +export const AI_VAULT_PARSE_CACHE_MAX_ENTRIES = 4_096 +export const AI_VAULT_PARSE_CACHE_KEY_MAX_UTF8_BYTES = 32 * 1024 +export const AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES = 256 * 1024 +export const AI_VAULT_PARSE_CACHE_MAX_RETAINED_UTF8_BYTES = 32 * 1024 * 1024 + +export type SessionParseCacheEntry = { + mtimeMs: number + sizeBytes: number | null + platform: NodeJS.Platform + session: AiVaultSession | null + resume: { + state: ResumableSessionParseState + // A trailing unterminated record must be reread after its writer completes it. + byteOffset: number + } | null +} + +export type PersistedSessionParseCacheEntry = Omit + +type RetainedEntry = { + entry: SessionParseCacheEntry + retainedUtf8Bytes: number +} + +const cache = new Map() +let retainedUtf8Bytes = 0 +let maxRetainedUtf8Bytes = AI_VAULT_PARSE_CACHE_MAX_RETAINED_UTF8_BYTES + +export function getSessionParseCacheEntry(path: string): SessionParseCacheEntry | undefined { + return cache.get(path)?.entry +} + +export function resetSessionParseCacheRetentionForTests(): void { + cache.clear() + retainedUtf8Bytes = 0 + maxRetainedUtf8Bytes = AI_VAULT_PARSE_CACHE_MAX_RETAINED_UTF8_BYTES +} + +export function setSessionParseCacheMaxRetainedBytesForTests(maxBytes: number): void { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Session parse cache byte limit must be a non-negative safe integer') + } + cache.clear() + retainedUtf8Bytes = 0 + maxRetainedUtf8Bytes = maxBytes +} + +export function inspectSessionParseCacheRetentionForTests(): { + paths: string[] + retainedUtf8Bytes: number +} { + return { paths: [...cache.keys()], retainedUtf8Bytes } +} + +export function snapshotSessionParseCacheForPersistence(): [ + string, + PersistedSessionParseCacheEntry +][] { + return [...cache].map(([path, retained]) => [path, persistedEntry(retained.entry)]) +} + +export function seedSessionParseCache( + entries: Iterable<[string, PersistedSessionParseCacheEntry]> +): void { + const tail = newestEntryTail(entries) + const selected: [string, SessionParseCacheEntry, number][] = [] + const selectedPaths = new Set() + let availableEntries = AI_VAULT_PARSE_CACHE_MAX_ENTRIES - cache.size + let availableBytes = maxRetainedUtf8Bytes - retainedUtf8Bytes + + for (let index = tail.length - 1; index >= 0 && availableEntries > 0; index -= 1) { + const [path, persisted] = tail[index] + // Live entries are fresher than disk, and the newest duplicate seed wins. + if (cache.has(path) || selectedPaths.has(path)) { + continue + } + selectedPaths.add(path) + const entry: SessionParseCacheEntry = { ...persisted, resume: null } + const bytes = retainedEntryUtf8Bytes(path, entry) + if (bytes === null || bytes > availableBytes) { + continue + } + selected.push([path, entry, bytes]) + availableEntries-- + availableBytes -= bytes + } + + for (let index = selected.length - 1; index >= 0; index -= 1) { + const [path, entry, bytes] = selected[index] + cache.set(path, { entry, retainedUtf8Bytes: bytes }) + retainedUtf8Bytes += bytes + } +} + +export function storeSessionParseCacheEntry(path: string, entry: SessionParseCacheEntry): void { + deleteSessionParseCacheEntry(path) + const bytes = retainedEntryUtf8Bytes(path, entry) + if (bytes === null || bytes > maxRetainedUtf8Bytes) { + return + } + + while ( + cache.size >= AI_VAULT_PARSE_CACHE_MAX_ENTRIES || + retainedUtf8Bytes + bytes > maxRetainedUtf8Bytes + ) { + const oldest = cache.keys().next().value + if (oldest === undefined) { + return + } + deleteSessionParseCacheEntry(oldest) + } + + cache.set(path, { entry, retainedUtf8Bytes: bytes }) + retainedUtf8Bytes += bytes +} + +function retainedEntryUtf8Bytes(path: string, entry: SessionParseCacheEntry): number | null { + const keyBytes = Buffer.byteLength(path, 'utf8') + if (keyBytes > AI_VAULT_PARSE_CACHE_KEY_MAX_UTF8_BYTES) { + return null + } + try { + const persistedBytes = stringifyJsonWithinByteLimit( + persistedEntry(entry), + AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES + ).byteLength + const resumeBytes = entry.resume?.state.retainedUtf8Bytes() ?? 0 + const valueBytes = persistedBytes + resumeBytes + if ( + resumeBytes < 0 || + !Number.isSafeInteger(valueBytes) || + valueBytes > AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES + ) { + return null + } + return keyBytes + valueBytes + } catch { + return null + } +} + +function persistedEntry(entry: SessionParseCacheEntry): PersistedSessionParseCacheEntry { + return { + mtimeMs: entry.mtimeMs, + sizeBytes: entry.sizeBytes, + platform: entry.platform, + session: entry.session + } +} + +function deleteSessionParseCacheEntry(path: string): void { + const retained = cache.get(path) + if (!retained) { + return + } + cache.delete(path) + retainedUtf8Bytes -= retained.retainedUtf8Bytes +} + +function newestEntryTail( + entries: Iterable<[string, PersistedSessionParseCacheEntry]> +): [string, PersistedSessionParseCacheEntry][] { + const ring = Array.from({ length: AI_VAULT_PARSE_CACHE_MAX_ENTRIES }) as [ + string, + PersistedSessionParseCacheEntry + ][] + let count = 0 + for (const entry of entries) { + ring[count % AI_VAULT_PARSE_CACHE_MAX_ENTRIES] = entry + count++ + } + const length = Math.min(count, AI_VAULT_PARSE_CACHE_MAX_ENTRIES) + const start = count > length ? count % AI_VAULT_PARSE_CACHE_MAX_ENTRIES : 0 + return Array.from({ length }, (_, index) => ring[(start + index) % ring.length]) +} diff --git a/src/main/ai-vault/session-scanner-accumulator.ts b/src/main/ai-vault/session-scanner-accumulator.ts index 48574818ff3..e268d547a73 100644 --- a/src/main/ai-vault/session-scanner-accumulator.ts +++ b/src/main/ai-vault/session-scanner-accumulator.ts @@ -52,6 +52,31 @@ export function cloneSessionAccumulator(accumulator: SessionAccumulator): Sessio return { ...accumulator, previewMessages: [...accumulator.previewMessages] } } +export function sessionAccumulatorRetainedUtf8Bytes(accumulator: SessionAccumulator): number { + let bytes = stringBytes( + accumulator.agent, + accumulator.sessionId, + accumulator.title, + accumulator.fallbackTitle, + accumulator.cwd, + accumulator.branch, + accumulator.model, + accumulator.filePath, + accumulator.createdAt, + accumulator.updatedAt, + accumulator.modifiedAt, + accumulator.lastUserPrompt + ) + for (const preview of accumulator.previewMessages) { + bytes += stringBytes(preview.role, preview.text, preview.timestamp) + } + return bytes +} + +function stringBytes(...values: (string | null)[]): number { + return values.reduce((total, value) => total + Buffer.byteLength(value ?? '', 'utf8'), 0) +} + // Resumable fold for parsers whose only parse state is the accumulator itself // (cursor, copilot, droid, openclaw/pi, gemini-jsonl). Parsers with extra // closure state (claude, codex) build their own ResumableSessionParseState. @@ -63,6 +88,7 @@ export function accumulatorFoldResumeState( consumeLine: (line) => consumeRecordLine(accumulator, line), clone: () => accumulatorFoldResumeState(cloneSessionAccumulator(accumulator), consumeRecordLine), + retainedUtf8Bytes: () => sessionAccumulatorRetainedUtf8Bytes(accumulator), touchFile: (file) => { accumulator.modifiedAt = file.modifiedAt }, diff --git a/src/main/ai-vault/session-scanner-antigravity-parser.ts b/src/main/ai-vault/session-scanner-antigravity-parser.ts index 2e3a3a08b4a..78c04af58f6 100644 --- a/src/main/ai-vault/session-scanner-antigravity-parser.ts +++ b/src/main/ai-vault/session-scanner-antigravity-parser.ts @@ -1,6 +1,5 @@ -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' import type { ExecutionHostId } from '../../shared/execution-host' import { accumulatorFoldResumeState, @@ -25,10 +24,7 @@ export async function parseAntigravitySessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseAntigravitySessionLines({ file, lines, platform }) } diff --git a/src/main/ai-vault/session-scanner-claude-subagents.test.ts b/src/main/ai-vault/session-scanner-claude-subagents.test.ts index eb0f90d9bc9..5a2a2587199 100644 --- a/src/main/ai-vault/session-scanner-claude-subagents.test.ts +++ b/src/main/ai-vault/session-scanner-claude-subagents.test.ts @@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' +import { AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES } from './session-list-retention' import { listClaudeSubagentSessions } from './session-scanner-claude-subagents' import { countSubagentTranscripts } from './session-scanner-subagent-transcripts' @@ -375,6 +376,111 @@ describe('listClaudeSubagentSessions', () => { ]) }) + it('caps an oversized subagent directory to the newest transcript files', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-subagent-cap-')) + tempRoots.push(root) + const parentFilePath = join(root, 'project', 'parent-session.jsonl') + const subagentsDir = join(root, 'project', 'parent-session', 'subagents') + + for (const [index, agentId] of ['oldest', 'middle', 'newest'].entries()) { + await writeSubagentTranscript({ + subagentsDir, + agentId, + taskPrompt: `Task for ${agentId}`, + timestamp: `2026-07-05T10:0${index + 1}:00.000Z` + }) + const mtime = new Date(1_700_000_000_000 + index * 1_000) + await utimes(join(subagentsDir, `agent-${agentId}.jsonl`), mtime, mtime) + } + + const result = await listClaudeSubagentSessions({ + parentFilePath, + platform: 'darwin', + maxSessions: 2 + }) + + expect(result.sessions.map((session) => session.title)).toEqual([ + 'Task for newest', + 'Task for middle' + ]) + expect(result.issues).toHaveLength(1) + expect(result.issues[0]?.message).toContain('newest 2 subagent transcripts') + }) + + it('omits a subagent whose retained session metadata exceeds its per-session limit', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-subagent-session-limit-')) + tempRoots.push(root) + const parentFilePath = join(root, 'project', 'parent-session.jsonl') + const subagentsDir = join(root, 'project', 'parent-session', 'subagents') + + await writeJsonlFile(join(subagentsDir, 'agent-oversized.jsonl'), [ + { + type: 'user', + sessionId: 'parent-session', + timestamp: '2026-07-05T10:00:00.000Z', + cwd: 'x'.repeat(AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES + 1), + message: { role: 'user', content: 'Oversized task' } + } + ]) + await writeSubagentTranscript({ + subagentsDir, + agentId: 'retained', + taskPrompt: 'Retained task', + timestamp: '2026-07-05T10:01:00.000Z' + }) + + const result = await listClaudeSubagentSessions({ parentFilePath, platform: 'darwin' }) + + expect(result.sessions.map((session) => session.title)).toEqual(['Retained task']) + expect(result.issues).toHaveLength(1) + expect(result.issues[0]?.message).toContain('working directory exceeds') + }) + + it('stops before parsing older transcripts when the aggregate memory limit is full', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-subagent-aggregate-limit-')) + tempRoots.push(root) + const parentFilePath = join(root, 'project', 'parent-session.jsonl') + const subagentsDir = join(root, 'project', 'parent-session', 'subagents') + + for (let index = 0; index < 8; index += 1) { + await writeSubagentTranscript({ + subagentsDir, + agentId: `task-${index}`, + taskPrompt: `Task for task-${index}`, + timestamp: `2026-07-05T10:${index.toString().padStart(2, '0')}:00.000Z` + }) + const mtime = new Date(1_700_000_000_000 + index * 1_000) + await utimes(join(subagentsDir, `agent-task-${index}.jsonl`), mtime, mtime) + } + + const unbounded = await listClaudeSubagentSessions({ parentFilePath, platform: 'darwin' }) + expect(unbounded.issues).toEqual([]) + const newestSessionBytes = Buffer.byteLength(JSON.stringify([unbounded.sessions[0]]), 'utf8') + + await writeJsonlFile(join(subagentsDir, 'agent-unparsed.jsonl'), [ + { + type: 'user', + sessionId: 'parent-session', + timestamp: '2026-07-05T09:00:00.000Z', + cwd: 'x'.repeat(AI_VAULT_SESSION_PATH_MAX_UTF8_BYTES + 1), + message: { role: 'user', content: 'This older task must not be parsed' } + } + ]) + const oldestMtime = new Date(1_699_999_999_000) + await utimes(join(subagentsDir, 'agent-unparsed.jsonl'), oldestMtime, oldestMtime) + + const result = await listClaudeSubagentSessions({ + parentFilePath, + platform: 'darwin', + maxAggregateBytes: newestSessionBytes + }) + + expect(result.sessions.map((session) => session.title)).toEqual(['Task for task-7']) + expect(result.issues).toHaveLength(1) + expect(result.issues[0]?.message).toContain('8 older entries were omitted') + expect(result.issues[0]?.message).toContain('memory limit') + }) + it('returns an empty list when the session never spawned subagents', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-subagent-none-')) tempRoots.push(root) diff --git a/src/main/ai-vault/session-scanner-claude-subagents.ts b/src/main/ai-vault/session-scanner-claude-subagents.ts index 5c269d91cf3..f00ca70de80 100644 --- a/src/main/ai-vault/session-scanner-claude-subagents.ts +++ b/src/main/ai-vault/session-scanner-claude-subagents.ts @@ -1,15 +1,18 @@ -import { createReadStream } from 'node:fs' -import { readdir, readFile, stat } from 'node:fs/promises' +import { opendir, stat } from 'node:fs/promises' import { basename, extname, join } from 'node:path' -import { createInterface } from 'node:readline' import type { AiVaultScanIssue, AiVaultSession, AiVaultSubagentListResult, AiVaultSubagentRunStatus } from '../../shared/ai-vault-types' -import { sessionIdFromFileName, sessionSortTime } from './session-scanner-accumulator' +import { retainClaudeSubagentSessionBatch } from './claude-subagent-list-retention' +import { retainAiVaultSession } from './session-list-retention' +import { sessionIdFromFileName } from './session-scanner-accumulator' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import { parseClaudeSessionFile } from './session-scanner-primary-parsers' +import type { FileWithMtime } from './session-scanner-types' import { isSubagentTranscriptFileName, subagentTranscriptsDirFor, @@ -29,6 +32,9 @@ const SUBAGENT_RUNNING_RECENCY_MS = 5 * 60_000 // Match the main scanner's deliberate parse batching (SESSION_PARSE_CONCURRENCY): // opening every subagent read stream at once stalls over SSH/WSL UNC paths. const SUBAGENT_PARSE_CONCURRENCY = 8 +// A single expanded parent stays bounded; oversized sessions show the newest transcripts. +export const CLAUDE_SUBAGENT_SESSION_MAX = 2_048 +const CLAUDE_SUBAGENT_ISSUE_MAX = 256 const TASK_NOTIFICATION_MARKER = '' const TOOL_USE_RESULT_MARKER = '"toolUseResult"' @@ -63,40 +69,40 @@ export async function listClaudeSubagentSessions(args: { parentFilePath: string platform?: NodeJS.Platform now?: number + maxSessions?: number + maxAggregateBytes?: number }): Promise { const platform = args.platform ?? process.platform const now = args.now ?? Date.now() const issues: AiVaultScanIssue[] = [] const subagentsDir = subagentTranscriptsDirFor(args.parentFilePath) - let entries - try { - entries = await readdir(subagentsDir, { withFileTypes: true }) - } catch { - return { sessions: [], issues } - } - - const transcriptNames = entries - .filter((entry) => isSubagentTranscriptFileName(entry.name, entry.isFile())) - .map((entry) => entry.name) - if (transcriptNames.length === 0) { + const maxSessions = Math.min( + CLAUDE_SUBAGENT_SESSION_MAX, + Math.max(0, Math.floor(args.maxSessions ?? CLAUDE_SUBAGENT_SESSION_MAX)) + ) + const transcriptFiles = await newestSubagentTranscriptFiles(subagentsDir, maxSessions, issues) + if (transcriptFiles.length === 0) { return { sessions: [], issues } } // One pass over the parent transcript resolves every subagent's status. - const statusByAgentId = await collectSubagentTaskStatuses(args.parentFilePath) + const agentIds = new Set( + transcriptFiles.map((file) => subagentIdFromFileName(basename(file.path))) + ) + const statusByAgentId = await collectSubagentTaskStatuses(args.parentFilePath, agentIds) // Why: the layout fixes the parent's sessionId (.jsonl -> subagents/); // deriving it here avoids a subagent transcript with no sessionId records // linking to its own filename-derived id instead of the parent. const parentSessionId = sessionIdFromFileName(args.parentFilePath) - const parsed: (AiVaultSession | null)[] = [] - for (let index = 0; index < transcriptNames.length; index += SUBAGENT_PARSE_CONCURRENCY) { - const batch = transcriptNames.slice(index, index + SUBAGENT_PARSE_CONCURRENCY) + let parsed: AiVaultSession[] = [] + for (let index = 0; index < transcriptFiles.length; index += SUBAGENT_PARSE_CONCURRENCY) { + const batch = transcriptFiles.slice(index, index + SUBAGENT_PARSE_CONCURRENCY) const batchResults = await Promise.all( - batch.map((name) => + batch.map((file) => parseSubagentTranscript({ - filePath: join(subagentsDir, name), - agentId: subagentIdFromFileName(name), + file, + agentId: subagentIdFromFileName(basename(file.path)), parentSessionId, statusByAgentId, now, @@ -105,14 +111,92 @@ export async function listClaudeSubagentSessions(args: { }) ) ) - parsed.push(...batchResults) + const retained = retainClaudeSubagentSessionBatch(parsed, batchResults, args.maxAggregateBytes) + parsed = retained.sessions + if (retained.omitted > 0) { + const unparsed = transcriptFiles.length - index - batch.length + addClaudeSubagentIssue( + issues, + subagentsDir, + `Showing the newest ${parsed.length.toLocaleString()} subagent transcripts; ${(retained.omitted + unparsed).toLocaleString()} older entries were omitted because the list reached its memory limit.` + ) + break + } } - return { - sessions: parsed - .filter((session): session is AiVaultSession => session !== null) - .sort((left, right) => sessionSortTime(right) - sessionSortTime(left)), - issues + return { sessions: parsed, issues } +} + +async function newestSubagentTranscriptFiles( + subagentsDir: string, + limit: number, + issues: AiVaultScanIssue[] +): Promise { + let directory + try { + directory = await opendir(subagentsDir) + } catch { + return [] + } + + const files: FileWithMtime[] = [] + let validFileCount = 0 + try { + for await (const entry of directory) { + if (!isSubagentTranscriptFileName(entry.name, entry.isFile())) { + continue + } + const path = join(subagentsDir, entry.name) + try { + const fileStat = await stat(path) + validFileCount += 1 + retainNewestSubagentFile(files, limit, { + path, + mtimeMs: fileStat.mtimeMs, + modifiedAt: fileStat.mtime.toISOString(), + sizeBytes: fileStat.size, + dev: fileStat.dev, + ino: fileStat.ino, + nlink: fileStat.nlink + }) + } catch (err) { + addClaudeSubagentIssue(issues, path, errorMessage(err)) + } + } + } catch { + // A directory disappearing mid-scan leaves the successfully retained prefix usable. + } + if (validFileCount > limit) { + addClaudeSubagentIssue( + issues, + subagentsDir, + `Showing the newest ${limit.toLocaleString()} subagent transcripts; older entries were omitted.` + ) + } + return files +} + +function retainNewestSubagentFile( + files: FileWithMtime[], + limit: number, + file: FileWithMtime +): void { + let low = 0 + let high = files.length + while (low < high) { + const middle = Math.floor((low + high) / 2) + if ((files[middle]?.mtimeMs ?? Number.NEGATIVE_INFINITY) >= file.mtimeMs) { + low = middle + 1 + } else { + high = middle + } + } + if (low >= limit) { + return + } + files.splice(low, 0, file) + if (files.length > limit) { + files.pop() } } @@ -121,7 +205,7 @@ function subagentIdFromFileName(name: string): string { } async function parseSubagentTranscript(args: { - filePath: string + file: FileWithMtime agentId: string parentSessionId: string statusByAgentId: ReadonlyMap @@ -130,20 +214,12 @@ async function parseSubagentTranscript(args: { issues: AiVaultScanIssue[] }): Promise { try { - const fileStat = await stat(args.filePath) - const session = await parseClaudeSessionFile( - { - path: args.filePath, - mtimeMs: fileStat.mtimeMs, - modifiedAt: fileStat.mtime.toISOString() - }, - args.platform - ) + const session = await parseClaudeSessionFile(args.file, args.platform) if (!session) { return null } - const meta = await readSubagentMeta(args.filePath) - return { + const meta = await readSubagentMeta(args.file.path) + return retainAiVaultSession({ ...session, // Why: the spawn description is the name the main agent gave this Task; // it beats the transcript-derived fallback (the raw Task prompt). @@ -155,13 +231,13 @@ async function parseSubagentTranscript(args: { agentType: meta.agentType, status: resolveSubagentStatus({ reportedStatus: args.statusByAgentId.get(args.agentId), - mtimeMs: fileStat.mtimeMs, + mtimeMs: args.file.mtimeMs, now: args.now }) } - } + }) } catch (err) { - args.issues.push({ agent: 'claude', path: args.filePath, message: errorMessage(err) }) + addClaudeSubagentIssue(args.issues, args.file.path, errorMessage(err)) return null } } @@ -185,13 +261,13 @@ function resolveSubagentStatus(args: { // launch with status 'async_launched' and finish with a // whose is the agentId. Last record wins, so interim statuses (e.g. // 'async_launched', notification 'running') are superseded by terminal ones. -async function collectSubagentTaskStatuses(parentFilePath: string): Promise> { +async function collectSubagentTaskStatuses( + parentFilePath: string, + requestedAgentIds: ReadonlySet +): Promise> { const statuses = new Map() try { - const lines = createInterface({ - input: createReadStream(parentFilePath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(parentFilePath) for await (const line of lines) { const hasNotification = line.includes(TASK_NOTIFICATION_MARKER) const hasTaskResult = @@ -217,7 +293,7 @@ async function collectSubagentTaskStatuses(parentFilePath: string): Promise marker follows / lines in real // notifications, so it sits well past the 96-char title cap — the notification // text must be read untruncated (unlike titles/previews). Both delivery shapes @@ -269,11 +351,13 @@ function taskNotificationBlockText(block: unknown): string { async function readSubagentMeta(transcriptPath: string): Promise { const metaPath = `${transcriptPath.slice(0, -extname(transcriptPath).length)}.meta.json` try { - const record = asRecord(JSON.parse(await readFile(metaPath, 'utf-8')) as unknown) - return { - description: normalizeTitleText(extractString(record?.description) ?? ''), - agentType: extractString(record?.agentType) - } + return await withAiVaultWholeJsonFile(metaPath, (content) => { + const record = parseJsonObject(content) + return { + description: normalizeTitleText(extractString(record?.description) ?? ''), + agentType: extractString(record?.agentType) + } + }) } catch { // The sidecar is optional; the transcript still yields a usable title. return { description: null, agentType: null } diff --git a/src/main/ai-vault/session-scanner-codex-parser.ts b/src/main/ai-vault/session-scanner-codex-parser.ts index 7e08ec9d92e..cbe9fcebe1a 100644 --- a/src/main/ai-vault/session-scanner-codex-parser.ts +++ b/src/main/ai-vault/session-scanner-codex-parser.ts @@ -1,7 +1,6 @@ -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' import { readCodexSessionIndexTitle } from './session-scanner-codex-title-index' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' import type { ExecutionHostId } from '../../shared/execution-host' import { normalizePromptField } from '../../shared/agent-status-field-normalization' import { @@ -9,6 +8,7 @@ import { cloneSessionAccumulator, createAccumulator, finalizeSession, + sessionAccumulatorRetainedUtf8Bytes, sessionIdFromFileName, updateTimeline } from './session-scanner-accumulator' @@ -38,10 +38,7 @@ export async function parseCodexSessionFile( codexHome: string | null = null, executionHostId?: ExecutionHostId ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseCodexSessionLines({ file, @@ -276,6 +273,7 @@ function codexResumeStateFromParseState( consumeLine: (line) => consumeCodexRecordLine(state, line), clone: () => codexResumeStateFromParseState(cloneCodexParseState(state), codexHome, titleReader), + retainedUtf8Bytes: () => sessionAccumulatorRetainedUtf8Bytes(state.accumulator), touchFile: (file) => { state.accumulator.modifiedAt = file.modifiedAt }, diff --git a/src/main/ai-vault/session-scanner-codex-title-index.test.ts b/src/main/ai-vault/session-scanner-codex-title-index.test.ts index 3c5b129b5fc..f011c0605da 100644 --- a/src/main/ai-vault/session-scanner-codex-title-index.test.ts +++ b/src/main/ai-vault/session-scanner-codex-title-index.test.ts @@ -4,14 +4,18 @@ import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { _getCodexSessionIndexTitleCacheSizeForTest, + _getCodexSessionIndexRetainedTitleCountForTest, _hasCodexSessionIndexTitleCacheEntryForTest, _readCachedCodexSessionIndexTitlesForTest, _storeCodexSessionIndexTitleCacheEntryForTest, + CODEX_SESSION_INDEX_RETAINED_UTF8_BYTES_MAX, + CODEX_SESSION_INDEX_SESSION_ID_MAX_UTF8_BYTES, readCodexSessionIndexTitle, resetCodexSessionIndexTitleCacheForTests } from './session-scanner-codex-title-index' const CACHE_LIMIT = 64 +const RETAINED_TITLE_LIMIT = 2_048 let tempRoots: string[] = [] @@ -42,6 +46,42 @@ async function readTitle(codexHome: string, index: number): Promise { + it('bounds retained titles without dropping an older requested title', async () => { + const codexHome = await createCodexHome(0) + const lines = Array.from({ length: RETAINED_TITLE_LIMIT + 1 }, (_, index) => + JSON.stringify({ id: `session-${index}`, thread_name: `Title ${index}` }) + ) + await writeFile(join(codexHome, 'session_index.jsonl'), `${lines.join('\n')}\n`) + + expect(await readTitle(codexHome, 0)).toBe('Title 0') + expect(await _getCodexSessionIndexRetainedTitleCountForTest(codexHome)).toBe( + RETAINED_TITLE_LIMIT + ) + }) + + it('bounds retained title-key bytes without dropping an oversized requested id', async () => { + const codexHome = await createCodexHome(0) + const ids = Array.from({ length: 9 }, (_, index) => + `${index}`.padEnd(CODEX_SESSION_INDEX_SESSION_ID_MAX_UTF8_BYTES, 'x') + ) + const oversizedId = `${ids[0]}x` + const lines = [ + JSON.stringify({ id: oversizedId, thread_name: 'Oversized requested title' }), + ...ids.map((id, index) => JSON.stringify({ id, thread_name: `Title ${index}` })) + ] + await writeFile(join(codexHome, 'session_index.jsonl'), `${lines.join('\n')}\n`) + + await expect( + readCodexSessionIndexTitle( + join(codexHome, 'sessions', `${oversizedId}.jsonl`), + codexHome, + oversizedId + ) + ).resolves.toBe('Oversized requested title') + expect(await _getCodexSessionIndexRetainedTitleCountForTest(codexHome)).toBeLessThan(9) + expect(CODEX_SESSION_INDEX_RETAINED_UTF8_BYTES_MAX).toBe(512 * 1024) + }) + it('caps cached title indexes by Codex home', async () => { const homes: string[] = [] diff --git a/src/main/ai-vault/session-scanner-codex-title-index.ts b/src/main/ai-vault/session-scanner-codex-title-index.ts index 8ef2d5645cb..67ee4f0b499 100644 --- a/src/main/ai-vault/session-scanner-codex-title-index.ts +++ b/src/main/ai-vault/session-scanner-codex-title-index.ts @@ -1,16 +1,22 @@ -import { createReadStream } from 'node:fs' import { stat } from 'node:fs/promises' import { basename, dirname, join } from 'node:path' -import { createInterface } from 'node:readline' import { extractString, normalizeTitleText, parseJsonObject } from './session-scanner-values' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' // Codex names threads lazily in /session_index.jsonl; transcripts // carry no title of their own, so parsers look the thread name up here. const CODEX_SESSION_INDEX_FILE = 'session_index.jsonl' +// Misses fall back to an exact streaming lookup, so this only trades memory for scan time. +const CODEX_SESSION_INDEX_RETAINED_TITLE_MAX = 2_048 +export const CODEX_SESSION_INDEX_CACHE_KEY_MAX_UTF8_BYTES = 32 * 1024 +export const CODEX_SESSION_INDEX_SESSION_ID_MAX_UTF8_BYTES = 64 * 1024 +export const CODEX_SESSION_INDEX_RETAINED_UTF8_BYTES_MAX = 512 * 1024 +const CODEX_SESSION_INDEX_TITLE_MAX_UTF8_BYTES = 4 * 1024 +const CODEX_SESSION_INDEX_ENTRY_OVERHEAD_BYTES = 64 // Why: custom and WSL Codex homes can vary over a long-lived main process; -// each cached home can retain a full session_index title map. -const CODEX_SESSION_INDEX_TITLE_CACHE_MAX = 64 +// each cached home can retain its own session_index title map. +export const CODEX_SESSION_INDEX_TITLE_CACHE_MAX = 64 type CodexSessionIndexTitleCacheEntry = { signature: string @@ -18,6 +24,7 @@ type CodexSessionIndexTitleCacheEntry = { } const codexSessionIndexTitleCache = new Map>() +const retainedBytesByTitleMap = new WeakMap, number>() export function resetCodexSessionIndexTitleCacheForTests(): void { codexSessionIndexTitleCache.clear() @@ -31,6 +38,12 @@ export function _hasCodexSessionIndexTitleCacheEntryForTest(codexHome: string): return codexSessionIndexTitleCache.has(codexHome) } +export async function _getCodexSessionIndexRetainedTitleCountForTest( + codexHome: string +): Promise { + return (await codexSessionIndexTitleCache.get(codexHome))?.titles.size ?? 0 +} + export function _storeCodexSessionIndexTitleCacheEntryForTest( codexHome: string, signature: string, @@ -59,7 +72,21 @@ export async function readCodexSessionIndexTitle( return null } const titleBySessionId = await readCodexSessionIndexTitles(resolvedCodexHome) - return titleBySessionId.get(sessionId) ?? null + const cachedTitle = titleBySessionId.get(sessionId) + if (cachedTitle) { + return cachedTitle + } + + // Why: the retained map is capped, but an older requested session must still + // get the same title as before the cap. + const title = await readCodexSessionIndexTitleFromDisk( + join(resolvedCodexHome, CODEX_SESSION_INDEX_FILE), + sessionId + ) + if (title) { + retainCodexSessionIndexTitle(titleBySessionId, sessionId, title) + } + return title } function codexHomeFromSessionFilePath(sessionFilePath: string): string | null { @@ -121,8 +148,15 @@ function storeCodexSessionIndexTitleCacheEntry( codexHome: string, pending: Promise ): void { + if (Buffer.byteLength(codexHome, 'utf8') > CODEX_SESSION_INDEX_CACHE_KEY_MAX_UTF8_BYTES) { + return + } + const boundedPending = pending.then((entry) => { + boundCodexSessionIndexTitles(entry.titles) + return entry + }) codexSessionIndexTitleCache.delete(codexHome) - codexSessionIndexTitleCache.set(codexHome, pending) + codexSessionIndexTitleCache.set(codexHome, boundedPending) if (codexSessionIndexTitleCache.size > CODEX_SESSION_INDEX_TITLE_CACHE_MAX) { const oldest = codexSessionIndexTitleCache.keys().next() if (!oldest.done) { @@ -136,10 +170,7 @@ async function readCodexSessionIndexTitlesFromDisk( ): Promise> { const titleBySessionId = new Map() try { - const lines = createInterface({ - input: createReadStream(indexPath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(indexPath) for await (const line of lines) { const record = parseJsonObject(line) if (!record) { @@ -148,7 +179,7 @@ async function readCodexSessionIndexTitlesFromDisk( const sessionId = extractString(record.id) const title = normalizeTitleText(extractString(record.thread_name) ?? '') if (sessionId && title) { - titleBySessionId.set(sessionId, title) + retainCodexSessionIndexTitle(titleBySessionId, sessionId, title) } } } catch { @@ -156,3 +187,78 @@ async function readCodexSessionIndexTitlesFromDisk( } return titleBySessionId } + +async function readCodexSessionIndexTitleFromDisk( + indexPath: string, + requestedSessionId: string +): Promise { + let requestedTitle: string | null = null + try { + for await (const line of iterateAiVaultJsonlLines(indexPath)) { + const record = parseJsonObject(line) + if (record && extractString(record.id) === requestedSessionId) { + requestedTitle = + normalizeTitleText(extractString(record.thread_name) ?? '') ?? requestedTitle + } + } + } catch { + // Match the best-effort behavior of the full index read. + } + return requestedTitle +} + +export function retainCodexSessionIndexTitle( + titles: Map, + sessionId: string, + title: string +): void { + boundCodexSessionIndexTitles(titles) + const sessionIdBytes = Buffer.byteLength(sessionId, 'utf8') + const titleBytes = Buffer.byteLength(title, 'utf8') + if ( + sessionIdBytes > CODEX_SESSION_INDEX_SESSION_ID_MAX_UTF8_BYTES || + titleBytes > CODEX_SESSION_INDEX_TITLE_MAX_UTF8_BYTES + ) { + return + } + let retainedBytes = retainedBytesByTitleMap.get(titles) ?? 0 + const existing = titles.get(sessionId) + if (existing !== undefined) { + retainedBytes -= codexTitleEntryBytes(sessionId, existing) + } + titles.delete(sessionId) + const entryBytes = sessionIdBytes + titleBytes + CODEX_SESSION_INDEX_ENTRY_OVERHEAD_BYTES + while ( + titles.size >= CODEX_SESSION_INDEX_RETAINED_TITLE_MAX || + retainedBytes + entryBytes > CODEX_SESSION_INDEX_RETAINED_UTF8_BYTES_MAX + ) { + const oldest = titles.entries().next().value + if (!oldest) { + break + } + titles.delete(oldest[0]) + retainedBytes -= codexTitleEntryBytes(oldest[0], oldest[1]) + } + titles.set(sessionId, title) + retainedBytesByTitleMap.set(titles, retainedBytes + entryBytes) +} + +function boundCodexSessionIndexTitles(titles: Map): void { + if (retainedBytesByTitleMap.has(titles)) { + return + } + const entries = [...titles] + titles.clear() + retainedBytesByTitleMap.set(titles, 0) + for (const [sessionId, title] of entries) { + retainCodexSessionIndexTitle(titles, sessionId, title) + } +} + +function codexTitleEntryBytes(sessionId: string, title: string): number { + return ( + Buffer.byteLength(sessionId, 'utf8') + + Buffer.byteLength(title, 'utf8') + + CODEX_SESSION_INDEX_ENTRY_OVERHEAD_BYTES + ) +} diff --git a/src/main/ai-vault/session-scanner-devin-parser.ts b/src/main/ai-vault/session-scanner-devin-parser.ts index 4ad6fa45f78..81a8bdbc860 100644 --- a/src/main/ai-vault/session-scanner-devin-parser.ts +++ b/src/main/ai-vault/session-scanner-devin-parser.ts @@ -1,6 +1,6 @@ -import { readFile } from 'node:fs/promises' import type { AiVaultSession } from '../../shared/ai-vault-types' import type { ExecutionHostId } from '../../shared/execution-host' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import type { FileWithMtime } from './session-scanner-types' import { addPreviewContent, @@ -15,7 +15,8 @@ import { extractContentText, extractString, normalizeTitleText, - numberValue + numberValue, + parseJsonObject } from './session-scanner-values' type ParserSessionOptions = { @@ -27,7 +28,9 @@ export async function parseDevinSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - return parseDevinSessionContent(file, await readFile(file.path, 'utf-8'), platform) + return withAiVaultWholeJsonFile(file.path, (content) => + parseDevinSessionContent(file, content, platform) + ) } export function parseDevinSessionContent( @@ -36,7 +39,7 @@ export function parseDevinSessionContent( platform: NodeJS.Platform = process.platform, options: ParserSessionOptions = {} ): AiVaultSession | null { - const record = asRecord(JSON.parse(content) as unknown) + const record = parseJsonObject(content) if (!record) { return null } diff --git a/src/main/ai-vault/session-scanner-discovery.test.ts b/src/main/ai-vault/session-scanner-discovery.test.ts new file mode 100644 index 00000000000..0422134ad3b --- /dev/null +++ b/src/main/ai-vault/session-scanner-discovery.test.ts @@ -0,0 +1,162 @@ +import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import type { AiVaultScanIssue } from '../../shared/ai-vault-types' +import { discoverFiles } from './session-scanner-discovery' + +let tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true }))) + tempRoots = [] +}) + +describe('AI Vault session discovery retention', () => { + it('retains only the newest requested files while traversing nested directories', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-discovery-')) + tempRoots.push(root) + const nested = join(root, 'nested') + await mkdir(nested) + + for (let index = 0; index < 40; index += 1) { + const path = join(index % 2 === 0 ? root : nested, `session-${index}.jsonl`) + await writeFile(path, '{}\n') + const timestamp = new Date(1_700_000_000_000 + index * 1_000) + await utimes(path, timestamp, timestamp) + } + + const result = await discoverFiles({ + rootDir: root, + limit: 3, + agent: 'codex', + issues: [], + extensions: ['.jsonl'] + }) + + expect(result.files).toHaveLength(3) + expect(result.files.map((file) => basename(file.path))).toEqual([ + 'session-39.jsonl', + 'session-38.jsonl', + 'session-37.jsonl' + ]) + }) + + it('returns no retained paths when the requested limit is zero', async () => { + const result = await discoverFiles({ + rootDir: join('path', 'that', 'does-not-need-to-be-read'), + limit: 0, + agent: 'codex', + issues: [], + extensions: ['.jsonl'] + }) + + expect(result.files).toEqual([]) + }) + + it('accepts the exact aggregate entry capacity without changing results', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-discovery-')) + tempRoots.push(root) + await Promise.all([ + writeFile(join(root, 'session-a.jsonl'), '{}\n'), + writeFile(join(root, 'session-b.jsonl'), '{}\n') + ]) + const issues: AiVaultScanIssue[] = [] + + const result = await discoverFiles({ + rootDir: root, + limit: 10, + agent: 'codex', + issues, + extensions: ['.jsonl'], + limits: { maxEntries: 2 } + }) + + expect(result.files.map((file) => basename(file.path)).sort()).toEqual([ + 'session-a.jsonl', + 'session-b.jsonl' + ]) + expect(issues).toEqual([]) + }) + + it('reports aggregate entry overflow and releases the open directory', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-discovery-')) + tempRoots.push(root) + await Promise.all([ + writeFile(join(root, 'session-a.jsonl'), '{}\n'), + writeFile(join(root, 'session-b.jsonl'), '{}\n'), + writeFile(join(root, 'session-c.jsonl'), '{}\n') + ]) + const issues: AiVaultScanIssue[] = [] + + const result = await discoverFiles({ + rootDir: root, + limit: 10, + agent: 'codex', + issues, + extensions: ['.jsonl'], + limits: { maxEntries: 2 } + }) + + const retainedNames = result.files.map((file) => basename(file.path)) + expect(retainedNames).toHaveLength(2) + expect( + retainedNames.every((name) => + ['session-a.jsonl', 'session-b.jsonl', 'session-c.jsonl'].includes(name) + ) + ).toBe(true) + expect(issues).toEqual([ + expect.objectContaining({ + agent: 'codex', + path: root, + message: expect.stringContaining('safety limit') + }) + ]) + await expect(rm(root, { recursive: true, force: true })).resolves.toBeUndefined() + tempRoots = tempRoots.filter((candidate) => candidate !== root) + }) + + it('reports a depth overflow before opening the over-limit directory', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-discovery-')) + tempRoots.push(root) + const overLimit = join(root, 'level-one', 'level-two') + await mkdir(overLimit, { recursive: true }) + await writeFile(join(overLimit, 'session.jsonl'), '{}\n') + const issues: AiVaultScanIssue[] = [] + + const result = await discoverFiles({ + rootDir: root, + limit: 10, + agent: 'codex', + issues, + extensions: ['.jsonl'], + limits: { maxDepth: 1 } + }) + + expect(result.files).toEqual([]) + expect(issues).toEqual([ + expect.objectContaining({ path: root, message: expect.stringContaining('safety limit') }) + ]) + }) + + it('accepts a file at the exact folder-depth capacity', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-discovery-')) + tempRoots.push(root) + const exactLimit = join(root, 'level-one') + await mkdir(exactLimit) + await writeFile(join(exactLimit, 'session.jsonl'), '{}\n') + const issues: AiVaultScanIssue[] = [] + + const result = await discoverFiles({ + rootDir: root, + limit: 10, + agent: 'codex', + issues, + extensions: ['.jsonl'], + limits: { maxDepth: 1 } + }) + + expect(result.files.map((file) => basename(file.path))).toEqual(['session.jsonl']) + expect(issues).toEqual([]) + }) +}) diff --git a/src/main/ai-vault/session-scanner-discovery.ts b/src/main/ai-vault/session-scanner-discovery.ts index d93d6f01fea..90622baa974 100644 --- a/src/main/ai-vault/session-scanner-discovery.ts +++ b/src/main/ai-vault/session-scanner-discovery.ts @@ -1,8 +1,15 @@ -import { readdir, stat } from 'node:fs/promises' +import { opendir, stat } from 'node:fs/promises' import { basename, delimiter, extname, join } from 'node:path' import type { AiVaultAgent, AiVaultScanIssue } from '../../shared/ai-vault-types' import type { FileWithMtime, SessionFileDiscovery } from './session-scanner-types' import { errorMessage } from './session-scanner-values' +import { + AiVaultSessionDiscoveryBudget, + AiVaultSessionDiscoveryCapacityError, + type AiVaultSessionDiscoveryLimits +} from './session-discovery-budget' + +const AI_VAULT_DISCOVERY_ISSUE_MAX = 256 export async function discoverFiles(args: { rootDir: string @@ -12,33 +19,106 @@ export async function discoverFiles(args: { extensions: string[] filePredicate?: (path: string) => boolean directoryPredicate?: (name: string, depth: number) => boolean + limits?: Partial }): Promise { - const paths = await walkSessionFiles(args.rootDir, args.agent, args.issues, { - extensions: new Set(args.extensions), - filePredicate: args.filePredicate, - directoryPredicate: args.directoryPredicate - }) const files: FileWithMtime[] = [] - for (const path of paths) { + const limit = Math.max(0, Math.floor(args.limit)) + if (limit > 0) { try { - const fileStat = await stat(path) - files.push({ - path, - mtimeMs: fileStat.mtimeMs, - modifiedAt: fileStat.mtime.toISOString(), - sizeBytes: fileStat.size, - dev: fileStat.dev, - ino: fileStat.ino, - nlink: fileStat.nlink - }) - } catch (err) { - args.issues.push({ agent: args.agent, path, message: errorMessage(err) }) + await visitSessionFiles( + args.rootDir, + { + extensions: new Set(args.extensions), + filePredicate: args.filePredicate, + directoryPredicate: args.directoryPredicate + }, + async (path) => { + try { + const fileStat = await stat(path) + retainNewestFile(files, limit, { + path, + mtimeMs: fileStat.mtimeMs, + modifiedAt: fileStat.mtime.toISOString(), + sizeBytes: fileStat.size, + dev: fileStat.dev, + ino: fileStat.ino, + nlink: fileStat.nlink + }) + } catch (err) { + addDiscoveryIssue(args.issues, args.agent, path, errorMessage(err)) + } + return true + }, + new AiVaultSessionDiscoveryBudget(args.limits) + ) + } catch (error) { + if (error instanceof AiVaultSessionDiscoveryCapacityError) { + addDiscoveryIssue(args.issues, args.agent, args.rootDir, error.message) + } else { + throw error + } } } - return { - agent: args.agent, - rootDir: args.rootDir, - files: files.sort((left, right) => right.mtimeMs - left.mtimeMs).slice(0, args.limit) + return { agent: args.agent, rootDir: args.rootDir, files } +} + +type SessionFileTraversalOptions = { + extensions: Set + filePredicate?: (path: string) => boolean + directoryPredicate?: (name: string, depth: number) => boolean +} + +export async function findFirstSessionFile( + dirPath: string, + options: SessionFileTraversalOptions +): Promise { + let found: string | null = null + try { + await visitSessionFiles( + dirPath, + options, + async (path) => { + found = path + return false + }, + new AiVaultSessionDiscoveryBudget() + ) + } catch (error) { + if (!(error instanceof AiVaultSessionDiscoveryCapacityError)) { + throw error + } + } + return found +} + +function addDiscoveryIssue( + issues: AiVaultScanIssue[], + agent: AiVaultAgent, + path: string, + message: string +): void { + if (issues.length < AI_VAULT_DISCOVERY_ISSUE_MAX) { + issues.push({ agent, path, message }) + } +} + +function retainNewestFile(files: FileWithMtime[], limit: number, file: FileWithMtime): void { + let low = 0 + let high = files.length + while (low < high) { + const middle = Math.floor((low + high) / 2) + if ((files[middle]?.mtimeMs ?? Number.NEGATIVE_INFINITY) >= file.mtimeMs) { + low = middle + 1 + } else { + high = middle + } + } + if (low >= limit) { + return + } + files.splice(low, 0, file) + if (files.length > limit) { + files.pop() } } @@ -47,63 +127,94 @@ export async function discoverOpenClawFiles(args: { limit: number issues: AiVaultScanIssue[] }): Promise { - const discoveries = await Promise.all( - args.rootDirs.map((rootDir) => - discoverFiles({ - rootDir: basename(rootDir) === 'agents' ? rootDir : join(rootDir, 'agents'), - limit: args.limit, - agent: 'openclaw', - issues: args.issues, - extensions: ['.jsonl'], - filePredicate: (path) => path.split(/[\\/]/).includes('sessions') - }) - ) - ) - const files = discoveries - .flatMap((discovery) => discovery.files) - .sort((left, right) => right.mtimeMs - left.mtimeMs) - .slice(0, args.limit) + const files: FileWithMtime[] = [] + const limit = Math.max(0, Math.floor(args.limit)) + for (const rootDir of args.rootDirs) { + const discovery = await discoverFiles({ + rootDir: basename(rootDir) === 'agents' ? rootDir : join(rootDir, 'agents'), + limit, + agent: 'openclaw', + issues: args.issues, + extensions: ['.jsonl'], + filePredicate: (path) => path.split(/[\\/]/).includes('sessions') + }) + for (const file of discovery.files) { + retainNewestFile(files, limit, file) + } + } return { agent: 'openclaw', rootDir: args.rootDirs.join(delimiter), files } } -export async function walkSessionFiles( +async function visitSessionFiles( dirPath: string, - agent: AiVaultAgent, - issues: AiVaultScanIssue[], - options: { - extensions: Set - filePredicate?: (path: string) => boolean - // Return false to skip descending into a directory; depth 0 is a child of - // rootDir, so pruned subtrees are never stat'd or parsed. - directoryPredicate?: (name: string, depth: number) => boolean - }, + options: SessionFileTraversalOptions, + visitFile: (path: string) => Promise, + budget: AiVaultSessionDiscoveryBudget, depth = 0 -): Promise { - let entries +): Promise { + budget.enterDirectory(depth) + let directory try { - entries = await readdir(dirPath, { withFileTypes: true }) + directory = await opendir(dirPath) } catch { - return [] + return true } - const files: string[] = [] - for (const entry of entries) { - const fullPath = join(dirPath, entry.name) - if (entry.isDirectory()) { - // Skip whole subtrees an agent never wants (e.g. subagent transcripts), - // avoiding the readdir cost of descending into them. - if (options.directoryPredicate?.(entry.name, depth) ?? true) { - files.push(...(await walkSessionFiles(fullPath, agent, issues, options, depth + 1))) + const tasks: { kind: 'directory' | 'file'; path: string }[] = [] + let capacityError: AiVaultSessionDiscoveryCapacityError | null = null + try { + while (true) { + let entry + try { + entry = await directory.read() + } catch { + // A disappearing directory has no safe remainder to visit. + break + } + if (!entry) { + break + } + const fullPath = join(dirPath, entry.name) + try { + budget.visitEntry(fullPath) + } catch (error) { + if (error instanceof AiVaultSessionDiscoveryCapacityError) { + capacityError = error + break + } + throw error + } + if (entry.isDirectory()) { + // Skip whole subtrees an agent never wants (e.g. subagent transcripts), + // avoiding the directory-read cost of descending into them. + if (options.directoryPredicate?.(entry.name, depth) ?? true) { + tasks.push({ kind: 'directory', path: fullPath }) + } + continue + } + if ( + entry.isFile() && + options.extensions.has(extname(entry.name).toLowerCase()) && + (options.filePredicate?.(fullPath) ?? true) + ) { + tasks.push({ kind: 'file', path: fullPath }) } - continue } - if ( - entry.isFile() && - options.extensions.has(extname(entry.name).toLowerCase()) && - (options.filePredicate?.(fullPath) ?? true) - ) { - files.push(fullPath) + } finally { + await directory.close().catch(() => undefined) + } + + for (const task of tasks) { + const shouldContinue = + task.kind === 'directory' + ? await visitSessionFiles(task.path, options, visitFile, budget, depth + 1) + : await visitFile(task.path) + if (!shouldContinue) { + return false } } - return files + if (capacityError) { + throw capacityError + } + return true } diff --git a/src/main/ai-vault/session-scanner-droid-parser.ts b/src/main/ai-vault/session-scanner-droid-parser.ts index 2abaa7b8193..bd1f83f954c 100644 --- a/src/main/ai-vault/session-scanner-droid-parser.ts +++ b/src/main/ai-vault/session-scanner-droid-parser.ts @@ -1,7 +1,6 @@ -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' import type { ExecutionHostId } from '../../shared/execution-host' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' import type { FileWithMtime, ResumableSessionParseState, @@ -33,10 +32,7 @@ export async function parseDroidSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseDroidSessionLines({ file, lines, platform }) } diff --git a/src/main/ai-vault/session-scanner-gemini-parsers.ts b/src/main/ai-vault/session-scanner-gemini-parsers.ts index 6f0b9fdc7c1..7b398d81783 100644 --- a/src/main/ai-vault/session-scanner-gemini-parsers.ts +++ b/src/main/ai-vault/session-scanner-gemini-parsers.ts @@ -1,7 +1,6 @@ -import { createReadStream } from 'node:fs' -import { readFile } from 'node:fs/promises' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import type { FileWithMtime, ResumableParseFinalizeOptions, @@ -33,7 +32,9 @@ export async function parseGeminiSessionFile( return parseGeminiJsonlSessionFile(file, platform) } - return parseGeminiJsonSessionContent(file, await readFile(file.path, 'utf-8'), platform) + return withAiVaultWholeJsonFile(file.path, (content) => + parseGeminiJsonSessionContent(file, content, platform) + ) } export async function parseGeminiSessionContent( @@ -59,7 +60,7 @@ function parseGeminiJsonSessionContent( platform: NodeJS.Platform, options: ResumableParseFinalizeOptions = {} ): AiVaultSession | null { - const record = asRecord(JSON.parse(content) as unknown) + const record = parseJsonObject(content) if (!record) { return null } @@ -80,10 +81,7 @@ export async function parseGeminiJsonlSessionFile( file: FileWithMtime, platform: NodeJS.Platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseGeminiJsonlSessionLines({ file, lines, platform }) } diff --git a/src/main/ai-vault/session-scanner-graph-parsers.ts b/src/main/ai-vault/session-scanner-graph-parsers.ts index 04601eacde5..d128cb89904 100644 --- a/src/main/ai-vault/session-scanner-graph-parsers.ts +++ b/src/main/ai-vault/session-scanner-graph-parsers.ts @@ -1,9 +1,8 @@ -import { createReadStream } from 'node:fs' -import { readFile } from 'node:fs/promises' import { basename, dirname, join } from 'node:path' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' import type { ExecutionHostId } from '../../shared/execution-host' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import type { FileWithMtime, ResumableSessionParseState, @@ -26,7 +25,6 @@ import { extractString, firstString, parseJsonObject, - readJsonObjectIfExists, tokenTotal } from './session-scanner-values' @@ -39,39 +37,45 @@ export async function parseRovoSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const metadata = asRecord(JSON.parse(await readFile(file.path, 'utf-8')) as unknown) - if (!metadata) { + const accumulator = await withAiVaultWholeJsonFile(file.path, (content) => { + const metadata = parseJsonObject(content) + if (!metadata) { + return null + } + const next = createAccumulator({ + agent: 'rovo', + file, + sessionId: basename(dirname(file.path)) + }) + next.title = firstString(metadata, ['title', 'name', 'summary']) + next.cwd = firstString(metadata, [ + 'workspace_path', + 'workspacePath', + 'workspace', + 'cwd', + 'working_directory', + 'workingDirectory', + 'project_path', + 'projectPath' + ]) + updateTimeline(next, extractString(metadata.created_at) ?? extractString(metadata.createdAt)) + updateTimeline(next, extractString(metadata.updated_at) ?? extractString(metadata.updatedAt)) + return next + }) + if (!accumulator) { return null } - const accumulator = createAccumulator({ - agent: 'rovo', - file, - sessionId: basename(dirname(file.path)) - }) - accumulator.title = firstString(metadata, ['title', 'name', 'summary']) - accumulator.cwd = firstString(metadata, [ - 'workspace_path', - 'workspacePath', - 'workspace', - 'cwd', - 'working_directory', - 'workingDirectory', - 'project_path', - 'projectPath' - ]) - updateTimeline( - accumulator, - extractString(metadata.created_at) ?? extractString(metadata.createdAt) - ) - updateTimeline( - accumulator, - extractString(metadata.updated_at) ?? extractString(metadata.updatedAt) - ) const contextPath = join(dirname(file.path), 'session_context.json') - const context = await readJsonObjectIfExists(contextPath) - if (context) { - consumeRovoSessionContext(accumulator, context) + try { + await withAiVaultWholeJsonFile(contextPath, (content) => { + const context = parseJsonObject(content) + if (context) { + consumeRovoSessionContext(accumulator, context) + } + }) + } catch { + // Optional context may not exist yet; metadata still yields a useful row. } return finalizeSession(accumulator, platform) @@ -172,10 +176,7 @@ export async function parseMessageGraphSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseMessageGraphSessionLines({ agent, file, lines, platform }) } diff --git a/src/main/ai-vault/session-scanner-grok-parser.ts b/src/main/ai-vault/session-scanner-grok-parser.ts index 13620007447..84d23f7a115 100644 --- a/src/main/ai-vault/session-scanner-grok-parser.ts +++ b/src/main/ai-vault/session-scanner-grok-parser.ts @@ -1,8 +1,7 @@ -import { createReadStream } from 'node:fs' -import { readFile } from 'node:fs/promises' import { dirname, join } from 'node:path' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import type { FileWithMtime, SessionAccumulator } from './session-scanner-types' import { addPreviewMessage, @@ -27,24 +26,29 @@ export async function parseGrokSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const record = asRecord(JSON.parse(await readFile(file.path, 'utf-8')) as unknown) - if (!record) { + const accumulator = await withAiVaultWholeJsonFile(file.path, (content) => { + const record = parseJsonObject(content) + if (!record) { + return null + } + const info = asRecord(record.info) + const sessionId = extractString(info?.id) ?? sessionIdFromFileName(dirname(file.path)) + const next = createAccumulator({ agent: 'grok', file, sessionId }) + next.cwd = extractString(info?.cwd) + next.title = + normalizeTitleText(extractString(record.generated_title) ?? '') ?? + normalizeTitleText(extractString(record.session_summary) ?? '') + next.model = extractString(record.current_model_id) + next.branch = extractString(record.head_branch) + next.messageCount = numberValue(record.num_chat_messages) || numberValue(record.num_messages) + updateTimeline(next, extractString(record.created_at)) + updateTimeline(next, extractString(record.updated_at)) + updateTimeline(next, extractString(record.last_active_at)) + return next + }) + if (!accumulator) { return null } - const info = asRecord(record.info) - const sessionId = extractString(info?.id) ?? sessionIdFromFileName(dirname(file.path)) - const accumulator = createAccumulator({ agent: 'grok', file, sessionId }) - accumulator.cwd = extractString(info?.cwd) - accumulator.title = - normalizeTitleText(extractString(record.generated_title) ?? '') ?? - normalizeTitleText(extractString(record.session_summary) ?? '') - accumulator.model = extractString(record.current_model_id) - accumulator.branch = extractString(record.head_branch) - accumulator.messageCount = - numberValue(record.num_chat_messages) || numberValue(record.num_messages) - updateTimeline(accumulator, extractString(record.created_at)) - updateTimeline(accumulator, extractString(record.updated_at)) - updateTimeline(accumulator, extractString(record.last_active_at)) await consumeGrokChatHistory(accumulator, dirname(file.path)) return finalizeSession(accumulator, platform) } @@ -54,10 +58,7 @@ async function consumeGrokChatHistory( sessionDir: string ): Promise { try { - const lines = createInterface({ - input: createReadStream(join(sessionDir, 'chat_history.jsonl'), { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(join(sessionDir, 'chat_history.jsonl')) for await (const line of lines) { const record = parseJsonObject(line) diff --git a/src/main/ai-vault/session-scanner-kimi-index-cache.test.ts b/src/main/ai-vault/session-scanner-kimi-index-cache.test.ts index 549c1a3c170..6ac75daa429 100644 --- a/src/main/ai-vault/session-scanner-kimi-index-cache.test.ts +++ b/src/main/ai-vault/session-scanner-kimi-index-cache.test.ts @@ -4,13 +4,18 @@ import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { KimiSessionIndexCache, + KIMI_WORK_DIR_CACHE_KEY_MAX_UTF8_BYTES, KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS, + KIMI_WORK_DIR_MAP_MAX_RETAINED_UTF8_BYTES, + KIMI_WORK_DIR_SESSION_ID_MAX_UTF8_BYTES, KIMI_WORK_DIR_CACHE_TTL_MS, type KimiSessionIndexIdentity } from './session-scanner-kimi-index-cache' import { clearKimiSessionIndexCache, hasKimiSessionIndexCacheEntryForTests, + KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX, + readKimiWorkDirForSessionId, readKimiWorkDirBySessionId } from './session-scanner-kimi-paths' @@ -121,9 +126,42 @@ describe('KimiSessionIndexCache', () => { expect(cache.has('index')).toBe(true) cache.clear() }) + + it('bounds index keys and retained map bytes without changing returned values', async () => { + const cache = new KimiSessionIndexCache() + const oversizedPath = 'x'.repeat(KIMI_WORK_DIR_CACHE_KEY_MAX_UTF8_BYTES + 1) + const oversizedId = 'x'.repeat(KIMI_WORK_DIR_SESSION_ID_MAX_UTF8_BYTES + 1) + const value = new Map([ + [oversizedId, '/oversized'], + ['session', '/repo'] + ]) + + await expect( + cache.get(oversizedPath, IDENTITY, cache.beginRead(), async () => value) + ).resolves.toBe(value) + expect(value).toEqual(new Map([['session', '/repo']])) + expect(cache.has(oversizedPath)).toBe(false) + expect(KIMI_WORK_DIR_MAP_MAX_RETAINED_UTF8_BYTES).toBe(512 * 1024) + }) }) describe('Kimi session index reader cache', () => { + it('bounds retained work dirs without dropping an older requested session', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-kimi-index-bounds-')) + tempDirs.push(root) + const indexPath = join(root, 'session_index.jsonl') + const lines = Array.from( + { length: KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX + 1 }, + (_, index) => JSON.stringify({ sessionId: `session-${index}`, workDir: `/repo/${index}` }) + ) + await writeFile(indexPath, `${lines.join('\n')}\n`) + + await expect(readKimiWorkDirForSessionId(indexPath, 'session-0')).resolves.toBe('/repo/0') + expect((await readKimiWorkDirBySessionId(indexPath)).size).toBe( + KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX + ) + }) + it('releases a retained map when its index file disappears', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-kimi-index-delete-')) tempDirs.push(root) diff --git a/src/main/ai-vault/session-scanner-kimi-index-cache.ts b/src/main/ai-vault/session-scanner-kimi-index-cache.ts index fb5f6926eb3..6f7bc951753 100644 --- a/src/main/ai-vault/session-scanner-kimi-index-cache.ts +++ b/src/main/ai-vault/session-scanner-kimi-index-cache.ts @@ -13,6 +13,12 @@ type KimiSessionIndexCacheEntry = { } export const KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS = 64 +export const KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX = 2_048 +export const KIMI_WORK_DIR_CACHE_KEY_MAX_UTF8_BYTES = 32 * 1024 +export const KIMI_WORK_DIR_SESSION_ID_MAX_UTF8_BYTES = 64 * 1024 +export const KIMI_WORK_DIR_VALUE_MAX_UTF8_BYTES = 128 * 1024 +export const KIMI_WORK_DIR_MAP_MAX_RETAINED_UTF8_BYTES = 512 * 1024 +const KIMI_WORK_DIR_ENTRY_OVERHEAD_BYTES = 64 // Active Vault scans refresh this window; closing the surface releases parsed // index maps soon without making a live Kimi session reread on every scan. export const KIMI_WORK_DIR_CACHE_TTL_MS = 5 * 60_000 @@ -52,8 +58,13 @@ export class KimiSessionIndexCache { generation: number, load: () => Promise> ): Promise> { - if (generation < this.minimumCacheGeneration) { - return load() + const loadBounded = (): Promise> => + load().then((value) => boundKimiWorkDirMap(value)) + if ( + generation < this.minimumCacheGeneration || + Buffer.byteLength(indexPath, 'utf8') > KIMI_WORK_DIR_CACHE_KEY_MAX_UTF8_BYTES + ) { + return loadBounded() } const cached = this.entries.get(indexPath) const now = Date.now() @@ -64,7 +75,7 @@ export class KimiSessionIndexCache { if (cached && cached.generation > generation) { // Why: a slower, older stat must not replace a newer file generation // that another concurrent scan already cached for the same path. - return load() + return loadBounded() } const entry: KimiSessionIndexCacheEntry = { @@ -72,7 +83,7 @@ export class KimiSessionIndexCache { generation, identity, timer: null, - value: load() + value: loadBounded() } this.remember(indexPath, entry, now) return entry.value @@ -120,6 +131,65 @@ export class KimiSessionIndexCache { } } +const retainedBytesByWorkDirMap = new WeakMap, number>() + +export function retainKimiWorkDir( + map: Map, + sessionId: string, + workDir: string +): void { + boundKimiWorkDirMap(map) + const sessionIdBytes = Buffer.byteLength(sessionId, 'utf8') + const workDirBytes = Buffer.byteLength(workDir, 'utf8') + if ( + sessionIdBytes > KIMI_WORK_DIR_SESSION_ID_MAX_UTF8_BYTES || + workDirBytes > KIMI_WORK_DIR_VALUE_MAX_UTF8_BYTES + ) { + return + } + let retainedBytes = retainedBytesByWorkDirMap.get(map) ?? 0 + const existing = map.get(sessionId) + if (existing !== undefined) { + retainedBytes -= kimiWorkDirEntryBytes(sessionId, existing) + } + map.delete(sessionId) + const entryBytes = sessionIdBytes + workDirBytes + KIMI_WORK_DIR_ENTRY_OVERHEAD_BYTES + while ( + map.size >= KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX || + retainedBytes + entryBytes > KIMI_WORK_DIR_MAP_MAX_RETAINED_UTF8_BYTES + ) { + const oldest = map.entries().next().value + if (!oldest) { + break + } + map.delete(oldest[0]) + retainedBytes -= kimiWorkDirEntryBytes(oldest[0], oldest[1]) + } + map.set(sessionId, workDir) + retainedBytesByWorkDirMap.set(map, retainedBytes + entryBytes) +} + +function boundKimiWorkDirMap(map: Map): Map { + if (retainedBytesByWorkDirMap.has(map)) { + return map + } + const entries = [...map] + map.clear() + retainedBytesByWorkDirMap.set(map, 0) + for (const [sessionId, workDir] of entries) { + retainKimiWorkDir(map, sessionId, workDir) + } + return map +} + +function kimiWorkDirEntryBytes(sessionId: string, workDir: string): number { + return ( + Buffer.byteLength(sessionId, 'utf8') + + Buffer.byteLength(workDir, 'utf8') + + KIMI_WORK_DIR_ENTRY_OVERHEAD_BYTES + ) +} + function identitiesMatch(left: KimiSessionIndexIdentity, right: KimiSessionIndexIdentity): boolean { return ( left.changeTimeMs === right.changeTimeMs && diff --git a/src/main/ai-vault/session-scanner-kimi-parser.test.ts b/src/main/ai-vault/session-scanner-kimi-parser.test.ts index 67748ae727f..a093e5d0ae5 100644 --- a/src/main/ai-vault/session-scanner-kimi-parser.test.ts +++ b/src/main/ai-vault/session-scanner-kimi-parser.test.ts @@ -209,6 +209,25 @@ describe('parseKimiSessionFile', () => { expect(assistantPreview.text.endsWith('...')).toBe(true) }) + it('bounds many streamed assistant chunks before the preview is flushed', async () => { + const wireLines = Array.from({ length: 600 }, () => ({ + type: 'context.append_loop_event', + event: { type: 'content.part', part: { type: 'text', text: 'x'.repeat(1024) } } + })) + wireLines.push({ + type: 'context.append_loop_event', + event: { type: 'step.end', part: { type: 'text', text: '' } } + }) + const { file } = await writeKimiSession({ wireLines }) + + const session = await parseKimiSessionFile(file, 'darwin') + + expect(session?.messageCount).toBe(1) + expect(session?.previewMessages).toEqual([ + { role: 'assistant', text: `${'x'.repeat(217)}...`, timestamp: null } + ]) + }) + it('falls back to lastPrompt when the title is empty', async () => { const { file } = await writeKimiSession({ state: { diff --git a/src/main/ai-vault/session-scanner-kimi-parser.ts b/src/main/ai-vault/session-scanner-kimi-parser.ts index 61341098e0d..7767d39f854 100644 --- a/src/main/ai-vault/session-scanner-kimi-parser.ts +++ b/src/main/ai-vault/session-scanner-kimi-parser.ts @@ -1,7 +1,6 @@ -import { createReadStream } from 'node:fs' -import { readFile } from 'node:fs/promises' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import { addPreviewContent, addPreviewMessage, @@ -13,7 +12,7 @@ import { kimiPrimaryAgentWirePath, kimiSessionIdFromStatePath, kimiSessionIndexPathFromStatePath, - readKimiWorkDirBySessionId + readKimiWorkDirForSessionId } from './session-scanner-kimi-paths' import type { FileWithMtime, SessionAccumulator } from './session-scanner-types' import { @@ -26,6 +25,15 @@ import { parseJsonObject } from './session-scanner-values' +// The rendered preview is 220 chars; this also covers hidden-context close-tag scanning. +const KIMI_ASSISTANT_PREVIEW_SOURCE_MAX_CHARS = 512 * 1024 +const KIMI_ASSISTANT_PREVIEW_CHUNK_MAX = 256 + +type KimiAssistantPreviewBuffer = { + chunks: string[] + charCount: number +} + // Parses a Kimi Code `state.json` plus its sibling `agents//wire.jsonl` // transcript into an AI Vault session. Metadata (title, timestamps, last prompt) // comes from state.json; the work directory comes from the top-level @@ -34,13 +42,30 @@ export async function parseKimiSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - let stateRecord: Record | null + let metadata: { + title: string | null + fallbackTitle: string | null + createdAt: string | null + updatedAt: string | null + wirePath: string + } | null try { - stateRecord = asRecord(JSON.parse(await readFile(file.path, 'utf-8')) as unknown) + metadata = await withAiVaultWholeJsonFile(file.path, (content) => { + const stateRecord = parseJsonObject(content) + return stateRecord + ? { + title: normalizeTitleText(extractString(stateRecord.title) ?? ''), + fallbackTitle: normalizeTitleText(extractString(stateRecord.lastPrompt) ?? ''), + createdAt: extractString(stateRecord.createdAt), + updatedAt: extractString(stateRecord.updatedAt), + wirePath: kimiPrimaryAgentWirePath(file.path, stateRecord) + } + : null + }) } catch { return null } - if (!stateRecord) { + if (!metadata) { return null } @@ -50,17 +75,17 @@ export async function parseKimiSessionFile( // Why: Kimi sessions are work-dir-scoped — the resume command must `cd` into // the original directory or the CLI rejects it. That path lives only in the // top-level session_index.jsonl, keyed by the (prefixed) session id. - const workDirBySessionId = await readKimiWorkDirBySessionId( - kimiSessionIndexPathFromStatePath(file.path) + accumulator.cwd = await readKimiWorkDirForSessionId( + kimiSessionIndexPathFromStatePath(file.path), + sessionId ) - accumulator.cwd = workDirBySessionId.get(sessionId) ?? null - accumulator.title = normalizeTitleText(extractString(stateRecord.title) ?? '') - accumulator.fallbackTitle = normalizeTitleText(extractString(stateRecord.lastPrompt) ?? '') - updateTimeline(accumulator, extractString(stateRecord.createdAt)) - updateTimeline(accumulator, extractString(stateRecord.updatedAt)) + accumulator.title = metadata.title + accumulator.fallbackTitle = metadata.fallbackTitle + updateTimeline(accumulator, metadata.createdAt) + updateTimeline(accumulator, metadata.updatedAt) - await consumeKimiWireTranscript(accumulator, kimiPrimaryAgentWirePath(file.path, stateRecord)) + await consumeKimiWireTranscript(accumulator, metadata.wirePath) return finalizeSession(accumulator, platform) } @@ -69,14 +94,15 @@ async function consumeKimiWireTranscript( accumulator: SessionAccumulator, wirePath: string ): Promise { - let pendingAssistantText: string[] = [] + const pendingAssistantText: KimiAssistantPreviewBuffer = { chunks: [], charCount: 0 } const flushAssistant = (): void => { // Why: previews use the 220-char limit (normalizePreviewText), not the // 96-char title limit — assistant replies are shown in full preview width // like every other agent's. Join raw chunks first so inter-chunk spacing // survives; normalizePreviewText then collapses whitespace and caps length. - const text = normalizePreviewText(pendingAssistantText.join('')) - pendingAssistantText = [] + const text = normalizePreviewText(pendingAssistantText.chunks.join('')) + pendingAssistantText.chunks = [] + pendingAssistantText.charCount = 0 if (text) { accumulator.messageCount++ addPreviewMessage(accumulator, { role: 'assistant', text }) @@ -84,10 +110,7 @@ async function consumeKimiWireTranscript( } try { - const lines = createInterface({ - input: createReadStream(wirePath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(wirePath) for await (const line of lines) { const record = parseJsonObject(line) if (!record) { @@ -134,7 +157,7 @@ function consumeKimiUserMessage(accumulator: SessionAccumulator, value: unknown) function consumeKimiLoopEvent( value: unknown, - pendingAssistantText: string[], + pendingAssistantText: KimiAssistantPreviewBuffer, flushAssistant: () => void ): void { const event = asRecord(value) @@ -146,7 +169,7 @@ function consumeKimiLoopEvent( // Push the raw chunk text; flushAssistant normalizes the joined result so // multi-chunk spacing is not lost to per-chunk trimming. if (part?.type === 'text' && typeof part.text === 'string') { - pendingAssistantText.push(part.text) + appendKimiAssistantPreviewText(pendingAssistantText, part.text) } return } @@ -157,6 +180,19 @@ function consumeKimiLoopEvent( } } +function appendKimiAssistantPreviewText(buffer: KimiAssistantPreviewBuffer, text: string): void { + const remaining = KIMI_ASSISTANT_PREVIEW_SOURCE_MAX_CHARS - buffer.charCount + if (remaining <= 0 || text.length === 0) { + return + } + const retained = text.length <= remaining ? text : text.slice(0, remaining) + buffer.chunks.push(retained) + buffer.charCount += retained.length + if (buffer.chunks.length >= KIMI_ASSISTANT_PREVIEW_CHUNK_MAX) { + buffer.chunks = [buffer.chunks.join('')] + } +} + // Kimi reports per-turn usage as {inputOther, output, inputCacheRead, // inputCacheCreation}; sum all four for a session total. Skip any future // cumulative ("session"-scoped) record so turn deltas are not double-counted. diff --git a/src/main/ai-vault/session-scanner-kimi-paths.ts b/src/main/ai-vault/session-scanner-kimi-paths.ts index 7ed457f6c07..2c18b51ecf8 100644 --- a/src/main/ai-vault/session-scanner-kimi-paths.ts +++ b/src/main/ai-vault/session-scanner-kimi-paths.ts @@ -1,16 +1,21 @@ import { stat } from 'node:fs/promises' -import { createReadStream } from 'node:fs' import { homedir } from 'node:os' import { basename, dirname, join } from 'node:path' -import { createInterface } from 'node:readline' -import { asRecord, extractString } from './session-scanner-values' +import { asRecord, extractString, parseJsonObject } from './session-scanner-values' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' import { KimiSessionIndexCache, KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS, - KIMI_WORK_DIR_CACHE_TTL_MS + KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX, + KIMI_WORK_DIR_CACHE_TTL_MS, + retainKimiWorkDir } from './session-scanner-kimi-index-cache' -export { KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS, KIMI_WORK_DIR_CACHE_TTL_MS } +export { + KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS, + KIMI_WORK_DIR_CACHE_MAX_SESSIONS_PER_INDEX, + KIMI_WORK_DIR_CACHE_TTL_MS +} // Why: Kimi Code stores sessions under /sessions/, mirroring the // CLI's own `KIMI_CODE_HOME ?? ~/.kimi-code` resolution (see kimi-fetcher.ts). @@ -99,31 +104,34 @@ export async function readKimiWorkDirBySessionId(indexPath: string): Promise { + const workDirs = await readKimiWorkDirBySessionId(indexPath) + const cached = workDirs.get(sessionId) + if (cached) { + return cached + } + const workDir = await readKimiWorkDirFromIndex(indexPath, sessionId) + if (workDir) { + retainKimiWorkDir(workDirs, sessionId, workDir) + } + return workDir +} + async function parseKimiSessionIndex(indexPath: string): Promise> { const map = new Map() // Why: never reject. This promise is memoized and shared by every session // under one Kimi home; a mid-read failure (file deleted after stat, EACCES) // must degrade to whatever was parsed so the other sessions still list. try { - const lines = createInterface({ - input: createReadStream(indexPath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(indexPath) for await (const line of lines) { - if (!line.trim()) { - continue - } - let record: Record | null - try { - record = asRecord(JSON.parse(line) as unknown) - } catch { - continue - } - const sessionId = extractString(record?.sessionId) - const workDir = extractString(record?.workDir) - if (sessionId && workDir) { + const entry = parseKimiSessionIndexEntry(line) + if (entry) { // Later lines win so a resumed session reflects its most recent workDir. - map.set(sessionId, workDir) + retainKimiWorkDir(map, entry.sessionId, entry.workDir) } } } catch { @@ -131,3 +139,35 @@ async function parseKimiSessionIndex(indexPath: string): Promise { + let requestedWorkDir: string | null = null + try { + for await (const line of iterateAiVaultJsonlLines(indexPath)) { + const entry = parseKimiSessionIndexEntry(line) + if (entry?.sessionId === requestedSessionId) { + requestedWorkDir = entry.workDir + } + } + } catch { + // Match the partial, best-effort full-index read. + } + return requestedWorkDir +} + +function parseKimiSessionIndexEntry(line: string): { sessionId: string; workDir: string } | null { + if (!line.trim()) { + return null + } + try { + const record = parseJsonObject(line) + const sessionId = extractString(record?.sessionId) + const workDir = extractString(record?.workDir) + return sessionId && workDir ? { sessionId, workDir } : null + } catch { + return null + } +} diff --git a/src/main/ai-vault/session-scanner-opencode-sources.ts b/src/main/ai-vault/session-scanner-opencode-sources.ts index e4934aa8fda..b8f14a63af7 100644 --- a/src/main/ai-vault/session-scanner-opencode-sources.ts +++ b/src/main/ai-vault/session-scanner-opencode-sources.ts @@ -1,7 +1,10 @@ -import { readdir } from 'node:fs/promises' import { homedir } from 'node:os' import { dirname, join } from 'node:path' import type { AiVaultScanIssue } from '../../shared/ai-vault-types' +import { + listOpenCodeDatabaseFiles, + OPENCODE_DATABASE_FILE_LIMIT +} from '../opencode/opencode-database-files' import { listOpenCodeDatabases } from '../opencode-usage/scanner' import { discoverOpenCodeSessions } from './session-scanner-opencode-sqlite-discovery' import type { AiVaultScanOptions, SessionFileDiscovery } from './session-scanner-types' @@ -10,7 +13,6 @@ const OPENCODE_STORAGE_DIR = join( process.env.OPENCODE_CONFIG_DIR?.trim() || join(homedir(), '.local', 'share', 'opencode'), 'storage' ) - export function opencodeDiscoveries( options: AiVaultScanOptions, wslHomeDirs: readonly string[], @@ -21,7 +23,7 @@ export function opencodeDiscoveries( return storageDirs.map(async (storageDir, index) => discoverOpenCodeSessions({ storageDir, - dbPaths: await opencodeDbPathsForSource(options, wslHomeDirs, storageDir, index), + dbPaths: await opencodeDbPathsForSource(options, wslHomeDirs, storageDir, index, issues), limitPerAgent: limit, issues }) @@ -42,32 +44,36 @@ async function opencodeDbPathsForSource( options: AiVaultScanOptions, wslHomeDirs: readonly string[], storageDir: string, - sourceIndex: number + sourceIndex: number, + issues: AiVaultScanIssue[] ): Promise { if (options.opencodeDbPaths) { return sourceIndex === 0 ? options.opencodeDbPaths : [] } // Why: custom OpenCode storage roots still keep SQLite DBs in the parent data dir. if (sourceIndex === 0 && options.opencodeStorageDir) { - return listOpenCodeDatabasesInDirectory(dirname(storageDir)) + return listOpenCodeDatabasesInDirectory(dirname(storageDir), issues) } if (sourceIndex === 0) { return listOpenCodeDatabases() } const wslHomeDir = wslHomeDirs[sourceIndex - 1] return wslHomeDir - ? listOpenCodeDatabasesInDirectory(join(wslHomeDir, '.local', 'share', 'opencode')) + ? listOpenCodeDatabasesInDirectory(join(wslHomeDir, '.local', 'share', 'opencode'), issues) : [] } -async function listOpenCodeDatabasesInDirectory(dataDir: string): Promise { - try { - const entries = await readdir(dataDir, { withFileTypes: true }) - return entries - .filter((entry) => entry.isFile() && /^opencode(?:-[A-Za-z0-9_.-]+)?\.db$/.test(entry.name)) - .map((entry) => join(dataDir, entry.name)) - .sort() - } catch { - return [] +async function listOpenCodeDatabasesInDirectory( + dataDir: string, + issues: AiVaultScanIssue[] +): Promise { + const result = await listOpenCodeDatabaseFiles(dataDir) + if (result.truncated) { + issues.push({ + agent: 'opencode', + path: dataDir, + message: `OpenCode database discovery stopped after ${OPENCODE_DATABASE_FILE_LIMIT} files.` + }) } + return result.paths } diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite-bounds.test.ts b/src/main/ai-vault/session-scanner-opencode-sqlite-bounds.test.ts index 7f50a220247..dd075baa36e 100644 --- a/src/main/ai-vault/session-scanner-opencode-sqlite-bounds.test.ts +++ b/src/main/ai-vault/session-scanner-opencode-sqlite-bounds.test.ts @@ -1,12 +1,18 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import type { AiVaultScanIssue } from '../../shared/ai-vault-types' import Database from '../sqlite/sync-database' import { buildOpenCodeSqliteCandidatePath } from './session-scanner-opencode-sqlite-paths' import { listOpenCodeSqliteSessions } from './session-scanner-opencode-sqlite-list' import { parseOpenCodeSqliteSession } from './session-scanner-opencode-sqlite' +import { + OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES, + OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES, + OPENCODE_SQLITE_PART_JSON_MAX_BYTES, + OPENCODE_SQLITE_SESSION_ID_MAX_BYTES +} from './session-scanner-opencode-sqlite-limits' // Part B (#8864) bounds: discovery reads only the newest session identities and // recency fields, while the preview join reads parts of only the newest 100 @@ -15,6 +21,7 @@ import { parseOpenCodeSqliteSession } from './session-scanner-opencode-sqlite' let tempDirs: string[] = [] afterEach(() => { + vi.restoreAllMocks() for (const dir of tempDirs) { rmSync(dir, { recursive: true, force: true }) } @@ -120,6 +127,22 @@ function insertMessageWithPart( } describe('listOpenCodeSqliteSessions — LIMIT-first discovery', () => { + it('keeps an exact-limit session id and omits limit +1 before materializing it', async () => { + const { db, path } = createTempDb() + applySchema(db) + const exactId = 'e'.repeat(OPENCODE_SQLITE_SESSION_ID_MAX_BYTES) + const oversizedId = 'o'.repeat(OPENCODE_SQLITE_SESSION_ID_MAX_BYTES + 1) + insertSession(db, exactId, 1_777_634_001_000) + insertSession(db, oversizedId, 1_777_634_002_000) + db.close() + + const candidates = await listOpenCodeSqliteSessions({ dbPaths: [path], limit: 10, issues: [] }) + + expect(candidates.map((candidate) => candidate.file.path)).toEqual([ + buildOpenCodeSqliteCandidatePath(path, exactId) + ]) + }) + it('returns only the newest `limit` sessions by time_updated', async () => { const { db, path } = createTempDb() applySchema(db) @@ -174,6 +197,88 @@ describe('listOpenCodeSqliteSessions — LIMIT-first discovery', () => { }) describe('parseOpenCodeSqliteSession — bounded preview window', () => { + it('preserves exact-limit model JSON and drops limit +1 before JSON.parse', async () => { + const { db, path } = createTempDb() + applySchema(db) + insertSession(db, 'ses_model_exact', 1_777_634_001_000) + insertSession(db, 'ses_model_oversized', 1_777_634_002_000) + const modelPrefix = '{"id":"' + const modelSuffix = '"}' + db.prepare(`UPDATE session SET model = ? WHERE id = 'ses_model_exact'`).run( + `${modelPrefix}${'m'.repeat( + OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES - modelPrefix.length - modelSuffix.length + )}${modelSuffix}` + ) + db.prepare(`UPDATE session SET model = ? WHERE id = 'ses_model_oversized'`).run( + 'x'.repeat(OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES + 1) + ) + db.close() + + const exact = await parseOpenCodeSqliteSession({ + dbPath: path, + sessionId: 'ses_model_exact', + platform: 'darwin' + }) + const parseSpy = vi.spyOn(JSON, 'parse') + const oversized = await parseOpenCodeSqliteSession({ + dbPath: path, + sessionId: 'ses_model_oversized', + platform: 'darwin' + }) + + expect(exact?.model).toHaveLength( + OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES - modelPrefix.length - modelSuffix.length + ) + expect(oversized?.model).toBeNull() + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('omits oversized message and part JSON before retaining previews', async () => { + const { db, path } = createTempDb() + applySchema(db) + insertSession(db, 'ses_oversized_rows', 1_777_634_001_000) + db.prepare(`INSERT INTO message (id, session_id, time_created, data) VALUES (?, ?, ?, ?)`).run( + 'msg_oversized', + 'ses_oversized_rows', + 1_777_634_000_100, + 'x'.repeat(OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES + 1) + ) + db.prepare( + `INSERT INTO part (id, message_id, session_id, time_created, data) VALUES (?, ?, ?, ?, ?)` + ).run( + 'part_for_oversized_message', + 'msg_oversized', + 'ses_oversized_rows', + 1_777_634_000_100, + JSON.stringify({ type: 'text', text: 'must not surface' }) + ) + db.prepare(`INSERT INTO message (id, session_id, time_created, data) VALUES (?, ?, ?, ?)`).run( + 'msg_normal', + 'ses_oversized_rows', + 1_777_634_000_200, + JSON.stringify({ role: 'user' }) + ) + db.prepare( + `INSERT INTO part (id, message_id, session_id, time_created, data) VALUES (?, ?, ?, ?, ?)` + ).run( + 'part_oversized', + 'msg_normal', + 'ses_oversized_rows', + 1_777_634_000_200, + 'x'.repeat(OPENCODE_SQLITE_PART_JSON_MAX_BYTES + 1) + ) + db.close() + + const session = await parseOpenCodeSqliteSession({ + dbPath: path, + sessionId: 'ses_oversized_rows', + platform: 'darwin' + }) + + expect(session?.messageCount).toBe(1) + expect(session?.previewMessages).toEqual([]) + }) + it('surfaces the newest text previews and retains the full message count', async () => { // Positive coverage of preview selection (newest OPENCODE_SQLITE_PREVIEW_LIMIT // by recency) + full-count retention. The window bound itself is pinned by the diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite-limits.ts b/src/main/ai-vault/session-scanner-opencode-sqlite-limits.ts new file mode 100644 index 00000000000..b0cd7c0f360 --- /dev/null +++ b/src/main/ai-vault/session-scanner-opencode-sqlite-limits.ts @@ -0,0 +1,5 @@ +export const OPENCODE_SQLITE_SESSION_ID_MAX_BYTES = 16 * 1024 +export const OPENCODE_SQLITE_SESSION_TEXT_MAX_BYTES = 1024 * 1024 +export const OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES = 1024 * 1024 +export const OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES = 1024 * 1024 +export const OPENCODE_SQLITE_PART_JSON_MAX_BYTES = 4 * 1024 * 1024 diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite-list.ts b/src/main/ai-vault/session-scanner-opencode-sqlite-list.ts index b89b97b229e..a398712bf47 100644 --- a/src/main/ai-vault/session-scanner-opencode-sqlite-list.ts +++ b/src/main/ai-vault/session-scanner-opencode-sqlite-list.ts @@ -5,6 +5,7 @@ import type { SessionFileCandidate } from './session-scanner-types' import { errorMessage } from './session-scanner-values' import SyncDatabase from '../sqlite/sync-database' import { columnExists, tableExists } from '../opencode-usage/schema-helpers' +import { OPENCODE_SQLITE_SESSION_ID_MAX_BYTES } from './session-scanner-opencode-sqlite-limits' // Why: the SQLite session-list query + reader lives in its own electron-free // module so both the worker entry and the main-thread worker client can import @@ -41,7 +42,9 @@ function buildSessionListQuery(db: SyncDatabase): string { // refresh; the parse path loads metadata only for candidates it actually uses. return `SELECT id, time_created, time_updated FROM session - WHERE 1=1 ${parentIdPredicate} ${archivedPredicate} + WHERE typeof(id) = 'text' + AND length(CAST(id AS BLOB)) <= ${OPENCODE_SQLITE_SESSION_ID_MAX_BYTES} + ${parentIdPredicate} ${archivedPredicate} ORDER BY CASE WHEN time_updated > 0 THEN time_updated ELSE time_created END DESC LIMIT ?` } diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.test.ts b/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.test.ts index ab04a455cb5..ee68f50a7ff 100644 --- a/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.test.ts +++ b/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.test.ts @@ -4,6 +4,7 @@ import { IDLE_TEARDOWN_MS, LIST_TIMEOUT_MS, MAX_CONSECUTIVE_DEATHS, + MAX_PENDING_CALLS, OpenCodeSqliteWorkerClient, PARSE_TIMEOUT_MS } from './session-scanner-opencode-sqlite-worker-client' @@ -122,6 +123,33 @@ describe('OpenCodeSqliteWorkerClient', () => { expect(workers).toHaveLength(1) }) + it('rejects overload beyond the bounded process-wide queue and recovers after draining', async () => { + const workers: FakeWorker[] = [] + const client = new OpenCodeSqliteWorkerClient({ workerFactory: makeFactory(workers), log() {} }) + const admitted = Array.from({ length: MAX_PENDING_CALLS }, (_, index) => + client.parse({ dbPath: `/db#${index}`, sessionId: `s${index}`, platform: 'darwin' }) + ) + const settled = admitted.map((promise) => promise.catch((error: unknown) => error)) + + await expect( + client.parse({ dbPath: '/db#overflow', sessionId: 'overflow', platform: 'darwin' }) + ).rejects.toThrow(/256-request capacity/) + expect(workers[0]!.postedRequests).toHaveLength(1) + + for (let index = 0; index < MAX_CONSECUTIVE_DEATHS; index += 1) { + workers.at(-1)!.emit('error', new Error(`drain ${index}`)) + } + await Promise.all(settled) + + const recovered = client.parse({ + dbPath: '/db#recovered', + sessionId: 'recovered', + platform: 'darwin' + }) + workers.at(-1)!.emit('message', { id: workers.at(-1)!.lastId(), ok: true, value: 'ok' }) + await expect(recovered).resolves.toBe('ok') + }) + it('times out only the active call, then respawns and drains the queue', async () => { vi.useFakeTimers() try { diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.ts b/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.ts index ef156b0b3f4..faa4ef77e97 100644 --- a/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.ts +++ b/src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.ts @@ -19,6 +19,7 @@ import { errorMessage } from './session-scanner-values' export const LIST_TIMEOUT_MS = 30_000 export const PARSE_TIMEOUT_MS = 15_000 export const IDLE_TEARDOWN_MS = 30_000 +export const MAX_PENDING_CALLS = 256 // After this many consecutive worker deaths, fail the remaining queued calls to // scan issues instead of respawning so a DB that reliably kills the worker can't // spin a crash loop. Reset on any successful response, after draining, and when a @@ -142,6 +143,11 @@ export class OpenCodeSqliteWorkerClient { } private dispatch(request: OpenCodeSqliteRequestBody, timeoutMs: number): Promise { + if (this.queue.length + (this.active ? 1 : 0) >= MAX_PENDING_CALLS) { + return Promise.reject( + new Error(`OpenCode SQLite worker reached its ${MAX_PENDING_CALLS}-request capacity`) + ) + } return new Promise((resolve, reject) => { const id = this.nextId++ // A fresh burst from full idle starts a new scan: clear any death count diff --git a/src/main/ai-vault/session-scanner-opencode-sqlite.ts b/src/main/ai-vault/session-scanner-opencode-sqlite.ts index ad3e9b76f41..898ca9624db 100644 --- a/src/main/ai-vault/session-scanner-opencode-sqlite.ts +++ b/src/main/ai-vault/session-scanner-opencode-sqlite.ts @@ -5,9 +5,15 @@ import { finalizeSession, updateTimeline } from './session-scanner-accumulator' -import { normalizeTitleText } from './session-scanner-values' +import { normalizeTitleText, parseAiVaultJsonText } from './session-scanner-values' import SyncDatabase from '../sqlite/sync-database' import { columnExists, tableExists } from '../opencode-usage/schema-helpers' +import { + OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES, + OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES, + OPENCODE_SQLITE_PART_JSON_MAX_BYTES, + OPENCODE_SQLITE_SESSION_TEXT_MAX_BYTES +} from './session-scanner-opencode-sqlite-limits' // Why: OpenCode 1.17.x migrated session storage from per-session JSON files // to a single SQLite DB at ~/.local/share/opencode/opencode.db. This module @@ -63,8 +69,16 @@ function canReadOpenCodeSessions(db: SyncDatabase): boolean { ) } -function sessionColumnSelect(db: SyncDatabase, columnName: string): string { - return columnExists(db, 'session', columnName) ? `s.${columnName}` : 'NULL' +function boundedSessionTextColumnSelect( + db: SyncDatabase, + columnName: string, + maxBytes: number +): string { + return columnExists(db, 'session', columnName) + ? `CASE WHEN typeof(s.${columnName}) = 'text' + AND length(CAST(s.${columnName} AS BLOB)) <= ${maxBytes} + THEN s.${columnName} ELSE NULL END` + : 'NULL' } function sessionNumberColumnSelect(db: SyncDatabase, columnName: string): string { @@ -83,15 +97,20 @@ function buildSessionQuery(db: SyncDatabase): string { const messageCountSubquery = canCountOpenCodeMessages(db) ? `(SELECT COUNT(*) FROM message m WHERE m.session_id = s.id - AND json_extract(m.data, '$.role') IN ('user','assistant'))` + AND CASE + WHEN typeof(m.data) = 'text' + AND length(CAST(m.data AS BLOB)) <= ${OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES} + THEN json_extract(m.data, '$.role') IN ('user','assistant') + ELSE 0 + END)` : '0' return `SELECT s.id, - ${sessionColumnSelect(db, 'title')} AS title, - ${sessionColumnSelect(db, 'directory')} AS directory, + ${boundedSessionTextColumnSelect(db, 'title', OPENCODE_SQLITE_SESSION_TEXT_MAX_BYTES)} AS title, + ${boundedSessionTextColumnSelect(db, 'directory', OPENCODE_SQLITE_SESSION_TEXT_MAX_BYTES)} AS directory, s.time_created, s.time_updated, - ${sessionColumnSelect(db, 'model')} AS model_json, - ${sessionColumnSelect(db, 'agent')} AS agent, + ${boundedSessionTextColumnSelect(db, 'model', OPENCODE_SQLITE_MODEL_JSON_MAX_BYTES)} AS model_json, + ${boundedSessionTextColumnSelect(db, 'agent', OPENCODE_SQLITE_SESSION_TEXT_MAX_BYTES)} AS agent, ${sessionNumberColumnSelect(db, 'tokens_input')} AS tokens_input, ${sessionNumberColumnSelect(db, 'tokens_output')} AS tokens_output, ${sessionNumberColumnSelect(db, 'tokens_reasoning')} AS tokens_reasoning, @@ -108,7 +127,7 @@ function extractModelId(modelJson: string | null): string | null { return null } try { - const parsed = JSON.parse(modelJson) as unknown + const parsed = parseAiVaultJsonText(modelJson) const record = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) @@ -137,7 +156,7 @@ function mapPreviewRole(role: string | null): AiVaultSessionPreviewMessage['role function extractPartText(partData: string): string | null { try { - const parsed = JSON.parse(partData) as unknown + const parsed = parseAiVaultJsonText(partData) const record = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) @@ -172,11 +191,18 @@ function buildPreviewQuery(db: SyncDatabase): string | null { json_extract(m.data, '$.summary.body') AS summary_body FROM (SELECT id, data FROM message WHERE session_id = ? + AND typeof(data) = 'text' + AND length(CAST(data AS BLOB)) <= ${OPENCODE_SQLITE_MESSAGE_JSON_MAX_BYTES} ORDER BY time_created DESC, id DESC LIMIT ${OPENCODE_SQLITE_PREVIEW_MESSAGE_WINDOW}) m JOIN part p ON p.message_id = m.id WHERE json_extract(m.data, '$.role') IN ('user','assistant') - AND json_extract(p.data, '$.type') = 'text' + AND CASE + WHEN typeof(p.data) = 'text' + AND length(CAST(p.data AS BLOB)) <= ${OPENCODE_SQLITE_PART_JSON_MAX_BYTES} + THEN json_extract(p.data, '$.type') = 'text' + ELSE 0 + END ORDER BY p.time_created DESC LIMIT ?` } diff --git a/src/main/ai-vault/session-scanner-parse-cache.test.ts b/src/main/ai-vault/session-scanner-parse-cache.test.ts index 6ba5e3266d6..1d1bbf58231 100644 --- a/src/main/ai-vault/session-scanner-parse-cache.test.ts +++ b/src/main/ai-vault/session-scanner-parse-cache.test.ts @@ -7,6 +7,7 @@ import { resetSessionParseCacheForTests, createSessionParseStats } from './session-scanner-parse-cache' +import { AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES } from './session-parse-cache-retention' import { parseClaudeSessionFile } from './session-scanner-primary-parsers' import type { FileWithMtime, SessionFileCandidate } from './session-scanner-types' @@ -92,6 +93,29 @@ describe('parseAgentSessionFileCached', () => { expect(stats.incremental).toBe(0) }) + it('returns oversized metadata unchanged without retaining it', async () => { + const root = await makeTempDir() + const path = join(root, 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee.jsonl') + const oversizedModel = 'é'.repeat(AI_VAULT_PARSE_CACHE_VALUE_MAX_UTF8_BYTES / 2) + await writeFile( + path, + `${userRecord(0, 'question')}\n${assistantRecord(1, 'answer').replace( + 'claude-fable-5', + oversizedModel + )}\n` + ) + const candidate = await claudeCandidate(path) + const stats = createSessionParseStats() + + const first = await parseAgentSessionFileCached(candidate, process.platform, stats) + const second = await parseAgentSessionFileCached(candidate, process.platform, stats) + + expect(first?.model).toBe(oversizedModel) + expect(second).toEqual(first) + expect(stats.fullParses).toBe(2) + expect(stats.reused).toBe(0) + }) + it('incrementally parses appended lines and matches a cold parse exactly', async () => { const root = await makeTempDir() const path = join(root, 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee.jsonl') diff --git a/src/main/ai-vault/session-scanner-parse-cache.ts b/src/main/ai-vault/session-scanner-parse-cache.ts index b7aab770432..64835da7b5e 100644 --- a/src/main/ai-vault/session-scanner-parse-cache.ts +++ b/src/main/ai-vault/session-scanner-parse-cache.ts @@ -1,4 +1,3 @@ -import { createReadStream } from 'node:fs' import { open } from 'node:fs/promises' import type { AiVaultSession } from '../../shared/ai-vault-types' import { createAntigravitySessionResumeState } from './session-scanner-antigravity-parser' @@ -14,28 +13,21 @@ import { } from './session-scanner-secondary-parsers' import { countSubagentTranscripts } from './session-scanner-subagent-transcripts' import type { ResumableSessionParseState, SessionFileCandidate } from './session-scanner-types' +import { consumeAiVaultJsonlLines } from './session-jsonl-line-reader' +import { + getSessionParseCacheEntry, + resetSessionParseCacheRetentionForTests, + storeSessionParseCacheEntry, + type SessionParseCacheEntry +} from './session-parse-cache-retention' -// Sized past the default recency cap (1000) plus the in-scope cap (2000) so a -// full steady-state result set stays resident between forced rescans. -const MAX_CACHE_ENTRIES = 4096 +export { + seedSessionParseCache, + snapshotSessionParseCacheForPersistence, + type PersistedSessionParseCacheEntry +} from './session-parse-cache-retention' const NEWLINE_BYTE = 0x0a -const CARRIAGE_RETURN_BYTE = 0x0d - -type ResumePoint = { - state: ResumableSessionParseState - // Byte offset just past the last complete ('\n'-terminated) line consumed; - // a trailing unterminated line is deliberately left before this point. - byteOffset: number -} - -type SessionParseCacheEntry = { - mtimeMs: number - sizeBytes: number | null - platform: NodeJS.Platform - session: AiVaultSession | null - resume: ResumePoint | null -} // Incremental append-parsing applies only to transcripts that are append-only // JSONL line-folds. Whole-JSON documents (grok/rovo/devin/hermes/gemini-json) @@ -91,67 +83,8 @@ export function createSessionParseStats(): SessionParseStats { return { reused: 0, incremental: 0, fullParses: 0, bytesRead: 0 } } -const cache = new Map() - export function resetSessionParseCacheForTests(): void { - cache.clear() -} - -// Persisted subset of a cache entry: the non-serializable `resume` parser -// state is dropped (see session-parse-cache-persistence.ts). -export type PersistedSessionParseCacheEntry = Omit - -export function snapshotSessionParseCacheForPersistence(): [ - string, - PersistedSessionParseCacheEntry -][] { - return [...cache].map(([path, entry]): [string, PersistedSessionParseCacheEntry] => [ - path, - { - mtimeMs: entry.mtimeMs, - sizeBytes: entry.sizeBytes, - platform: entry.platform, - session: entry.session - } - ]) -} - -// Seeded entries carry `resume: null`: after a restart an unchanged file is a -// cache hit; a file that changed while the app was closed pays one full -// (not incremental) re-parse. -export function seedSessionParseCache( - entries: Iterable<[string, PersistedSessionParseCacheEntry]> -): void { - const list = [...entries] - // Snapshot order is oldest→newest (LRU); an over-cap list keeps the newest - // tail rather than seeding the oldest entries and dropping the tail. - for (const [path, entry] of list.slice(Math.max(0, list.length - MAX_CACHE_ENTRIES))) { - if (cache.size >= MAX_CACHE_ENTRIES) { - return - } - // In-process entries are always fresher than persisted ones; never clobber. - if (cache.has(path)) { - continue - } - cache.set(path, { - mtimeMs: entry.mtimeMs, - sizeBytes: entry.sizeBytes, - platform: entry.platform, - session: entry.session, - resume: null - }) - } -} - -function storeEntry(path: string, entry: SessionParseCacheEntry): void { - cache.delete(path) - cache.set(path, entry) - if (cache.size > MAX_CACHE_ENTRIES) { - const oldest = cache.keys().next() - if (!oldest.done) { - cache.delete(oldest.value) - } - } + resetSessionParseCacheRetentionForTests() } /** @@ -169,7 +102,7 @@ export async function parseAgentSessionFileCached( stats?: SessionParseStats ): Promise { const { file } = candidate - const entry = cache.get(file.path) + const entry = getSessionParseCacheEntry(file.path) const unchanged = entry !== undefined && @@ -190,7 +123,7 @@ export async function parseAgentSessionFileCached( entry.session = { ...entry.session, subagentTranscriptCount } } } - storeEntry(file.path, entry) + storeSessionParseCacheEntry(file.path, entry) return entry.session } @@ -203,7 +136,7 @@ export async function parseAgentSessionFileCached( stats, stateFactory }) - storeEntry(file.path, parsed) + storeSessionParseCacheEntry(file.path, parsed) return parsed.session } @@ -212,7 +145,7 @@ export async function parseAgentSessionFileCached( stats.bytesRead += file.sizeBytes ?? 0 } const session = await parseAgentSessionFile(candidate, platform) - storeEntry(file.path, { + storeSessionParseCacheEntry(file.path, { mtimeMs: file.mtimeMs, sizeBytes: file.sizeBytes ?? null, platform, @@ -250,7 +183,7 @@ async function parseResumableCandidate(args: { } } - const readResult = await consumeCompleteJsonlLines({ + const readResult = await consumeAiVaultJsonlLines({ path: file.path, start: startOffset, onLine: (line) => state.consumeLine(line) @@ -294,48 +227,3 @@ async function endsWithNewlineAt(path: string, offset: number): Promise await handle.close() } } - -type JsonlReadResult = { - consumedThrough: number - trailingPartialLine: string | null - bytesRead: number -} - -// Byte-accurate replacement for readline: offsets must count bytes (not -// UTF-8-decoded characters) so a resumed read starts exactly where the last -// complete line ended. -async function consumeCompleteJsonlLines(args: { - path: string - start: number - onLine: (line: string) => void -}): Promise { - let consumedThrough = args.start - let bytesRead = 0 - let remainder: Buffer | null = null - - const stream = createReadStream(args.path, { start: args.start }) - for await (const chunk of stream as AsyncIterable) { - bytesRead += chunk.length - const data = remainder ? Buffer.concat([remainder, chunk]) : chunk - let lineStart = 0 - let newlineIndex = data.indexOf(NEWLINE_BYTE, lineStart) - while (newlineIndex !== -1) { - let lineEnd = newlineIndex - if (lineEnd > lineStart && data[lineEnd - 1] === CARRIAGE_RETURN_BYTE) { - lineEnd-- - } - args.onLine(data.toString('utf-8', lineStart, lineEnd)) - lineStart = newlineIndex + 1 - newlineIndex = data.indexOf(NEWLINE_BYTE, lineStart) - } - consumedThrough += lineStart - // Copy the tail so retaining it doesn't pin the whole chunk buffer. - remainder = lineStart < data.length ? Buffer.from(data.subarray(lineStart)) : null - } - - return { - consumedThrough, - trailingPartialLine: remainder && remainder.length > 0 ? remainder.toString('utf-8') : null, - bytesRead - } -} diff --git a/src/main/ai-vault/session-scanner-primary-parsers.ts b/src/main/ai-vault/session-scanner-primary-parsers.ts index c7e67ec463b..cbd36f74c36 100644 --- a/src/main/ai-vault/session-scanner-primary-parsers.ts +++ b/src/main/ai-vault/session-scanner-primary-parsers.ts @@ -1,5 +1,3 @@ -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' import { LOCAL_EXECUTION_HOST_ID, type ExecutionHostId } from '../../shared/execution-host' import { isKnownHarnessInjectedUserTurnText } from '../../shared/harness-injected-user-turns' @@ -13,11 +11,13 @@ import { addPreviewContent, createAccumulator, finalizeSession, + sessionAccumulatorRetainedUtf8Bytes, sessionIdFromFileName, updateLatestLocation, updateTimeline } from './session-scanner-accumulator' import { countSubagentTranscripts } from './session-scanner-subagent-transcripts' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' import { asRecord, claudeUsageTotal, @@ -193,6 +193,11 @@ function claudeResumeStateFromParseState( return { consumeLine: (line) => consumeClaudeSessionLine(state, line), clone: () => claudeResumeStateFromParseState(cloneClaudeSessionParseState(state)), + retainedUtf8Bytes: () => + sessionAccumulatorRetainedUtf8Bytes(state.accumulator) + + Buffer.byteLength(state.metaTitle ?? '', 'utf8') + + Buffer.byteLength(state.generatedTitle ?? '', 'utf8') + + Buffer.byteLength(state.firstUserTitle ?? '', 'utf8'), touchFile: (file) => { state.accumulator.modifiedAt = file.modifiedAt }, @@ -204,10 +209,7 @@ export async function parseClaudeSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseClaudeSessionLines({ file, lines, platform }) } diff --git a/src/main/ai-vault/session-scanner-retention.test.ts b/src/main/ai-vault/session-scanner-retention.test.ts new file mode 100644 index 00000000000..4c3f63dfe00 --- /dev/null +++ b/src/main/ai-vault/session-scanner-retention.test.ts @@ -0,0 +1,41 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { AI_VAULT_SESSION_ID_MAX_UTF8_BYTES } from './session-list-retention' +import { scanAiVaultSessions } from './session-scanner' +import { isolatedScanRoots, writeJsonlFile } from './session-scanner-test-fixtures' + +const tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('local AI Vault result retention', () => { + it('omits a resume-critical overflow and surfaces a bounded scan issue', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-vault-result-retention-')) + tempRoots.push(root) + const roots = isolatedScanRoots(root) + const sessionId = 'x'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES + 1) + await writeJsonlFile(join(roots.claudeProjectsDir, 'repo', 'session.jsonl'), [ + { + type: 'user', + sessionId, + timestamp: '2026-01-01T00:00:00.000Z', + cwd: '/repo', + message: { role: 'user', content: 'Retain safely' } + } + ]) + + const result = await scanAiVaultSessions(roots) + + expect(result.sessions).toEqual([]) + expect(result.issues).toEqual([ + expect.objectContaining({ + path: join(roots.claudeProjectsDir, 'repo', 'session.jsonl'), + message: expect.stringContaining('session id exceeds the 65536 byte limit') + }) + ]) + }) +}) diff --git a/src/main/ai-vault/session-scanner-scope-discovery.ts b/src/main/ai-vault/session-scanner-scope-discovery.ts index 41aac5f2700..525ef6d4c9c 100644 --- a/src/main/ai-vault/session-scanner-scope-discovery.ts +++ b/src/main/ai-vault/session-scanner-scope-discovery.ts @@ -1,6 +1,4 @@ -import { createReadStream } from 'node:fs' -import { readdir, stat } from 'node:fs/promises' -import { createInterface } from 'node:readline' +import { opendir, stat } from 'node:fs/promises' import { extname, join } from 'node:path' import { isPathInsideOrEqual, @@ -8,6 +6,8 @@ import { } from '../../shared/cross-platform-path' import type { AiVaultScanIssue } from '../../shared/ai-vault-types' import { parseWslUncPath } from '../../shared/wsl-paths' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { AiVaultScopeCwdCache } from './session-scope-cwd-cache' import type { FileWithMtime } from './session-scanner-types' import { errorMessage, extractString, parseJsonObject } from './session-scanner-values' @@ -19,8 +19,7 @@ const CLAUDE_EXTENSIONS = new Set(['.jsonl']) // A Claude project dir encodes exactly one cwd, so a resolved cwd never // changes; caching it spares each rescan the transcript-head reads. -const PROJECT_DIR_CWD_CACHE_MAX = 2048 -const projectDirCwdCache = new Map() +const projectDirCwdCache = new AiVaultScopeCwdCache() export function resetProjectDirCwdCacheForTests(): void { projectDirCwdCache.clear() @@ -29,19 +28,10 @@ export function resetProjectDirCwdCacheForTests(): void { async function cachedProjectDirCwd(projectDir: string): Promise { const cached = projectDirCwdCache.get(projectDir) if (cached !== undefined) { - // Refresh recency so hot in-scope dirs outlive one-off ones at the cap. - projectDirCwdCache.delete(projectDir) - projectDirCwdCache.set(projectDir, cached) return cached } const cwd = await readProjectDirCwd(projectDir) if (cwd) { - if (projectDirCwdCache.size >= PROJECT_DIR_CWD_CACHE_MAX) { - const oldest = projectDirCwdCache.keys().next() - if (!oldest.done) { - projectDirCwdCache.delete(oldest.value) - } - } projectDirCwdCache.set(projectDir, cwd) } return cwd @@ -70,7 +60,7 @@ export async function discoverInScopeClaudeFiles(args: { const scopeProjectPrefixes = claudeProjectScopePrefixes(args.scopePaths) const collected = new Map() for (const rootDir of args.rootDirs) { - for (const projectDir of await listProjectDirs(rootDir, scopeProjectPrefixes)) { + for await (const projectDir of iterateProjectDirs(rootDir, scopeProjectPrefixes)) { const cwd = await cachedProjectDirCwd(projectDir) if (!cwd || !args.scopePaths.some((scopePath) => isCwdInsideScopePath(scopePath, cwd))) { continue @@ -130,21 +120,20 @@ function isCwdInsideScopePath(scopePath: string, cwd: string): boolean { return isPathInsideOrEqual(wslScopePath.linuxPath, cwd) } -async function listProjectDirs( +async function* iterateProjectDirs( rootDir: string, scopeProjectPrefixes: ReadonlySet -): Promise { - let entries +): AsyncGenerator { try { - entries = await readdir(rootDir, { withFileTypes: true }) + const directory = await opendir(rootDir) + for await (const entry of directory) { + if (entry.isDirectory() && isClaudeProjectDirInScope(entry.name, scopeProjectPrefixes)) { + yield join(rootDir, entry.name) + } + } } catch { - return [] + // Missing or unreadable roots have no project directories to yield. } - return entries - .filter( - (entry) => entry.isDirectory() && isClaudeProjectDirInScope(entry.name, scopeProjectPrefixes) - ) - .map((entry) => join(rootDir, entry.name)) } async function readProjectDirCwd(projectDir: string): Promise { @@ -159,34 +148,31 @@ async function readProjectDirCwd(projectDir: string): Promise { } async function newestClaudeFilesInDir(projectDir: string): Promise { - let entries + const newest: { path: string; mtimeMs: number }[] = [] try { - entries = await readdir(projectDir, { withFileTypes: true }) + const directory = await opendir(projectDir) + for await (const entry of directory) { + if (!entry.isFile() || !CLAUDE_EXTENSIONS.has(extname(entry.name).toLowerCase())) { + continue + } + const path = join(projectDir, entry.name) + try { + addBoundedPath(newest, REPRESENTATIVE_FILE_LIMIT, { + path, + mtimeMs: (await stat(path)).mtimeMs + }) + } catch { + // Best effort: unreadable candidates are ignored here and reported during full collection. + } + } } catch { return [] } - const newest: { path: string; mtimeMs: number }[] = [] - for (const entry of entries) { - if (!entry.isFile() || !CLAUDE_EXTENSIONS.has(extname(entry.name).toLowerCase())) { - continue - } - const path = join(projectDir, entry.name) - try { - addBoundedPath(newest, REPRESENTATIVE_FILE_LIMIT, { - path, - mtimeMs: (await stat(path)).mtimeMs - }) - } catch { - // Best effort: unreadable candidates are ignored here and reported during - // full collection if the project directory proves in-scope. - } - } return newest.sort((left, right) => right.mtimeMs - left.mtimeMs).map((value) => value.path) } async function readFirstCwd(filePath: string): Promise { - const input = createReadStream(filePath, { encoding: 'utf-8' }) - const lines = createInterface({ input, crlfDelay: Infinity }) + const lines = iterateAiVaultJsonlLines(filePath) let read = 0 try { for await (const line of lines) { @@ -200,11 +186,6 @@ async function readFirstCwd(filePath: string): Promise { } } catch { return null - } finally { - // readline.close() leaves the underlying stream open; destroy it so the early - // break/catch paths don't leak a file descriptor (this runs per project dir). - lines.close() - input.destroy() } return null } @@ -216,31 +197,30 @@ async function collectClaudeFiles(args: { limit: number excludedFilePaths: ReadonlySet }): Promise { - let entries try { - entries = await readdir(args.projectDir, { withFileTypes: true }) + const directory = await opendir(args.projectDir) + for await (const entry of directory) { + if (!entry.isFile() || !CLAUDE_EXTENSIONS.has(extname(entry.name).toLowerCase())) { + continue + } + const path = join(args.projectDir, entry.name) + if (args.collected.has(path) || args.excludedFilePaths.has(path)) { + continue + } + try { + const fileStat = await stat(path) + addBoundedFile(args.collected, args.limit, { + path, + mtimeMs: fileStat.mtimeMs, + modifiedAt: fileStat.mtime.toISOString(), + sizeBytes: fileStat.size + }) + } catch (err) { + args.issues.push({ agent: 'claude', path, message: errorMessage(err) }) + } + } } catch { - return - } - for (const entry of entries) { - if (!entry.isFile() || !CLAUDE_EXTENSIONS.has(extname(entry.name).toLowerCase())) { - continue - } - const path = join(args.projectDir, entry.name) - if (args.collected.has(path) || args.excludedFilePaths.has(path)) { - continue - } - try { - const fileStat = await stat(path) - addBoundedFile(args.collected, args.limit, { - path, - mtimeMs: fileStat.mtimeMs, - modifiedAt: fileStat.mtime.toISOString(), - sizeBytes: fileStat.size - }) - } catch (err) { - args.issues.push({ agent: 'claude', path, message: errorMessage(err) }) - } + // Missing or unreadable project directories contribute no sessions. } } diff --git a/src/main/ai-vault/session-scanner-secondary-parsers.ts b/src/main/ai-vault/session-scanner-secondary-parsers.ts index 364c25b5d85..b7c86b0adef 100644 --- a/src/main/ai-vault/session-scanner-secondary-parsers.ts +++ b/src/main/ai-vault/session-scanner-secondary-parsers.ts @@ -1,12 +1,12 @@ -import { createReadStream } from 'node:fs' -import { readFile, readdir } from 'node:fs/promises' +import { opendir } from 'node:fs/promises' import { join } from 'node:path' -import { createInterface } from 'node:readline' import type { AiVaultSession } from '../../shared/ai-vault-types' -import type { ExecutionHostId } from '../../shared/execution-host' +import { iterateAiVaultJsonlLines } from './session-jsonl-line-reader' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' import type { FileWithMtime, ResumableSessionParseState, + ResumableParseFinalizeOptions, SessionAccumulator } from './session-scanner-types' import { @@ -35,19 +35,11 @@ import { tokenTotal } from './session-scanner-values' -type ParserSessionOptions = { - executionHostId?: ExecutionHostId - executionHostPlatform?: NodeJS.Platform | null -} - export async function parseCopilotSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseCopilotSessionLines({ file, lines, platform }) } @@ -55,7 +47,7 @@ export async function parseCopilotSessionContent( file: FileWithMtime, content: string, platform: NodeJS.Platform = process.platform, - options: ParserSessionOptions = {} + options: ResumableParseFinalizeOptions = {} ): Promise { return parseCopilotSessionLines({ file, @@ -127,7 +119,7 @@ async function parseCopilotSessionLines(args: { file: FileWithMtime lines: AsyncIterable | Iterable platform: NodeJS.Platform - options?: ParserSessionOptions + options?: ResumableParseFinalizeOptions }): Promise { const state = createCopilotSessionResumeState(args.file) for await (const line of args.lines) { @@ -140,10 +132,7 @@ export async function parseCursorSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) + const lines = iterateAiVaultJsonlLines(file.path) return parseCursorSessionLines({ file, lines, platform }) } @@ -151,7 +140,7 @@ export async function parseCursorSessionContent( file: FileWithMtime, content: string, platform: NodeJS.Platform = process.platform, - options: ParserSessionOptions = {} + options: ResumableParseFinalizeOptions = {} ): Promise { return parseCursorSessionLines({ file, @@ -193,7 +182,7 @@ async function parseCursorSessionLines(args: { file: FileWithMtime lines: AsyncIterable | Iterable platform: NodeJS.Platform - options?: ParserSessionOptions + options?: ResumableParseFinalizeOptions }): Promise { const state = createCursorSessionResumeState(args.file) for await (const line of args.lines) { @@ -206,18 +195,28 @@ export async function parseOpenCodeSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - const record = asRecord(JSON.parse(await readFile(file.path, 'utf-8')) as unknown) - if (!record) { + const parsed = await withAiVaultWholeJsonFile(file.path, (content) => { + const record = parseJsonObject(content) + if (!record) { + return null + } + const sessionId = extractString(record.id) ?? sessionIdFromFileName(file.path) + const accumulator = createAccumulator({ agent: 'opencode', file, sessionId }) + accumulator.title = normalizeTitleText(extractString(record.title) ?? '') + accumulator.cwd = extractString(record.directory) + updateTimeline(accumulator, timeObjectValue(record.time, 'created')) + updateTimeline(accumulator, timeObjectValue(record.time, 'updated')) + return { accumulator, sessionId } + }) + if (!parsed) { return null } - const sessionId = extractString(record.id) ?? sessionIdFromFileName(file.path) - const accumulator = createAccumulator({ agent: 'opencode', file, sessionId }) - accumulator.title = normalizeTitleText(extractString(record.title) ?? '') - accumulator.cwd = extractString(record.directory) - updateTimeline(accumulator, timeObjectValue(record.time, 'created')) - updateTimeline(accumulator, timeObjectValue(record.time, 'updated')) - await consumeOpenCodeMessages(accumulator, findOpenCodeStorageRoot(file.path), sessionId) - return finalizeSession(accumulator, platform) + await consumeOpenCodeMessages( + parsed.accumulator, + findOpenCodeStorageRoot(file.path), + parsed.sessionId + ) + return finalizeSession(parsed.accumulator, platform) } export async function consumeOpenCodeMessages( @@ -229,44 +228,43 @@ export async function consumeOpenCodeMessages( return } const messageDir = join(storageRoot, 'message', sessionId) - let entries try { - entries = await readdir(messageDir, { withFileTypes: true }) - } catch { - return - } - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.json')) { - continue - } - const message = asRecord( - JSON.parse(await readFile(join(messageDir, entry.name), 'utf-8')) as unknown - ) - if (!message) { - continue - } - const role = extractString(message.role) - if (role === 'user' || role === 'assistant') { - accumulator.messageCount++ - updateTimeline(accumulator, timeObjectValue(message.time, 'created')) - if (role === 'user') { - accumulator.title ??= extractString(asRecord(message.summary)?.title) - accumulator.title ??= extractString(asRecord(message.summary)?.body) + const directory = await opendir(messageDir) + for await (const entry of directory) { + if (!entry.isFile() || !entry.name.endsWith('.json')) { + continue } - addPreviewMessage(accumulator, { - role, - text: - extractPreviewContentText(message.content) ?? - extractString(asRecord(message.summary)?.body) ?? - extractString(asRecord(message.summary)?.title), - timestamp: timeObjectValue(message.time, 'created') + await withAiVaultWholeJsonFile(join(messageDir, entry.name), (content) => { + const message = parseJsonObject(content) + if (!message) { + return + } + const role = extractString(message.role) + if (role === 'user' || role === 'assistant') { + accumulator.messageCount++ + updateTimeline(accumulator, timeObjectValue(message.time, 'created')) + if (role === 'user') { + accumulator.title ??= extractString(asRecord(message.summary)?.title) + accumulator.title ??= extractString(asRecord(message.summary)?.body) + } + addPreviewMessage(accumulator, { + role, + text: + extractPreviewContentText(message.content) ?? + extractString(asRecord(message.summary)?.body) ?? + extractString(asRecord(message.summary)?.title), + timestamp: timeObjectValue(message.time, 'created') + }) + accumulator.model = + extractString(asRecord(message.model)?.modelID) || + extractString(message.modelID) || + accumulator.model + accumulator.totalTokens += tokenTotal(message.tokens) + } }) - accumulator.model = - extractString(asRecord(message.model)?.modelID) || - extractString(message.modelID) || - accumulator.model - accumulator.totalTokens += tokenTotal(message.tokens) } + } catch { + // Missing or unreadable message directories contribute no messages. } } @@ -274,16 +272,18 @@ export async function parseHermesSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform ): Promise { - return parseHermesSessionContent(file, await readFile(file.path, 'utf-8'), platform) + return withAiVaultWholeJsonFile(file.path, (content) => + parseHermesSessionContent(file, content, platform) + ) } export async function parseHermesSessionContent( file: FileWithMtime, content: string, platform: NodeJS.Platform = process.platform, - options: ParserSessionOptions = {} + options: ResumableParseFinalizeOptions = {} ): Promise { - const record = asRecord(JSON.parse(content) as unknown) + const record = parseJsonObject(content) if (!record) { return null } diff --git a/src/main/ai-vault/session-scanner-subagent-transcripts.ts b/src/main/ai-vault/session-scanner-subagent-transcripts.ts index 0477f9dc229..bede82c0986 100644 --- a/src/main/ai-vault/session-scanner-subagent-transcripts.ts +++ b/src/main/ai-vault/session-scanner-subagent-transcripts.ts @@ -1,4 +1,4 @@ -import { readdir } from 'node:fs/promises' +import { opendir } from 'node:fs/promises' import { basename, dirname, extname, join } from 'node:path' // Exported so discovery can prune these subtrees using the same literal that @@ -36,13 +36,23 @@ export function subagentTranscriptsDirFor(transcriptFilePath: string): string { * transcripts and are excluded. */ export async function countSubagentTranscripts(transcriptFilePath: string): Promise { - let entries + let directory try { - entries = await readdir(subagentTranscriptsDirFor(transcriptFilePath), { withFileTypes: true }) + directory = await opendir(subagentTranscriptsDirFor(transcriptFilePath)) } catch { return 0 } - return entries.filter((entry) => isSubagentTranscriptFileName(entry.name, entry.isFile())).length + let count = 0 + try { + for await (const entry of directory) { + if (isSubagentTranscriptFileName(entry.name, entry.isFile())) { + count += 1 + } + } + } catch { + // A disappearing directory still has the successfully counted prefix. + } + return count } // Direct child of a subagents dir: `//subagents/agent-.jsonl`. diff --git a/src/main/ai-vault/session-scanner-types.ts b/src/main/ai-vault/session-scanner-types.ts index f9af3f20897..91cd6c5c0db 100644 --- a/src/main/ai-vault/session-scanner-types.ts +++ b/src/main/ai-vault/session-scanner-types.ts @@ -87,6 +87,8 @@ export type ResumableParseFinalizeOptions = { export type ResumableSessionParseState = { consumeLine(line: string): void clone(): ResumableSessionParseState + // The cache budget must include fold state that persistence intentionally omits. + retainedUtf8Bytes(): number // Refresh per-scan file metadata (mtime display string) without re-parsing. touchFile(file: FileWithMtime): void finalize( diff --git a/src/main/ai-vault/session-scanner-values.test.ts b/src/main/ai-vault/session-scanner-values.test.ts index a9954ae8d26..bb93c4681fc 100644 --- a/src/main/ai-vault/session-scanner-values.test.ts +++ b/src/main/ai-vault/session-scanner-values.test.ts @@ -1,12 +1,22 @@ import { describe, expect, it, vi } from 'vitest' import { + AI_VAULT_JSON_STRUCTURE_LIMITS, extractPreviewContentText, normalizeAgentSessionsDir, normalizePreviewText, - normalizeTitleText + normalizeTitleText, + parseJsonObject } from './session-scanner-values' describe('AI Vault session scanner text values', () => { + it('rejects structurally amplified JSON before parsing', () => { + const parseSpy = vi.spyOn(JSON, 'parse') + const amplified = `[${'0,'.repeat(AI_VAULT_JSON_STRUCTURE_LIMITS.structuralTokens)}0]` + + expect(parseJsonObject(amplified)).toBeNull() + expect(parseSpy).not.toHaveBeenCalled() + }) + it('normalizes compact title text without surfacing hidden context blocks', () => { expect( normalizeTitleText( diff --git a/src/main/ai-vault/session-scanner-values.ts b/src/main/ai-vault/session-scanner-values.ts index 39006bd68a4..d954c8d4a28 100644 --- a/src/main/ai-vault/session-scanner-values.ts +++ b/src/main/ai-vault/session-scanner-values.ts @@ -1,6 +1,22 @@ import { homedir } from 'node:os' import { basename, dirname, join } from 'node:path' -import { readFile } from 'node:fs/promises' +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from '../../shared/json-text-structure-limit' + +export const AI_VAULT_JSON_STRUCTURE_LIMITS: JsonTextStructureLimits = { + structuralTokens: 1_000_000, + nestingDepth: 256 +} + +export function parseAiVaultJsonText( + content: string, + limits: JsonTextStructureLimits = AI_VAULT_JSON_STRUCTURE_LIMITS +): unknown { + assertJsonTextStructureWithinLimits(content, limits) + return JSON.parse(content) as unknown +} export function timestampMs(value: unknown): number { if (typeof value === 'string') { @@ -18,7 +34,7 @@ export function parseJsonObject(line: string): Record | null { return null } try { - const parsed = JSON.parse(line) as unknown + const parsed = parseAiVaultJsonText(line) return asRecord(parsed) } catch { return null @@ -69,16 +85,6 @@ export function extractGitBranch(value: unknown): string | null { return extractString(git.branch) || extractString(git.current_branch) } -export async function readJsonObjectIfExists( - filePath: string -): Promise | null> { - try { - return asRecord(JSON.parse(await readFile(filePath, 'utf-8')) as unknown) - } catch { - return null - } -} - export function arrayValue(value: unknown): unknown[] { return Array.isArray(value) ? value : [] } diff --git a/src/main/ai-vault/session-scanner.ts b/src/main/ai-vault/session-scanner.ts index f1979c67626..99ead01f352 100644 --- a/src/main/ai-vault/session-scanner.ts +++ b/src/main/ai-vault/session-scanner.ts @@ -1,4 +1,3 @@ -import { readFile } from 'node:fs/promises' import type { AiVaultListResult, AiVaultScanIssue, @@ -7,6 +6,11 @@ import type { import { LOCAL_EXECUTION_HOST_ID, type ExecutionHostId } from '../../shared/execution-host' import { withSpan } from '../observability/tracer' import { sessionSortTime } from './session-scanner-accumulator' +import { + boundAiVaultListResult, + retainAiVaultSession, + retainAiVaultSessionsWithinAggregate +} from './session-list-retention' import { codexRolloutHardlinkIdentity, dedupeCodexRolloutFileAliases, @@ -39,6 +43,7 @@ import type { SessionParseResult } from './session-scanner-types' import { clampPositiveInteger, errorMessage } from './session-scanner-values' +import { withAiVaultWholeJsonFile } from './session-whole-json-reader' const DEFAULT_LIMIT = 1000 const DEFAULT_SCAN_LIMIT_PER_AGENT = 1000 @@ -138,11 +143,11 @@ export async function scanAiVaultSessions( scheduleSessionParseCachePersist(parseStats) - return { + return boundAiVaultListResult({ sessions: mergeSessions(cappedSessions, scopeSessions), issues: issues.map((issue) => ({ executionHostId, ...issue })), scannedAt: new Date().toISOString() - } + }) }) } @@ -250,7 +255,17 @@ async function parseSessionCandidates(args: { // Why: cross-volume backfill copies have no shared inode, so collapse // parsed aliases before they can crowd the unique-session parse budget. const uniqueSessions = dedupeCodexSessionsBySessionId(sessions) - sessions.splice(0, sessions.length, ...uniqueSessions) + const retained = retainAiVaultSessionsWithinAggregate(uniqueSessions) + sessions.splice(0, sessions.length, ...retained.sessions) + if (retained.omitted > 0) { + args.issues.push({ + executionHostId: args.executionHostId, + agent: 'codex', + path: 'AI Vault session list', + message: `AI Vault stopped after omitting ${retained.omitted} sessions at its memory limit.` + }) + break + } index += batchSize } @@ -270,8 +285,9 @@ async function parseSessionCandidate( if (session && candidate.antigravityHistoryPath && antigravityWorkspaceResolver) { session = await antigravityWorkspaceResolver.enrich(session, candidate.antigravityHistoryPath) } + const stamped = session ? withSessionExecutionHost(session, executionHostId) : null return { - session: session ? withSessionExecutionHost(session, executionHostId) : null, + session: stamped ? retainAiVaultSession(stamped) : null, issue: null } } catch (err) { @@ -289,7 +305,7 @@ async function parseSessionCandidate( async function readOptionalTextFile(path: string): Promise { try { - return await readFile(path, 'utf-8') + return await withAiVaultWholeJsonFile(path, (content) => content) } catch { return null } diff --git a/src/main/ai-vault/session-scope-cwd-cache.test.ts b/src/main/ai-vault/session-scope-cwd-cache.test.ts new file mode 100644 index 00000000000..02eb96a9737 --- /dev/null +++ b/src/main/ai-vault/session-scope-cwd-cache.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { + AI_VAULT_SCOPE_CWD_CACHE_KEY_MAX_UTF8_BYTES, + AI_VAULT_SCOPE_CWD_CACHE_VALUE_MAX_UTF8_BYTES, + AiVaultScopeCwdCache +} from './session-scope-cwd-cache' + +describe('AI Vault scope cwd cache', () => { + it('rejects keys and values one byte over their UTF-8 limits', () => { + const cache = new AiVaultScopeCwdCache() + const exactKey = 'é'.repeat(AI_VAULT_SCOPE_CWD_CACHE_KEY_MAX_UTF8_BYTES / 2) + const exactValue = 'é'.repeat(AI_VAULT_SCOPE_CWD_CACHE_VALUE_MAX_UTF8_BYTES / 2) + + cache.set(exactKey, exactValue) + cache.set(`${exactKey}x`, '/overflow-key') + cache.set('/overflow-value', `${exactValue}x`) + + expect(cache.inspectForTests().keys).toEqual([exactKey]) + }) + + it('evicts least-recently-used entries at the aggregate byte boundary', () => { + const entryBytes = Buffer.byteLength('/a/repo', 'utf8') + Buffer.byteLength('/a', 'utf8') + 128 + const cache = new AiVaultScopeCwdCache({ maxEntries: 3, maxRetainedBytes: entryBytes * 2 }) + + cache.set('/a/repo', '/a') + cache.set('/b/repo', '/b') + expect(cache.get('/a/repo')).toBe('/a') + cache.set('/c/repo', '/c') + + expect(cache.inspectForTests().keys).toEqual(['/a/repo', '/c/repo']) + }) +}) diff --git a/src/main/ai-vault/session-scope-cwd-cache.ts b/src/main/ai-vault/session-scope-cwd-cache.ts new file mode 100644 index 00000000000..d562728ab9d --- /dev/null +++ b/src/main/ai-vault/session-scope-cwd-cache.ts @@ -0,0 +1,77 @@ +export const AI_VAULT_SCOPE_CWD_CACHE_MAX_ENTRIES = 2_048 +export const AI_VAULT_SCOPE_CWD_CACHE_KEY_MAX_UTF8_BYTES = 32 * 1024 +export const AI_VAULT_SCOPE_CWD_CACHE_VALUE_MAX_UTF8_BYTES = 128 * 1024 +export const AI_VAULT_SCOPE_CWD_CACHE_MAX_RETAINED_UTF8_BYTES = 8 * 1024 * 1024 + +const ENTRY_OVERHEAD_BYTES = 128 + +type RetainedCwd = { cwd: string; bytes: number } + +export class AiVaultScopeCwdCache { + private readonly entries = new Map() + private retainedBytes = 0 + + constructor( + private readonly limits: { + maxEntries: number + maxRetainedBytes: number + } = { + maxEntries: AI_VAULT_SCOPE_CWD_CACHE_MAX_ENTRIES, + maxRetainedBytes: AI_VAULT_SCOPE_CWD_CACHE_MAX_RETAINED_UTF8_BYTES + } + ) {} + + get(projectDir: string): string | undefined { + const retained = this.entries.get(projectDir) + if (!retained) { + return undefined + } + this.entries.delete(projectDir) + this.entries.set(projectDir, retained) + return retained.cwd + } + + set(projectDir: string, cwd: string): void { + this.delete(projectDir) + const keyBytes = Buffer.byteLength(projectDir, 'utf8') + const valueBytes = Buffer.byteLength(cwd, 'utf8') + const bytes = keyBytes + valueBytes + ENTRY_OVERHEAD_BYTES + if ( + keyBytes > AI_VAULT_SCOPE_CWD_CACHE_KEY_MAX_UTF8_BYTES || + valueBytes > AI_VAULT_SCOPE_CWD_CACHE_VALUE_MAX_UTF8_BYTES || + bytes > this.limits.maxRetainedBytes + ) { + return + } + while ( + this.entries.size >= this.limits.maxEntries || + this.retainedBytes + bytes > this.limits.maxRetainedBytes + ) { + const oldest = this.entries.keys().next().value + if (oldest === undefined) { + return + } + this.delete(oldest) + } + this.entries.set(projectDir, { cwd, bytes }) + this.retainedBytes += bytes + } + + clear(): void { + this.entries.clear() + this.retainedBytes = 0 + } + + inspectForTests(): { keys: string[]; retainedBytes: number } { + return { keys: [...this.entries.keys()], retainedBytes: this.retainedBytes } + } + + private delete(projectDir: string): void { + const retained = this.entries.get(projectDir) + if (!retained) { + return + } + this.entries.delete(projectDir) + this.retainedBytes -= retained.bytes + } +} diff --git a/src/main/ai-vault/session-whole-json-reader.test.ts b/src/main/ai-vault/session-whole-json-reader.test.ts new file mode 100644 index 00000000000..1c524d988d3 --- /dev/null +++ b/src/main/ai-vault/session-whole-json-reader.test.ts @@ -0,0 +1,93 @@ +import { mkdtemp, open, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { + AI_VAULT_WHOLE_JSON_MAX_BYTES, + AI_VAULT_JSON_READ_MAX_ACTIVE, + AI_VAULT_JSON_READ_MAX_WAITERS, + AiVaultJsonReadBudget, + withAiVaultWholeJsonFile +} from './session-whole-json-reader' + +describe('AI Vault whole-JSON read budget', () => { + it('queues aggregate bytes and releases admission idempotently', async () => { + const budget = new AiVaultJsonReadBudget(10) + const releaseSix = await budget.acquire(6) + let admitted = false + const second = budget.acquire(5).then((release) => { + admitted = true + return release + }) + + await Promise.resolve() + expect(admitted).toBe(false) + releaseSix() + const releaseFive = await second + expect(admitted).toBe(true) + + releaseSix() + releaseFive() + await expect(budget.acquire(10)).resolves.toEqual(expect.any(Function)) + }) + + it('rejects a single document larger than the process budget', () => { + const budget = new AiVaultJsonReadBudget(10) + + expect(() => budget.acquire(11)).toThrow('exceeds 10 byte limit') + }) + + it('bounds queued readers and admits them in FIFO order', async () => { + const budget = new AiVaultJsonReadBudget(1) + const releaseActive = await budget.acquire(1) + const queued = Array.from({ length: AI_VAULT_JSON_READ_MAX_WAITERS }, () => budget.acquire(1)) + + expect(() => budget.acquire(1)).toThrow('reader is busy') + releaseActive() + for (const pending of queued) { + const release = await pending + release() + } + await expect(budget.acquire(1)).resolves.toEqual(expect.any(Function)) + }) + + it('uses a fixed active-reader pool even when byte capacity remains', async () => { + const budget = new AiVaultJsonReadBudget(1024) + const releases = await Promise.all( + Array.from({ length: AI_VAULT_JSON_READ_MAX_ACTIVE }, () => budget.acquire(1)) + ) + let admitted = false + const queued = budget.acquire(1).then((release) => { + admitted = true + return release + }) + + await Promise.resolve() + expect(admitted).toBe(false) + releases[0]() + const releaseQueued = await queued + expect(admitted).toBe(true) + for (const release of releases.slice(1)) { + release() + } + releaseQueued() + }) + + it('rejects an oversized sparse file before reading or parsing it', async () => { + const directory = await mkdtemp(join(tmpdir(), 'orca-vault-whole-json-')) + const path = join(directory, 'oversized.json') + const handle = await open(path, 'w') + await handle.truncate(AI_VAULT_WHOLE_JSON_MAX_BYTES + 1) + await handle.close() + const consume = vi.fn() + + try { + await expect(withAiVaultWholeJsonFile(path, consume)).rejects.toThrow( + `exceeds ${AI_VAULT_WHOLE_JSON_MAX_BYTES} byte limit` + ) + expect(consume).not.toHaveBeenCalled() + } finally { + await rm(directory, { recursive: true }) + } + }) +}) diff --git a/src/main/ai-vault/session-whole-json-reader.ts b/src/main/ai-vault/session-whole-json-reader.ts new file mode 100644 index 00000000000..ab3c6c2d659 --- /dev/null +++ b/src/main/ai-vault/session-whole-json-reader.ts @@ -0,0 +1,87 @@ +import { stat } from 'node:fs/promises' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' + +export const AI_VAULT_WHOLE_JSON_MAX_BYTES = 64 * 1024 * 1024 +export const AI_VAULT_JSON_READ_MAX_ACTIVE = 8 +export const AI_VAULT_JSON_READ_MAX_WAITERS = 256 +const WAIT_QUEUE_COMPACTION_HEAD = 64 + +type AdmissionWaiter = { + bytes: number + resolve: (release: () => void) => void +} + +export class AiVaultJsonReadBudget { + private retainedBytes = 0 + private activeReaders = 0 + private waiters: (AdmissionWaiter | undefined)[] = [] + private waiterHead = 0 + + constructor(readonly maxBytes: number) { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('AI Vault JSON read budget must be a non-negative safe integer') + } + } + + acquire(bytes: number): Promise<() => void> { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.maxBytes) { + throw new RangeError(`AI Vault JSON document exceeds ${this.maxBytes} byte limit`) + } + if (this.waiters.length - this.waiterHead >= AI_VAULT_JSON_READ_MAX_WAITERS) { + throw new Error('AI Vault JSON reader is busy; retry after active reads finish') + } + return new Promise((resolve) => { + this.waiters.push({ bytes, resolve }) + this.admitWaiters() + }) + } + + private admitWaiters(): void { + while (this.waiterHead < this.waiters.length) { + const waiter = this.waiters[this.waiterHead]! + if ( + this.activeReaders >= AI_VAULT_JSON_READ_MAX_ACTIVE || + waiter.bytes > this.maxBytes - this.retainedBytes + ) { + return + } + this.waiters[this.waiterHead] = undefined + this.waiterHead += 1 + if (this.waiterHead >= WAIT_QUEUE_COMPACTION_HEAD) { + this.waiters.splice(0, this.waiterHead) + this.waiterHead = 0 + } + this.retainedBytes += waiter.bytes + this.activeReaders += 1 + let released = false + waiter.resolve(() => { + if (released) { + return + } + released = true + this.retainedBytes -= waiter.bytes + this.activeReaders -= 1 + this.admitWaiters() + }) + } + this.waiters.length = 0 + this.waiterHead = 0 + } +} + +const jsonReadBudget = new AiVaultJsonReadBudget(AI_VAULT_WHOLE_JSON_MAX_BYTES) + +export async function withAiVaultWholeJsonFile( + filePath: string, + consume: (content: string) => T | Promise +): Promise { + const fileStats = await stat(filePath) + const release = await jsonReadBudget.acquire(fileStats.size) + try { + // Reject concurrent growth: the admission covers exactly the statted bytes. + const { buffer } = await readNodeFileWithinLimit(filePath, fileStats.size) + return await consume(buffer.toString('utf8')) + } finally { + release() + } +} diff --git a/src/main/amp/hook-service.test.ts b/src/main/amp/hook-service.test.ts index 5ed0f6c5e3e..9b902a1f459 100644 --- a/src/main/amp/hook-service.test.ts +++ b/src/main/amp/hook-service.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' +import { GENERATED_NODE_MANAGED_FILE_MAX_BYTES } from '../generated-node-bounded-file-reader' const { homedirMock } = vi.hoisted(() => ({ homedirMock: vi.fn<() => string>() @@ -59,6 +60,11 @@ describe('AmpHookService', () => { expect(source).not.toContain('postQueue = postQueue.then') expect(source).toContain('process.env.ORCA_PANE_KEY') expect(source).toContain('process.env.ORCA_AGENT_HOOK_ENDPOINT') + expect(source).toContain( + `function readOrcaManagedFileWithinLimit(fs: any, path: string, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES})` + ) + expect(source).toContain('readOrcaManagedFileWithinLimit(fs, endpointPath)') + expect(source).not.toContain('readFileSync') }) it('does not overwrite an existing user-authored Amp plugin file', () => { diff --git a/src/main/amp/hook-service.ts b/src/main/amp/hook-service.ts index 83ed74d1665..45d2abd5449 100644 --- a/src/main/amp/hook-service.ts +++ b/src/main/amp/hook-service.ts @@ -3,12 +3,18 @@ emitted plugin bytes drift from the installer checks that protect user plugin files from being overwritten. */ import { randomUUID } from 'node:crypto' -import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' import type { SFTPWrapper } from 'ssh2' import type { AgentHookInstallStatus } from '../../shared/agent-hook-types' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' +import { getGeneratedNodeBoundedFileReaderSourceLines } from '../generated-node-bounded-file-reader' +import { AGENT_HOOK_PLUGIN_MAX_BYTES } from '../agent-hooks/agent-hook-file-limits' import { readTextFileRemote, writeTextFileRemoteAtomic @@ -53,7 +59,10 @@ function readLocalPluginState(pluginPath: string): PluginFileState { return { kind: 'absent' } } try { - const content = readFileSync(pluginPath, 'utf-8') + const content = readNodeFileSyncWithinLimit( + pluginPath, + AGENT_HOOK_PLUGIN_MAX_BYTES + ).buffer.toString('utf8') if (!isManagedPlugin(content)) { return { kind: 'unmanaged' } } @@ -105,10 +114,17 @@ function writeTextFileAtomic(filePath: string, content: string): void { mkdirSync(dir, { recursive: true }) if (existsSync(filePath)) { try { - if (readFileSync(filePath, 'utf-8') === content) { + const existing = readNodeFileSyncWithinLimit( + filePath, + AGENT_HOOK_PLUGIN_MAX_BYTES + ).buffer.toString('utf8') + if (existing === content) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // Fall through to the atomic write path. } } @@ -130,12 +146,14 @@ function writeTextFileAtomic(filePath: string, content: string): void { function getAmpPluginSource(): string { return [ - "import { readFileSync, statSync } from 'fs'", + "import * as fs from 'fs'", "import type { PluginAPI } from '@ampcode/plugin'", '', `// ${AMP_PLUGIN_MARKER}`, 'type HookCoords = { port?: string; token?: string; env?: string; version?: string }', '', + ...getGeneratedNodeBoundedFileReaderSourceLines({ typed: true }), + '', 'let warnedBadEndpoint = false', "let cachedEndpointKey = ''", 'let cachedEndpointValues: HookCoords | null = null', @@ -144,12 +162,12 @@ function getAmpPluginSource(): string { ' const endpointPath = process.env.ORCA_AGENT_HOOK_ENDPOINT', ' if (!endpointPath) return null', ' try {', - ' const stat = statSync(endpointPath)', + ' const stat = fs.statSync(endpointPath)', ' const cacheKey = `${stat.mtimeMs}:${stat.size}:${stat.ino}`', ' if (cacheKey === cachedEndpointKey && cachedEndpointValues) {', ' return cachedEndpointValues', ' }', - " const contents = readFileSync(endpointPath, 'utf8')", + ' const contents = readOrcaManagedFileWithinLimit(fs, endpointPath)', ' const out: HookCoords = {}', ' for (const line of contents.split(/\\r?\\n/)) {', ' const match = line.match(/^(?:set\\s+)?([A-Z0-9_]+)=(.*)$/)', @@ -346,11 +364,16 @@ export class AmpHookService { async installRemote(sftp: SFTPWrapper, remoteHome: string): Promise { const remotePluginPath = getRemotePluginPath(remoteHome) try { - const existing = await readTextFileRemote(sftp, remotePluginPath) + const existing = await readTextFileRemote(sftp, remotePluginPath, AGENT_HOOK_PLUGIN_MAX_BYTES) if (existing !== null && !isManagedPlugin(existing)) { return statusFromState(remotePluginPath, { kind: 'unmanaged' }) } - await writeTextFileRemoteAtomic(sftp, remotePluginPath, getAmpPluginSource()) + await writeTextFileRemoteAtomic( + sftp, + remotePluginPath, + getAmpPluginSource(), + AGENT_HOOK_PLUGIN_MAX_BYTES + ) return { agent: 'amp', state: 'installed', diff --git a/src/main/attribution/terminal-attribution-bounds.test.ts b/src/main/attribution/terminal-attribution-bounds.test.ts new file mode 100644 index 00000000000..b357565eaad --- /dev/null +++ b/src/main/attribution/terminal-attribution-bounds.test.ts @@ -0,0 +1,178 @@ +import { spawnSync } from 'node:child_process' +import { + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE, + ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES, + ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES, + applyTerminalAttributionEnv +} from './terminal-attribution' + +const roots: string[] = [] +const posixIt = process.platform === 'win32' ? it.skip : it + +function makeRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-attribution-bounds-')) + roots.push(root) + return root +} + +function attributionEnv(root: string, binDir: string): Record { + const env = { ...process.env, PATH: `${binDir}:${process.env.PATH ?? ''}` } + applyTerminalAttributionEnv(env as Record, { + enabled: true, + userDataPath: join(root, 'user-data') + }) + return env as Record +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('terminal attribution payload bounds', () => { + posixIt('accepts a commit message file at the exact byte boundary', () => { + const root = makeRoot() + const binDir = join(root, 'bin') + const messagePath = join(root, 'message.txt') + const receivedPath = join(root, 'received-message.txt') + mkdirSync(binDir) + writeFileSync(messagePath, 'm'.repeat(ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES)) + writeFileSync( + join(binDir, 'git'), + `#!/usr/bin/env bash +set -euo pipefail +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-F" || "$1" == "--file" ]]; then + cp "$2" "${receivedPath}" + exit 0 + fi + shift +done +exit 2 +` + ) + chmodSync(join(binDir, 'git'), 0o755) + + const result = spawnSync('git', ['commit', '-F', messagePath], { + encoding: 'utf8', + env: attributionEnv(root, binDir) + }) + + expect(result.status).toBe(0) + expect(readFileSync(receivedPath, 'utf8')).toBe( + `${'m'.repeat(ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES)}\n\nCo-authored-by: Orca \n` + ) + }) + + posixIt('rejects a sparse commit message file one byte over the boundary', () => { + const root = makeRoot() + const binDir = join(root, 'bin') + const messagePath = join(root, 'message.txt') + const calledPath = join(root, 'git-called') + mkdirSync(binDir) + writeFileSync(messagePath, 'm') + truncateSync(messagePath, ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES + 1) + writeFileSync( + join(binDir, 'git'), + `#!/usr/bin/env bash +touch "${calledPath}" +exit 0 +` + ) + chmodSync(join(binDir, 'git'), 0o755) + + const result = spawnSync('git', ['commit', '-F', messagePath], { + encoding: 'utf8', + env: attributionEnv(root, binDir) + }) + + expect(result.status).toBe(ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE) + expect(result.stderr).toContain( + `${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES + 1} bytes exceeds the ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}-byte limit` + ) + expect(existsSync(calledPath)).toBe(false) + }) + + posixIt('preserves gh output at the exact byte boundary', () => { + const root = makeRoot() + const binDir = join(root, 'bin') + const payloadPath = join(root, 'gh-output.txt') + mkdirSync(binDir) + writeFileSync(payloadPath, 'o'.repeat(ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES)) + writeFileSync( + join(binDir, 'gh'), + `#!/usr/bin/env bash +cat "${payloadPath}" +` + ) + chmodSync(join(binDir, 'gh'), 0o755) + + const result = spawnSync('gh', ['pr', 'create', '--fill'], { + encoding: 'utf8', + env: attributionEnv(root, binDir), + maxBuffer: ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES + 64 * 1024 + }) + + expect(result.status).toBe(0) + expect(result.stdout).toBe('o'.repeat(ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES)) + expect(result.stderr).toBe('') + }) + + posixIt('fails clearly when gh output is one byte over the boundary', () => { + const root = makeRoot() + const binDir = join(root, 'bin') + const payloadPath = join(root, 'gh-output.txt') + mkdirSync(binDir) + writeFileSync(payloadPath, 'o'.repeat(ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES + 1)) + writeFileSync( + join(binDir, 'gh'), + `#!/usr/bin/env bash +cat "${payloadPath}" +` + ) + chmodSync(join(binDir, 'gh'), 0o755) + + const result = spawnSync('gh', ['pr', 'create', '--fill'], { + encoding: 'utf8', + env: attributionEnv(root, binDir) + }) + + expect(result.status).toBe(ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE) + expect(result.stdout).toBe('') + expect(result.stderr).toContain( + `${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES + 1} bytes exceeds the ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}-byte limit` + ) + }) + + it('emits inclusive PowerShell byte boundaries for commit files and gh captures', () => { + const root = makeRoot() + applyTerminalAttributionEnv( + { PATH: process.env.PATH ?? '' }, + { enabled: true, userDataPath: join(root, 'user-data') } + ) + const shimDir = join(root, 'user-data', 'orca-terminal-attribution', 'win32') + const gitWrapper = readFileSync(join(shimDir, 'git-wrapper.ps1'), 'utf8') + const ghWrapper = readFileSync(join(shimDir, 'gh-wrapper.ps1'), 'utf8') + + expect(gitWrapper).toContain('$guard.Length -gt $MaxBytes') + expect(gitWrapper).toContain(`'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}`) + expect(gitWrapper).toContain(`exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE}`) + expect(ghWrapper).toContain(`$stdoutBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}`) + expect(ghWrapper).toContain(`$stderrBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}`) + expect(ghWrapper).toContain(`exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE}`) + }) +}) diff --git a/src/main/attribution/terminal-attribution-retention.test.ts b/src/main/attribution/terminal-attribution-retention.test.ts new file mode 100644 index 00000000000..f127e29f55e --- /dev/null +++ b/src/main/attribution/terminal-attribution-retention.test.ts @@ -0,0 +1,46 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + _getAttributionWrittenRootCountForTests, + _isAttributionWrittenRootRetainableForTests, + _resetAttributionWrittenRootsForTests, + applyTerminalAttributionEnv, + ATTRIBUTION_WRITTEN_ROOT_MAX_BYTES, + ATTRIBUTION_WRITTEN_ROOT_MAX_ENTRIES +} from './terminal-attribution' + +let fixtureRoot: string | null = null + +afterEach(() => { + _resetAttributionWrittenRootsForTests() + if (fixtureRoot) { + rmSync(fixtureRoot, { recursive: true, force: true }) + fixtureRoot = null + } +}) + +describe('terminal attribution root retention', () => { + it('bounds successfully ensured roots', () => { + fixtureRoot = mkdtempSync(join(tmpdir(), 'orca-attribution-roots-')) + for (let index = 0; index <= ATTRIBUTION_WRITTEN_ROOT_MAX_ENTRIES; index += 1) { + applyTerminalAttributionEnv( + { PATH: process.env.PATH ?? '' }, + { enabled: true, userDataPath: join(fixtureRoot, String(index)) } + ) + } + expect(_getAttributionWrittenRootCountForTests()).toBe(ATTRIBUTION_WRITTEN_ROOT_MAX_ENTRIES) + }) + + it('measures retained roots by UTF-8 bytes', () => { + expect( + _isAttributionWrittenRootRetainableForTests('x'.repeat(ATTRIBUTION_WRITTEN_ROOT_MAX_BYTES)) + ).toBe(true) + expect( + _isAttributionWrittenRootRetainableForTests( + '😀'.repeat(ATTRIBUTION_WRITTEN_ROOT_MAX_BYTES / 4 + 1) + ) + ).toBe(false) + }) +}) diff --git a/src/main/attribution/terminal-attribution.test.ts b/src/main/attribution/terminal-attribution.test.ts index 86096168f0c..f981762e06a 100644 --- a/src/main/attribution/terminal-attribution.test.ts +++ b/src/main/attribution/terminal-attribution.test.ts @@ -6,12 +6,18 @@ import { mkdtempSync, readFileSync, rmSync, + statSync, + truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { applyTerminalAttributionEnv, resolveAttributionShellFamily } from './terminal-attribution' +import { + ATTRIBUTION_SHIM_VERSION_MAX_BYTES, + applyTerminalAttributionEnv, + resolveAttributionShellFamily +} from './terminal-attribution' describe('applyTerminalAttributionEnv', () => { let tmpRoot: string | null = null @@ -81,6 +87,42 @@ describe('applyTerminalAttributionEnv', () => { ) }) + it('accepts an attribution version marker at the exact byte boundary', () => { + const root = makeTmpRoot() + const userDataPath = join(root, 'user-data') + const shimRoot = join(userDataPath, 'orca-terminal-attribution') + const sentinelPath = join(shimRoot, 'posix', 'git') + mkdirSync(join(shimRoot, 'posix'), { recursive: true }) + mkdirSync(join(shimRoot, 'win32'), { recursive: true }) + writeFileSync(sentinelPath, 'leave-existing-wrapper') + writeFileSync( + join(shimRoot, 'VERSION'), + `7${' '.repeat(ATTRIBUTION_SHIM_VERSION_MAX_BYTES - 1)}` + ) + + applyTerminalAttributionEnv({}, { enabled: true, userDataPath }) + + expect(readFileSync(sentinelPath, 'utf8')).toBe('leave-existing-wrapper') + expect(statSync(join(shimRoot, 'VERSION')).size).toBe(ATTRIBUTION_SHIM_VERSION_MAX_BYTES) + }) + + it('rebuilds attribution shims for a sparse version marker over the boundary', () => { + const root = makeTmpRoot() + const userDataPath = join(root, 'user-data') + const shimRoot = join(userDataPath, 'orca-terminal-attribution') + const versionPath = join(shimRoot, 'VERSION') + mkdirSync(shimRoot, { recursive: true }) + writeFileSync(versionPath, '7') + truncateSync(versionPath, ATTRIBUTION_SHIM_VERSION_MAX_BYTES + 1) + + applyTerminalAttributionEnv({}, { enabled: true, userDataPath }) + + expect(readFileSync(versionPath, 'utf8')).toBe('7\n') + expect(readFileSync(join(shimRoot, 'posix', 'git'), 'utf8')).toContain( + 'ORCA_GIT_COMMIT_TRAILER' + ) + }) + posixSubprocessIt('does not amend HEAD when git commit --dry-run exits successfully', () => { const root = makeTmpRoot() const repo = join(root, 'repo') diff --git a/src/main/attribution/terminal-attribution.ts b/src/main/attribution/terminal-attribution.ts index d1c4dd099fe..ba2b54b07f4 100644 --- a/src/main/attribution/terminal-attribution.ts +++ b/src/main/attribution/terminal-attribution.ts @@ -2,12 +2,18 @@ scripts for both POSIX shells and Windows shells. Keeping the scripts adjacent to the env injection code makes the attribution behavior auditable as one unit instead of scattering generated shell fragments across files. */ -import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs' import { join, win32 as pathWin32 } from 'node:path' import { ORCA_GIT_COMMIT_TRAILER } from '../../shared/orca-attribution' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' const ATTRIBUTION_ROOT_DIR = 'orca-terminal-attribution' -const ATTRIBUTION_SHIM_VERSION = '6' +const ATTRIBUTION_SHIM_VERSION = '7' +export const ATTRIBUTION_SHIM_VERSION_MAX_BYTES = 1024 +export const ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES = 1024 * 1024 +export const ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024 +export const ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE = 74 const ORCA_PRODUCT_URL = 'https://github.com/stablyai/orca' const ORCA_GH_FOOTER = `Made with [Orca](${ORCA_PRODUCT_URL}) 🐋` const SHELL_DOLLAR = '$' @@ -23,6 +29,38 @@ const ATTRIBUTION_ENV_KEYS = [ ] as const const writtenRoots = new Set() +export const ATTRIBUTION_WRITTEN_ROOT_MAX_ENTRIES = 64 +export const ATTRIBUTION_WRITTEN_ROOT_MAX_BYTES = 64 * 1024 + +function hasWrittenRoot(rootDir: string): boolean { + if (!writtenRoots.has(rootDir)) { + return false + } + writtenRoots.delete(rootDir) + writtenRoots.add(rootDir) + return true +} + +function rememberWrittenRoot(rootDir: string): void { + if (!isRetainableWrittenRoot(rootDir)) { + return + } + writtenRoots.delete(rootDir) + writtenRoots.add(rootDir) + while (writtenRoots.size > ATTRIBUTION_WRITTEN_ROOT_MAX_ENTRIES) { + const oldest = writtenRoots.values().next().value + if (oldest === undefined) { + return + } + writtenRoots.delete(oldest) + } +} + +function isRetainableWrittenRoot(rootDir: string): boolean { + return !measureUtf8ByteLength(rootDir, { + stopAfterBytes: ATTRIBUTION_WRITTEN_ROOT_MAX_BYTES + }).exceededLimit +} type AttributionShimPaths = { posixDir: string @@ -155,12 +193,12 @@ function ensureAttributionShims(userDataPath: string): AttributionShimPaths { const win32Dir = join(rootDir, 'win32') const versionFile = join(rootDir, 'VERSION') - if (writtenRoots.has(rootDir)) { + if (hasWrittenRoot(rootDir)) { return { posixDir, win32Dir } } if (readShimVersion(versionFile) === ATTRIBUTION_SHIM_VERSION) { - writtenRoots.add(rootDir) + rememberWrittenRoot(rootDir) return { posixDir, win32Dir } } @@ -176,14 +214,28 @@ function ensureAttributionShims(userDataPath: string): AttributionShimPaths { writeExecutable(join(win32Dir, 'gh-wrapper.ps1'), WIN32_GH_PS_WRAPPER) writeFileSync(versionFile, `${ATTRIBUTION_SHIM_VERSION}\n`, 'utf8') - writtenRoots.add(rootDir) + rememberWrittenRoot(rootDir) return { posixDir, win32Dir } } +export function _resetAttributionWrittenRootsForTests(): void { + writtenRoots.clear() +} + +export function _getAttributionWrittenRootCountForTests(): number { + return writtenRoots.size +} + +export function _isAttributionWrittenRootRetainableForTests(rootDir: string): boolean { + return isRetainableWrittenRoot(rootDir) +} + function readShimVersion(versionFile: string): string | null { try { - return readFileSync(versionFile, 'utf8').trim() + return readNodeFileSyncWithinLimit(versionFile, ATTRIBUTION_SHIM_VERSION_MAX_BYTES) + .buffer.toString('utf8') + .trim() } catch { return null } @@ -346,6 +398,60 @@ has_unsupported_commit_message_source() { return 1 } +validate_commit_message_file_sizes() { + local arg source_file byte_count + while [[ $# -gt 0 ]]; do + arg="$1" + source_file="" + case "$arg" in + -F|--file) + shift + source_file="${SHELL_DOLLAR}{1:-}" + ;; + --file=*) + source_file="${SHELL_DOLLAR}{arg#--file=}" + ;; + -F?*) + source_file="${SHELL_DOLLAR}{arg:2}" + ;; + esac + if [[ -n "$source_file" && -f "$source_file" ]]; then + if ! byte_count="$( + LC_ALL=C head -c ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES + 1} <"$source_file" | + LC_ALL=C wc -c + )"; then + printf 'Orca attribution wrapper could not inspect commit message file %q.\n' "$source_file" >&2 + return 1 + fi + if (( byte_count > ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES} )); then + printf 'Orca attribution wrapper refused commit message file %q: %s bytes exceeds the ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}-byte limit.\n' "$source_file" "$byte_count" >&2 + return 1 + fi + fi + shift + done + return 0 +} + +read_bounded_commit_message() { + local source_file="$1" + local snapshot byte_count + snapshot="$(mktemp)" + if ! LC_ALL=C head -c ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES + 1} <"$source_file" >"$snapshot"; then + rm -f "$snapshot" + printf 'Orca attribution wrapper could not read commit message file %q.\n' "$source_file" >&2 + return 1 + fi + byte_count="$(LC_ALL=C wc -c <"$snapshot")" + if (( byte_count > ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES} )); then + rm -f "$snapshot" + printf 'Orca attribution wrapper refused commit message file %q: more than ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES} bytes.\n' "$source_file" >&2 + return 1 + fi + cat "$snapshot" + rm -f "$snapshot" +} + message_already_has_trailer() { local arg next_arg while [[ $# -gt 0 ]]; do @@ -386,7 +492,13 @@ message_already_has_trailer() { return 1 } -if ! has_explicit_commit_message "$@" || has_unsupported_commit_message_source "$@" || message_already_has_trailer "$@"; then +if ! has_explicit_commit_message "$@" || has_unsupported_commit_message_source "$@"; then + PATH="$real_path" exec "$real_git" "$@" +fi +if ! validate_commit_message_file_sizes "$@"; then + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} +fi +if message_already_has_trailer "$@"; then PATH="$real_path" exec "$real_git" "$@" fi @@ -409,7 +521,10 @@ while [[ $# -gt 0 ]]; do source_file="${SHELL_DOLLAR}{1:-}" tmp_file="$(mktemp)" if [[ -n "$source_file" && -f "$source_file" ]]; then - printf '%s\n\n%s\n' "$(cat "$source_file")" "$trailer" >"$tmp_file" + if ! source_message="$(read_bounded_commit_message "$source_file")"; then + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + fi + printf '%s\n\n%s\n' "$source_message" "$trailer" >"$tmp_file" attributed_args+=("$arg" "$tmp_file") replaced_file_message=1 else @@ -424,7 +539,10 @@ while [[ $# -gt 0 ]]; do source_file="${SHELL_DOLLAR}{arg#--file=}" tmp_file="$(mktemp)" if [[ -f "$source_file" ]]; then - printf '%s\n\n%s\n' "$(cat "$source_file")" "$trailer" >"$tmp_file" + if ! source_message="$(read_bounded_commit_message "$source_file")"; then + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + fi + printf '%s\n\n%s\n' "$source_message" "$trailer" >"$tmp_file" attributed_args+=("--file=$tmp_file") replaced_file_message=1 else @@ -439,7 +557,10 @@ while [[ $# -gt 0 ]]; do source_file="${SHELL_DOLLAR}{arg:2}" tmp_file="$(mktemp)" if [[ -f "$source_file" ]]; then - printf '%s\n\n%s\n' "$(cat "$source_file")" "$trailer" >"$tmp_file" + if ! source_message="$(read_bounded_commit_message "$source_file")"; then + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + fi + printf '%s\n\n%s\n' "$source_message" "$trailer" >"$tmp_file" attributed_args+=("-F$tmp_file") replaced_file_message=1 else @@ -474,6 +595,21 @@ if [[ -z "$real_gh" ]]; then exit 127 fi +read_bounded_capture() { + local capture_path="$1" + local stream_name="$2" + local byte_count + if ! byte_count="$(LC_ALL=C wc -c <"$capture_path")"; then + printf 'Orca attribution wrapper could not inspect gh %s capture.\n' "$stream_name" >&2 + return 1 + fi + if (( byte_count > ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES} )); then + printf 'Orca attribution wrapper refused gh %s capture: %s bytes exceeds the ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}-byte limit.\n' "$stream_name" "$byte_count" >&2 + return 1 + fi + cat "$capture_path" +} + append_footer() { local kind="$1" local url_pattern="$2" @@ -587,8 +723,15 @@ if [[ "\${1:-}" == "pr" && "\${2:-}" == "create" ]]; then else status=$? fi - stdout_capture="$(cat "$stdout_file")" - stderr_capture="$(cat "$stderr_file")" + capture_overflow=0 + stdout_capture="$(read_bounded_capture "$stdout_file" stdout)" || capture_overflow=1 + stderr_capture="$(read_bounded_capture "$stderr_file" stderr)" || capture_overflow=1 + if [[ $capture_overflow -ne 0 ]]; then + cleanup_capture + trap - EXIT + [[ $status -ne 0 ]] && exit $status + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + fi cat "$stderr_file" >&2 cat "$stdout_file" if [[ $status -eq 0 ]]; then @@ -621,8 +764,15 @@ if [[ "\${1:-}" == "issue" && "\${2:-}" == "create" ]]; then else status=$? fi - stdout_capture="$(cat "$stdout_file")" - stderr_capture="$(cat "$stderr_file")" + capture_overflow=0 + stdout_capture="$(read_bounded_capture "$stdout_file" stdout)" || capture_overflow=1 + stderr_capture="$(read_bounded_capture "$stderr_file" stderr)" || capture_overflow=1 + if [[ $capture_overflow -ne 0 ]]; then + cleanup_capture + trap - EXIT + [[ $status -ne 0 ]] && exit $status + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + fi cat "$stderr_file" >&2 cat "$stdout_file" if [[ $status -eq 0 ]]; then @@ -702,6 +852,23 @@ const WIN32_GIT_PS_WRAPPER = String.raw`$ErrorActionPreference = 'Stop' $realGit = if ($env:ORCA_REAL_GIT) { $env:ORCA_REAL_GIT } else { 'git' } $trailer = if ($env:ORCA_GIT_COMMIT_TRAILER) { $env:ORCA_GIT_COMMIT_TRAILER } else { 'Co-authored-by: Orca ' } +function Get-OrcaBoundedFileContent { + param([string]$Path, [string]$Description, [long]$MaxBytes) + $guard = $null + try { + $guard = [System.IO.File]::Open($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::Read) + if ($guard.Length -gt $MaxBytes) { + [Console]::Error.WriteLine("Orca attribution wrapper refused $($Description): $($guard.Length) bytes exceeds the $MaxBytes-byte limit.") + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} + } + return (Get-Content -LiteralPath $Path -Raw) + } finally { + if ($null -ne $guard) { + $guard.Dispose() + } + } +} + if ($args -contains '--dry-run') { & $realGit @args exit $LASTEXITCODE @@ -807,17 +974,17 @@ function Test-CommitMessageHasTrailer { } } elseif ($arg -eq '-F' -or $arg -eq '--file') { $i++ - if ($i -lt $CommandArgs.Count -and (Test-Path -LiteralPath $CommandArgs[$i]) -and (Get-Content -LiteralPath $CommandArgs[$i] -Raw) -match [Regex]::Escape($trailer)) { + if ($i -lt $CommandArgs.Count -and (Test-Path -LiteralPath $CommandArgs[$i]) -and (Get-OrcaBoundedFileContent $CommandArgs[$i] 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}) -match [Regex]::Escape($trailer)) { return $true } } elseif ($arg.StartsWith('--file=')) { $path = $arg.Substring('--file='.Length) - if ((Test-Path -LiteralPath $path) -and (Get-Content -LiteralPath $path -Raw) -match [Regex]::Escape($trailer)) { + if ((Test-Path -LiteralPath $path) -and (Get-OrcaBoundedFileContent $path 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}) -match [Regex]::Escape($trailer)) { return $true } } elseif ($arg.StartsWith('-F') -and $arg.Length -gt 2) { $path = $arg.Substring(2) - if ((Test-Path -LiteralPath $path) -and (Get-Content -LiteralPath $path -Raw) -match [Regex]::Escape($trailer)) { + if ((Test-Path -LiteralPath $path) -and (Get-OrcaBoundedFileContent $path 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}) -match [Regex]::Escape($trailer)) { return $true } } @@ -840,7 +1007,7 @@ for ($i = 0; $i -lt $args.Count; $i++) { $sourceFile = if ($i -lt $args.Count) { $args[$i] } else { '' } if ($sourceFile -and (Test-Path -LiteralPath $sourceFile)) { $tmpFile = [System.IO.Path]::GetTempFileName() - Set-Content -LiteralPath $tmpFile -Value ((Get-Content -LiteralPath $sourceFile -Raw).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline + Set-Content -LiteralPath $tmpFile -Value ((Get-OrcaBoundedFileContent $sourceFile 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline $attributedArgs.Add($arg) $attributedArgs.Add($tmpFile) $replacedFileMessage = $true @@ -852,7 +1019,7 @@ for ($i = 0; $i -lt $args.Count; $i++) { $sourceFile = $arg.Substring('--file='.Length) if (Test-Path -LiteralPath $sourceFile) { $tmpFile = [System.IO.Path]::GetTempFileName() - Set-Content -LiteralPath $tmpFile -Value ((Get-Content -LiteralPath $sourceFile -Raw).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline + Set-Content -LiteralPath $tmpFile -Value ((Get-OrcaBoundedFileContent $sourceFile 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline $attributedArgs.Add("--file=$tmpFile") $replacedFileMessage = $true } else { @@ -862,7 +1029,7 @@ for ($i = 0; $i -lt $args.Count; $i++) { $sourceFile = $arg.Substring(2) if (Test-Path -LiteralPath $sourceFile) { $tmpFile = [System.IO.Path]::GetTempFileName() - Set-Content -LiteralPath $tmpFile -Value ((Get-Content -LiteralPath $sourceFile -Raw).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline + Set-Content -LiteralPath $tmpFile -Value ((Get-OrcaBoundedFileContent $sourceFile 'commit message file' ${ATTRIBUTION_COMMIT_MESSAGE_MAX_BYTES}).TrimEnd("${POWERSHELL_TICK}r", "${POWERSHELL_TICK}n") + "${POWERSHELL_TICK}r${POWERSHELL_TICK}n${POWERSHELL_TICK}r${POWERSHELL_TICK}n" + $trailer) -NoNewline $attributedArgs.Add("-F$tmpFile") $replacedFileMessage = $true } else { @@ -951,6 +1118,21 @@ $stdoutFile = [System.IO.Path]::GetTempFileName() $stderrFile = [System.IO.Path]::GetTempFileName() & $realGh @args > $stdoutFile 2> $stderrFile $status = $LASTEXITCODE +$stdoutBytes = if (Test-Path -LiteralPath $stdoutFile) { (Get-Item -LiteralPath $stdoutFile).Length } else { 0 } +$stderrBytes = if (Test-Path -LiteralPath $stderrFile) { (Get-Item -LiteralPath $stderrFile).Length } else { 0 } +if ($stdoutBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES} -or $stderrBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}) { + if ($stdoutBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}) { + [Console]::Error.WriteLine("Orca attribution wrapper refused gh stdout capture: $stdoutBytes bytes exceeds the ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}-byte limit.") + } + if ($stderrBytes -gt ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}) { + [Console]::Error.WriteLine("Orca attribution wrapper refused gh stderr capture: $stderrBytes bytes exceeds the ${ATTRIBUTION_COMMAND_OUTPUT_MAX_BYTES}-byte limit.") + } + Remove-Item -LiteralPath $stdoutFile, $stderrFile -Force -ErrorAction SilentlyContinue + if ($status -ne 0) { + exit $status + } + exit ${ATTRIBUTION_BOUND_EXCEEDED_EXIT_CODE} +} $stdoutCapture = if (Test-Path -LiteralPath $stdoutFile) { Get-Content -LiteralPath $stdoutFile -Raw } else { '' } $stderrCapture = if (Test-Path -LiteralPath $stderrFile) { Get-Content -LiteralPath $stderrFile -Raw } else { '' } if ($stderrCapture) { diff --git a/src/main/automations/dispatch-tokens.test.ts b/src/main/automations/dispatch-tokens.test.ts new file mode 100644 index 00000000000..87775f5b592 --- /dev/null +++ b/src/main/automations/dispatch-tokens.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + _getAutomationDispatchTokenCountForTests, + _resetAutomationDispatchTokensForTests, + beginAutomationDispatchTokenUse, + clearAutomationDispatchTokens, + createAutomationDispatchToken, + DISPATCH_TOKEN_MAX_ENTRIES, + finishAutomationDispatchTokenUse, + releaseAutomationDispatchTokenUse +} from './dispatch-tokens' + +beforeEach(() => { + _resetAutomationDispatchTokensForTests() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('automation dispatch tokens', () => { + it('preserves reservation, release, and one-time finish behavior', () => { + const token = createAutomationDispatchToken('automation', 'run') + const args = { token, automationId: 'automation', runId: 'run', reservationId: 'request-1' } + + expect(beginAutomationDispatchTokenUse(args)).toBe(true) + expect(beginAutomationDispatchTokenUse(args)).toBe(false) + releaseAutomationDispatchTokenUse({ token, reservationId: 'request-1' }) + expect(beginAutomationDispatchTokenUse(args)).toBe(true) + finishAutomationDispatchTokenUse({ token, reservationId: 'request-1' }) + expect(beginAutomationDispatchTokenUse(args)).toBe(false) + }) + + it('keeps delimiter-containing identities distinct', () => { + const token = createAutomationDispatchToken('a:b', 'c') + expect( + beginAutomationDispatchTokenUse({ + token, + automationId: 'a', + runId: 'b:c', + reservationId: 'request' + }) + ).toBe(false) + }) + + it('clears only the matching automation run', () => { + const removed = createAutomationDispatchToken('automation', 'run-1') + const retained = createAutomationDispatchToken('automation', 'run-2') + clearAutomationDispatchTokens('automation', 'run-1') + + expect( + beginAutomationDispatchTokenUse({ + token: removed, + automationId: 'automation', + runId: 'run-1', + reservationId: 'request' + }) + ).toBe(false) + expect( + beginAutomationDispatchTokenUse({ + token: retained, + automationId: 'automation', + runId: 'run-2', + reservationId: 'request' + }) + ).toBe(true) + }) + + it('physically prunes expired records', () => { + vi.useFakeTimers() + const expired = createAutomationDispatchToken('automation', 'old') + vi.advanceTimersByTime(30 * 60_000 + 1) + createAutomationDispatchToken('automation', 'new') + + expect(_getAutomationDispatchTokenCountForTests()).toBe(1) + expect( + beginAutomationDispatchTokenUse({ + token: expired, + automationId: 'automation', + runId: 'old', + reservationId: 'request' + }) + ).toBe(false) + }) + + it('evicts the oldest unused token at the entry cap', () => { + const oldest = createAutomationDispatchToken('automation', 'run-0') + for (let index = 1; index <= DISPATCH_TOKEN_MAX_ENTRIES; index += 1) { + createAutomationDispatchToken('automation', `run-${index}`) + } + + expect(_getAutomationDispatchTokenCountForTests()).toBe(DISPATCH_TOKEN_MAX_ENTRIES) + expect( + beginAutomationDispatchTokenUse({ + token: oldest, + automationId: 'automation', + runId: 'run-0', + reservationId: 'request' + }) + ).toBe(false) + }) +}) diff --git a/src/main/automations/dispatch-tokens.ts b/src/main/automations/dispatch-tokens.ts index b8856a16220..075794fc686 100644 --- a/src/main/automations/dispatch-tokens.ts +++ b/src/main/automations/dispatch-tokens.ts @@ -1,10 +1,10 @@ -import { randomUUID } from 'node:crypto' +import { createHash, randomUUID } from 'node:crypto' const DISPATCH_TOKEN_TTL_MS = 30 * 60_000 +export const DISPATCH_TOKEN_MAX_ENTRIES = 1024 type DispatchTokenRecord = { - automationId: string - runId: string + dispatchIdentity: string expiresAt: number reservedBy?: string inFlight: boolean @@ -20,12 +20,34 @@ function pruneExpiredDispatchTokens(now = Date.now()): void { } } +function identityDigest(...parts: string[]): string { + const digest = createHash('sha256') + for (const part of parts) { + digest.update(`${part.length}:`) + digest.update(part) + } + return digest.digest('base64url') +} + +function evictOldestUnusedDispatchTokens(): void { + while (dispatchTokens.size >= DISPATCH_TOKEN_MAX_ENTRIES) { + const oldestUnused = Array.from(dispatchTokens).find(([, record]) => !record.inFlight) + if (!oldestUnused) { + return + } + dispatchTokens.delete(oldestUnused[0]) + } +} + export function createAutomationDispatchToken(automationId: string, runId: string): string { pruneExpiredDispatchTokens() + evictOldestUnusedDispatchTokens() const token = randomUUID() + if (dispatchTokens.size >= DISPATCH_TOKEN_MAX_ENTRIES) { + return token + } dispatchTokens.set(token, { - automationId, - runId, + dispatchIdentity: identityDigest(automationId, runId), expiresAt: Date.now() + DISPATCH_TOKEN_TTL_MS, inFlight: false }) @@ -41,19 +63,19 @@ export function beginAutomationDispatchTokenUse(args: { pruneExpiredDispatchTokens() const record = dispatchTokens.get(args.token) const valid = - record?.automationId === args.automationId && - record.runId === args.runId && + record?.dispatchIdentity === identityDigest(args.automationId, args.runId) && record.expiresAt > Date.now() if (!valid) { return false } - if (record.reservedBy !== undefined && record.reservedBy !== args.reservationId) { + const reservationIdentity = identityDigest(args.reservationId) + if (record.reservedBy !== undefined && record.reservedBy !== reservationIdentity) { return false } if (record.inFlight) { return false } - record.reservedBy = args.reservationId + record.reservedBy = reservationIdentity record.inFlight = true return true } @@ -63,7 +85,7 @@ export function releaseAutomationDispatchTokenUse(args: { reservationId: string }): void { const record = dispatchTokens.get(args.token) - if (record?.reservedBy === args.reservationId) { + if (record?.reservedBy === identityDigest(args.reservationId)) { record.inFlight = false } } @@ -73,15 +95,24 @@ export function finishAutomationDispatchTokenUse(args: { reservationId: string }): void { const record = dispatchTokens.get(args.token) - if (record?.reservedBy === args.reservationId) { + if (record?.reservedBy === identityDigest(args.reservationId)) { dispatchTokens.delete(args.token) } } export function clearAutomationDispatchTokens(automationId: string, runId: string): void { + const dispatchIdentity = identityDigest(automationId, runId) for (const [token, record] of dispatchTokens) { - if (record.automationId === automationId && record.runId === runId) { + if (record.dispatchIdentity === dispatchIdentity) { dispatchTokens.delete(token) } } } + +export function _resetAutomationDispatchTokensForTests(): void { + dispatchTokens.clear() +} + +export function _getAutomationDispatchTokenCountForTests(): number { + return dispatchTokens.size +} diff --git a/src/main/automations/external-job-mappers.ts b/src/main/automations/external-job-mappers.ts index d3648af446a..6a1f99783b3 100644 --- a/src/main/automations/external-job-mappers.ts +++ b/src/main/automations/external-job-mappers.ts @@ -200,6 +200,7 @@ export function mapHermesJobs(managerId: string, rawJobs: unknown): ExternalAuto lastError: asString(job.last_error) ?? asString(job.last_delivery_error), workdir: asString(job.workdir), runCount: asNumber(job.run_count) ?? (Array.isArray(job.runs) ? job.runs.length : 0), + ...(job.run_count_saturated === true ? { runCountSaturated: true as const } : {}), runs: mapExternalRuns({ managerId, provider: 'hermes', diff --git a/src/main/automations/external-manager-remote-fanout.test.ts b/src/main/automations/external-manager-remote-fanout.test.ts new file mode 100644 index 00000000000..15353016e66 --- /dev/null +++ b/src/main/automations/external-manager-remote-fanout.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as NodeFs from 'node:fs' +import type { Store } from '../persistence' +import type { SshTarget } from '../../shared/ssh-types' +import { getActiveMultiplexer } from '../ipc/ssh' +import { listExternalAutomationManagers } from './external-manager' + +const execFileMock = vi.hoisted(() => + vi.fn( + ( + _command: string, + _args: string[], + _options: unknown, + callback: (error: Error | null) => void + ) => { + callback(new Error('not installed')) + return { kill: vi.fn() } + } + ) +) + +vi.mock('node:child_process', () => ({ execFile: execFileMock })) +vi.mock('node:fs', async () => { + const actual = await vi.importActual('node:fs') + return { ...actual, existsSync: vi.fn(() => false) } +}) +vi.mock('../ipc/ssh', () => ({ getActiveMultiplexer: vi.fn() })) + +beforeEach(() => { + vi.mocked(getActiveMultiplexer).mockReset() +}) + +describe('remote external automation manager fanout', () => { + it('bounds probes for a large target list and preserves provider order', async () => { + const targets: SshTarget[] = Array.from({ length: 100 }, (_, index) => ({ + id: `ssh-${index}`, + label: `Target ${index}`, + host: `host-${index}`, + port: 22, + username: 'orca' + })) + let inFlight = 0 + let peak = 0 + const request = vi.fn(async (_method: string, input: { provider: string }) => { + inFlight += 1 + peak = Math.max(peak, inFlight) + await Promise.resolve() + inFlight -= 1 + return { + jobs: [], + hermesAvailable: input.provider === 'hermes', + openclawAvailable: input.provider === 'openclaw' + } + }) + vi.mocked(getActiveMultiplexer).mockReturnValue({ + isDisposed: () => false, + request + } as never) + + const managers = await listExternalAutomationManagers({ + getSshTargets: () => targets + } as Store) + + expect(peak).toBe(4) + expect(request).toHaveBeenCalledTimes(200) + expect(managers.map((manager) => manager.id)).toEqual( + targets.flatMap((target) => [`hermes:ssh:${target.id}`, `openclaw:ssh:${target.id}`]) + ) + }) +}) diff --git a/src/main/automations/external-manager.test.ts b/src/main/automations/external-manager.test.ts index dc96b9b964a..59c104197a3 100644 --- a/src/main/automations/external-manager.test.ts +++ b/src/main/automations/external-manager.test.ts @@ -344,6 +344,7 @@ describe('listExternalAutomationRuns', () => { it('requests paginated Hermes runs from the remote relay', async () => { const request = vi.fn().mockResolvedValue({ total: 42, + totalSaturated: true, runs: [ { id: 'job-1:2026-05-15_09-00-00.md', @@ -375,6 +376,7 @@ describe('listExternalAutomationRuns', () => { page: 2, pageSize: 10, total: 42, + totalSaturated: true, runs: [ { id: 'job-1:2026-05-15_09-00-00.md', diff --git a/src/main/automations/external-manager.ts b/src/main/automations/external-manager.ts index 8c4d0f0d53e..b5f9d698782 100644 --- a/src/main/automations/external-manager.ts +++ b/src/main/automations/external-manager.ts @@ -2,7 +2,6 @@ * and lifecycle routing share provider/target validation and remote relay fallbacks. */ import { execFile } from 'node:child_process' import { existsSync } from 'node:fs' -import { readFile } from 'node:fs/promises' import { homedir } from 'node:os' import { join } from 'node:path' import type { @@ -15,6 +14,8 @@ import type { ExternalAutomationRunsPage, ExternalAutomationUpdateInput } from '../../shared/automations-types' +import { readExternalAutomationJobsFile } from '../../shared/external-automation-jobs-file' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' import type { SshTarget } from '../../shared/ssh-types' import type { Store } from '../persistence' import { getActiveMultiplexer } from '../ipc/ssh' @@ -32,6 +33,8 @@ const OPENCLAW_JOBS_FILE = join(homedir(), '.openclaw', 'cron', 'jobs.json') const EXTERNAL_JOB_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/ const LOCAL_COMMAND_LOOKUP_TIMEOUT_MS = 5_000 const LOCAL_AUTOMATION_COMMAND_TIMEOUT_MS = 30_000 +const HERMES_JOB_RUN_COUNT_CONCURRENCY = 4 +const REMOTE_AUTOMATION_MANAGER_FETCH_CONCURRENCY = 4 function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) @@ -130,39 +133,30 @@ async function readLocalHermesJobs(): Promise { if (!existsSync(HERMES_JOBS_FILE)) { return [] } - const content = await readFile(HERMES_JOBS_FILE, 'utf-8') - const parsed = JSON.parse(content) as unknown - const jobs = Array.isArray(parsed) - ? parsed - : isRecord(parsed) && Array.isArray(parsed.jobs) - ? parsed.jobs - : [] - return Promise.all( - jobs.map(async (job) => { - if (!isRecord(job)) { - return job - } - const jobId = typeof job.id === 'string' ? job.id : null - if (!jobId) { - return job - } - const runsPage = await readHermesCronOutputRunsPage(jobId, { page: 1, pageSize: 0 }) - return { - ...job, - run_count: runsPage.total, - runs: [] - } - }) - ) + const jobs = await readExternalAutomationJobsFile(HERMES_JOBS_FILE, { allowRootArray: true }) + return mapWithConcurrency(jobs, HERMES_JOB_RUN_COUNT_CONCURRENCY, async (job) => { + if (!isRecord(job)) { + return job + } + const jobId = typeof job.id === 'string' ? job.id : null + if (!jobId) { + return job + } + const runsPage = await readHermesCronOutputRunsPage(jobId, { page: 1, pageSize: 0 }) + return { + ...job, + run_count: runsPage.total, + ...(runsPage.totalSaturated ? { run_count_saturated: true } : {}), + runs: [] + } + }) } async function readLocalOpenClawJobs(): Promise { if (!existsSync(OPENCLAW_JOBS_FILE)) { return [] } - const content = await readFile(OPENCLAW_JOBS_FILE, 'utf-8') - const parsed = JSON.parse(content) as unknown - return isRecord(parsed) && Array.isArray(parsed.jobs) ? parsed.jobs : [] + return readExternalAutomationJobsFile(OPENCLAW_JOBS_FILE, { allowRootArray: false }) } async function listLocalHermesManager(): Promise { @@ -300,16 +294,22 @@ async function listRemoteManager( export async function listExternalAutomationManagers( store: Store ): Promise { + const remoteManagerRequests = store + .getSshTargets() + // Why: runtime-owned hidden targets are excluded from SSH/run-target surfaces. + .filter((target) => !isRuntimeOwnedSshTarget(target)) + .flatMap((target) => [ + { provider: 'hermes' as const, target }, + { provider: 'openclaw' as const, target } + ]) const [localHermes, localOpenClaw, remote] = await Promise.all([ listLocalHermesManager(), listLocalOpenClawManager(), - Promise.all( - store - .getSshTargets() - // Why: runtime-owned hidden targets are excluded from SSH/run-target - // surfaces; don't probe them for external automations either. - .filter((target) => !isRuntimeOwnedSshTarget(target)) - .flatMap((target) => [listRemoteHermesManager(target), listRemoteOpenClawManager(target)]) + mapWithConcurrency( + remoteManagerRequests, + REMOTE_AUTOMATION_MANAGER_FETCH_CONCURRENCY, + ({ provider, target }) => + provider === 'hermes' ? listRemoteHermesManager(target) : listRemoteOpenClawManager(target) ) ]) return [ @@ -351,6 +351,7 @@ export async function listExternalAutomationRuns( page, pageSize, total: result.total, + ...(result.totalSaturated ? { totalSaturated: true } : {}), runs: mapHermesJobs(input.managerId, [{ id: input.jobId, runs: result.runs }])[0]?.runs ?? [] } } @@ -363,7 +364,7 @@ export async function listExternalAutomationRuns( jobId: input.jobId, page, pageSize - })) as { total?: number; runs?: unknown[] } + })) as { total?: number; totalSaturated?: boolean; runs?: unknown[] } return { managerId: input.managerId, provider: input.provider, @@ -372,6 +373,7 @@ export async function listExternalAutomationRuns( page, pageSize, total: typeof result.total === 'number' && Number.isFinite(result.total) ? result.total : 0, + ...(result.totalSaturated === true ? { totalSaturated: true } : {}), runs: mapHermesJobs(input.managerId, [{ id: input.jobId, runs: result.runs ?? [] }])[0]?.runs ?? [] } diff --git a/src/main/automations/headless-dispatch.ts b/src/main/automations/headless-dispatch.ts index 359673bc573..00cca368e35 100644 --- a/src/main/automations/headless-dispatch.ts +++ b/src/main/automations/headless-dispatch.ts @@ -4,6 +4,7 @@ import type { AutomationRunOutputSnapshot } from '../../shared/automations-types' import type { AutomationRunTargetResult } from './run-target-resolution' +import { appendCompactedStringChunk } from '../../shared/string-chunk-compaction' const MAX_HEADLESS_OUTPUT_SNAPSHOT_CHARS = 256 * 1024 @@ -39,7 +40,7 @@ export function createHeadlessAutomationOutputSnapshotBuffer(): { if (!chunk) { return } - chunks.push(chunk) + appendCompactedStringChunk(chunks, chunk) totalChars += chunk.length let overflowChars = totalChars - MAX_HEADLESS_OUTPUT_SNAPSHOT_CHARS while (overflowChars > 0 && chunks.length > 0) { diff --git a/src/main/automations/hermes-cron-output.test.ts b/src/main/automations/hermes-cron-output.test.ts index 5bf9b2fddb5..2419863ae2e 100644 --- a/src/main/automations/hermes-cron-output.test.ts +++ b/src/main/automations/hermes-cron-output.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, mkdir, rm, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -11,6 +11,7 @@ const fakeDbRows = vi.hoisted(() => ({ messages: [] as Record[] })) const fakePrepareSqls = vi.hoisted(() => [] as string[]) +const fakeIteratedSqls = vi.hoisted(() => [] as string[]) const fakeDatabase = vi.hoisted(() => vi.fn(function FakeDatabase() { return { @@ -26,7 +27,14 @@ const fakeDatabase = vi.hoisted(() => sql.includes('FROM sessions') ? fakeDbRows.sessions : fakeDbRows.messages.filter((message) => message.session_id === param) - ) + ), + iterate: vi.fn(function* (param: string) { + fakeIteratedSqls.push(sql) + const rows = sql.includes('FROM sessions') + ? fakeDbRows.sessions + : fakeDbRows.messages.filter((message) => message.session_id === param) + yield* rows + }) } }), close: vi.fn() @@ -53,6 +61,7 @@ beforeEach(() => { fakeDbRows.sessions = [] fakeDbRows.messages = [] fakePrepareSqls.length = 0 + fakeIteratedSqls.length = 0 fakeDatabase.mockClear() }) @@ -141,6 +150,26 @@ Run summary: monitor automation completed successfully. expect((page.runs[0] as { output_content?: string }).output_content).toContain( 'full command output line' ) + const transcriptSql = fakeIteratedSqls.find((sql) => sql.includes('FROM messages')) + expect(transcriptSql).toContain('substr(CAST(content AS BLOB)') + expect(transcriptSql).toContain('LIMIT 10001') + }) + + it('omits an oversized sparse markdown output without reading it wholesale', async () => { + const home = await createHermesHome() + const outputDir = join(home, 'cron', 'output', 'job-1') + const outputPath = join(outputDir, '2026-05-15_09-02-00.md') + await mkdir(outputDir, { recursive: true }) + await writeFile(outputPath, '', 'utf-8') + await truncate(outputPath, 64 * 1024 * 1024) + + const { readHermesCronOutputRunsPage } = await loadReader() + const page = await readHermesCronOutputRunsPage('job-1', { page: 1, pageSize: 25 }) + + expect(page.runs[0]).toMatchObject({ + output_content: null, + error: expect.stringContaining('File too large') + }) }) it('builds large response previews without broad regex captures', async () => { @@ -277,6 +306,47 @@ Run summary: monitor automation completed successfully. expect(fakePrepareSqls.some((sql) => sql.includes('FROM messages'))).toBe(false) }) + it('caps one hydrated history page at 100 runs', async () => { + const home = await createHermesHome() + const outputDir = join(home, 'cron', 'output', 'job-1') + await mkdir(outputDir, { recursive: true }) + await Promise.all( + Array.from({ length: 101 }, async (_, index) => { + const minute = String(Math.floor(index / 60)).padStart(2, '0') + const second = String(index % 60).padStart(2, '0') + await writeFile( + join(outputDir, `2026-05-15_09-${minute}-${second}.md`), + `run ${index}`, + 'utf-8' + ) + }) + ) + + const { readHermesCronOutputRunsPage } = await loadReader() + const page = await readHermesCronOutputRunsPage('job-1', { + page: 1, + pageSize: Number.MAX_SAFE_INTEGER + }) + + expect(page.total).toBe(101) + expect(page.runs).toHaveLength(100) + }) + + it('bounds session run refs and reports a saturated total', async () => { + const home = await createHermesHome() + await writeFile(join(home, 'state.db'), '', 'utf-8') + fakeDbRows.sessions = Array.from({ length: 10_001 }, (_, index) => ({ + id: `cron_job-1_20260515_${String(index).padStart(6, '0')}`, + started_at: Date.UTC(2026, 4, 15, 9, 0, 0) / 1000 - index + })) + + const { readHermesCronOutputRunsPage } = await loadReader() + const page = await readHermesCronOutputRunsPage('job-1', { page: 1, pageSize: 0 }) + + expect(page).toEqual({ total: 10_000, totalSaturated: true, runs: [] }) + expect(fakeIteratedSqls.find((sql) => sql.includes('FROM sessions'))).toContain('LIMIT 10001') + }) + it('caches count-only reads until the cache is cleared', async () => { const home = await createHermesHome() const outputDir = join(home, 'cron', 'output', 'job-1') diff --git a/src/main/automations/hermes-cron-output.ts b/src/main/automations/hermes-cron-output.ts index db1e054f7e8..62eb2b22b85 100644 --- a/src/main/automations/hermes-cron-output.ts +++ b/src/main/automations/hermes-cron-output.ts @@ -1,9 +1,25 @@ /* eslint-disable max-lines -- Why: Hermes run history has to reconcile * markdown output files with SQLite session transcripts from separate stores. */ import { existsSync } from 'node:fs' -import { open, readdir, readFile, realpath, stat } from 'node:fs/promises' +import { open, opendir, realpath, stat } from 'node:fs/promises' import { homedir } from 'node:os' import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { + type BoundedHermesRunRefs, + HERMES_RUN_REF_MAX_ENTRIES, + HERMES_SESSION_RUN_REFS_SELECT_SQL, + HermesRunRefRetainer +} from '../../shared/hermes-run-ref-retention' +import { + formatHermesSessionMessagesWithinLimits, + HERMES_PRIMARY_OUTPUT_MAX_BYTES, + HERMES_RUN_PAGE_MAX_RUNS, + HERMES_SESSION_RUN_SELECT_SQL, + HERMES_SESSION_TRANSCRIPT_SELECT_SQL, + HERMES_SESSION_TRANSCRIPT_TRUNCATED_ERROR, + hydrateHermesRunPageWithinLimits +} from '../../shared/hermes-run-output-limits' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' import Database from '../sqlite/sync-database' const HERMES_HOME = process.env.HERMES_HOME?.trim() || join(homedir(), '.hermes') @@ -22,6 +38,7 @@ const LATEST_LOG_PATH_PATTERN = export type HermesCronOutputRunsPage = { total: number + totalSaturated?: true runs: unknown[] } @@ -362,9 +379,10 @@ async function readReferencedLogFile(content: string): Promise<{ return null } if (logStat.size <= MAX_REFERENCED_LOG_BYTES) { + const { buffer } = await readNodeFileWithinLimit(logPath, MAX_REFERENCED_LOG_BYTES) return { path: logPath, - content: await readFile(logPath, 'utf-8'), + content: buffer.toString('utf-8'), truncated: false } } @@ -411,31 +429,6 @@ async function appendReferencedLogFile(content: string): Promise { .join('\n\n') } -function formatSessionMessages(messages: Record[]): string | null { - if (messages.length === 0) { - return null - } - return messages - .map((message) => { - const role = typeof message.role === 'string' ? message.role : 'message' - const content = typeof message.content === 'string' ? message.content.trim() : '' - const toolName = typeof message.tool_name === 'string' ? message.tool_name.trim() : '' - const reasoning = - typeof message.reasoning_content === 'string' - ? message.reasoning_content.trim() - : typeof message.reasoning === 'string' - ? message.reasoning.trim() - : '' - const parts = [ - `## ${role}${toolName ? ` / ${toolName}` : ''}`, - reasoning ? `### Reasoning\n\n${reasoning}` : null, - content || '(empty)' - ].filter(Boolean) - return parts.join('\n\n') - }) - .join('\n\n---\n\n') -} - function getRunKey(run: unknown): string | null { return isRecord(run) ? asString(run.run_key) : null } @@ -579,18 +572,20 @@ export async function readHermesCronOutputRuns(jobId: string): Promise { +async function readHermesCronOutputRunRefs( + jobId: string +): Promise> { const outputRuns = await readHermesOutputFileRunRefs(jobId) - return mergeHermesOutputAndSessionRunRefs(outputRuns, readHermesSessionDbRunRefs(jobId)).sort( - (a, b) => { - const aTime = getRawRunTime(a) - const bTime = getRawRunTime(b) - if (Number.isFinite(aTime) && Number.isFinite(bTime)) { - return bTime - aTime - } - return getRawRunId(b).localeCompare(getRawRunId(a)) - } - ) + const sessionRuns = readHermesSessionDbRunRefs(jobId) + const mergedRetainer = new HermesRunRefRetainer() + for (const ref of mergeHermesOutputAndSessionRunRefs(outputRuns.refs, sessionRuns.refs)) { + mergedRetainer.add(ref) + } + const merged = mergedRetainer.finish() + return { + refs: merged.refs, + saturated: outputRuns.saturated || sessionRuns.saturated || merged.saturated + } } // Why: opening the Automations page calls readHermesCronOutputRunsPage with @@ -600,9 +595,10 @@ async function readHermesCronOutputRunRefs(jobId: string): Promise + promise: Promise expiresAt: number } +type HermesRunCount = { total: number; totalSaturated?: true } const hermesRunCountCache = new Map() export function clearHermesCronOutputRunCountCache(jobId?: string): void { @@ -628,7 +624,7 @@ function pruneHermesRunCountCache(now: number): void { } } -async function readHermesCronOutputRunCount(jobId: string): Promise { +async function readHermesCronOutputRunCount(jobId: string): Promise { const now = Date.now() const cached = hermesRunCountCache.get(jobId) if (cached && cached.expiresAt > now) { @@ -641,7 +637,10 @@ async function readHermesCronOutputRunCount(jobId: string): Promise { // size bound and expired sweep, a long session can pin stale job ids forever. pruneHermesRunCountCache(now) const entry: HermesRunCountCacheEntry = { - promise: readHermesCronOutputRunRefs(jobId).then((refs) => refs.length), + promise: readHermesCronOutputRunRefs(jobId).then((result) => ({ + total: result.refs.length, + ...(result.saturated ? { totalSaturated: true as const } : {}) + })), expiresAt: Number.POSITIVE_INFINITY } hermesRunCountCache.set(jobId, entry) @@ -685,57 +684,54 @@ export async function readHermesCronOutputRunsPage( return { total: 0, runs: [] } } const safePage = Math.max(1, Math.floor(page)) - const safePageSize = Math.max(0, Math.floor(pageSize)) + const safePageSize = Math.min(HERMES_RUN_PAGE_MAX_RUNS, Math.max(0, Math.floor(pageSize))) if (safePageSize === 0) { // Why: manager listing only needs a badge count; hydrating markdown logs // and full session transcripts can make opening Automations very slow. - return { total: await readHermesCronOutputRunCount(jobId), runs: [] } + return { ...(await readHermesCronOutputRunCount(jobId)), runs: [] } } const runRefs = await readHermesCronOutputRunRefs(jobId) const start = (safePage - 1) * safePageSize - const pageRefs = runRefs.slice(start, start + safePageSize) + const pageRefs = runRefs.refs.slice(start, start + safePageSize) return { - total: runRefs.length, - runs: await Promise.all(pageRefs.map((ref) => hydrateHermesRunRef(jobId, ref))) + total: runRefs.refs.length, + ...(runRefs.saturated ? { totalSaturated: true } : {}), + runs: await hydrateHermesRunPageWithinLimits(pageRefs, (ref) => hydrateHermesRunRef(jobId, ref)) } } -function getRawRunId(run: unknown): string { - if (typeof run === 'object' && run !== null && 'id' in run) { - return String((run as { id: unknown }).id) - } - return '' -} - -function getRawRunTime(run: unknown): number { - if (typeof run !== 'object' || run === null || !('run_at' in run)) { - return Number.NaN - } - const runAt = (run as { run_at: unknown }).run_at - return typeof runAt === 'string' ? Date.parse(runAt) : Number.NaN -} - -async function readHermesOutputFileRunRefs(jobId: string): Promise { +async function readHermesOutputFileRunRefs( + jobId: string +): Promise> { const outputDir = join(HERMES_OUTPUT_DIR, jobId) if (!existsSync(outputDir)) { - return [] + return { refs: [], saturated: false } } - const entries = await readdir(outputDir, { withFileTypes: true }) - return entries - .filter((entry) => entry.isFile() && HERMES_OUTPUT_FILE_PATTERN.test(entry.name)) - .map((entry) => ({ + const retainer = new HermesRunRefRetainer() + const directory = await opendir(outputDir) + for await (const entry of directory) { + if (!entry.isFile() || !HERMES_OUTPUT_FILE_PATTERN.test(entry.name)) { + continue + } + retainer.add({ kind: 'output' as const, id: `${jobId}:${entry.name}`, job_id: jobId, run_at: runAtFromHermesOutputFile(entry.name), run_key: runKeyFromHermesOutputFile(entry.name), output_path: join(outputDir, entry.name) - })) + }) + } + return retainer.finish() } async function readHermesOutputFileRun(ref: HermesOutputRunRef): Promise { try { - const content = await readFile(ref.output_path, 'utf-8') + const { buffer } = await readNodeFileWithinLimit( + ref.output_path, + HERMES_PRIMARY_OUTPUT_MAX_BYTES + ) + const content = buffer.toString('utf-8') const parsed = parseHermesOutput(content) const outputContent = await appendReferencedLogFile(parsed.outputContent) return { @@ -764,37 +760,41 @@ async function readHermesOutputFileRun(ref: HermesOutputRunRef): Promise { if (!existsSync(HERMES_STATE_DB)) { - return [] + return { refs: [], saturated: false } } try { const db = new Database(HERMES_STATE_DB, { readonly: true, fileMustExist: true }) try { const pattern = `cron\\_${escapeSqlLike(jobId)}\\_%` - const rows = db - .prepare( - `SELECT id, started_at - FROM sessions - WHERE id LIKE ? ESCAPE '\\' - ORDER BY started_at DESC` - ) - .all(pattern) as Record[] - return rows.map((row) => { + const statement = db.prepare(HERMES_SESSION_RUN_REFS_SELECT_SQL) + const rows = + typeof statement.iterate === 'function' + ? (statement.iterate(pattern) as Iterable>) + : (statement.all(pattern) as Record[]) + const retainer = new HermesRunRefRetainer() + let read = 0 + for (const row of rows) { + if (read >= HERMES_RUN_REF_MAX_ENTRIES + 1) { + break + } + read += 1 const runId = typeof row.id === 'string' ? row.id : `${jobId}:${String(row.started_at)}` - return { + retainer.add({ kind: 'session', id: runId, job_id: jobId, run_at: runAtFromUnixSeconds(row.started_at), run_key: runId.split(`${jobId}_`).at(-1) ?? null - } - }) + }) + } + return retainer.finish() } finally { db.close() } } catch { - return [] + return { refs: [], saturated: false } } } @@ -805,25 +805,18 @@ function readHermesSessionDbRunById(jobId: string, runId: string): unknown | nul try { const db = new Database(HERMES_STATE_DB, { readonly: true, fileMustExist: true }) try { - const row = db - .prepare( - `SELECT id, title, started_at, ended_at, end_reason, model, message_count, - input_tokens, output_tokens, estimated_cost_usd - FROM sessions - WHERE id = ?` - ) - .get(runId) as Record | undefined + const row = db.prepare(HERMES_SESSION_RUN_SELECT_SQL).get(runId) as + | Record + | undefined if (!row) { return null } - const messages = db - .prepare( - `SELECT role, content, tool_name, reasoning, reasoning_content - FROM messages - WHERE session_id = ? - ORDER BY timestamp, id` - ) - .all(runId) as Record[] + const messageStatement = db.prepare(HERMES_SESSION_TRANSCRIPT_SELECT_SQL) + const messages = + typeof messageStatement.iterate === 'function' + ? (messageStatement.iterate(runId) as Iterable>) + : (messageStatement.all(runId) as Record[]) + const formattedMessages = formatHermesSessionMessagesWithinLimits(messages) const title = typeof row.title === 'string' && row.title.trim() ? row.title.trim() : null const model = typeof row.model === 'string' && row.model.trim() ? row.model.trim() : null const messageCount = typeof row.message_count === 'number' ? row.message_count : null @@ -843,8 +836,8 @@ function readHermesSessionDbRunById(jobId: string, runId: string): unknown | nul run_key: runId.split(`${jobId}_`).at(-1) ?? null, status: typeof row.ended_at === 'number' ? 'completed' : 'unknown', output_preview: summaryParts.join(' · ') || null, - output_content: formatSessionMessages(messages), - error: null, + output_content: formattedMessages.content, + error: formattedMessages.truncated ? HERMES_SESSION_TRANSCRIPT_TRUNCATED_ERROR : null, output_path: HERMES_STATE_DB } } finally { diff --git a/src/main/azure-devops/azure-devops-api-request.ts b/src/main/azure-devops/azure-devops-api-request.ts index 5fc8f256793..d5dfe3e9a33 100644 --- a/src/main/azure-devops/azure-devops-api-request.ts +++ b/src/main/azure-devops/azure-devops-api-request.ts @@ -1,6 +1,7 @@ import { Buffer } from 'node:buffer' import type { AzureDevOpsRepoRef } from './repository-ref' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' const REQUEST_TIMEOUT_MS = 5000 @@ -95,7 +96,7 @@ export async function requestAzureDevOpsJsonAtBase( } return null } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } catch (error) { if (throwOnFailure) { throw error diff --git a/src/main/azure-devops/repository-ref.ts b/src/main/azure-devops/repository-ref.ts index 868d0d8c9bc..a826681b297 100644 --- a/src/main/azure-devops/repository-ref.ts +++ b/src/main/azure-devops/repository-ref.ts @@ -1,5 +1,9 @@ import { gitExecFileAsync } from '../git/runner' import { getSshGitProvider } from '../providers/ssh-git-dispatch' +import { + buildRepositoryRefCacheKey, + RepositoryRefCache +} from '../source-control/repository-ref-cache' export type AzureDevOpsRepoRef = { host: string @@ -14,8 +18,7 @@ type LocalGitExecOptions = { wslDistro?: string } -const REPO_REF_CACHE_MAX_ENTRIES = 512 -const repoRefCache = new Map() +const repoRefCache = new RepositoryRefCache() /** @internal - exposed for tests only */ export function _resetAzureDevOpsRepoRefCache(): void { @@ -27,17 +30,6 @@ export function _getAzureDevOpsRepoRefCacheSize(): number { return repoRefCache.size } -function rememberRepoRefCacheEntry(cacheKey: string, value: AzureDevOpsRepoRef | null): void { - repoRefCache.set(cacheKey, value) - while (repoRefCache.size > REPO_REF_CACHE_MAX_ENTRIES) { - const oldestKey = repoRefCache.keys().next().value - if (oldestKey === undefined) { - return - } - repoRefCache.delete(oldestKey) - } -} - function decodeSegment(value: string): string { try { return decodeURIComponent(value) @@ -212,9 +204,10 @@ export async function getAzureDevOpsRepoRefForRemote( localGitOptions: LocalGitExecOptions = {} ): Promise { const runtimeKey = connectionId ?? `local:${localGitOptions.wslDistro ?? 'host'}` - const cacheKey = `${runtimeKey}\0${repoPath}\0${remoteName}` - if (repoRefCache.has(cacheKey)) { - return repoRefCache.get(cacheKey)! + const cacheKey = buildRepositoryRefCacheKey([runtimeKey, repoPath, remoteName]) + const cached = repoRefCache.get(cacheKey) + if (cached.found) { + return cached.value } try { const sshGitProvider = connectionId ? getSshGitProvider(connectionId) : null @@ -228,7 +221,20 @@ export async function getAzureDevOpsRepoRefForRemote( ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) }) const result = parseAzureDevOpsRepoRef(stdout) - rememberRepoRefCacheEntry(cacheKey, result) + repoRefCache.remember( + cacheKey, + result, + result + ? [ + result.host, + result.project, + result.repository, + result.apiBaseUrl, + result.webBaseUrl, + result.organization ?? '' + ] + : [] + ) return result } catch { if (connectionId) { @@ -236,7 +242,7 @@ export async function getAzureDevOpsRepoRefForRemote( // caching them as "not Azure DevOps" would poison the repo for the session. return null } - rememberRepoRefCacheEntry(cacheKey, null) + repoRefCache.remember(cacheKey, null, []) return null } } diff --git a/src/main/bitbucket/client.ts b/src/main/bitbucket/client.ts index e5163bd525a..ca124bb07ef 100644 --- a/src/main/bitbucket/client.ts +++ b/src/main/bitbucket/client.ts @@ -15,6 +15,7 @@ import { type HostedReviewExecutionOptions } from '../source-control/hosted-review-git-options' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' const DEFAULT_API_BASE_URL = 'https://api.bitbucket.org/2.0' const REQUEST_TIMEOUT_MS = 5000 @@ -113,7 +114,7 @@ async function requestJson( } return null } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } catch (error) { if (throwOnFailure) { throw error diff --git a/src/main/bitbucket/repository-ref.ts b/src/main/bitbucket/repository-ref.ts index fb07c32d4bc..a4b743157de 100644 --- a/src/main/bitbucket/repository-ref.ts +++ b/src/main/bitbucket/repository-ref.ts @@ -1,5 +1,9 @@ import { gitExecFileAsync } from '../git/runner' import { getSshGitProvider } from '../providers/ssh-git-dispatch' +import { + buildRepositoryRefCacheKey, + RepositoryRefCache +} from '../source-control/repository-ref-cache' export type BitbucketRepoRef = { workspace: string @@ -10,8 +14,7 @@ type LocalGitExecOptions = { wslDistro?: string } -const REPO_REF_CACHE_MAX_ENTRIES = 512 -const repoRefCache = new Map() +const repoRefCache = new RepositoryRefCache() /** @internal - exposed for tests only */ export function _resetBitbucketRepoRefCache(): void { @@ -23,17 +26,6 @@ export function _getBitbucketRepoRefCacheSize(): number { return repoRefCache.size } -function rememberRepoRefCacheEntry(cacheKey: string, value: BitbucketRepoRef | null): void { - repoRefCache.set(cacheKey, value) - while (repoRefCache.size > REPO_REF_CACHE_MAX_ENTRIES) { - const oldestKey = repoRefCache.keys().next().value - if (oldestKey === undefined) { - return - } - repoRefCache.delete(oldestKey) - } -} - function decodeSegment(value: string): string { try { return decodeURIComponent(value) @@ -87,9 +79,10 @@ export async function getBitbucketRepoRefForRemote( localGitOptions: LocalGitExecOptions = {} ): Promise { const runtimeKey = connectionId ?? `local:${localGitOptions.wslDistro ?? 'host'}` - const cacheKey = `${runtimeKey}\0${repoPath}\0${remoteName}` - if (repoRefCache.has(cacheKey)) { - return repoRefCache.get(cacheKey)! + const cacheKey = buildRepositoryRefCacheKey([runtimeKey, repoPath, remoteName]) + const cached = repoRefCache.get(cacheKey) + if (cached.found) { + return cached.value } try { const sshGitProvider = connectionId ? getSshGitProvider(connectionId) : null @@ -103,7 +96,7 @@ export async function getBitbucketRepoRefForRemote( ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) }) const result = parseBitbucketRepoRef(stdout) - rememberRepoRefCacheEntry(cacheKey, result) + repoRefCache.remember(cacheKey, result, result ? [result.workspace, result.repoSlug] : []) return result } catch { if (connectionId) { @@ -111,7 +104,7 @@ export async function getBitbucketRepoRefForRemote( // caching them as "not Bitbucket" would poison the repo for the session. return null } - rememberRepoRefCacheEntry(cacheKey, null) + repoRefCache.remember(cacheKey, null, []) return null } } diff --git a/src/main/browser/agent-browser-bridge.test.ts b/src/main/browser/agent-browser-bridge.test.ts index dc804a2dbe1..6a708937382 100644 --- a/src/main/browser/agent-browser-bridge.test.ts +++ b/src/main/browser/agent-browser-bridge.test.ts @@ -1,23 +1,25 @@ /* eslint-disable max-lines */ import { describe, it, expect, vi, beforeEach } from 'vitest' -const { execFileMock, webContentsFromIdMock, existsSyncMock, readFileSyncMock, stdinWrites } = +const { execFileMock, webContentsFromIdMock, existsSyncMock, readScreenshotFileMock, stdinWrites } = vi.hoisted(() => ({ execFileMock: vi.fn(), webContentsFromIdMock: vi.fn(), existsSyncMock: vi.fn(() => false), - readFileSyncMock: vi.fn(() => Buffer.from('')), + readScreenshotFileMock: vi.fn(() => Buffer.from('')), stdinWrites: [] as string[] })) vi.mock('child_process', () => ({ execFile: execFileMock })) vi.mock('fs', () => ({ existsSync: existsSyncMock, - readFileSync: readFileSyncMock, accessSync: vi.fn(), chmodSync: vi.fn(), constants: { X_OK: 1 } })) +vi.mock('./browser-screenshot-file-reader', () => ({ + readBrowserScreenshotFile: readScreenshotFileMock +})) vi.mock('os', () => ({ platform: () => 'darwin', arch: () => 'arm64' })) vi.mock('electron', () => { return { @@ -53,10 +55,16 @@ vi.mock('./cdp-bridge', () => ({ import { AGENT_BROWSER_CLIPBOARD_WRITE_MAX_BYTES, + AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION, AGENT_BROWSER_TEXT_ARGUMENT_MAX_BYTES, AgentBrowserBridge } from './agent-browser-bridge' import type { BrowserManager } from './browser-manager' +import { + BROWSER_SCREENSHOT_MAX_DIMENSION_PX, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' +import { CDP_PDF_MAX_RETAINED_BYTES, CDP_PDF_MEMORY_LIMIT_ERROR } from './cdp-print-to-pdf' import { CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS, CLIPBOARD_TEXT_WRITE_MAX_BYTES, @@ -298,7 +306,7 @@ describe('AgentBrowserBridge', () => { stdinWrites.length = 0 CdpWsProxyMock.instances.length = 0 existsSyncMock.mockReturnValue(false) - readFileSyncMock.mockReturnValue(Buffer.from('')) + readScreenshotFileMock.mockReturnValue(Buffer.from('')) const wc = mockWebContents(100) webContentsFromIdMock.mockReturnValue(wc) bridge = new AgentBrowserBridge(mockBrowserManager()) @@ -763,6 +771,68 @@ describe('AgentBrowserBridge', () => { expect(snapshotIdx).toBeLessThan(clickIdx) }) + it('rejects overload instead of retaining an unbounded per-session command queue', async () => { + let releaseFirst!: () => void + const firstTurn = new Promise((resolve) => { + releaseFirst = resolve + }) + const internals = bridge as unknown as { + enqueueTargetedCommand: ( + worktreeId: string | undefined, + browserPageId: string | undefined, + execute: () => Promise, + options: { ensureSession: boolean; ensureVisible: boolean } + ) => Promise + } + const enqueue = (execute: () => Promise) => + internals.enqueueTargetedCommand(undefined, undefined, execute, { + ensureSession: false, + ensureVisible: false + }) + + const active = enqueue(() => firstTurn) + await vi.waitFor(() => + expect((bridge as unknown as { processingQueues: Set }).processingQueues.size).toBe(1) + ) + const queued = Array.from({ length: AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION }, () => + enqueue(async () => {}) + ) + + await expect(enqueue(async () => {})).rejects.toMatchObject({ code: 'browser_busy' }) + releaseFirst() + await expect(Promise.all([active, ...queued])).resolves.toHaveLength( + AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION + 1 + ) + }) + + it('bounds callers waiting for hung browser session setup before they reach the queue', async () => { + let releaseSetup!: () => void + const setup = new Promise((resolve) => { + releaseSetup = resolve + }) + const internals = bridge as unknown as { + ensureSession: () => Promise + enqueueTargetedCommand: ( + worktreeId: string | undefined, + browserPageId: string | undefined, + execute: () => Promise, + options: { ensureVisible: boolean } + ) => Promise + } + internals.ensureSession = vi.fn(() => setup) + const enqueue = () => + internals.enqueueTargetedCommand(undefined, undefined, async () => {}, { + ensureVisible: false + }) + const pending = Array.from({ length: AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION }, enqueue) + + await expect(enqueue()).rejects.toMatchObject({ code: 'browser_busy' }) + releaseSetup() + await expect(Promise.all(pending)).resolves.toHaveLength( + AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION + ) + }) + it('acquires an automation visibility lease while running snapshot commands', async () => { const lifecycleEvents: string[] = [] const restore = vi.fn(() => { @@ -1012,7 +1082,7 @@ describe('AgentBrowserBridge', () => { ) existsSyncMock.mockReturnValue(true) const screenshotBytes = Buffer.from('serialized-screenshot') - readFileSyncMock.mockReturnValue(screenshotBytes) + readScreenshotFileMock.mockReturnValue(screenshotBytes) const b = new AgentBrowserBridge( mockBrowserManager(tabs, worktrees, { @@ -1084,6 +1154,22 @@ describe('AgentBrowserBridge', () => { } }) + it('rejects an oversized screenshot file before reading or base64-copying it', () => { + existsSyncMock.mockReturnValue(true) + readScreenshotFileMock.mockImplementation(() => { + throw new Error(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + }) + + expect(() => + ( + bridge as unknown as { + readScreenshotFromResult: (raw: unknown, format?: string) => unknown + } + ).readScreenshotFromResult({ path: '/tmp/oversized.png' }, 'png') + ).toThrow(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + expect(readScreenshotFileMock).toHaveBeenCalledWith('/tmp/oversized.png') + }) + it('captures full-page screenshots directly through CDP using CSS layout bounds', async () => { vi.useFakeTimers() try { @@ -1131,6 +1217,88 @@ describe('AgentBrowserBridge', () => { } }) + it('reports oversized full-page geometry with a stable browser error code', async () => { + vi.useFakeTimers() + try { + const wc = mockWebContents(100) + wc.debugger.sendCommand.mockResolvedValueOnce({ + cssContentSize: { width: BROWSER_SCREENSHOT_MAX_DIMENSION_PX + 1, height: 1 } + }) + webContentsFromIdMock.mockReturnValue(wc) + + const screenshotPromise = bridge.fullPageScreenshot('png') + const rejection = expect(screenshotPromise).rejects.toMatchObject({ + code: 'browser_screenshot_too_large', + message: BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + }) + await vi.advanceTimersByTimeAsync(500) + + await rejection + expect(wc.debugger.sendCommand).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) + + it('rejects a direct PDF before making an oversized base64 copy', async () => { + succeedWith(null) + const toString = vi.fn() + const wc = { + ...mockWebContents(100), + printToPDF: vi.fn(async () => ({ + length: CDP_PDF_MAX_RETAINED_BYTES + 1, + toString + })) + } + webContentsFromIdMock.mockReturnValue(wc) + + await expect(bridge.pdf()).rejects.toThrow(CDP_PDF_MEMORY_LIMIT_ERROR) + expect(toString).not.toHaveBeenCalled() + }) + + it('shares one native PDF print cap across browser sessions', async () => { + const tabs = new Map([ + ['tab-1', 1], + ['tab-2', 2], + ['tab-3', 3] + ]) + const worktrees = new Map([ + ['tab-1', 'wt-1'], + ['tab-2', 'wt-2'], + ['tab-3', 'wt-3'] + ]) + const resolvers: ((buffer: Buffer) => void)[] = [] + const webContentsById = new Map( + [1, 2, 3].map((id) => [ + id, + { + ...mockWebContents(id), + printToPDF: vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + } + ]) + ) + webContentsFromIdMock.mockImplementation((id: number) => webContentsById.get(id) ?? null) + succeedWith(null) + const boundedBridge = new AgentBrowserBridge(mockBrowserManager(tabs, worktrees)) + boundedBridge.setActiveTab(1, 'wt-1') + boundedBridge.setActiveTab(2, 'wt-2') + boundedBridge.setActiveTab(3, 'wt-3') + + const active = [boundedBridge.pdf('wt-1'), boundedBridge.pdf('wt-2')] + await vi.waitFor(() => expect(resolvers).toHaveLength(2)) + await expect(boundedBridge.pdf('wt-3')).rejects.toMatchObject({ code: 'browser_busy' }) + + for (const resolve of resolvers) { + resolve(Buffer.from('%PDF-bounded')) + } + await expect(Promise.all(active)).resolves.toHaveLength(2) + }) + // ── Timeout escalation ── it('destroys session after 3 consecutive timeouts', async () => { @@ -2522,6 +2690,19 @@ describe('AgentBrowserBridge', () => { expect(viewportCall).toBeUndefined() }) + it('rejects an oversized viewport before asking Chromium to allocate it', async () => { + const wc = mockWebContents(100) + webContentsFromIdMock.mockReturnValue(wc) + + await expect( + bridge.setViewport(BROWSER_SCREENSHOT_MAX_DIMENSION_PX + 1, 1, 1) + ).rejects.toMatchObject({ + code: 'browser_screenshot_too_large', + message: BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + }) + expect(wc.debugger.sendCommand).not.toHaveBeenCalled() + }) + it('normalizes selector wait state=visible to the default supported semantics', async () => { succeedWith({ selector: 'h1', waited: 'selector' }) diff --git a/src/main/browser/agent-browser-bridge.ts b/src/main/browser/agent-browser-bridge.ts index 565352d3d50..1063d7b6843 100644 --- a/src/main/browser/agent-browser-bridge.ts +++ b/src/main/browser/agent-browser-bridge.ts @@ -1,11 +1,19 @@ /* eslint-disable max-lines */ import { execFile, type ChildProcess } from 'node:child_process' -import { existsSync, accessSync, chmodSync, readFileSync, constants } from 'node:fs' +import { existsSync, accessSync, chmodSync, constants } from 'node:fs' import { join } from 'node:path' import { platform, arch } from 'node:os' import { app, type WebContents } from 'electron' import { CdpWsProxy } from './cdp-ws-proxy' import { captureFullPageScreenshot } from './cdp-screenshot' +import { assertCdpPdfWithinMemoryLimit } from './cdp-print-to-pdf' +import { + assertBrowserScreenshotGeometry, + BROWSER_SCREENSHOT_BUSY_ERROR, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' +import { readBrowserScreenshotFile } from './browser-screenshot-file-reader' +import { BROWSER_PDF_BUSY_ERROR, startBrowserPdfPrint } from './browser-pdf-admission' import { acquireElectronDebugger } from './electron-debugger-lease' import type { BrowserManager } from './browser-manager' import { BrowserError } from './cdp-bridge' @@ -59,6 +67,8 @@ const STALE_SESSION_CLOSE_TIMEOUT_MS = 3_000 const EMBEDDED_NAVIGATION_TIMEOUT_MS = 30_000 export const AGENT_BROWSER_TEXT_ARGUMENT_MAX_BYTES = 8 * 1024 export const AGENT_BROWSER_CLIPBOARD_WRITE_MAX_BYTES = AGENT_BROWSER_TEXT_ARGUMENT_MAX_BYTES +export const AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION = 64 +const AGENT_BROWSER_MAX_QUEUED_COMMANDS_TOTAL = 512 type SessionState = { proxy: CdpWsProxy @@ -575,6 +585,9 @@ export class AgentBrowserBridge { private readonly sessions = new Map() private readonly commandQueues = new Map() private readonly processingQueues = new Set() + private queuedCommandCount = 0 + private readonly pendingEnqueueCounts = new Map() + private pendingEnqueueCount = 0 // Why: screenshot prep mutates shared paintability across tabs; serialize globally so concurrent captures don't blank each other. private screenshotTurn: Promise = Promise.resolve() private readonly agentBrowserBin: string @@ -1483,7 +1496,17 @@ export class AgentBrowserBridge { if (!existsSync(parsed.path)) { throw new BrowserError('browser_error', `Screenshot file not found: ${parsed.path}`) } - const data = readFileSync(parsed.path).toString('base64') + let bytes: Buffer + try { + bytes = readBrowserScreenshotFile(parsed.path) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unable to read screenshot file' + if (message !== BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) { + throw new BrowserError('browser_error', message) + } + throw new BrowserError('browser_screenshot_too_large', BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + } + const data = bytes.toString('base64') return { data, format: format === 'jpeg' ? 'jpeg' : 'png' } as BrowserScreenshotResult } @@ -1529,7 +1552,17 @@ export class AgentBrowserBridge { } return await captureFullPageScreenshot(wc, format) } catch (error) { - throw new BrowserError('browser_error', (error as Error).message) + if (error instanceof BrowserError) { + throw error + } + const message = error instanceof Error ? error.message : 'Screenshot failed' + if (message === BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) { + throw new BrowserError('browser_screenshot_too_large', message) + } + if (message === BROWSER_SCREENSHOT_BUSY_ERROR) { + throw new BrowserError('browser_busy', message) + } + throw new BrowserError('browser_error', message) } finally { restore() } @@ -1780,10 +1813,15 @@ export class AgentBrowserBridge { if (!wc) { throw new BrowserError('browser_no_tab', 'Tab is no longer available') } - const buffer = await wc.printToPDF({ + const print = startBrowserPdfPrint(wc, { printBackground: true, preferCSSPageSize: true }) + if (!print) { + throw new BrowserError('browser_busy', BROWSER_PDF_BUSY_ERROR) + } + const buffer = await print + assertCdpPdfWithinMemoryLimit(buffer) return { data: buffer.toString('base64') } }) } @@ -1861,6 +1899,14 @@ export class AgentBrowserBridge { worktreeId?: string, browserPageId?: string ): Promise { + try { + assertBrowserScreenshotGeometry(width, height, scale) + } catch (error) { + throw new BrowserError( + 'browser_screenshot_too_large', + error instanceof Error ? error.message : BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + ) + } return this.enqueueTargetedCommand(worktreeId, browserPageId, async (_sessionName, target) => { const wc = this.getWebContents(target.webContentsId) if (!wc) { @@ -2069,12 +2115,29 @@ export class AgentBrowserBridge { const target = this.resolveCommandTarget(worktreeId, browserPageId, options.requireScopedTarget) const sessionName = `orca-tab-${target.browserPageId}` - if (options.ensureSession !== false) { - await this.ensureSession(sessionName, target.browserPageId, target.webContentsId) + this.acquirePendingEnqueue(sessionName) + try { + if (options.ensureSession !== false) { + await this.ensureSession(sessionName, target.browserPageId, target.webContentsId) + } + } finally { + this.releasePendingEnqueue(sessionName) } return new Promise((resolve, reject) => { let queue = this.commandQueues.get(sessionName) + if ( + (queue?.length ?? 0) >= AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION || + this.queuedCommandCount >= AGENT_BROWSER_MAX_QUEUED_COMMANDS_TOTAL + ) { + reject( + new BrowserError( + 'browser_busy', + 'Browser command queue is full; retry after the current commands finish' + ) + ) + return + } if (!queue) { queue = [] this.commandQueues.set(sessionName, queue) @@ -2091,10 +2154,37 @@ export class AgentBrowserBridge { resolve: resolve as (value: unknown) => void, reject }) + this.queuedCommandCount += 1 this.processQueue(sessionName) }) } + private acquirePendingEnqueue(sessionName: string): void { + const pendingForSession = this.pendingEnqueueCounts.get(sessionName) ?? 0 + const queuedForSession = this.commandQueues.get(sessionName)?.length ?? 0 + if ( + pendingForSession + queuedForSession >= AGENT_BROWSER_MAX_QUEUED_COMMANDS_PER_SESSION || + this.pendingEnqueueCount + this.queuedCommandCount >= AGENT_BROWSER_MAX_QUEUED_COMMANDS_TOTAL + ) { + throw new BrowserError( + 'browser_busy', + 'Browser command queue is full; retry after the current commands finish' + ) + } + this.pendingEnqueueCounts.set(sessionName, pendingForSession + 1) + this.pendingEnqueueCount += 1 + } + + private releasePendingEnqueue(sessionName: string): void { + const remaining = (this.pendingEnqueueCounts.get(sessionName) ?? 1) - 1 + if (remaining > 0) { + this.pendingEnqueueCounts.set(sessionName, remaining) + } else { + this.pendingEnqueueCounts.delete(sessionName) + } + this.pendingEnqueueCount = Math.max(0, this.pendingEnqueueCount - 1) + } + private async executeWithVisibleTarget( sessionName: string, worktreeId: string | undefined, @@ -2159,6 +2249,7 @@ export class AgentBrowserBridge { const queue = this.commandQueues.get(sessionName) while (queue && queue.length > 0) { const cmd = queue.shift()! + this.queuedCommandCount = Math.max(0, this.queuedCommandCount - 1) try { const result = await cmd.execute() cmd.resolve(result) @@ -2456,6 +2547,7 @@ export class AgentBrowserBridge { this.commandQueues.delete(sessionName) this.processingQueues.delete(sessionName) if (queue) { + this.queuedCommandCount = Math.max(0, this.queuedCommandCount - queue.length) const err = new BrowserError( 'browser_tab_closed', 'Tab was closed while commands were queued' diff --git a/src/main/browser/browser-cookie-import.comet.test.ts b/src/main/browser/browser-cookie-import.comet.test.ts index 47e84db1607..1bb906bb2c2 100644 --- a/src/main/browser/browser-cookie-import.comet.test.ts +++ b/src/main/browser/browser-cookie-import.comet.test.ts @@ -1,11 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as childProcessModule from 'node:child_process' import type * as fsModule from 'node:fs' +import type * as boundedFileReaderModule from '../../shared/node-bounded-file-reader' -const { sessionFromPartitionMock, dialogShowOpenDialogMock } = vi.hoisted(() => ({ - sessionFromPartitionMock: vi.fn(), - dialogShowOpenDialogMock: vi.fn() -})) +const { sessionFromPartitionMock, dialogShowOpenDialogMock, boundedLocalStateReadMock } = + vi.hoisted(() => ({ + sessionFromPartitionMock: vi.fn(), + dialogShowOpenDialogMock: vi.fn(), + boundedLocalStateReadMock: vi.fn<(filePath: string) => string>() + })) vi.mock('electron', () => ({ BrowserWindow: { fromWebContents: vi.fn() }, @@ -13,12 +16,32 @@ vi.mock('electron', () => ({ session: { fromPartition: sessionFromPartitionMock } })) +vi.mock('../../shared/node-bounded-file-reader', async () => { + const actual = await vi.importActual( + '../../shared/node-bounded-file-reader' + ) + return { + ...actual, + readNodeFileSyncWithinLimit: (filePath: string, maxBytes: number) => { + const buffer = Buffer.from(boundedLocalStateReadMock(filePath)) + if (buffer.byteLength > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.byteLength, maxBytes) + } + return { buffer, stats: { size: buffer.byteLength } } + } + } +}) + import { BROWSER_FAMILY_LABELS } from '../../shared/constants' function slashPath(pathValue: string): string { return pathValue.replaceAll('\\', '/') } +function mockLocalState(value: unknown): void { + boundedLocalStateReadMock.mockReturnValue(JSON.stringify(value)) +} + describe('detectInstalledBrowsers — Comet', () => { const originalPlatform = process.platform const originalHome = process.env.HOME @@ -32,6 +55,10 @@ describe('detectInstalledBrowsers — Comet', () => { vi.resetModules() Object.defineProperty(process, 'platform', { value: 'darwin' }) process.env.HOME = '/Users/test' + boundedLocalStateReadMock.mockReset() + boundedLocalStateReadMock.mockImplementation(() => { + throw new Error('ENOENT') + }) }) afterEach(() => { @@ -41,6 +68,7 @@ describe('detectInstalledBrowsers — Comet', () => { }) it('detects Comet when its data directory and Cookies DB exist', async () => { + mockLocalState({ profile: { info_cache: { Default: { name: 'Default' } } } }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { @@ -88,6 +116,15 @@ describe('detectInstalledBrowsers — Comet', () => { }) it('enumerates all Comet profiles from Local State info_cache', async () => { + mockLocalState({ + profile: { + info_cache: { + Default: { name: 'Personal' }, + 'Profile 1': { name: 'Work' }, + 'Profile 2': { name: 'Research' } + } + } + }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { @@ -130,6 +167,13 @@ describe('detectInstalledBrowsers — Comet', () => { }) it('ignores Comet profile directories that escape the browser root', async () => { + mockLocalState({ + profile: { + info_cache: { + '../Outside': { name: 'Outside' } + } + } + }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { @@ -195,6 +239,7 @@ describe('detectInstalledBrowsers — Comet', () => { }) it('skips Comet when the data directory exists but no Cookies DB is present', async () => { + mockLocalState({ profile: { info_cache: { Default: { name: 'Default' } } } }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { diff --git a/src/main/browser/browser-cookie-import.helium.test.ts b/src/main/browser/browser-cookie-import.helium.test.ts index 1c551ac004a..4dc44ee3ea8 100644 --- a/src/main/browser/browser-cookie-import.helium.test.ts +++ b/src/main/browser/browser-cookie-import.helium.test.ts @@ -1,11 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as childProcessModule from 'node:child_process' import type * as fsModule from 'node:fs' +import type * as boundedFileReaderModule from '../../shared/node-bounded-file-reader' -const { sessionFromPartitionMock, dialogShowOpenDialogMock } = vi.hoisted(() => ({ - sessionFromPartitionMock: vi.fn(), - dialogShowOpenDialogMock: vi.fn() -})) +const { sessionFromPartitionMock, dialogShowOpenDialogMock, boundedLocalStateReadMock } = + vi.hoisted(() => ({ + sessionFromPartitionMock: vi.fn(), + dialogShowOpenDialogMock: vi.fn(), + boundedLocalStateReadMock: vi.fn<(filePath: string) => string>() + })) vi.mock('electron', () => ({ BrowserWindow: { fromWebContents: vi.fn() }, @@ -13,12 +16,32 @@ vi.mock('electron', () => ({ session: { fromPartition: sessionFromPartitionMock } })) +vi.mock('../../shared/node-bounded-file-reader', async () => { + const actual = await vi.importActual( + '../../shared/node-bounded-file-reader' + ) + return { + ...actual, + readNodeFileSyncWithinLimit: (filePath: string, maxBytes: number) => { + const buffer = Buffer.from(boundedLocalStateReadMock(filePath)) + if (buffer.byteLength > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.byteLength, maxBytes) + } + return { buffer, stats: { size: buffer.byteLength } } + } + } +}) + import { BROWSER_FAMILY_LABELS } from '../../shared/constants' function slashPath(pathValue: string): string { return pathValue.replaceAll('\\', '/') } +function mockLocalState(value: unknown): void { + boundedLocalStateReadMock.mockReturnValue(JSON.stringify(value)) +} + describe('detectInstalledBrowsers — Helium', () => { const originalPlatform = process.platform const originalHome = process.env.HOME @@ -30,6 +53,10 @@ describe('detectInstalledBrowsers — Helium', () => { vi.resetModules() Object.defineProperty(process, 'platform', { value: 'darwin' }) process.env.HOME = '/Users/test' + boundedLocalStateReadMock.mockReset() + boundedLocalStateReadMock.mockImplementation(() => { + throw new Error('ENOENT') + }) }) afterEach(() => { @@ -39,6 +66,7 @@ describe('detectInstalledBrowsers — Helium', () => { }) it('detects Helium under its bundle-id data dir via the legacy Cookies path', async () => { + mockLocalState({ profile: { info_cache: { Default: { name: 'Default' } } } }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { @@ -93,6 +121,14 @@ describe('detectInstalledBrowsers — Helium', () => { }) it('enumerates all Helium profiles from Local State info_cache', async () => { + mockLocalState({ + profile: { + info_cache: { + Default: { name: 'Personal' }, + 'Profile 1': { name: 'Work' } + } + } + }) vi.doMock('node:fs', async () => { const actual = await vi.importActual('node:fs') return { diff --git a/src/main/browser/browser-cookie-import.test.ts b/src/main/browser/browser-cookie-import.test.ts index 27663017097..1c1f2b9c85b 100644 --- a/src/main/browser/browser-cookie-import.test.ts +++ b/src/main/browser/browser-cookie-import.test.ts @@ -36,6 +36,7 @@ vi.mock('electron', () => ({ import { buildChromiumCookieInsertParams, + CHROMIUM_LOCAL_STATE_MAX_BYTES, importCookiesFromFile, importCookiesFromBrowser, detectInstalledBrowsers, @@ -47,7 +48,20 @@ import { createChromiumCookieTestDatabase, encryptMacChromiumCookie } from './browser-cookie-import-test-database' -import { existsSync, writeFileSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs' +import { COOKIE_JSON_FILE_MAX_BYTES } from './browser-cookie-json-file-parser' +import { INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES } from './installed-browser-cookie-store-limits' +import { SAFARI_COOKIE_STORE_MAX_FILE_BYTES } from './safari-cookie-store-decoder' +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { DatabaseSync } from 'node:sqlite' import { join } from 'node:path' import { tmpdir } from 'node:os' @@ -63,6 +77,40 @@ function chromeBrowser(cookiesPath: string): DetectedBrowser { } } +function firefoxBrowser(cookiesPath: string): DetectedBrowser { + return { + family: 'firefox', + label: 'Firefox', + cookiesPath, + profiles: [{ name: 'default', directory: 'default' }], + selectedProfile: 'default' + } +} + +function createFirefoxCookieTestDatabase( + databasePath: string, + rows: { name: string; value: string; host: string }[] +): void { + const database = new DatabaseSync(databasePath) + database.exec(` + CREATE TABLE moz_cookies ( + name TEXT, + value TEXT, + host TEXT, + path TEXT, + expiry INTEGER, + isSecure INTEGER, + isHttpOnly INTEGER, + sameSite INTEGER + ) + `) + const insert = database.prepare('INSERT INTO moz_cookies VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + for (const row of rows) { + insert.run(row.name, row.value, row.host, '/', 2_000_000_000, 0, 0, 0) + } + database.close() +} + const LARGE_SAFARI_COOKIE_COUNT = 150_000 describe('summarizeCookieImportError', () => { @@ -281,6 +329,20 @@ describe('importCookiesFromFile', () => { expect(result.reason).toContain('Could not read') }) + it('rejects an oversized JSON file before reading its bytes', async () => { + const filePath = join(tmpDir, 'oversized.json') + writeFileSync(filePath, '') + truncateSync(filePath, COOKIE_JSON_FILE_MAX_BYTES + 1) + + const result = await importCookiesFromFile(filePath, 'persist:test') + + expect(result).toEqual({ + ok: false, + reason: 'Cookie file is too large to import safely (64 MiB file limit).' + }) + expect(sessionFromPartitionMock).not.toHaveBeenCalled() + }) + it('normalizes sameSite values', async () => { const filePath = writeCookieFile([ { domain: '.test.com', name: 'a', value: '1', sameSite: 'None' }, @@ -364,6 +426,94 @@ describe('importCookiesFromBrowser Safari', () => { expect(result).toEqual({ ok: false, reason: 'All Safari cookies are expired.' }) expect(cookiesSetMock).not.toHaveBeenCalled() }) + + it('rejects an oversized Safari store before reading its bytes', async () => { + const cookiesPath = join(tmpDir, 'Cookies.binarycookies') + writeFileSync(cookiesPath, '') + truncateSync(cookiesPath, SAFARI_COOKIE_STORE_MAX_FILE_BYTES + 1) + const browser: DetectedBrowser = { + family: 'safari', + label: 'Safari', + cookiesPath, + profiles: [], + selectedProfile: 'Default' + } + + const result = await importCookiesFromBrowser(browser, 'persist:test') + + expect(result).toEqual({ + ok: false, + reason: 'Safari cookie store is too large to import safely (64 MiB file limit).' + }) + expect(sessionFromPartitionMock).not.toHaveBeenCalled() + }) +}) + +describe('importCookiesFromBrowser Firefox', () => { + let tmpDir: string + let cookiesSetMock: ReturnType + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'orca-firefox-cookie-test-')) + cookiesSetMock = vi.fn().mockResolvedValue(undefined) + sessionFromPartitionMock.mockReset() + sessionFromPartitionMock.mockReturnValue({ + cookies: { set: cookiesSetMock } + }) + }) + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }) + }) + + it('streams ordinary cookies in source order without changing the summary', async () => { + const cookiesPath = join(tmpDir, 'cookies.sqlite') + createFirefoxCookieTestDatabase(cookiesPath, [ + { name: 'first', value: 'one', host: '.first.example.com' }, + { name: 'second', value: 'two', host: '.second.example.com' } + ]) + + const result = await importCookiesFromBrowser(firefoxBrowser(cookiesPath), 'persist:test') + + expect(result).toEqual({ + ok: true, + profileId: '', + summary: { + totalCookies: 2, + importedCookies: 2, + skippedCookies: 0, + domains: ['first.example.com', 'second.example.com'] + } + }) + expect(cookiesSetMock.mock.calls.map(([cookie]) => cookie.name)).toEqual(['first', 'second']) + }) + + it('rejects oversized cookie data before iterating or mutating the target session', async () => { + const cookiesPath = join(tmpDir, 'cookies.sqlite') + const database = new DatabaseSync(cookiesPath) + database.exec(` + CREATE VIEW moz_cookies AS + SELECT + 'sid' AS name, + zeroblob(${INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES + 1}) AS value, + '.example.com' AS host, + '/' AS path, + 0 AS expiry, + 0 AS isSecure, + 0 AS isHttpOnly, + 0 AS sameSite + `) + database.close() + + const result = await importCookiesFromBrowser(firefoxBrowser(cookiesPath), 'persist:test') + + expect(result).toEqual({ + ok: false, + reason: + 'Firefox cookie store is too large to import safely (250,000-cookie and 64 MiB cookie-data limits).' + }) + expect(sessionFromPartitionMock).not.toHaveBeenCalled() + }) }) describe('importCookiesFromBrowser Chromium', () => { @@ -530,9 +680,79 @@ describe('importCookiesFromBrowser Chromium', () => { expect(result).toEqual({ ok: false, reason: 'Could not create staging cookie database.' }) expect(readdirSync(join(tmpDir, 'userData', 'cookie-import-staging'))).toEqual([]) }) + + it('rejects oversized source data before decrypting or clearing target cookies', async () => { + const sourceCookiesPath = join(tmpDir, 'Chrome', 'Default', 'Network', 'Cookies') + const targetCookiesPath = join(tmpDir, 'userData', 'Partitions', 'test', 'Network', 'Cookies') + mkdirSync(join(sourceCookiesPath, '..'), { recursive: true }) + const sourceDatabase = new DatabaseSync(sourceCookiesPath) + sourceDatabase.exec(` + CREATE VIEW cookies AS + SELECT + 1 AS creation_utc, + '.example.com' AS host_key, + '' AS top_frame_site_key, + 'sid' AS name, + zeroblob(${INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES + 1}) AS value, + X'' AS encrypted_value, + '/' AS path, + 0 AS expires_utc, + 0 AS is_secure, + 0 AS is_httponly, + 0 AS samesite, + 0 AS source_scheme, + -1 AS source_port, + 0 AS last_update_utc, + 0 AS has_cross_site_ancestor + `) + sourceDatabase.close() + createChromiumCookieTestDatabase(targetCookiesPath, []).close() + + const result = await importCookiesFromBrowser(chromeBrowser(sourceCookiesPath), 'persist:test') + + expect(result).toEqual({ + ok: false, + reason: + 'Google Chrome cookie store is too large to import safely (250,000-cookie and 64 MiB cookie-data limits).' + }) + expect(clearStorageDataMock).not.toHaveBeenCalled() + expect(cookiesSetMock).not.toHaveBeenCalled() + expect(readdirSync(join(tmpDir, 'userData', 'cookie-import-staging'))).toEqual([]) + }) }) describe('detectInstalledBrowsers', () => { + it('falls back safely when Chromium Local State exceeds its read cap', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-browser-detection-bounds-')) + const originalConfigHome = process.env.XDG_CONFIG_HOME + const originalPlatform = process.platform + try { + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) + process.env.XDG_CONFIG_HOME = root + const browserRoot = join(root, 'google-chrome') + mkdirSync(join(browserRoot, 'Default'), { recursive: true }) + writeFileSync(join(browserRoot, 'Default', 'Cookies'), '') + writeFileSync(join(browserRoot, 'Local State'), '') + truncateSync(join(browserRoot, 'Local State'), CHROMIUM_LOCAL_STATE_MAX_BYTES + 1) + + const chrome = detectInstalledBrowsers().find((browser) => browser.family === 'chrome') + + expect(chrome?.profiles).toEqual([{ name: 'Default', directory: 'Default' }]) + expect(chrome?.selectedProfile).toBe('Default') + } finally { + Object.defineProperty(process, 'platform', { + configurable: true, + value: originalPlatform + }) + if (originalConfigHome === undefined) { + delete process.env.XDG_CONFIG_HOME + } else { + process.env.XDG_CONFIG_HOME = originalConfigHome + } + rmSync(root, { recursive: true, force: true }) + } + }) + it('returns an array of detected browsers', () => { const browsers = detectInstalledBrowsers() expect(Array.isArray(browsers)).toBe(true) diff --git a/src/main/browser/browser-cookie-import.ts b/src/main/browser/browser-cookie-import.ts index 7bcfd4ea247..5e1e30b9a18 100644 --- a/src/main/browser/browser-cookie-import.ts +++ b/src/main/browser/browser-cookie-import.ts @@ -8,12 +8,10 @@ import { existsSync, mkdtempSync, mkdirSync, - readFileSync, - readdirSync, + opendirSync, rmSync, unlinkSync } from 'node:fs' -import { readFile } from 'node:fs/promises' import { DatabaseSync } from 'node:sqlite' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -73,13 +71,42 @@ import type { BrowserCookieImportSummary, BrowserSessionProfileSource } from '../../shared/types' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' import { browserSessionRegistry } from './browser-session-registry' import { setupClientHintsOverride } from './browser-session-ua' +import { + COOKIE_JSON_FILE_MAX_BYTES, + COOKIE_JSON_FILE_MAX_DEPTH, + COOKIE_JSON_FILE_MAX_ENTRIES, + COOKIE_JSON_FILE_MAX_RETAINED_BYTES, + CookieJsonFileFormatError, + CookieJsonFileLimitError, + enforceCookieJsonRetainedBytes, + visitCookieJsonFileObjects +} from './browser-cookie-json-file-parser' import { createChromiumCookieSnapshot, type ChromiumCookieSnapshot } from './chromium-cookie-snapshot' import { resolveChromiumCookiesPath } from './chromium-cookie-path' +import { + assertInstalledBrowserCookieStoreWithinLimits, + InstalledBrowserCookieStoreLimitError, + installedBrowserCookieStoreLimitReason +} from './installed-browser-cookie-store-limits' +import { + decodeSafariCookieStore, + SAFARI_COOKIE_STORE_MAX_COOKIES, + SAFARI_COOKIE_STORE_MAX_FILE_BYTES, + SAFARI_COOKIE_STORE_MAX_PAGES, + SAFARI_COOKIE_STORE_MAX_PARSED_BYTES, + SafariCookieStoreLimitError, + removeExpiredSafariCookiesInPlace +} from './safari-cookie-store-decoder' // --------------------------------------------------------------------------- // Browser detection @@ -90,6 +117,10 @@ export type BrowserProfile = { directory: string } +export const CHROMIUM_LOCAL_STATE_MAX_BYTES = 16 * 1024 * 1024 +const BROWSER_PROFILE_SCAN_MAX_ENTRIES = 10_000 +const BROWSER_PROFILE_RETAINED_NAME_MAX_BYTES = 4 * 1024 * 1024 + export type DetectedBrowser = { family: BrowserSessionProfileSource['browserFamily'] label: string @@ -210,19 +241,38 @@ function discoverProfiles(browserRoot: string): BrowserProfile[] { if (!existsSync(localStatePath)) { return [{ name: 'Default', directory: 'Default' }] } - const raw = readFileSync(localStatePath, 'utf-8') - const localState = JSON.parse(raw) + const localState = JSON.parse( + readNodeFileSyncWithinLimit(localStatePath, CHROMIUM_LOCAL_STATE_MAX_BYTES).buffer.toString( + 'utf8' + ) + ) const infoCache = localState?.profile?.info_cache if (!infoCache || typeof infoCache !== 'object') { return [{ name: 'Default', directory: 'Default' }] } const profiles: BrowserProfile[] = [] - for (const [dir, info] of Object.entries(infoCache)) { + let retainedNameBytes = 0 + let visitedEntries = 0 + for (const dir in infoCache) { + if (!Object.hasOwn(infoCache, dir)) { + continue + } + visitedEntries++ + if (visitedEntries > BROWSER_PROFILE_SCAN_MAX_ENTRIES) { + return [{ name: 'Default', directory: 'Default' }] + } // Why: Local State is external metadata, but profile dirs become path segments. if (!isSafeBrowserProfileDirectory(dir)) { continue } - const profileName = (info as { name?: string })?.name ?? dir + const info = (infoCache as Record)[dir] + const configuredName = + info && typeof info === 'object' ? (info as { name?: unknown }).name : undefined + const profileName = typeof configuredName === 'string' ? configuredName : dir + retainedNameBytes += Buffer.byteLength(dir, 'utf8') + Buffer.byteLength(profileName, 'utf8') + if (retainedNameBytes > BROWSER_PROFILE_RETAINED_NAME_MAX_BYTES) { + return [{ name: 'Default', directory: 'Default' }] + } profiles.push({ name: profileName, directory: dir }) } return profiles.length > 0 ? profiles : [{ name: 'Default', directory: 'Default' }] @@ -257,9 +307,30 @@ function discoverFirefoxProfiles(): BrowserProfile[] { if (!existsSync(profilesRoot)) { return [] } - const entries = readdirSync(profilesRoot, { withFileTypes: true }) - .filter((e) => e.isDirectory()) - .map((e) => e.name) + const entries: string[] = [] + let retainedNameBytes = 0 + const directory = opendirSync(profilesRoot) + try { + for (let visitedEntries = 0; ; visitedEntries++) { + const entry = directory.readSync() + if (!entry) { + break + } + if (visitedEntries >= BROWSER_PROFILE_SCAN_MAX_ENTRIES) { + return [] + } + if (!entry.isDirectory()) { + continue + } + retainedNameBytes += Buffer.byteLength(entry.name, 'utf8') + if (retainedNameBytes > BROWSER_PROFILE_RETAINED_NAME_MAX_BYTES) { + return [] + } + entries.push(entry.name) + } + } finally { + directory.closeSync() + } // Why: Firefox dirs are named .; prefer 'default-release' as the primary profile on most installs. const sorted = entries.sort((a, b) => { if (a.includes('default-release')) { @@ -310,8 +381,6 @@ function detectFirefox(): DetectedBrowser | null { // Safari detection // --------------------------------------------------------------------------- -const MAC_EPOCH_DELTA = 978_307_200 - function detectSafari(): DetectedBrowser | null { if (process.platform !== 'darwin') { return null @@ -645,41 +714,62 @@ export async function importCookiesFromFile( ): Promise { let rawContent: string try { - rawContent = await readFile(filePath, 'utf-8') - } catch { + rawContent = ( + await readNodeFileWithinLimit(filePath, COOKIE_JSON_FILE_MAX_BYTES) + ).buffer.toString('utf8') + } catch (err) { + if (err instanceof NodeFileReadTooLargeError) { + return { + ok: false, + reason: `Cookie file is too large to import safely (${COOKIE_JSON_FILE_MAX_BYTES / 1024 / 1024} MiB file limit).` + } + } return { ok: false, reason: 'Could not read the selected file.' } } - let parsed: unknown + const validated: ValidatedCookie[] = [] + let totalEntries = 0 + let retainedBytes = 0 try { - parsed = JSON.parse(rawContent) - } catch { + totalEntries = visitCookieJsonFileObjects(rawContent, (entry) => { + const cookie = validateCookieEntry(entry as RawCookieEntry) + if (!cookie) { + return + } + retainedBytes += + Buffer.byteLength(cookie.url) + + Buffer.byteLength(cookie.name) + + Buffer.byteLength(cookie.value) + + Buffer.byteLength(cookie.domain) + + Buffer.byteLength(cookie.path) + enforceCookieJsonRetainedBytes(retainedBytes) + validated.push(cookie) + }) + } catch (err) { + if (err instanceof CookieJsonFileFormatError) { + return err.kind === 'root' + ? { ok: false, reason: 'Expected a JSON array of cookie objects.' } + : { ok: false, reason: 'File is not valid JSON.' } + } + if (err instanceof CookieJsonFileLimitError) { + const limit = { + depth: `${COOKIE_JSON_FILE_MAX_DEPTH} levels of nesting`, + entries: `${COOKIE_JSON_FILE_MAX_ENTRIES.toLocaleString('en-US')} entries`, + 'retained-bytes': `${COOKIE_JSON_FILE_MAX_RETAINED_BYTES / 1024 / 1024} MiB parsed cookie data` + }[err.kind] + return { + ok: false, + reason: `Cookie file is too large to import safely (${limit} limit).` + } + } return { ok: false, reason: 'File is not valid JSON.' } } - if (!Array.isArray(parsed)) { - return { ok: false, reason: 'Expected a JSON array of cookie objects.' } - } - - if (parsed.length === 0) { + if (totalEntries === 0) { return { ok: false, reason: 'Cookie file is empty.' } } - const validated: ValidatedCookie[] = [] - let skipped = 0 - for (const entry of parsed) { - if (typeof entry !== 'object' || entry === null) { - skipped++ - continue - } - const cookie = validateCookieEntry(entry as RawCookieEntry) - if (cookie) { - validated.push(cookie) - } else { - skipped++ - } - } - + const skipped = totalEntries - validated.length if (validated.length === 0) { return { ok: false, @@ -687,7 +777,7 @@ export async function importCookiesFromFile( } } - return importValidatedCookies(validated, parsed.length, targetPartition) + return importValidatedCookies(validated, totalEntries, targetPartition) } // --------------------------------------------------------------------------- @@ -986,8 +1076,11 @@ function getWindowsEncryptionKey(browser: DetectedBrowser): EncryptionKeyResult } try { - const raw = readFileSync(localStatePath, 'utf-8') - const localState = JSON.parse(raw) + const localState = JSON.parse( + readNodeFileSyncWithinLimit(localStatePath, CHROMIUM_LOCAL_STATE_MAX_BYTES).buffer.toString( + 'utf8' + ) + ) const encryptedKeyB64 = localState?.os_crypt?.encrypted_key if (typeof encryptedKeyB64 !== 'string') { return null @@ -1103,144 +1196,6 @@ function decryptAes256Gcm(payload: Buffer, key: Buffer): Buffer | null { } } -// --------------------------------------------------------------------------- -// Safari binary cookie parser -// --------------------------------------------------------------------------- - -function decodeSafariBinaryCookies(buffer: Buffer): ValidatedCookie[] { - if (buffer.length < 8) { - return [] - } - if (buffer.subarray(0, 4).toString('utf8') !== 'cook') { - return [] - } - - const pageCount = buffer.readUInt32BE(4) - let cursor = 8 - if (cursor + pageCount * 4 > buffer.length) { - return [] - } - const pageSizes: number[] = [] - for (let i = 0; i < pageCount; i++) { - pageSizes.push(buffer.readUInt32BE(cursor)) - cursor += 4 - } - - const cookies: ValidatedCookie[] = [] - for (const pageSize of pageSizes) { - const page = buffer.subarray(cursor, cursor + pageSize) - cursor += pageSize - appendSafariCookies(cookies, decodeSafariPage(page)) - } - return cookies -} - -function appendSafariCookies(target: ValidatedCookie[], cookies: readonly ValidatedCookie[]): void { - // Why: pages can hold large cookie lists; push per-item to avoid exceeding the spread argument limit. - for (const cookie of cookies) { - target.push(cookie) - } -} - -function decodeSafariPage(page: Buffer): ValidatedCookie[] { - if (page.length < 16) { - return [] - } - if (page.readUInt32BE(0) !== 0x00000100) { - return [] - } - - const cookieCount = page.readUInt32LE(4) - if (8 + cookieCount * 4 > page.length) { - return [] - } - const offsets: number[] = [] - let cursor = 8 - for (let i = 0; i < cookieCount; i++) { - offsets.push(page.readUInt32LE(cursor)) - cursor += 4 - } - - const cookies: ValidatedCookie[] = [] - for (const offset of offsets) { - const cookie = decodeSafariCookie(page.subarray(offset)) - if (cookie) { - cookies.push(cookie) - } - } - return cookies -} - -function decodeSafariCookie(buf: Buffer): ValidatedCookie | null { - if (buf.length < 48) { - return null - } - // Why: size comes from the file and could be attacker-controlled; clamp so readCString can't escape the subarray. - const size = Math.min(buf.readUInt32LE(0), buf.length) - if (size < 48) { - return null - } - - const flags = buf.readUInt32LE(8) - const secure = (flags & 1) !== 0 - const httpOnly = (flags & 4) !== 0 - - const urlOffset = buf.readUInt32LE(16) - const nameOffset = buf.readUInt32LE(20) - const pathOffset = buf.readUInt32LE(24) - const valueOffset = buf.readUInt32LE(28) - - // Why: Safari stores dates as Mac absolute time (seconds since 2001-01-01). - const expiration = buf.length >= 48 ? buf.readDoubleLE(40) : 0 - - const name = readCString(buf, nameOffset, size) - if (!name) { - return null - } - const value = readCString(buf, valueOffset, size) ?? '' - const path = readCString(buf, pathOffset, size) ?? '/' - const rawUrl = readCString(buf, urlOffset, size) ?? '' - - // Why: Safari stores the domain in the URL field, not as a separate domain column. - const domain = rawUrl.startsWith('.') ? rawUrl : rawUrl || null - if (!domain) { - return null - } - - const url = deriveUrl(domain, secure) - if (!url) { - return null - } - - const expirationDate = expiration > 0 ? Math.round(expiration + MAC_EPOCH_DELTA) : undefined - - return { - url, - name, - value, - domain, - path, - secure, - httpOnly, - sameSite: 'unspecified', - expirationDate - } -} - -function readCString(buf: Buffer, offset: number, end: number): string | null { - if (offset < 0 || offset >= end) { - return null - } - let cursor = offset - while (cursor < end && buf[cursor] !== 0) { - cursor++ - } - if (cursor >= end) { - return null - } - return buf.toString('utf8', offset, cursor) -} - // --------------------------------------------------------------------------- // Firefox import // --------------------------------------------------------------------------- @@ -1274,8 +1229,9 @@ async function importCookiesFromFirefox( } } + let db: InstanceType | null = null try { - const db = new DatabaseSync(tmpCookiesPath, { readOnly: true }) + db = new DatabaseSync(tmpCookiesPath, { readOnly: true }) type FirefoxRow = { name: string value: string @@ -1286,18 +1242,37 @@ async function importCookiesFromFirefox( isHttpOnly: number sameSite: number } + const sourceStats = db + .prepare( + `SELECT + COUNT(*) AS cookie_count, + COALESCE(SUM( + length(CAST(COALESCE(name, '') AS BLOB)) + + length(CAST(COALESCE(value, '') AS BLOB)) + + length(CAST(COALESCE(host, '') AS BLOB)) + + length(CAST(COALESCE(path, '') AS BLOB)) + ), 0) AS cookie_bytes + FROM moz_cookies` + ) + .get() as { cookie_count: number | bigint; cookie_bytes: number | bigint } + const totalRows = assertInstalledBrowserCookieStoreWithinLimits( + sourceStats.cookie_count, + sourceStats.cookie_bytes + ) + + diag(` Firefox source has ${totalRows} cookies`) + if (totalRows === 0) { + db.close() + db = null + rmSync(tmpDir, { recursive: true, force: true }) + return { ok: false, reason: 'No cookies found in Firefox.' } + } + const rows = db .prepare( 'SELECT name, value, host, path, expiry, isSecure, isHttpOnly, sameSite FROM moz_cookies' ) - .all() as FirefoxRow[] - db.close() - - diag(` Firefox source has ${rows.length} cookies`) - if (rows.length === 0) { - rmSync(tmpDir, { recursive: true, force: true }) - return { ok: false, reason: 'No cookies found in Firefox.' } - } + .iterate() as IterableIterator const now = Math.floor(Date.now() / 1000) const validated: ValidatedCookie[] = [] @@ -1328,6 +1303,8 @@ async function importCookiesFromFirefox( expirationDate: row.expiry > 0 ? row.expiry : undefined }) } + db.close() + db = null rmSync(tmpDir, { recursive: true, force: true }) @@ -1335,10 +1312,18 @@ async function importCookiesFromFirefox( return { ok: false, reason: 'No valid cookies found in Firefox.' } } - return importValidatedCookies(validated, rows.length, targetPartition) + return importValidatedCookies(validated, totalRows, targetPartition) } catch (err) { + try { + db?.close() + } catch { + /* best-effort */ + } rmSync(tmpDir, { recursive: true, force: true }) diag(` Firefox import failed: ${err}`) + if (err instanceof InstalledBrowserCookieStoreLimitError) { + return { ok: false, reason: installedBrowserCookieStoreLimitReason(browser.label) } + } return { ok: false, reason: 'Could not import cookies from Firefox. Try closing Firefox first.' @@ -1350,6 +1335,17 @@ async function importCookiesFromFirefox( // Safari import // --------------------------------------------------------------------------- +function safariCookieStoreLimitReason(error: SafariCookieStoreLimitError): string { + switch (error.kind) { + case 'pages': + return `Safari cookie store is too large to import safely (${SAFARI_COOKIE_STORE_MAX_PAGES.toLocaleString('en-US')}-page limit).` + case 'cookies': + return `Safari cookie store is too large to import safely (${SAFARI_COOKIE_STORE_MAX_COOKIES.toLocaleString('en-US')}-cookie limit).` + case 'parsed-bytes': + return `Safari cookie store is too large to import safely (${SAFARI_COOKIE_STORE_MAX_PARSED_BYTES / 1024 / 1024} MiB parsed cookie-data limit).` + } +} + async function importCookiesFromSafari( browser: DetectedBrowser, targetPartition: string @@ -1358,9 +1354,18 @@ async function importCookiesFromSafari( let data: Buffer try { - data = readFileSync(browser.cookiesPath) + data = readNodeFileSyncWithinLimit( + browser.cookiesPath, + SAFARI_COOKIE_STORE_MAX_FILE_BYTES + ).buffer } catch (err) { diag(` Safari read failed: ${err}`) + if (err instanceof NodeFileReadTooLargeError) { + return { + ok: false, + reason: `Safari cookie store is too large to import safely (${SAFARI_COOKIE_STORE_MAX_FILE_BYTES / 1024 / 1024} MiB file limit).` + } + } // Why: Safari's Cookies.binarycookies is in a sandbox container; reading it needs Full Disk Access. const isPermError = err instanceof Error && 'code' in err && (err as NodeJS.ErrnoException).code === 'EPERM' @@ -1375,23 +1380,25 @@ async function importCookiesFromSafari( } try { - const cookies = decodeSafariBinaryCookies(data) - diag(` Safari source has ${cookies.length} cookies`) + const decoded = decodeSafariCookieStore(data) + diag(` Safari source has ${decoded.totalCookies} cookies`) - if (cookies.length === 0) { + if (decoded.totalCookies === 0) { return { ok: false, reason: 'No cookies found in Safari.' } } const now = Math.floor(Date.now() / 1000) - const valid = cookies.filter((c) => !c.expirationDate || c.expirationDate > now) - - if (valid.length === 0) { + removeExpiredSafariCookiesInPlace(decoded.cookies, now) + if (decoded.cookies.length === 0) { return { ok: false, reason: 'All Safari cookies are expired.' } } - return importValidatedCookies(valid, cookies.length, targetPartition) + return importValidatedCookies(decoded.cookies, decoded.totalCookies, targetPartition) } catch (err) { diag(` Safari import failed: ${err}`) + if (err instanceof SafariCookieStoreLimitError) { + return { ok: false, reason: safariCookieStoreLimitReason(err) } + } return { ok: false, reason: 'Could not import cookies from Safari.' } } } @@ -1498,16 +1505,30 @@ export async function importCookiesFromBrowser( stagingDb.exec('DELETE FROM cookies') - const sourceRows = sourceDb.prepare('SELECT * FROM cookies ORDER BY rowid').all() as Record< - string, - unknown - >[] - sourceDb.close() - sourceDb = null + const sourceStats = sourceDb + .prepare( + `SELECT + COUNT(*) AS cookie_count, + COALESCE(SUM( + length(CAST(COALESCE(host_key, '') AS BLOB)) + + length(CAST(COALESCE(name, '') AS BLOB)) + + length(CAST(COALESCE(value, '') AS BLOB)) + + length(COALESCE(encrypted_value, X'')) + + length(CAST(COALESCE(path, '') AS BLOB)) + ), 0) AS cookie_bytes + FROM cookies` + ) + .get() as { cookie_count: number | bigint; cookie_bytes: number | bigint } + const totalSourceRows = assertInstalledBrowserCookieStoreWithinLimits( + sourceStats.cookie_count, + sourceStats.cookie_bytes + ) - diag(` source has ${sourceRows.length} cookies`) + diag(` source has ${totalSourceRows} cookies`) - if (sourceRows.length === 0) { + if (totalSourceRows === 0) { + sourceDb.close() + sourceDb = null stagingDb.close() stagingDb = null try { @@ -1518,14 +1539,16 @@ export async function importCookiesFromBrowser( return { ok: false, reason: `No cookies found in ${browser.label}.` } } - const needsSourceKey = sourceRows.some((sourceRow) => { - const encRaw = sourceRow.encrypted_value - return encRaw instanceof Uint8Array && encRaw.length > 0 - }) + const needsSourceKey = + sourceDb + .prepare('SELECT 1 AS needed FROM cookies WHERE length(encrypted_value) > 0 LIMIT 1') + .get() !== undefined const sourceKey = needsSourceKey ? getEncryptionKey(browser.keychainService!, browser.keychainAccount!, browser) : null if (needsSourceKey && !sourceKey) { + sourceDb.close() + sourceDb = null stagingDb.close() stagingDb = null // Why: key denial happens after staging, so clean up the target DB copy or retries pile up. @@ -1564,7 +1587,6 @@ export async function importCookiesFromBrowser( const domainSet = new Set() type DecryptedCookie = { - decryptedValue: Buffer value: string domain: string name: string @@ -1584,6 +1606,9 @@ export async function importCookiesFromBrowser( stagingDb.exec('BEGIN TRANSACTION') + const sourceRows = sourceDb + .prepare('SELECT * FROM cookies ORDER BY rowid') + .iterate() as IterableIterator> for (const sourceRow of sourceRows) { const encRaw = sourceRow.encrypted_value // Why: node:sqlite returns BLOBs as Uint8Array; treat any other type as missing, not an empty buffer that would silently blank the cookie value. @@ -1626,7 +1651,6 @@ export async function importCookiesFromBrowser( const value = decryptedValue.toString('latin1') decryptedCookies.push({ - decryptedValue, value, domain, name, @@ -1641,6 +1665,8 @@ export async function importCookiesFromBrowser( insertStmt.run(...params) imported++ } + sourceDb.close() + sourceDb = null diag(` skipped ${integritySkipped} Google integrity cookies (SIDCC/STRP/AEC)`) stagingDb.exec('COMMIT') @@ -1706,7 +1732,7 @@ export async function importCookiesFromBrowser( } const summary: BrowserCookieImportSummary = { - totalCookies: sourceRows.length, + totalCookies: totalSourceRows, importedCookies: imported, skippedCookies: skipped, domains: [...domainSet].sort() @@ -1731,6 +1757,9 @@ export async function importCookiesFromBrowser( /* may not exist yet */ } diag(` SQLite import failed: ${err}`) + if (err instanceof InstalledBrowserCookieStoreLimitError) { + return { ok: false, reason: installedBrowserCookieStoreLimitReason(browser.label) } + } return { ok: false, reason: reasonWithDiagLog( diff --git a/src/main/browser/browser-cookie-json-file-parser.test.ts b/src/main/browser/browser-cookie-json-file-parser.test.ts new file mode 100644 index 00000000000..40ad84e35ee --- /dev/null +++ b/src/main/browser/browser-cookie-json-file-parser.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it, vi } from 'vitest' +import { + COOKIE_JSON_FILE_MAX_BYTES, + COOKIE_JSON_FILE_MAX_DEPTH, + COOKIE_JSON_FILE_MAX_ENTRIES, + COOKIE_JSON_FILE_MAX_RETAINED_BYTES, + CookieJsonFileFormatError, + CookieJsonFileLimitError, + enforceCookieJsonRetainedBytes, + visitCookieJsonFileObjects +} from './browser-cookie-json-file-parser' + +describe('visitCookieJsonFileObjects', () => { + it('visits top-level objects in order and counts every entry', () => { + const onObject = vi.fn() + const total = visitCookieJsonFileObjects( + '[{"name":"first"}, 42, ["ignored"], {"name":"last","nested":{"x":1}}]', + onObject + ) + + expect(total).toBe(4) + expect(onObject.mock.calls.map(([entry]) => entry)).toEqual([ + { name: 'first' }, + { name: 'last' } + ]) + }) + + it('accepts the exact entry boundary and rejects one over it', () => { + expect(visitCookieJsonFileObjects('[{},{}]', () => undefined, 2)).toBe(2) + expect(() => visitCookieJsonFileObjects('[{},{},{}]', () => undefined, 2)).toThrow( + expect.objectContaining({ + name: CookieJsonFileLimitError.name, + kind: 'entries', + observed: 3, + limit: 2 + }) + ) + }) + + it('accepts the retained-byte boundary and rejects one byte over it', () => { + expect(() => enforceCookieJsonRetainedBytes(COOKIE_JSON_FILE_MAX_RETAINED_BYTES)).not.toThrow() + expect(() => enforceCookieJsonRetainedBytes(COOKIE_JSON_FILE_MAX_RETAINED_BYTES + 1)).toThrow( + expect.objectContaining({ + name: CookieJsonFileLimitError.name, + kind: 'retained-bytes', + observed: COOKIE_JSON_FILE_MAX_RETAINED_BYTES + 1 + }) + ) + }) + + it('accepts the nesting boundary and rejects one level over it', () => { + expect( + visitCookieJsonFileObjects(`${'['.repeat(4)}0${']'.repeat(4)}`, () => undefined, 10, 4) + ).toBe(1) + expect(() => + visitCookieJsonFileObjects(`${'['.repeat(5)}0${']'.repeat(5)}`, () => undefined, 10, 4) + ).toThrow( + expect.objectContaining({ + name: CookieJsonFileLimitError.name, + kind: 'depth', + observed: 5, + limit: 4 + }) + ) + }) + + it.each([ + { raw: '{"name":"not-an-array"}', kind: 'root' }, + { raw: '[{"name":}]', kind: 'syntax' }, + { raw: '[{"name":"cookie"},]', kind: 'syntax' }, + { raw: '[/* comment */ {"name":"cookie"}]', kind: 'syntax' } + ])('rejects $kind input without returning partial data', ({ raw, kind }) => { + expect(() => visitCookieJsonFileObjects(raw, () => undefined)).toThrow( + expect.objectContaining({ name: CookieJsonFileFormatError.name, kind }) + ) + }) + + it('publishes the production file, entry, and retained-data limits', () => { + expect({ + fileBytes: COOKIE_JSON_FILE_MAX_BYTES, + depth: COOKIE_JSON_FILE_MAX_DEPTH, + entries: COOKIE_JSON_FILE_MAX_ENTRIES, + retainedBytes: COOKIE_JSON_FILE_MAX_RETAINED_BYTES + }).toEqual({ + fileBytes: 64 * 1024 * 1024, + depth: 128, + entries: 250_000, + retainedBytes: 64 * 1024 * 1024 + }) + }) +}) diff --git a/src/main/browser/browser-cookie-json-file-parser.ts b/src/main/browser/browser-cookie-json-file-parser.ts new file mode 100644 index 00000000000..5c33bddecb8 --- /dev/null +++ b/src/main/browser/browser-cookie-json-file-parser.ts @@ -0,0 +1,184 @@ +import { visit, type JSONPath } from 'jsonc-parser' + +export const COOKIE_JSON_FILE_MAX_BYTES = 64 * 1024 * 1024 +export const COOKIE_JSON_FILE_MAX_ENTRIES = 250_000 +export const COOKIE_JSON_FILE_MAX_DEPTH = 128 +export const COOKIE_JSON_FILE_MAX_RETAINED_BYTES = 64 * 1024 * 1024 + +export type CookieJsonFileLimitKind = 'depth' | 'entries' | 'retained-bytes' +export type CookieJsonFileFormatKind = 'syntax' | 'root' + +export class CookieJsonFileLimitError extends Error { + constructor( + readonly kind: CookieJsonFileLimitKind, + readonly observed: number, + readonly limit: number + ) { + super(`Cookie JSON file exceeds the ${kind} limit`) + this.name = 'CookieJsonFileLimitError' + } +} + +export class CookieJsonFileFormatError extends Error { + constructor(readonly kind: CookieJsonFileFormatKind) { + super(`Cookie JSON file has invalid ${kind}`) + this.name = 'CookieJsonFileFormatError' + } +} + +type Container = { + kind: 'array' | 'object' + topLevelEntry: boolean +} + +const COOKIE_PROPERTY_NAMES = new Set([ + 'domain', + 'name', + 'value', + 'path', + 'secure', + 'httpOnly', + 'sameSite', + 'expirationDate' +]) + +export function visitCookieJsonFileObjects( + rawContent: string, + onObject: (entry: Record) => void, + maxEntries = COOKIE_JSON_FILE_MAX_ENTRIES, + maxDepth = COOKIE_JSON_FILE_MAX_DEPTH +): number { + enforceJsonNestingDepth(rawContent, maxDepth) + const containers: Container[] = [] + let rootKind: 'array' | 'other' | null = null + let currentEntry: Record | null = null + let entryCount = 0 + let syntaxError = false + + const countEntry = (): void => { + entryCount += 1 + if (entryCount > maxEntries) { + throw new CookieJsonFileLimitError('entries', entryCount, maxEntries) + } + } + const isTopLevelEntry = (path: JSONPath): boolean => rootKind === 'array' && path.length === 1 + const setDirectProperty = (path: JSONPath, value: unknown): void => { + const property = path[1] + if ( + currentEntry && + path.length === 2 && + typeof property === 'string' && + COOKIE_PROPERTY_NAMES.has(property) + ) { + currentEntry[property] = value + } + } + + // Why: retain only fields validation uses instead of materializing the entire external JSON tree. + visit( + rawContent, + { + onArrayBegin: (_offset, _length, _line, _character, pathSupplier) => { + const path = pathSupplier() + if (path.length === 0) { + rootKind = 'array' + } + const topLevelEntry = isTopLevelEntry(path) + if (topLevelEntry) { + countEntry() + } else { + setDirectProperty(path, undefined) + } + containers.push({ kind: 'array', topLevelEntry }) + }, + onArrayEnd: () => { + containers.pop() + }, + onObjectBegin: (_offset, _length, _line, _character, pathSupplier) => { + const path = pathSupplier() + if (path.length === 0) { + rootKind = 'other' + } + const topLevelEntry = isTopLevelEntry(path) + if (topLevelEntry) { + countEntry() + currentEntry = {} + } else { + setDirectProperty(path, undefined) + } + containers.push({ kind: 'object', topLevelEntry }) + }, + onObjectEnd: () => { + const container = containers.pop() + if (!container || container.kind !== 'object' || !container.topLevelEntry) { + return + } + if (currentEntry) { + onObject(currentEntry) + } + currentEntry = null + }, + onLiteralValue: (value, _offset, _length, _line, _character, pathSupplier) => { + const path = pathSupplier() + if (path.length === 0) { + rootKind = 'other' + } else if (isTopLevelEntry(path)) { + countEntry() + } else { + setDirectProperty(path, value) + } + }, + onError: () => { + syntaxError = true + } + }, + { disallowComments: true, allowTrailingComma: false, allowEmptyContent: false } + ) + + if (syntaxError) { + throw new CookieJsonFileFormatError('syntax') + } + if (rootKind !== 'array') { + throw new CookieJsonFileFormatError('root') + } + return entryCount +} + +function enforceJsonNestingDepth(rawContent: string, maxDepth: number): void { + let depth = 0 + let inString = false + let escaped = false + for (let index = 0; index < rawContent.length; index += 1) { + const character = rawContent[index] + if (inString) { + if (escaped) { + escaped = false + } else if (character === '\\') { + escaped = true + } else if (character === '"') { + inString = false + } + continue + } + if (character === '"') { + inString = true + } else if (character === '[' || character === '{') { + depth += 1 + if (depth > maxDepth) { + throw new CookieJsonFileLimitError('depth', depth, maxDepth) + } + } else if (character === ']' || character === '}') { + depth = Math.max(0, depth - 1) + } + } +} + +export function enforceCookieJsonRetainedBytes(observed: number): void { + if (observed > COOKIE_JSON_FILE_MAX_RETAINED_BYTES) { + throw new CookieJsonFileLimitError( + 'retained-bytes', + observed, + COOKIE_JSON_FILE_MAX_RETAINED_BYTES + ) + } +} diff --git a/src/main/browser/browser-manager.test.ts b/src/main/browser/browser-manager.test.ts index c49088569f4..6432e8b08da 100644 --- a/src/main/browser/browser-manager.test.ts +++ b/src/main/browser/browser-manager.test.ts @@ -54,7 +54,7 @@ vi.mock('./popup-origin-bar-window', () => ({ openPopupWithOriginBar: openPopupWithOriginBarMock })) -import { browserManager } from './browser-manager' +import { MAX_ACTIVE_BROWSER_DOWNLOADS, browserManager } from './browser-manager' describe('browserManager', () => { const rendererWebContentsId = 5001 @@ -1927,6 +1927,24 @@ describe('browserManager', () => { ) }) + it('cancels excess concurrent downloads before retaining another item', () => { + const item = createDownloadItem() + const managerState = browserManager as unknown as { downloadsById: Map } + for (let index = 0; index < MAX_ACTIVE_BROWSER_DOWNLOADS; index += 1) { + managerState.downloadsById.set(`active-${index}`, {}) + } + + try { + browserManager.handleGuestWillDownload({ guestWebContentsId: 999, item }) + + expect(item.cancel).toHaveBeenCalledTimes(1) + expect(item.setSavePath).not.toHaveBeenCalled() + expect(managerState.downloadsById.size).toBe(MAX_ACTIVE_BROWSER_DOWNLOADS) + } finally { + managerState.downloadsById.clear() + } + }) + it('flushes started and terminal snapshots for downloads that finish before registration', () => { const rendererSendMock = vi.fn() const guest = { diff --git a/src/main/browser/browser-manager.ts b/src/main/browser/browser-manager.ts index bd0e5698cd8..0c7856d070f 100644 --- a/src/main/browser/browser-manager.ts +++ b/src/main/browser/browser-manager.ts @@ -198,6 +198,8 @@ type ActiveDownload = { cleanup: (() => void) | null } +export const MAX_ACTIVE_BROWSER_DOWNLOADS = 64 + function safeOrigin(rawUrl: string): string { const external = normalizeExternalBrowserUrl(rawUrl) const urlToParse = external ?? rawUrl @@ -1257,6 +1259,14 @@ export class BrowserManager { handleGuestWillDownload(args: { guestWebContentsId: number; item: Electron.DownloadItem }): void { const { guestWebContentsId, item } = args + if (this.downloadsById.size >= MAX_ACTIVE_BROWSER_DOWNLOADS) { + try { + item.cancel() + } catch { + // Why: rejecting excess admission must remain safe if Chromium already finalized the item. + } + return + } const downloadId = randomUUID() const requestedFilename = (() => { try { diff --git a/src/main/browser/browser-pdf-admission.test.ts b/src/main/browser/browser-pdf-admission.test.ts new file mode 100644 index 00000000000..f68771f9907 --- /dev/null +++ b/src/main/browser/browser-pdf-admission.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from 'vitest' +import { + acquireBrowserPdfAdmission, + BROWSER_PDF_MAX_CONCURRENT_PRINTS, + startBrowserPdfPrint +} from './browser-pdf-admission' + +function createWebContents() { + return { printToPDF: vi.fn<() => Promise>() } +} + +describe('browser PDF admission', () => { + it('caps native prints process-wide and releases only after native settlement', async () => { + const resolvers: ((data: Buffer) => void)[] = [] + const webContents = createWebContents() + webContents.printToPDF.mockImplementation( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + + const active = Array.from({ length: BROWSER_PDF_MAX_CONCURRENT_PRINTS }, () => + startBrowserPdfPrint(webContents as never, {}) + ) + expect(active.every(Boolean)).toBe(true) + expect(startBrowserPdfPrint(createWebContents() as never, {})).toBeNull() + + resolvers[0]!(Buffer.from('first')) + await Promise.resolve() + const nextWebContents = createWebContents() + nextWebContents.printToPDF.mockResolvedValue(Buffer.from('next')) + await expect(startBrowserPdfPrint(nextWebContents as never, {})).resolves.toEqual( + Buffer.from('next') + ) + + resolvers[1]!(Buffer.from('second')) + await Promise.all(active) + }) + + it('lets pre-render work release unused admission', () => { + const first = acquireBrowserPdfAdmission() + const second = acquireBrowserPdfAdmission() + expect(first).not.toBeNull() + expect(second).not.toBeNull() + expect(acquireBrowserPdfAdmission()).toBeNull() + + first!.releaseIfIdle() + const replacement = acquireBrowserPdfAdmission() + expect(replacement).not.toBeNull() + second!.releaseIfIdle() + replacement!.releaseIfIdle() + }) + + it('releases process-wide print slots after synchronous and asynchronous failures', async () => { + const synchronous = createWebContents() + synchronous.printToPDF.mockImplementation(() => { + throw new Error('sync print failure') + }) + await expect(startBrowserPdfPrint(synchronous as never, {})).rejects.toThrow( + 'sync print failure' + ) + + const asynchronous = createWebContents() + asynchronous.printToPDF.mockRejectedValue(new Error('async print failure')) + await expect(startBrowserPdfPrint(asynchronous as never, {})).rejects.toThrow( + 'async print failure' + ) + + const replacement = createWebContents() + replacement.printToPDF.mockResolvedValue(Buffer.from('replacement')) + await expect(startBrowserPdfPrint(replacement as never, {})).resolves.toEqual( + Buffer.from('replacement') + ) + }) +}) diff --git a/src/main/browser/browser-pdf-admission.ts b/src/main/browser/browser-pdf-admission.ts new file mode 100644 index 00000000000..8ef66e39415 --- /dev/null +++ b/src/main/browser/browser-pdf-admission.ts @@ -0,0 +1,59 @@ +import type { PrintToPDFOptions, WebContents } from 'electron' + +export const BROWSER_PDF_MAX_CONCURRENT_PRINTS = 2 +export const BROWSER_PDF_BUSY_ERROR = 'Too many PDF print requests are already running' + +let activePrints = 0 + +export type BrowserPdfAdmission = { + print(webContents: WebContents, options: PrintToPDFOptions): Promise + releaseIfIdle(): void +} + +export function acquireBrowserPdfAdmission(): BrowserPdfAdmission | null { + if (activePrints >= BROWSER_PDF_MAX_CONCURRENT_PRINTS) { + return null + } + activePrints += 1 + let printStarted = false + let released = false + const release = (): void => { + if (released) { + return + } + released = true + activePrints = Math.max(0, activePrints - 1) + } + + return { + print(webContents, options) { + if (printStarted) { + return Promise.reject(new Error('PDF admission has already started a print')) + } + printStarted = true + let print: Promise + try { + print = webContents.printToPDF(options) + } catch (error) { + release() + return Promise.reject(error) + } + // Why: native PDF work cannot be cancelled when its caller times out or disconnects. + void print.then(release, release) + return print + }, + releaseIfIdle() { + if (!printStarted) { + release() + } + } + } +} + +export function startBrowserPdfPrint( + webContents: WebContents, + options: PrintToPDFOptions +): Promise | null { + const admission = acquireBrowserPdfAdmission() + return admission?.print(webContents, options) ?? null +} diff --git a/src/main/browser/browser-screencast-stream.test.ts b/src/main/browser/browser-screencast-stream.test.ts index d1fc838807d..7929ba611a3 100644 --- a/src/main/browser/browser-screencast-stream.test.ts +++ b/src/main/browser/browser-screencast-stream.test.ts @@ -3,7 +3,11 @@ import { EventEmitter } from 'node:events' import { describe, expect, it, vi } from 'vitest' import { decodeBrowserScreencastFrame } from '../../shared/browser-screencast-protocol' -import { startBrowserScreencast } from './browser-screencast-stream' +import { + decodeBrowserScreencastImage, + MAX_BROWSER_SCREENCAST_BASE64_CHARACTERS, + startBrowserScreencast +} from './browser-screencast-stream' function createMockWebContents() { let attached = false @@ -57,6 +61,16 @@ function jpegWithSize(width: number, height: number): Buffer { } describe('startBrowserScreencast', () => { + it('rejects oversized CDP image text before base64 decoding', () => { + const decode = vi.spyOn(Buffer, 'from') + + expect( + decodeBrowserScreencastImage('A'.repeat(MAX_BROWSER_SCREENCAST_BASE64_CHARACTERS + 1)) + ).toBeNull() + expect(decode).not.toHaveBeenCalled() + decode.mockRestore() + }) + it('emits an initial captured frame before CDP produces screencast events', async () => { const webContents = createMockWebContents() const firstFrame = Buffer.from('first-frame') diff --git a/src/main/browser/browser-screencast-stream.ts b/src/main/browser/browser-screencast-stream.ts index ad2e1e27e62..19f9d57d423 100644 --- a/src/main/browser/browser-screencast-stream.ts +++ b/src/main/browser/browser-screencast-stream.ts @@ -10,9 +10,12 @@ import { import { BrowserError } from './cdp-bridge' import { acquireElectronDebugger, type ElectronDebuggerLease } from './electron-debugger-lease' import { readBrowserScreencastImageSize } from './browser-screencast-image-size' +import { REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES } from '../../shared/remote-runtime-memory-limits' const DEBUGGER_COMMAND_TIMEOUT_MS = 8_000 const BACKPRESSURE_RETRY_MS = 50 +export const MAX_BROWSER_SCREENCAST_BASE64_CHARACTERS = + Math.ceil(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES / 3) * 4 export type BrowserScreencastOptions = { format: BrowserScreencastFormat @@ -452,7 +455,10 @@ export async function startBrowserScreencast( } try { - const image = new Uint8Array(Buffer.from(data, 'base64')) + const image = decodeBrowserScreencastImage(data) + if (!image) { + throw new Error('Browser screencast frame is too large') + } // Why: image dimension parsing happens for every live frame; share the // result between stale-frame rejection and metadata enrichment. const imageSize = readBrowserScreencastImageSize(image, options.format) @@ -558,7 +564,10 @@ export async function startBrowserScreencast( if (!data) { return } - image = new Uint8Array(Buffer.from(data, 'base64')) + image = decodeBrowserScreencastImage(data) + if (!image) { + return + } } if (isSnapshotStale(initialOnly, generation)) { return @@ -632,3 +641,11 @@ export async function startBrowserScreencast( done } } + +export function decodeBrowserScreencastImage(data: string): Uint8Array | null { + if (data.length > MAX_BROWSER_SCREENCAST_BASE64_CHARACTERS) { + return null + } + const image = new Uint8Array(Buffer.from(data, 'base64')) + return image.byteLength <= REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES ? image : null +} diff --git a/src/main/browser/browser-screenshot-admission.test.ts b/src/main/browser/browser-screenshot-admission.test.ts new file mode 100644 index 00000000000..f6b74b5d37e --- /dev/null +++ b/src/main/browser/browser-screenshot-admission.test.ts @@ -0,0 +1,95 @@ +import { EventEmitter } from 'node:events' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + resetBrowserScreenshotAdmissionForTests, + startBrowserFallbackCapture, + startBrowserScreenshotCommand +} from './browser-screenshot-admission' +import { BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES } from './browser-screenshot-limits' + +function deferred(): { + promise: Promise + reject: (error: Error) => void + resolve: (value: T) => void +} { + let reject!: (error: Error) => void + let resolve!: (value: T) => void + const promise = new Promise((accept, fail) => { + resolve = accept + reject = fail + }) + return { promise, reject, resolve } +} + +function createWebContents() { + const events = new EventEmitter() + return Object.assign(events, { + capturePage: vi.fn(), + debugger: { sendCommand: vi.fn() } + }) +} + +afterEach(() => { + resetBrowserScreenshotAdmissionForTests() +}) + +describe('browser screenshot admission', () => { + it('caps native capture work across different WebContents and releases on settlement', async () => { + const operations = Array.from({ length: BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES }, () => + deferred() + ) + const contents = operations.map((operation) => { + const webContents = createWebContents() + webContents.debugger.sendCommand.mockReturnValue(operation.promise) + return webContents + }) + const active = contents.map((webContents) => + startBrowserScreenshotCommand(webContents as never, 'Page.captureScreenshot', {}) + ) + + const overflow = createWebContents() + overflow.debugger.sendCommand.mockResolvedValue({ data: 'later' }) + expect( + startBrowserScreenshotCommand(overflow as never, 'Page.captureScreenshot', {}) + ).toBeNull() + + operations[0]!.resolve({ data: 'first' }) + await expect(active[0]).resolves.toEqual({ data: 'first' }) + await expect( + startBrowserScreenshotCommand(overflow as never, 'Page.captureScreenshot', {}) + ).resolves.toEqual({ data: 'later' }) + + operations[1]!.reject(new Error('capture failed')) + await expect(active[1]).rejects.toThrow('capture failed') + }) + + it('shares admission with fallback capture and releases synchronous failures', async () => { + const commandContents = createWebContents() + commandContents.debugger.sendCommand.mockImplementation(() => { + throw new Error('sync failure') + }) + await expect( + startBrowserScreenshotCommand(commandContents as never, 'Page.captureScreenshot', {}) + ).rejects.toThrow('sync failure') + + const fallbackContents = createWebContents() + const image = { isEmpty: () => false } + fallbackContents.capturePage.mockResolvedValue(image) + await expect(startBrowserFallbackCapture(fallbackContents as never)).resolves.toBe(image) + }) + + it('releases a hung operation when its WebContents is destroyed', async () => { + const hung = createWebContents() + hung.debugger.sendCommand.mockImplementation(() => new Promise(() => {})) + expect( + startBrowserScreenshotCommand(hung as never, 'Page.captureScreenshot', {}) + ).not.toBeNull() + hung.emit('destroyed') + + const replacement = createWebContents() + replacement.debugger.sendCommand.mockResolvedValue({ data: 'replacement' }) + await expect( + startBrowserScreenshotCommand(replacement as never, 'Page.captureScreenshot', {}) + ).resolves.toEqual({ data: 'replacement' }) + }) +}) diff --git a/src/main/browser/browser-screenshot-admission.ts b/src/main/browser/browser-screenshot-admission.ts new file mode 100644 index 00000000000..4442da7d18c --- /dev/null +++ b/src/main/browser/browser-screenshot-admission.ts @@ -0,0 +1,64 @@ +import type { WebContents } from 'electron' +import { BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES } from './browser-screenshot-limits' + +let activeCaptures = 0 +let admissionEpoch = 0 + +function startBrowserScreenshotOperation( + webContents: WebContents, + start: () => Promise +): Promise | null { + if (activeCaptures >= BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES) { + return null + } + activeCaptures += 1 + const operationEpoch = admissionEpoch + + let released = false + const onDestroyed = (): void => release() + const release = (): void => { + if (released) { + return + } + released = true + webContents.removeListener?.('destroyed', onDestroyed) + if (operationEpoch === admissionEpoch) { + activeCaptures = Math.max(0, activeCaptures - 1) + } + } + // Why: destroying the native target releases its capture resources even if Chromium omits rejection. + webContents.once?.('destroyed', onDestroyed) + + let command: Promise + try { + command = start() + } catch (error) { + release() + return Promise.reject(error) + } + // Why: a timed-out native command cannot be cancelled, so it owns its slot until it truly settles. + void command.then(release, release) + return command +} + +export function startBrowserScreenshotCommand( + webContents: WebContents, + method: 'Page.captureScreenshot' | 'Page.getLayoutMetrics', + params: Record +): Promise | null { + return startBrowserScreenshotOperation( + webContents, + () => webContents.debugger.sendCommand(method, params) as Promise + ) +} + +export function startBrowserFallbackCapture( + webContents: WebContents +): Promise | null { + return startBrowserScreenshotOperation(webContents, () => webContents.capturePage()) +} + +export function resetBrowserScreenshotAdmissionForTests(): void { + admissionEpoch += 1 + activeCaptures = 0 +} diff --git a/src/main/browser/browser-screenshot-file-reader.test.ts b/src/main/browser/browser-screenshot-file-reader.test.ts new file mode 100644 index 00000000000..a608d2d3dfa --- /dev/null +++ b/src/main/browser/browser-screenshot-file-reader.test.ts @@ -0,0 +1,36 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { readBrowserScreenshotFile } from './browser-screenshot-file-reader' +import { BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR } from './browser-screenshot-limits' + +const temporaryDirectories: string[] = [] + +function createFile(bytes: Buffer): string { + const directory = mkdtempSync(join(tmpdir(), 'orca-screenshot-reader-')) + temporaryDirectories.push(directory) + const path = join(directory, 'screenshot.png') + writeFileSync(path, bytes) + return path +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('readBrowserScreenshotFile', () => { + it('reads a screenshot at the byte boundary', () => { + const path = createFile(Buffer.from('1234')) + + expect(readBrowserScreenshotFile(path, 4)).toEqual(Buffer.from('1234')) + }) + + it('rejects an oversized screenshot before allocating its contents', () => { + const path = createFile(Buffer.from('12345')) + + expect(() => readBrowserScreenshotFile(path, 4)).toThrow(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + }) +}) diff --git a/src/main/browser/browser-screenshot-file-reader.ts b/src/main/browser/browser-screenshot-file-reader.ts new file mode 100644 index 00000000000..85a24734ed0 --- /dev/null +++ b/src/main/browser/browser-screenshot-file-reader.ts @@ -0,0 +1,54 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { + BROWSER_SCREENSHOT_MAX_ENCODED_BYTES, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' + +const MIN_GROWTH_BYTES = 64 * 1024 + +function throwIfOverLimit(bytes: number, maxBytes: number): void { + if (bytes > maxBytes) { + throw new Error(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + } +} + +export function readBrowserScreenshotFile( + path: string, + maxBytes = BROWSER_SCREENSHOT_MAX_ENCODED_BYTES +): Buffer { + const descriptor = openSync(path, 'r') + try { + const initialSize = fstatSync(descriptor).size + throwIfOverLimit(initialSize, maxBytes) + let bytes = Buffer.allocUnsafe(initialSize) + let offset = 0 + + while (true) { + while (offset < bytes.length) { + const read = readSync(descriptor, bytes, offset, bytes.length - offset, null) + if (read === 0) { + return bytes.subarray(0, offset) + } + offset += read + } + + const probe = Buffer.allocUnsafe(1) + if (readSync(descriptor, probe, 0, 1, null) === 0) { + return bytes.subarray(0, offset) + } + throwIfOverLimit(offset + 1, maxBytes) + + const nextCapacity = Math.min( + maxBytes, + Math.max(MIN_GROWTH_BYTES, bytes.length * 2, offset + 1) + ) + const expanded = Buffer.allocUnsafe(nextCapacity) + bytes.copy(expanded, 0, 0, offset) + expanded[offset] = probe[0]! + bytes = expanded + offset += 1 + } + } finally { + closeSync(descriptor) + } +} diff --git a/src/main/browser/browser-screenshot-limits.test.ts b/src/main/browser/browser-screenshot-limits.test.ts new file mode 100644 index 00000000000..1379905e127 --- /dev/null +++ b/src/main/browser/browser-screenshot-limits.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { + assertBrowserScreenshotEncodedBytes, + assertBrowserScreenshotGeometry, + BROWSER_SCREENSHOT_MAX_EFFECTIVE_PIXELS, + BROWSER_SCREENSHOT_MAX_ENCODED_BYTES, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' + +describe('browser screenshot memory limits', () => { + it('accepts ordinary HiDPI viewport geometry', () => { + expect(() => assertBrowserScreenshotGeometry(1_440, 900, 2)).not.toThrow() + }) + + it('counts device scale in the effective pixel budget', () => { + const side = Math.floor(Math.sqrt(BROWSER_SCREENSHOT_MAX_EFFECTIVE_PIXELS)) + + expect(() => assertBrowserScreenshotGeometry(side, side, 2)).toThrow( + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + ) + }) + + it('accepts the encoded-byte boundary and rejects the next byte', () => { + expect(() => + assertBrowserScreenshotEncodedBytes(BROWSER_SCREENSHOT_MAX_ENCODED_BYTES) + ).not.toThrow() + expect(() => + assertBrowserScreenshotEncodedBytes(BROWSER_SCREENSHOT_MAX_ENCODED_BYTES + 1) + ).toThrow(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + }) +}) diff --git a/src/main/browser/browser-screenshot-limits.ts b/src/main/browser/browser-screenshot-limits.ts new file mode 100644 index 00000000000..c440369badd --- /dev/null +++ b/src/main/browser/browser-screenshot-limits.ts @@ -0,0 +1,37 @@ +export const BROWSER_SCREENSHOT_MAX_DIMENSION_PX = 32_768 +export const BROWSER_SCREENSHOT_MAX_EFFECTIVE_PIXELS = 32 * 1024 * 1024 +export const BROWSER_SCREENSHOT_MAX_ENCODED_BYTES = 32 * 1024 * 1024 +export const BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES = 2 +export const BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR = + 'Screenshot exceeds the browser automation memory limit' +export const BROWSER_SCREENSHOT_BUSY_ERROR = 'Too many screenshot requests are already running' + +export function assertBrowserScreenshotGeometry(width: number, height: number, scale = 1): void { + const effectiveWidth = width * scale + const effectiveHeight = height * scale + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + !Number.isFinite(scale) || + width <= 0 || + height <= 0 || + scale <= 0 || + effectiveWidth > BROWSER_SCREENSHOT_MAX_DIMENSION_PX || + effectiveHeight > BROWSER_SCREENSHOT_MAX_DIMENSION_PX || + effectiveWidth * effectiveHeight > BROWSER_SCREENSHOT_MAX_EFFECTIVE_PIXELS + ) { + throw new Error(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + } +} + +export function assertBrowserScreenshotEncodedBytes(bytes: number): void { + if (!Number.isFinite(bytes) || bytes < 0 || bytes > BROWSER_SCREENSHOT_MAX_ENCODED_BYTES) { + throw new Error(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + } +} + +export function assertBrowserScreenshotBase64(data: string): void { + const padding = data.endsWith('==') ? 2 : data.endsWith('=') ? 1 : 0 + const decodedBytes = Math.max(0, Math.floor((data.length * 3) / 4) - padding) + assertBrowserScreenshotEncodedBytes(decodedBytes) +} diff --git a/src/main/browser/browser-session-registry.persistence.test.ts b/src/main/browser/browser-session-registry.persistence.test.ts index 174b5aa7715..1d3b47ca08b 100644 --- a/src/main/browser/browser-session-registry.persistence.test.ts +++ b/src/main/browser/browser-session-registry.persistence.test.ts @@ -105,6 +105,19 @@ function installModuleMocks( fsState.present.add(key) }) })) + vi.doMock('../../shared/node-bounded-file-reader', () => ({ + readNodeFileSyncWithinLimit: vi.fn((p: string, maxBytes: number) => { + const value = fsState.files.get(fsKey(p)) + if (value === undefined) { + throw new Error('ENOENT') + } + const buffer = Buffer.from(value) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: {} } + }) + })) vi.doMock('./browser-manager', () => ({ browserManager: { @@ -207,6 +220,73 @@ describe('BrowserSessionRegistry persistence', () => { }) }) + it('ignores oversized browser-session metadata', async () => { + const fsState = createFsState() + fsState.files.set(META_PATH, ' '.repeat(1024 * 1024 + 1)) + fsState.present.add(META_PATH) + + installModuleMocks(fsState) + const { browserSessionRegistry } = await import('./browser-session-registry') + browserSessionRegistry.initializeBrowserSessionsFromPersistedState() + + expect(browserSessionRegistry.listProfiles()).toHaveLength(1) + expect(browserSessionRegistry.listProfiles()[0]?.id).toBe('default') + }) + + it('rejects an oversized profile label without retaining or persisting it', async () => { + const fsState = createFsState() + installModuleMocks(fsState) + const { browserSessionRegistry, MAX_BROWSER_SESSION_LABEL_BYTES } = + await import('./browser-session-registry') + + expect( + browserSessionRegistry.createProfile( + 'isolated', + 'x'.repeat(MAX_BROWSER_SESSION_LABEL_BYTES + 1) + ) + ).toBeNull() + expect(browserSessionRegistry.listProfiles()).toHaveLength(1) + expect(fsState.files.has(META_PATH)).toBe(false) + }) + + it('keeps profile creation bounded while preserving the last readable metadata snapshot', async () => { + const fsState = createFsState() + installModuleMocks(fsState) + const { browserSessionRegistry, MAX_BROWSER_SESSION_LABEL_BYTES } = + await import('./browser-session-registry') + const label = 'x'.repeat(MAX_BROWSER_SESSION_LABEL_BYTES - 8) + for (let index = 0; index < 256; index += 1) { + browserSessionRegistry.createProfile('isolated', `${index}-${label}`) + } + + const durableProfiles = JSON.parse(fsState.files.get(META_PATH) ?? '{}').profiles + expect(browserSessionRegistry.listProfiles().length).toBeGreaterThan(durableProfiles.length + 1) + expect(Buffer.byteLength(fsState.files.get(META_PATH) ?? '')).toBeLessThanOrEqual(1024 * 1024) + }) + + it('preserves metadata when a pending import path would exceed the byte limit', async () => { + const fsState = createFsState() + seedMeta(fsState, { + defaultSource: null, + userAgent: null, + userAgentByPartition: {}, + pendingCookieDbPath: null, + pendingCookieImports: {}, + profiles: [] + }) + const before = fsState.files.get(META_PATH) + installModuleMocks(fsState) + const { browserSessionRegistry, MAX_BROWSER_SESSION_META_FILE_BYTES } = + await import('./browser-session-registry') + + browserSessionRegistry.setPendingCookieImport( + 'persist:orca-browser', + 'x'.repeat(MAX_BROWSER_SESSION_META_FILE_BYTES) + ) + + expect(fsState.files.get(META_PATH)).toBe(before) + }) + it('merges partition-keyed pending entries without clobbering unrelated entries', async () => { const fsState = createFsState() seedMeta(fsState, { diff --git a/src/main/browser/browser-session-registry.ts b/src/main/browser/browser-session-registry.ts index 72a888edb62..fdd15140abf 100644 --- a/src/main/browser/browser-session-registry.ts +++ b/src/main/browser/browser-session-registry.ts @@ -2,17 +2,11 @@ import { app, session } from 'electron' import type { Session } from 'electron' import { randomUUID } from 'node:crypto' -import { - copyFileSync, - existsSync, - mkdirSync, - readFileSync, - renameSync, - unlinkSync, - writeFileSync -} from 'node:fs' +import { copyFileSync, existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { dirname, join } from 'node:path' import { ORCA_BROWSER_PARTITION } from '../../shared/constants' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' import { DEFAULT_LOCAL_ORCA_PROFILE_ID, getOrcaProfileBrowserDefaultPartition, @@ -46,6 +40,9 @@ export type BrowserSessionRegistryProfileOptions = { } const BROWSER_SESSION_META_FILE_NAME = 'browser-session-meta.json' +export const MAX_BROWSER_SESSION_META_FILE_BYTES = 1024 * 1024 +export const MAX_BROWSER_SESSION_PROFILES = 256 +export const MAX_BROWSER_SESSION_LABEL_BYTES = 16 * 1024 const LEGACY_BROWSER_SESSION_PARTITION_RE = /^persist:orca-browser-session-[\da-f-]{8}-[\da-f-]{4}-[\da-f-]{4}-[\da-f-]{4}-[\da-f-]{12}$/ @@ -103,8 +100,12 @@ class BrowserSessionRegistry { try { const existing = this.loadPersistedMeta() const tmpPath = `${this.metadataPath}.tmp` + const serialized = stringifyJsonWithinByteLimit( + { ...existing, ...updates }, + MAX_BROWSER_SESSION_META_FILE_BYTES + ).serialized mkdirSync(dirname(this.metadataPath), { recursive: true }) - writeFileSync(tmpPath, JSON.stringify({ ...existing, ...updates })) + writeFileSync(tmpPath, serialized) renameSync(tmpPath, this.metadataPath) } catch { // best-effort @@ -126,7 +127,10 @@ class BrowserSessionRegistry { private loadPersistedMeta(): BrowserSessionMeta { try { - const raw = readFileSync(this.metadataPath, 'utf-8') + const raw = readNodeFileSyncWithinLimit( + this.metadataPath, + MAX_BROWSER_SESSION_META_FILE_BYTES + ).buffer.toString('utf8') const data = JSON.parse(raw) const legacyUserAgent = typeof data?.userAgent === 'string' ? data.userAgent : null const userAgentByPartition: Record = @@ -340,7 +344,11 @@ class BrowserSessionRegistry { createProfile(scope: BrowserSessionProfileScope, label: string): BrowserSessionProfile | null { // Why: block scope:'default' here — only the constructor makes the default profile; a second one sharing the partition breaks delete. - if (scope === 'default') { + if ( + scope === 'default' || + Buffer.byteLength(label, 'utf8') > MAX_BROWSER_SESSION_LABEL_BYTES || + this.profiles.size >= MAX_BROWSER_SESSION_PROFILES + ) { return null } const id = randomUUID() diff --git a/src/main/browser/cdp-bridge-memory-bounds.test.ts b/src/main/browser/cdp-bridge-memory-bounds.test.ts new file mode 100644 index 00000000000..80a0ef50c68 --- /dev/null +++ b/src/main/browser/cdp-bridge-memory-bounds.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { webContentsFromId } = vi.hoisted(() => ({ webContentsFromId: vi.fn() })) + +vi.mock('electron', () => ({ + webContents: { fromId: webContentsFromId } +})) + +import { CdpBridge } from './cdp-bridge' +import { + CDP_MAX_CONSOLE_TEXT_CODE_UNITS, + CDP_MAX_IFRAME_SESSIONS, + CDP_MAX_PAUSED_REQUESTS +} from './cdp-event-memory-bounds' + +function createHarness() { + let attached = false + const messageListeners: ((event: unknown, method: string, params: unknown) => void)[] = [] + const sendCommand = vi.fn(async (_method: string, _params?: unknown) => ({})) + const guest = { + id: 1, + isDestroyed: () => false, + getURL: () => 'https://example.com', + getTitle: () => 'Example', + debugger: { + isAttached: () => attached, + attach: () => { + attached = true + }, + sendCommand, + on: (event: string, listener: (event: unknown, method: string, params: unknown) => void) => { + if (event === 'message') { + messageListeners.push(listener) + } + }, + removeListener: vi.fn() + } + } + const browserManager = { + webContentsIdByTabId: new Map([['page-1', guest.id]]) + } + const bridge = new CdpBridge(browserManager as never) + bridge.setActiveTab(guest.id) + return { + bridge, + guest, + sendCommand, + emit(method: string, params: unknown) { + for (const listener of messageListeners) { + listener({}, method, params) + } + } + } +} + +describe('CdpBridge event retention', () => { + beforeEach(() => { + webContentsFromId.mockReset() + }) + + it('bounds paused requests, iframe sessions, and console text', async () => { + const harness = createHarness() + webContentsFromId.mockReturnValue(harness.guest) + const internals = harness.bridge as unknown as { + ensureDebuggerAttached: (guest: unknown) => Promise + tabState: Map< + string, + { + capturing: boolean + intercepting: boolean + consoleLog: { text: string }[] + pausedRequests: Map + iframeSessions: Map + } + > + } + await internals.ensureDebuggerAttached(harness.guest) + const state = internals.tabState.get('page-1')! + state.capturing = true + state.intercepting = true + + harness.emit('Runtime.consoleAPICalled', { + args: [{ value: 'x'.repeat(CDP_MAX_CONSOLE_TEXT_CODE_UNITS * 2) }] + }) + for (let index = 0; index < CDP_MAX_PAUSED_REQUESTS + 10; index++) { + harness.emit('Fetch.requestPaused', { + requestId: `request-${index}`, + request: { url: `https://example.com/${index}`, method: 'GET', headers: {} } + }) + } + for (let index = 0; index < CDP_MAX_IFRAME_SESSIONS + 1; index++) { + harness.emit('Target.attachedToTarget', { + sessionId: `session-${index}`, + targetInfo: { type: 'iframe', targetId: `frame-${index}` } + }) + } + + expect(state.consoleLog[0]?.text).toHaveLength(CDP_MAX_CONSOLE_TEXT_CODE_UNITS) + expect(state.pausedRequests.size).toBe(CDP_MAX_PAUSED_REQUESTS) + expect(state.iframeSessions.size).toBe(CDP_MAX_IFRAME_SESSIONS) + expect( + harness.sendCommand.mock.calls.filter(([method]) => method === 'Fetch.continueRequest') + ).toHaveLength(10) + expect( + harness.sendCommand.mock.calls.filter(([method]) => method === 'Target.detachFromTarget') + ).toHaveLength(1) + }) +}) diff --git a/src/main/browser/cdp-bridge.ts b/src/main/browser/cdp-bridge.ts index 47d263ac371..2d6ac9f8dd4 100644 --- a/src/main/browser/cdp-bridge.ts +++ b/src/main/browser/cdp-bridge.ts @@ -48,8 +48,16 @@ import { import { insertTextThroughCdp } from './browser-text-insertion' import type { BrowserManager } from './browser-manager' import { ANTI_DETECTION_SCRIPT } from './anti-detection' - -const CAPTURE_LOG_LIMIT = 1000 +import { CdpCommandQueue } from './cdp-command-queue' +import { + buildBoundedCdpConsoleEntry, + buildBoundedCdpInterceptedRequest, + buildBoundedCdpNetworkEntry, + CDP_CAPTURE_LOG_LIMIT, + CDP_MAX_IFRAME_SESSIONS, + CDP_MAX_PAUSED_REQUESTS, + isBoundedCdpIframeSession +} from './cdp-event-memory-bounds' export class BrowserError extends Error { constructor( @@ -79,21 +87,21 @@ type TabState = { networkRequestMap: Map } -type QueuedCommand = { - execute: () => Promise - resolve: (value: unknown) => void - reject: (reason: unknown) => void -} - export class CdpBridge { private activeWebContentsId: number | null = null private readonly tabState = new Map() - private readonly commandQueues = new Map() - private readonly processingQueues = new Set() + private readonly commandQueue: CdpCommandQueue private readonly browserManager: BrowserManager constructor(browserManager: BrowserManager) { this.browserManager = browserManager + this.commandQueue = new CdpCommandQueue( + () => + new BrowserError( + 'browser_busy', + 'Browser command queue is full; retry after the current commands finish' + ) + ) } setActiveTab(webContentsId: number): void { @@ -1010,7 +1018,13 @@ export class CdpBridge { this.removeDebuggerListeners(guest, state) } this.tabState.delete(tabId) - this.commandQueues.delete(tabId) + this.commandQueue.closeTab( + tabId, + new BrowserError( + 'browser_debugger_detached', + 'Browser tab closed before command execution.' + ) + ) } } @@ -1195,10 +1209,21 @@ export class CdpBridge { } | undefined if (p?.sessionId && p.targetInfo?.type === 'iframe' && p.targetInfo.targetId) { - state.iframeSessions.set(p.targetInfo.targetId, p.sessionId) - guest.debugger.sendCommand('DOM.enable', {}, p.sessionId).catch(() => {}) - guest.debugger.sendCommand('Accessibility.enable', {}, p.sessionId).catch(() => {}) - guest.debugger.sendCommand('Runtime.enable', {}, p.sessionId).catch(() => {}) + const frameId = p.targetInfo.targetId + const canRetainSession = + isBoundedCdpIframeSession(frameId, p.sessionId) && + (state.iframeSessions.has(frameId) || + state.iframeSessions.size < CDP_MAX_IFRAME_SESSIONS) + if (canRetainSession) { + state.iframeSessions.set(frameId, p.sessionId) + guest.debugger.sendCommand('DOM.enable', {}, p.sessionId).catch(() => {}) + guest.debugger.sendCommand('Accessibility.enable', {}, p.sessionId).catch(() => {}) + guest.debugger.sendCommand('Runtime.enable', {}, p.sessionId).catch(() => {}) + } else { + guest.debugger + .sendCommand('Target.detachFromTarget', { sessionId: p.sessionId }) + .catch(() => {}) + } } } if (method === 'Target.detachedFromTarget') { @@ -1224,15 +1249,8 @@ export class CdpBridge { } | undefined if (p) { - const text = (p.args ?? []).map((a) => a.value ?? a.description ?? '').join(' ') - state.consoleLog.push({ - level: p.type ?? 'log', - text, - timestamp: p.timestamp ?? Date.now(), - url: p.stackTrace?.callFrames?.[0]?.url, - line: p.stackTrace?.callFrames?.[0]?.lineNumber - }) - if (state.consoleLog.length > CAPTURE_LOG_LIMIT) { + state.consoleLog.push(buildBoundedCdpConsoleEntry(p)) + if (state.consoleLog.length > CDP_CAPTURE_LOG_LIMIT) { state.consoleLog.shift() } } @@ -1252,20 +1270,13 @@ export class CdpBridge { } | undefined if (p?.response) { - const entry: BrowserNetworkEntry = { - url: p.response.url ?? '', - method: '', - status: p.response.status ?? 0, - mimeType: p.response.mimeType ?? '', - size: 0, - timestamp: p.timestamp ?? Date.now() - } + const entry = buildBoundedCdpNetworkEntry(p.response, p.timestamp) state.networkLog.push(entry) // Why: map requestId→entry so loadingFinished attributes size to the right response, not the latest one. if (p.requestId) { state.networkRequestMap.set(p.requestId, entry) } - if (state.networkLog.length > CAPTURE_LOG_LIMIT) { + if (state.networkLog.length > CDP_CAPTURE_LOG_LIMIT) { const evicted = state.networkLog.shift() if (evicted) { for (const [requestId, requestEntry] of state.networkRequestMap) { @@ -1299,13 +1310,23 @@ export class CdpBridge { } | undefined if (p?.requestId && p.request) { - state.pausedRequests.set(p.requestId, { - id: p.requestId, - url: p.request.url ?? '', - method: p.request.method ?? 'GET', - headers: (p.request.headers ?? {}) as Record, - resourceType: p.resourceType ?? 'Other' + const boundedRequest = buildBoundedCdpInterceptedRequest({ + requestId: p.requestId, + request: p.request, + resourceType: p.resourceType }) + if ( + boundedRequest && + (state.pausedRequests.has(p.requestId) || + state.pausedRequests.size < CDP_MAX_PAUSED_REQUESTS) + ) { + state.pausedRequests.set(p.requestId, boundedRequest) + } else { + state.pausedRequests.delete(p.requestId) + guest.debugger + .sendCommand('Fetch.continueRequest', { requestId: p.requestId }) + .catch(() => {}) + } } } } @@ -1681,40 +1702,7 @@ export class CdpBridge { private async enqueueCommand(execute: () => Promise): Promise { const guest = this.getActiveGuest() const tabId = this.resolveTabId(guest.id) - - return new Promise((resolve, reject) => { - let queue = this.commandQueues.get(tabId) - if (!queue) { - queue = [] - this.commandQueues.set(tabId, queue) - } - queue.push({ - execute: execute as () => Promise, - resolve: resolve as (value: unknown) => void, - reject - }) - this.processQueue(tabId) - }) - } - - private async processQueue(tabId: string): Promise { - if (this.processingQueues.has(tabId)) { - return - } - this.processingQueues.add(tabId) - - const queue = this.commandQueues.get(tabId) - while (queue && queue.length > 0) { - const cmd = queue.shift()! - try { - const result = await cmd.execute() - cmd.resolve(result) - } catch (error) { - cmd.reject(error) - } - } - - this.processingQueues.delete(tabId) + return this.commandQueue.enqueue(tabId, execute) } } diff --git a/src/main/browser/cdp-command-queue.test.ts b/src/main/browser/cdp-command-queue.test.ts new file mode 100644 index 00000000000..04f919b900a --- /dev/null +++ b/src/main/browser/cdp-command-queue.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from 'vitest' +import { CdpCommandQueue } from './cdp-command-queue' + +function deferred(): { + promise: Promise + resolve: (value: T | PromiseLike) => void +} { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +describe('CdpCommandQueue', () => { + it('preserves ordinary per-tab command order', async () => { + const first = deferred() + const calls: number[] = [] + const queue = new CdpCommandQueue(() => new Error('full'), 4, 8) + const one = queue.enqueue('tab-1', async () => { + calls.push(1) + await first.promise + return 'one' + }) + const two = queue.enqueue('tab-1', async () => { + calls.push(2) + return 'two' + }) + + expect(calls).toEqual([1]) + first.resolve() + + await expect(Promise.all([one, two])).resolves.toEqual(['one', 'two']) + expect(calls).toEqual([1, 2]) + }) + + it('rejects commands beyond per-tab and aggregate queue caps', async () => { + const first = deferred() + const second = deferred() + const queue = new CdpCommandQueue(() => new Error('full'), 2, 3) + const active = queue.enqueue('tab-1', () => first.promise) + const activeTwo = queue.enqueue('tab-2', () => second.promise) + const queued = [ + queue.enqueue('tab-1', async () => 1), + queue.enqueue('tab-1', async () => 2), + queue.enqueue('tab-2', async () => 3) + ] + + await expect(queue.enqueue('tab-1', async () => 4)).rejects.toThrow('full') + await expect(queue.enqueue('tab-3', async () => 5)).rejects.toThrow('full') + + first.resolve() + second.resolve() + await expect(Promise.all([active, activeTwo, ...queued])).resolves.toEqual([ + undefined, + undefined, + 1, + 2, + 3 + ]) + }) + + it('rejects retained commands when a tab closes and continues a replacement queue', async () => { + const first = deferred() + const executeAfterClose = vi.fn(async () => 'stale') + const queue = new CdpCommandQueue(() => new Error('full'), 4, 8) + const active = queue.enqueue('tab-1', () => first.promise) + const stale = queue.enqueue('tab-1', executeAfterClose) + + queue.closeTab('tab-1', new Error('closed')) + await expect(stale).rejects.toThrow('closed') + expect(executeAfterClose).not.toHaveBeenCalled() + + const replacement = queue.enqueue('tab-1', async () => 'fresh') + first.resolve() + await expect(active).resolves.toBeUndefined() + await expect(replacement).resolves.toBe('fresh') + }) +}) diff --git a/src/main/browser/cdp-command-queue.ts b/src/main/browser/cdp-command-queue.ts new file mode 100644 index 00000000000..71ea74e68fc --- /dev/null +++ b/src/main/browser/cdp-command-queue.ts @@ -0,0 +1,80 @@ +export const CDP_MAX_QUEUED_COMMANDS_PER_TAB = 64 +export const CDP_MAX_QUEUED_COMMANDS_TOTAL = 512 + +type QueuedCdpCommand = { + execute: () => Promise + resolve: (value: unknown) => void + reject: (reason: unknown) => void +} + +export class CdpCommandQueue { + private readonly queues = new Map() + private readonly processing = new Set() + private queuedCount = 0 + + constructor( + private readonly createOverflowError: () => Error, + private readonly maxQueuedPerTab = CDP_MAX_QUEUED_COMMANDS_PER_TAB, + private readonly maxQueuedTotal = CDP_MAX_QUEUED_COMMANDS_TOTAL + ) {} + + enqueue(tabId: string, execute: () => Promise): Promise { + return new Promise((resolve, reject) => { + let queue = this.queues.get(tabId) + if ((queue?.length ?? 0) >= this.maxQueuedPerTab || this.queuedCount >= this.maxQueuedTotal) { + reject(this.createOverflowError()) + return + } + if (!queue) { + queue = [] + this.queues.set(tabId, queue) + } + queue.push({ + execute: execute as () => Promise, + resolve: resolve as (value: unknown) => void, + reject + }) + this.queuedCount += 1 + void this.process(tabId) + }) + } + + closeTab(tabId: string, error: Error): void { + const queue = this.queues.get(tabId) + if (!queue) { + return + } + this.queues.delete(tabId) + for (const command of queue.splice(0)) { + this.queuedCount = Math.max(0, this.queuedCount - 1) + command.reject(error) + } + } + + private async process(tabId: string): Promise { + if (this.processing.has(tabId)) { + return + } + this.processing.add(tabId) + const queue = this.queues.get(tabId) + try { + while (queue && queue.length > 0) { + const command = queue.shift()! + this.queuedCount = Math.max(0, this.queuedCount - 1) + try { + command.resolve(await command.execute()) + } catch (error) { + command.reject(error) + } + } + if (this.queues.get(tabId) === queue) { + this.queues.delete(tabId) + } + } finally { + this.processing.delete(tabId) + if ((this.queues.get(tabId)?.length ?? 0) > 0) { + void this.process(tabId) + } + } + } +} diff --git a/src/main/browser/cdp-event-memory-bounds.test.ts b/src/main/browser/cdp-event-memory-bounds.test.ts new file mode 100644 index 00000000000..7d30f6c0bfb --- /dev/null +++ b/src/main/browser/cdp-event-memory-bounds.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { + buildBoundedCdpConsoleEntry, + buildBoundedCdpInterceptedRequest, + buildBoundedCdpNetworkEntry, + CDP_MAX_CAPTURE_URL_CODE_UNITS, + CDP_MAX_CONSOLE_ARGUMENTS, + CDP_MAX_CONSOLE_TEXT_CODE_UNITS, + CDP_MAX_INTERCEPTED_HEADERS, + CDP_MAX_INTERCEPTED_METADATA_CODE_UNITS +} from './cdp-event-memory-bounds' + +describe('CDP event memory bounds', () => { + it('preserves ordinary console and network capture fields', () => { + expect( + buildBoundedCdpConsoleEntry({ + type: 'warning', + args: [{ value: 'hello' }, { value: 42 }], + timestamp: 123, + stackTrace: { callFrames: [{ url: 'https://example.com/app.js', lineNumber: 7 }] } + }) + ).toEqual({ + level: 'warning', + text: 'hello 42', + timestamp: 123, + url: 'https://example.com/app.js', + line: 7 + }) + expect( + buildBoundedCdpNetworkEntry( + { url: 'https://example.com/data', status: 200, mimeType: 'application/json' }, + 456 + ) + ).toEqual({ + url: 'https://example.com/data', + method: '', + status: 200, + mimeType: 'application/json', + size: 0, + timestamp: 456 + }) + }) + + it('bounds console argument count, joined text, and capture URLs', () => { + const entry = buildBoundedCdpConsoleEntry({ + args: Array.from({ length: CDP_MAX_CONSOLE_ARGUMENTS + 100 }, () => ({ + value: 'x'.repeat(100) + })), + stackTrace: { callFrames: [{ url: 'u'.repeat(CDP_MAX_CAPTURE_URL_CODE_UNITS + 100) }] } + }) + + expect(entry.text.length).toBeLessThanOrEqual(CDP_MAX_CONSOLE_TEXT_CODE_UNITS) + expect(entry.url).toHaveLength(CDP_MAX_CAPTURE_URL_CODE_UNITS) + }) + + it('rejects intercepted metadata beyond aggregate and header-count caps', () => { + expect( + buildBoundedCdpInterceptedRequest({ + requestId: 'request-1', + request: { url: 'https://example.com', headers: { accept: '*/*' } } + }) + ).toEqual({ + id: 'request-1', + url: 'https://example.com', + method: 'GET', + headers: { accept: '*/*' }, + resourceType: 'Other' + }) + + expect( + buildBoundedCdpInterceptedRequest({ + requestId: 'request-2', + request: { url: 'u'.repeat(CDP_MAX_INTERCEPTED_METADATA_CODE_UNITS + 1) } + }) + ).toBeNull() + + const headers = Object.fromEntries( + Array.from({ length: CDP_MAX_INTERCEPTED_HEADERS + 1 }, (_, index) => [`x-${index}`, 'v']) + ) + expect( + buildBoundedCdpInterceptedRequest({ requestId: 'request-3', request: { headers } }) + ).toBeNull() + }) +}) diff --git a/src/main/browser/cdp-event-memory-bounds.ts b/src/main/browser/cdp-event-memory-bounds.ts new file mode 100644 index 00000000000..c2fdfa61649 --- /dev/null +++ b/src/main/browser/cdp-event-memory-bounds.ts @@ -0,0 +1,145 @@ +import type { + BrowserConsoleEntry, + BrowserInterceptedRequest, + BrowserNetworkEntry +} from '../../shared/runtime-types' + +export const CDP_CAPTURE_LOG_LIMIT = 1000 +export const CDP_MAX_PAUSED_REQUESTS = 256 +export const CDP_MAX_IFRAME_SESSIONS = 1024 +export const CDP_MAX_CONSOLE_ARGUMENTS = 1024 +export const CDP_MAX_CONSOLE_TEXT_CODE_UNITS = 16 * 1024 +export const CDP_MAX_CAPTURE_URL_CODE_UNITS = 16 * 1024 +export const CDP_MAX_INTERCEPTED_HEADERS = 256 +export const CDP_MAX_INTERCEPTED_METADATA_CODE_UNITS = 64 * 1024 +const CDP_MAX_LEVEL_CODE_UNITS = 32 +const CDP_MAX_MIME_TYPE_CODE_UNITS = 1024 +const CDP_MAX_SESSION_ID_CODE_UNITS = 1024 + +type ConsoleEvent = { + type?: string + args?: { value?: unknown; description?: unknown }[] + timestamp?: number + stackTrace?: { callFrames?: { url?: string; lineNumber?: number }[] } +} + +type InterceptedRequestEvent = { + requestId: string + request: { url?: string; method?: string; headers?: Record } + resourceType?: string +} + +export function buildBoundedCdpConsoleEntry(event: ConsoleEvent): BrowserConsoleEntry { + const parts: string[] = [] + let remaining = CDP_MAX_CONSOLE_TEXT_CODE_UNITS + const args = event.args ?? [] + const argumentCount = Math.min(args.length, CDP_MAX_CONSOLE_ARGUMENTS) + for (let index = 0; index < argumentCount && remaining > 0; index++) { + if (index > 0) { + parts.push(' ') + remaining -= 1 + if (remaining <= 0) { + break + } + } + const raw = consoleArgumentText(args[index]) + const value = truncateCdpField(raw, remaining) + parts.push(value) + remaining -= value.length + } + return { + level: truncateCdpField(event.type ?? 'log', CDP_MAX_LEVEL_CODE_UNITS), + text: parts.join(''), + timestamp: event.timestamp ?? Date.now(), + url: truncateOptionalCdpField( + event.stackTrace?.callFrames?.[0]?.url, + CDP_MAX_CAPTURE_URL_CODE_UNITS + ), + line: event.stackTrace?.callFrames?.[0]?.lineNumber + } +} + +export function buildBoundedCdpNetworkEntry( + response: { url?: string; status?: number; mimeType?: string }, + timestamp?: number +): BrowserNetworkEntry { + return { + url: truncateCdpField(response.url ?? '', CDP_MAX_CAPTURE_URL_CODE_UNITS), + method: '', + status: response.status ?? 0, + mimeType: truncateCdpField(response.mimeType ?? '', CDP_MAX_MIME_TYPE_CODE_UNITS), + size: 0, + timestamp: timestamp ?? Date.now() + } +} + +export function buildBoundedCdpInterceptedRequest( + event: InterceptedRequestEvent +): BrowserInterceptedRequest | null { + let remaining = CDP_MAX_INTERCEPTED_METADATA_CODE_UNITS + const reserve = (value: string): boolean => { + if (value.length > remaining) { + return false + } + remaining -= value.length + return true + } + const url = event.request.url ?? '' + const method = event.request.method ?? 'GET' + const resourceType = event.resourceType ?? 'Other' + if (![event.requestId, url, method, resourceType].every(reserve)) { + return null + } + + const headers: Record = Object.create(null) as Record + let headerCount = 0 + for (const name in event.request.headers ?? {}) { + if (!Object.hasOwn(event.request.headers ?? {}, name)) { + continue + } + const value = event.request.headers![name] + if ( + headerCount >= CDP_MAX_INTERCEPTED_HEADERS || + typeof value !== 'string' || + !reserve(name) || + !reserve(value) + ) { + return null + } + headers[name] = value + headerCount += 1 + } + return { id: event.requestId, url, method, headers, resourceType } +} + +export function isBoundedCdpIframeSession(frameId: string, sessionId: string): boolean { + return ( + frameId.length <= CDP_MAX_SESSION_ID_CODE_UNITS && + sessionId.length <= CDP_MAX_SESSION_ID_CODE_UNITS + ) +} + +function consoleArgumentText(arg: { value?: unknown; description?: unknown } | undefined): string { + const value = arg?.value ?? arg?.description ?? '' + if (typeof value === 'string') { + return value + } + if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') { + return String(value) + } + return '' +} + +function truncateOptionalCdpField( + value: string | undefined, + maxLength: number +): string | undefined { + return value === undefined ? undefined : truncateCdpField(value, maxLength) +} + +function truncateCdpField(value: string, maxLength: number): string { + if (value.length <= maxLength) { + return value + } + return `${value.slice(0, Math.max(0, maxLength - 1))}…` +} diff --git a/src/main/browser/cdp-print-to-pdf.test.ts b/src/main/browser/cdp-print-to-pdf.test.ts index e6f2163b5dd..8391599fd9a 100644 --- a/src/main/browser/cdp-print-to-pdf.test.ts +++ b/src/main/browser/cdp-print-to-pdf.test.ts @@ -1,8 +1,22 @@ import { describe, it, expect, vi, afterEach } from 'vitest' -import { buildPrintToPdfOptions, CdpPdfStreamStore } from './cdp-print-to-pdf' +import { + buildPrintToPdfOptions, + CDP_PDF_MAX_RETAINED_STREAMS, + CDP_PDF_MEMORY_LIMIT_ERROR, + CdpPdfRetentionBudget, + CdpPdfStreamStore +} from './cdp-print-to-pdf' +import type { CdpPdfStreamStoreOptions } from './cdp-print-to-pdf' const DEFAULT_MARGIN_INCHES = 1 / 2.54 const TTL_MS = 5 * 60 * 1000 +const stores = new Set() + +function createStore(options: CdpPdfStreamStoreOptions = {}): CdpPdfStreamStore { + const store = new CdpPdfStreamStore(options) + stores.add(store) + return store +} describe('buildPrintToPdfOptions', () => { it('returns empty options for empty params', () => { @@ -78,11 +92,15 @@ describe('buildPrintToPdfOptions', () => { describe('CdpPdfStreamStore', () => { afterEach(() => { + for (const store of stores) { + store.clear() + } + stores.clear() vi.useRealTimers() }) it('claims only its own handles', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('pdf')) expect(store.ownsHandle({ handle })).toBe(true) @@ -93,12 +111,12 @@ describe('CdpPdfStreamStore', () => { }) it('mints distinct handles per stream', () => { - const store = new CdpPdfStreamStore() + const store = createStore() expect(store.create(Buffer.from('a'))).not.toBe(store.create(Buffer.from('b'))) }) it('reads sequential chunks and reports eof', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abcdef')) expect(store.read({ handle, size: 2 })).toEqual({ @@ -112,7 +130,7 @@ describe('CdpPdfStreamStore', () => { }) it('honors an explicit read offset', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abcdef')) expect(store.read({ handle, offset: 4 })).toEqual({ @@ -122,20 +140,20 @@ describe('CdpPdfStreamStore', () => { }) it('returns an empty eof chunk when reading past the end', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abc')) expect(store.read({ handle, offset: 99 })).toEqual({ data: '', eof: true }) }) it('returns null for unknown handles', () => { - const store = new CdpPdfStreamStore() + const store = createStore() expect(store.read({ handle: 'nope' })).toBeNull() expect(store.read({})).toBeNull() }) it('drops a stream on close', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abc')) store.close({ handle }) @@ -143,7 +161,7 @@ describe('CdpPdfStreamStore', () => { }) it('drops all streams on clear', () => { - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abc')) store.clear() @@ -152,7 +170,7 @@ describe('CdpPdfStreamStore', () => { it('evicts an abandoned stream after the TTL', () => { vi.useFakeTimers() - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abc')) vi.advanceTimersByTime(TTL_MS + 1) @@ -161,7 +179,7 @@ describe('CdpPdfStreamStore', () => { it('refreshes the TTL on each read', () => { vi.useFakeTimers() - const store = new CdpPdfStreamStore() + const store = createStore() const handle = store.create(Buffer.from('abcdef')) vi.advanceTimersByTime(TTL_MS - 1) @@ -169,4 +187,67 @@ describe('CdpPdfStreamStore', () => { vi.advanceTimersByTime(TTL_MS - 1) expect(store.read({ handle, size: 1 })).not.toBeNull() }) + + it('rejects aggregate stream retention past its count or byte budget', () => { + const store = createStore({ maxStreams: 2, maxRetainedBytes: 5 }) + const first = store.create(Buffer.from('ab')) + store.create(Buffer.from('cde')) + + expect(() => store.create(Buffer.alloc(0))).toThrow(CDP_PDF_MEMORY_LIMIT_ERROR) + store.close({ handle: first }) + expect(() => store.create(Buffer.from('fg'))).not.toThrow() + expect(() => createStore({ maxRetainedBytes: 5 }).create(Buffer.alloc(6))).toThrow( + CDP_PDF_MEMORY_LIMIT_ERROR + ) + store.clear() + }) + + it('clamps oversized IO.read requests to a bounded base64 chunk', () => { + const store = createStore({ maxReadChunkBytes: 2 }) + const handle = store.create(Buffer.from('abcdef')) + + expect(store.read({ handle, size: Number.MAX_SAFE_INTEGER })).toEqual({ + data: Buffer.from('ab').toString('base64'), + eof: false + }) + store.clear() + }) + + it('shares the default retained-stream cap across independent proxy stores', () => { + const admitted = Array.from({ length: CDP_PDF_MAX_RETAINED_STREAMS }, () => { + const store = createStore() + return { handle: store.create(Buffer.alloc(0)), store } + }) + const overflow = createStore() + + expect(() => overflow.create(Buffer.alloc(0))).toThrow(CDP_PDF_MEMORY_LIMIT_ERROR) + + admitted[0]!.store.close({ handle: admitted[0]!.handle }) + expect(() => overflow.create(Buffer.alloc(0))).not.toThrow() + }) + + it('releases a shared byte budget on close, expiry, clear, and failed admission', () => { + vi.useFakeTimers() + const budget = new CdpPdfRetentionBudget(4, 5) + const first = createStore({ maxRetainedBytes: 5, retentionBudget: budget }) + const second = createStore({ maxRetainedBytes: 5, retentionBudget: budget }) + const firstHandle = first.create(Buffer.from('abc')) + const secondHandle = second.create(Buffer.from('de')) + + expect(budget.inspect()).toEqual({ retainedBytes: 5, retainedStreams: 2 }) + expect(() => second.create(Buffer.from('f'))).toThrow(CDP_PDF_MEMORY_LIMIT_ERROR) + expect(budget.inspect()).toEqual({ retainedBytes: 5, retainedStreams: 2 }) + + first.close({ handle: firstHandle }) + expect(budget.inspect()).toEqual({ retainedBytes: 2, retainedStreams: 1 }) + second.close({ handle: secondHandle }) + const expiring = first.create(Buffer.from('xy')) + expect(first.read({ handle: expiring })).not.toBeNull() + vi.advanceTimersByTime(TTL_MS + 1) + expect(budget.inspect()).toEqual({ retainedBytes: 0, retainedStreams: 0 }) + + first.create(Buffer.from('z')) + first.clear() + expect(budget.inspect()).toEqual({ retainedBytes: 0, retainedStreams: 0 }) + }) }) diff --git a/src/main/browser/cdp-print-to-pdf.ts b/src/main/browser/cdp-print-to-pdf.ts index 643013377d8..bc0999012cb 100644 --- a/src/main/browser/cdp-print-to-pdf.ts +++ b/src/main/browser/cdp-print-to-pdf.ts @@ -5,6 +5,9 @@ const PDF_DEFAULT_MARGIN_INCHES = 1 / 2.54 const PDF_STREAM_CHUNK_BYTES = 1024 * 1024 const PDF_STREAM_HANDLE_PREFIX = 'orca-pdf-' const PDF_STREAM_TTL_MS = 5 * 60 * 1000 +export const CDP_PDF_MAX_RETAINED_STREAMS = 8 +export const CDP_PDF_MAX_RETAINED_BYTES = 64 * 1024 * 1024 +export const CDP_PDF_MEMORY_LIMIT_ERROR = 'PDF exceeds the browser automation memory limit' function finiteNumber(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null @@ -80,6 +83,7 @@ type PdfStream = { data: Buffer offset: number cleanupTimer: ReturnType + releaseRetention: () => void } export type PdfStreamChunk = { @@ -87,6 +91,57 @@ export type PdfStreamChunk = { eof: boolean } +export type CdpPdfStreamStoreOptions = { + maxStreams?: number + maxRetainedBytes?: number + maxReadChunkBytes?: number + retentionBudget?: CdpPdfRetentionBudget +} + +export function assertCdpPdfWithinMemoryLimit(data: Buffer): void { + if (data.length > CDP_PDF_MAX_RETAINED_BYTES) { + throw new Error(CDP_PDF_MEMORY_LIMIT_ERROR) + } +} + +export class CdpPdfRetentionBudget { + private retainedBytes = 0 + private retainedStreams = 0 + + constructor( + private readonly maxStreams = CDP_PDF_MAX_RETAINED_STREAMS, + private readonly maxRetainedBytes = CDP_PDF_MAX_RETAINED_BYTES + ) {} + + retain(bytes: number): (() => void) | null { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + this.retainedStreams >= this.maxStreams || + this.retainedBytes + bytes > this.maxRetainedBytes + ) { + return null + } + this.retainedStreams += 1 + this.retainedBytes += bytes + let released = false + return () => { + if (released) { + return + } + released = true + this.retainedStreams = Math.max(0, this.retainedStreams - 1) + this.retainedBytes = Math.max(0, this.retainedBytes - bytes) + } + } + + inspect(): { retainedBytes: number; retainedStreams: number } { + return { retainedBytes: this.retainedBytes, retainedStreams: this.retainedStreams } + } +} + +const processPdfRetentionBudget = new CdpPdfRetentionBudget() + /** * Holds the PDF buffers produced for CDP `transferMode: "ReturnAsStream"` and * serves them back through `IO.read` / `IO.close`, the path Playwright's @@ -97,7 +152,25 @@ export type PdfStreamChunk = { export class CdpPdfStreamStore { private readonly streams = new Map() private readonly handlePrefix = `${PDF_STREAM_HANDLE_PREFIX}${randomUUID()}-` + private readonly maxStreams: number + private readonly maxRetainedBytes: number + private readonly maxReadChunkBytes: number + private readonly retentionBudget: CdpPdfRetentionBudget private nextId = 0 + private retainedBytes = 0 + + constructor(options: CdpPdfStreamStoreOptions = {}) { + this.maxStreams = Math.max(0, Math.floor(options.maxStreams ?? CDP_PDF_MAX_RETAINED_STREAMS)) + this.maxRetainedBytes = Math.max( + 0, + Math.floor(options.maxRetainedBytes ?? CDP_PDF_MAX_RETAINED_BYTES) + ) + this.maxReadChunkBytes = Math.max( + 1, + Math.floor(options.maxReadChunkBytes ?? PDF_STREAM_CHUNK_BYTES) + ) + this.retentionBudget = options.retentionBudget ?? processPdfRetentionBudget + } /** True when `params.handle` names one of this store's streams. */ ownsHandle(params: Record): boolean { @@ -105,12 +178,30 @@ export class CdpPdfStreamStore { } create(data: Buffer): string { + if ( + data.length > this.maxRetainedBytes || + this.streams.size >= this.maxStreams || + this.retainedBytes + data.length > this.maxRetainedBytes + ) { + throw new Error(CDP_PDF_MEMORY_LIMIT_ERROR) + } + const releaseRetention = this.retentionBudget.retain(data.length) + if (!releaseRetention) { + throw new Error(CDP_PDF_MEMORY_LIMIT_ERROR) + } const handle = `${this.handlePrefix}${++this.nextId}` - this.streams.set(handle, { - data, - offset: 0, - cleanupTimer: this.scheduleCleanup(handle) - }) + let cleanupTimer: ReturnType | null = null + try { + cleanupTimer = this.scheduleCleanup(handle) + this.streams.set(handle, { data, offset: 0, cleanupTimer, releaseRetention }) + } catch (error) { + if (cleanupTimer) { + clearTimeout(cleanupTimer) + } + releaseRetention() + throw error + } + this.retainedBytes += data.length return handle } @@ -130,8 +221,8 @@ export class CdpPdfStreamStore { const requestedSize = finiteNumber(params.size) const size = requestedSize !== null && requestedSize > 0 - ? Math.floor(requestedSize) - : PDF_STREAM_CHUNK_BYTES + ? Math.min(Math.floor(requestedSize), this.maxReadChunkBytes) + : this.maxReadChunkBytes const start = Math.min(stream.offset, stream.data.length) const end = Math.min(start + size, stream.data.length) const chunk = stream.data.subarray(start, end) @@ -146,10 +237,9 @@ export class CdpPdfStreamStore { } clear(): void { - for (const stream of this.streams.values()) { - clearTimeout(stream.cleanupTimer) + for (const handle of this.streams.keys()) { + this.delete(handle) } - this.streams.clear() } private scheduleCleanup(handle: string): ReturnType { @@ -173,5 +263,7 @@ export class CdpPdfStreamStore { } clearTimeout(stream.cleanupTimer) this.streams.delete(handle) + this.retainedBytes = Math.max(0, this.retainedBytes - stream.data.length) + stream.releaseRetention() } } diff --git a/src/main/browser/cdp-screenshot.test.ts b/src/main/browser/cdp-screenshot.test.ts index 97859f7297f..23409d48598 100644 --- a/src/main/browser/cdp-screenshot.test.ts +++ b/src/main/browser/cdp-screenshot.test.ts @@ -1,9 +1,18 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { captureFullPageScreenshot, captureScreenshot } from './cdp-screenshot' +import { resetBrowserScreenshotAdmissionForTests } from './browser-screenshot-admission' +import { + BROWSER_SCREENSHOT_BUSY_ERROR, + BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES, + BROWSER_SCREENSHOT_MAX_DIMENSION_PX, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' function createMockWebContents() { return { + once: vi.fn(), + removeListener: vi.fn(), isDestroyed: vi.fn(() => false), invalidate: vi.fn(), capturePage: vi.fn(), @@ -16,6 +25,7 @@ function createMockWebContents() { describe('captureScreenshot', () => { afterEach(() => { + resetBrowserScreenshotAdmissionForTests() vi.useRealTimers() }) @@ -36,6 +46,102 @@ describe('captureScreenshot', () => { expect(onError).not.toHaveBeenCalled() }) + it('rejects concurrent capture overload without retaining another native promise', async () => { + const webContents = createMockWebContents() + const resolvers: ((result: { data: string }) => void)[] = [] + webContents.debugger.sendCommand.mockImplementation( + () => + new Promise<{ data: string }>((resolve) => { + resolvers.push(resolve) + }) + ) + const results = Array.from({ length: BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES + 1 }, () => + vi.fn() + ) + const errors = results.map(() => vi.fn()) + + for (let index = 0; index < results.length; index += 1) { + captureScreenshot(webContents as never, { format: 'png' }, results[index]!, errors[index]!) + } + await Promise.resolve() + await Promise.resolve() + + expect(webContents.debugger.sendCommand).toHaveBeenCalledTimes( + BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES + ) + expect(errors.at(-1)).toHaveBeenCalledWith(BROWSER_SCREENSHOT_BUSY_ERROR) + for (const resolve of resolvers) { + resolve({ data: 'cG5n' }) + } + await Promise.resolve() + await Promise.resolve() + expect( + results + .slice(0, BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES) + .every((callback) => callback.mock.calls.length === 1) + ).toBe(true) + }) + + it('keeps timed-out native captures admitted until Chromium actually settles', async () => { + vi.useFakeTimers() + const webContents = createMockWebContents() + webContents.debugger.sendCommand.mockImplementation(() => new Promise(() => {})) + webContents.capturePage.mockResolvedValue({ + isEmpty: () => false, + getSize: () => ({ width: 800, height: 600 }), + toPNG: () => Buffer.from('fallback-png') + }) + const results = Array.from({ length: BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES }, () => + vi.fn() + ) + const errors = results.map(() => vi.fn()) + + for (let index = 0; index < results.length; index += 1) { + captureScreenshot(webContents as never, { format: 'png' }, results[index]!, errors[index]!) + } + await vi.advanceTimersByTimeAsync(8000) + + expect(results.every((callback) => callback.mock.calls.length === 0)).toBe(true) + expect( + errors.every((callback) => callback.mock.calls[0]?.[0] === BROWSER_SCREENSHOT_BUSY_ERROR) + ).toBe(true) + expect(webContents.capturePage).not.toHaveBeenCalled() + const overloadError = vi.fn() + captureScreenshot(webContents as never, { format: 'png' }, vi.fn(), overloadError) + expect(overloadError).toHaveBeenCalledWith(BROWSER_SCREENSHOT_BUSY_ERROR) + expect(webContents.debugger.sendCommand).toHaveBeenCalledTimes( + BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES + ) + }) + + it('retains admission for a fallback that outlives the failed CDP capture', async () => { + vi.useFakeTimers() + const webContents = createMockWebContents() + const commandRejectors: ((error: Error) => void)[] = [] + webContents.debugger.sendCommand.mockImplementation( + () => + new Promise((_, reject) => { + commandRejectors.push(reject) + }) + ) + webContents.capturePage.mockImplementation(() => new Promise(() => {})) + const firstError = vi.fn() + + captureScreenshot(webContents as never, { format: 'png' }, vi.fn(), firstError) + await vi.advanceTimersByTimeAsync(8000) + commandRejectors[0]!(new Error('CDP failed after timeout')) + await Promise.resolve() + await Promise.resolve() + + expect(firstError).toHaveBeenCalledWith('CDP failed after timeout') + const secondError = vi.fn() + captureScreenshot(webContents as never, { format: 'png' }, vi.fn(), secondError) + await vi.advanceTimersByTimeAsync(8000) + + expect(secondError).toHaveBeenCalledWith(BROWSER_SCREENSHOT_BUSY_ERROR) + expect(webContents.capturePage).toHaveBeenCalledOnce() + }) + it('falls back to capturePage when Page.captureScreenshot stalls', async () => { vi.useFakeTimers() @@ -43,6 +149,7 @@ describe('captureScreenshot', () => { webContents.debugger.sendCommand.mockImplementation(() => new Promise(() => {})) webContents.capturePage.mockResolvedValueOnce({ isEmpty: () => false, + getSize: () => ({ width: 800, height: 600 }), toPNG: () => Buffer.from('fallback-png') }) const onResult = vi.fn() @@ -63,6 +170,7 @@ describe('captureScreenshot', () => { const croppedImage = { isEmpty: () => false, + getSize: () => ({ width: 60, height: 80 }), toPNG: () => Buffer.from('cropped-png') } const webContents = createMockWebContents() @@ -127,6 +235,47 @@ describe('captureScreenshot', () => { ) }) + it('rejects an oversized clip before asking Chromium to capture it', () => { + const webContents = createMockWebContents() + const onResult = vi.fn() + const onError = vi.fn() + + captureScreenshot( + webContents as never, + { + format: 'png', + clip: { x: 0, y: 0, width: BROWSER_SCREENSHOT_MAX_DIMENSION_PX + 1, height: 1 } + }, + onResult, + onError + ) + + expect(onError).toHaveBeenCalledWith(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + expect(onResult).not.toHaveBeenCalled() + expect(webContents.debugger.sendCommand).not.toHaveBeenCalled() + }) + + it('rejects an oversized fallback bitmap before encoding it', async () => { + vi.useFakeTimers() + const toPNG = vi.fn(() => Buffer.from('unused')) + const webContents = createMockWebContents() + webContents.debugger.sendCommand.mockImplementation(() => new Promise(() => {})) + webContents.capturePage.mockResolvedValueOnce({ + isEmpty: () => false, + getSize: () => ({ width: BROWSER_SCREENSHOT_MAX_DIMENSION_PX + 1, height: 1 }), + toPNG + }) + const onResult = vi.fn() + const onError = vi.fn() + + captureScreenshot(webContents as never, { format: 'png' }, onResult, onError) + await vi.advanceTimersByTimeAsync(8000) + + expect(onError).toHaveBeenCalledWith(BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + expect(onResult).not.toHaveBeenCalled() + expect(toPNG).not.toHaveBeenCalled() + }) + it('ignores the fallback result when CDP settles first after the timeout fires', async () => { vi.useFakeTimers() @@ -237,6 +386,11 @@ describe('captureScreenshot', () => { }) describe('captureFullPageScreenshot', () => { + afterEach(() => { + resetBrowserScreenshotAdmissionForTests() + vi.useRealTimers() + }) + it('uses cssContentSize so HiDPI pages are captured at the real page size', async () => { const webContents = createMockWebContents() webContents.debugger.sendCommand.mockImplementation((method: string) => { @@ -288,4 +442,43 @@ describe('captureFullPageScreenshot', () => { clip: { x: 0, y: 0, width: 800, height: 1600, scale: 1 } }) }) + + it('rejects oversized full-page layout bounds before capture allocation', async () => { + const webContents = createMockWebContents() + webContents.debugger.sendCommand.mockResolvedValueOnce({ + cssContentSize: { width: BROWSER_SCREENSHOT_MAX_DIMENSION_PX + 1, height: 1 } + }) + + await expect(captureFullPageScreenshot(webContents as never, 'png')).rejects.toThrow( + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + ) + expect(webContents.debugger.sendCommand).toHaveBeenCalledOnce() + expect(webContents.debugger.sendCommand).toHaveBeenCalledWith('Page.getLayoutMetrics', {}) + }) + + it('bounds hung full-page metric commands across logical timeouts', async () => { + vi.useFakeTimers() + const webContents = createMockWebContents() + webContents.debugger.sendCommand.mockImplementation(() => new Promise(() => {})) + const active = Array.from({ length: BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES }, () => + captureFullPageScreenshot(webContents as never, 'png') + ) + const activeAssertions = active.map((capture) => + expect(capture).rejects.toThrow( + 'Screenshot timed out — the browser tab may not be visible or the window may not have focus.' + ) + ) + + await expect(captureFullPageScreenshot(webContents as never, 'png')).rejects.toThrow( + BROWSER_SCREENSHOT_BUSY_ERROR + ) + await vi.advanceTimersByTimeAsync(8000) + await Promise.all(activeAssertions) + await expect(captureFullPageScreenshot(webContents as never, 'png')).rejects.toThrow( + BROWSER_SCREENSHOT_BUSY_ERROR + ) + expect(webContents.debugger.sendCommand).toHaveBeenCalledTimes( + BROWSER_SCREENSHOT_MAX_CONCURRENT_CAPTURES + ) + }) }) diff --git a/src/main/browser/cdp-screenshot.ts b/src/main/browser/cdp-screenshot.ts index 7ba4315d7d9..2abba87c152 100644 --- a/src/main/browser/cdp-screenshot.ts +++ b/src/main/browser/cdp-screenshot.ts @@ -1,83 +1,20 @@ import type { WebContents } from 'electron' +import { + assertBrowserScreenshotBase64, + assertBrowserScreenshotGeometry, + BROWSER_SCREENSHOT_BUSY_ERROR, + BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR +} from './browser-screenshot-limits' +import { + startBrowserFallbackCapture, + startBrowserScreenshotCommand +} from './browser-screenshot-admission' +import { encodeNativeImageScreenshot } from './native-image-screenshot-encoder' const SCREENSHOT_TIMEOUT_MS = 8000 const FALLBACK_CAPTURE_TIMEOUT_MS = 1000 const SCREENSHOT_TIMEOUT_MESSAGE = 'Screenshot timed out — the browser tab may not be visible or the window may not have focus.' - -function applyFallbackClip( - image: Electron.NativeImage, - params: Record | undefined -): Electron.NativeImage | null { - if (params?.captureBeyondViewport) { - // Why: capturePage() can only see the currently painted viewport. If the - // caller asked for beyond-viewport pixels, returning a viewport-sized image - // would silently lie about what was captured. - return null - } - - const clip = params?.clip - if (!clip || typeof clip !== 'object') { - return image - } - const clipRect = clip as Record - - const x = typeof clipRect.x === 'number' ? clipRect.x : Number.NaN - const y = typeof clipRect.y === 'number' ? clipRect.y : Number.NaN - const width = typeof clipRect.width === 'number' ? clipRect.width : Number.NaN - const height = typeof clipRect.height === 'number' ? clipRect.height : Number.NaN - const scale = - typeof clipRect.scale === 'number' && Number.isFinite(clipRect.scale) && clipRect.scale > 0 - ? clipRect.scale - : 1 - - if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) { - return null - } - - const cropRect = { - x: Math.round(x * scale), - y: Math.round(y * scale), - width: Math.round(width * scale), - height: Math.round(height * scale) - } - const imageSize = image.getSize() - if ( - cropRect.x < 0 || - cropRect.y < 0 || - cropRect.width <= 0 || - cropRect.height <= 0 || - cropRect.x + cropRect.width > imageSize.width || - cropRect.y + cropRect.height > imageSize.height - ) { - return null - } - - return image.crop(cropRect) -} - -function encodeNativeImageScreenshot( - image: Electron.NativeImage, - params: Record | undefined -): { data: string } | null { - if (image.isEmpty()) { - return null - } - - const clippedImage = applyFallbackClip(image, params) - if (!clippedImage || clippedImage.isEmpty()) { - return null - } - - const format = params?.format === 'jpeg' ? 'jpeg' : 'png' - const quality = - typeof params?.quality === 'number' && Number.isFinite(params.quality) - ? Math.max(0, Math.min(100, Math.round(params.quality))) - : undefined - const buffer = format === 'jpeg' ? clippedImage.toJPEG(quality ?? 90) : clippedImage.toPNG() - return { data: buffer.toString('base64') } -} - function getLayoutClip(metrics: { cssContentSize?: { width?: number; height?: number } contentSize?: { width?: number; height?: number } @@ -100,25 +37,31 @@ function getLayoutClip(metrics: { return null } - return { + const clip = { x: 0, y: 0, width: Math.ceil(width), height: Math.ceil(height), scale: 1 } + assertBrowserScreenshotGeometry(clip.width, clip.height) + return clip } async function sendCommandWithTimeout( webContents: WebContents, - method: string, + method: 'Page.captureScreenshot' | 'Page.getLayoutMetrics', params: Record | undefined, timeoutMessage: string ): Promise { + const command = startBrowserScreenshotCommand(webContents, method, params ?? {}) + if (!command) { + throw new Error(BROWSER_SCREENSHOT_BUSY_ERROR) + } let timer: NodeJS.Timeout | null = null try { return await Promise.race([ - webContents.debugger.sendCommand(method, params ?? {}) as Promise, + command, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(timeoutMessage)), SCREENSHOT_TIMEOUT_MS) }) @@ -160,17 +103,27 @@ export async function captureFullPageScreenshot( const { data } = await sendCommandWithTimeout<{ data: string }>( webContents, 'Page.captureScreenshot', - { - format, - captureBeyondViewport: true, - clip - }, + { format, captureBeyondViewport: true, clip }, SCREENSHOT_TIMEOUT_MESSAGE ) + assertBrowserScreenshotBase64(data) return { data, format } } +function assertCaptureClipWithinLimit(params: Record | undefined): void { + const clip = params?.clip + if (!clip || typeof clip !== 'object') { + return + } + const values = clip as Record + if (typeof values.width !== 'number' || typeof values.height !== 'number') { + return + } + const scale = typeof values.scale === 'number' ? values.scale : 1 + assertBrowserScreenshotGeometry(values.width, values.height, scale) +} + // Why: Electron's capturePage() is unreliable on webview guests — the compositor // may not produce frames when the webview panel is inactive, unfocused, or in a // split-pane layout. Instead, use the debugger's Page.captureScreenshot which @@ -192,7 +145,12 @@ export function captureScreenshot( onError('Debugger not attached') return } - + try { + assertCaptureClipWithinLimit(params) + } catch (error) { + onError(error instanceof Error ? error.message : BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + return + } const screenshotParams: Record = {} if (params?.format) { screenshotParams.format = params.format @@ -209,6 +167,15 @@ export function captureScreenshot( if (params?.fromSurface != null) { screenshotParams.fromSurface = params.fromSurface } + const capture = startBrowserScreenshotCommand( + webContents, + 'Page.captureScreenshot', + screenshotParams + ) + if (!capture) { + onError(BROWSER_SCREENSHOT_BUSY_ERROR) + return + } let settled = false let timeoutTimer: ReturnType | null = null @@ -227,6 +194,16 @@ export function captureScreenshot( if (settled) { return } + const data = + result && typeof result === 'object' ? (result as { data?: unknown }).data : undefined + try { + if (typeof data === 'string') { + assertBrowserScreenshotBase64(data) + } + } catch (error) { + settleError(error instanceof Error ? error.message : BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR) + return + } settled = true clearTimers() onResult(result) @@ -257,42 +234,48 @@ export function captureScreenshot( () => settleError(SCREENSHOT_TIMEOUT_MESSAGE), FALLBACK_CAPTURE_TIMEOUT_MS ) - void Promise.resolve() - .then(() => webContents.capturePage()) - .then( - (image) => { - if (settled) { - return - } - if (fallbackTimer) { - clearTimeout(fallbackTimer) - fallbackTimer = null - } - let fallback: { data: string } | null = null - try { - fallback = encodeNativeImageScreenshot(image, params) - } catch { - settleError(SCREENSHOT_TIMEOUT_MESSAGE) - return - } - if (fallback) { - settleResult(fallback) - return - } - settleError(SCREENSHOT_TIMEOUT_MESSAGE) - }, - () => { - if (fallbackTimer) { - clearTimeout(fallbackTimer) - fallbackTimer = null - } - settleError(SCREENSHOT_TIMEOUT_MESSAGE) + const fallbackCapture = startBrowserFallbackCapture(webContents) + if (!fallbackCapture) { + settleError(BROWSER_SCREENSHOT_BUSY_ERROR) + return + } + void fallbackCapture.then( + (image) => { + if (settled) { + return } - ) + if (fallbackTimer) { + clearTimeout(fallbackTimer) + fallbackTimer = null + } + let fallback: { data: string } | null = null + try { + fallback = encodeNativeImageScreenshot(image, params) + } catch (error) { + settleError( + error instanceof Error && error.message === BROWSER_SCREENSHOT_MEMORY_LIMIT_ERROR + ? error.message + : SCREENSHOT_TIMEOUT_MESSAGE + ) + return + } + if (fallback) { + settleResult(fallback) + return + } + settleError(SCREENSHOT_TIMEOUT_MESSAGE) + }, + () => { + if (fallbackTimer) { + clearTimeout(fallbackTimer) + fallbackTimer = null + } + settleError(SCREENSHOT_TIMEOUT_MESSAGE) + } + ) }, SCREENSHOT_TIMEOUT_MS) - dbg - .sendCommand('Page.captureScreenshot', screenshotParams) + void capture .then((result) => settleResult(result)) .catch((err) => settleError((err as Error).message)) } diff --git a/src/main/browser/cdp-ws-proxy.test.ts b/src/main/browser/cdp-ws-proxy.test.ts index 5c366eda727..522efddf022 100644 --- a/src/main/browser/cdp-ws-proxy.test.ts +++ b/src/main/browser/cdp-ws-proxy.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import WebSocket from 'ws' -import { CdpWsProxy } from './cdp-ws-proxy' +import { + CDP_PROXY_MAX_MESSAGE_BYTES, + CDP_PROXY_MAX_TCP_CONNECTIONS, + CdpWsProxy +} from './cdp-ws-proxy' import { connect, createMockWebContents, @@ -52,6 +56,16 @@ describe('CdpWsProxy', () => { expect(server.listenerCount('error')).toBeLessThanOrEqual(1) }) + it('caps pre-upgrade sockets and WebSocket message materialization', () => { + const internals = proxy as unknown as { + httpServer: { maxConnections: number } + wss: { options: { maxPayload: number } } + } + + expect(internals.httpServer.maxConnections).toBe(CDP_PROXY_MAX_TCP_CONNECTIONS) + expect(internals.wss.options.maxPayload).toBe(CDP_PROXY_MAX_MESSAGE_BYTES) + }) + it('attaches debugger on start', () => { expect(mock.webContents.debugger.attach).toHaveBeenCalledWith('1.3') }) @@ -744,6 +758,37 @@ describe('CdpWsProxy', () => { createSpy.mockRestore() }) + it('rejects PDF print overload instead of retaining unlimited native promises', async () => { + const resolvePrints: ((buffer: Buffer) => void)[] = [] + mock.webContents.printToPDF.mockImplementation( + () => + new Promise>((resolve) => { + resolvePrints.push(resolve) + }) + ) + const client = await connect(endpoint) + const responses: Record[] = [] + client.on('message', (data) => responses.push(JSON.parse(data.toString()))) + + for (let id = 31; id <= 33; id += 1) { + client.send(JSON.stringify({ id, method: 'Page.printToPDF' })) + } + + await vi.waitFor(() => + expect(responses.find((response) => response.id === 33)).toEqual({ + id: 33, + error: { code: -32000, message: 'Too many PDF print requests are already running' } + }) + ) + expect(mock.webContents.printToPDF).toHaveBeenCalledTimes(2) + + for (const resolve of resolvePrints) { + resolve(Buffer.from('%PDF-bounded')) + } + await vi.waitFor(() => expect(responses).toHaveLength(3)) + client.close() + }) + it('forwards non-PDF IO streams to the debugger', async () => { mock.webContents.debugger.sendCommand .mockResolvedValueOnce({ data: 'trace-data', eof: false }) diff --git a/src/main/browser/cdp-ws-proxy.ts b/src/main/browser/cdp-ws-proxy.ts index 99c1d554654..1e7277d2598 100644 --- a/src/main/browser/cdp-ws-proxy.ts +++ b/src/main/browser/cdp-ws-proxy.ts @@ -3,11 +3,18 @@ import { WebSocketServer, WebSocket } from 'ws' import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http' import type { WebContents } from 'electron' import { captureScreenshot } from './cdp-screenshot' -import { buildPrintToPdfOptions, CdpPdfStreamStore } from './cdp-print-to-pdf' +import { + assertCdpPdfWithinMemoryLimit, + buildPrintToPdfOptions, + CdpPdfStreamStore +} from './cdp-print-to-pdf' import { ANTI_DETECTION_SCRIPT } from './anti-detection' import { acquireElectronDebugger, type ElectronDebuggerLease } from './electron-debugger-lease' +import { BROWSER_PDF_BUSY_ERROR, startBrowserPdfPrint } from './browser-pdf-admission' const LIFECYCLE_PRIMING_TIMEOUT_MS = 1_000 +export const CDP_PROXY_MAX_MESSAGE_BYTES = 16 * 1024 * 1024 +export const CDP_PROXY_MAX_TCP_CONNECTIONS = 16 export class CdpWsProxy { // Why: holds each session's last DOM.focus params to replay right before the next @@ -40,7 +47,12 @@ export class CdpWsProxy { await this.attachDebugger() return new Promise((resolve, reject) => { this.httpServer = createServer((req, res) => this.handleHttpRequest(req, res)) - this.wss = new WebSocketServer({ server: this.httpServer }) + // Why: raw sockets that never upgrade must not bypass the proxy's single active WebSocket. + this.httpServer.maxConnections = CDP_PROXY_MAX_TCP_CONNECTIONS + this.wss = new WebSocketServer({ + server: this.httpServer, + maxPayload: CDP_PROXY_MAX_MESSAGE_BYTES + }) const failStart = (error: Error): void => { this.httpServer?.removeListener('error', onListenError) this.wss?.close() @@ -635,8 +647,13 @@ export class CdpWsProxy { this.sendError(clientId, 'Browser tab is no longer available', client) return } + const print = startBrowserPdfPrint(this.webContents, buildPrintToPdfOptions(params)) + if (!print) { + this.sendError(clientId, BROWSER_PDF_BUSY_ERROR, client) + return + } try { - const pdf = await this.webContents.printToPDF(buildPrintToPdfOptions(params)) + const pdf = await print // Why: printToPDF can resolve after the client disconnected (or was // replaced). Bail before registering a stream so its buffer isn't // orphaned in pdfStreams past the disconnect's clear() until the TTL. @@ -644,6 +661,7 @@ export class CdpWsProxy { return } const buffer = Buffer.isBuffer(pdf) ? pdf : Buffer.from(pdf) + assertCdpPdfWithinMemoryLimit(buffer) if (params.transferMode === 'ReturnAsStream') { const handle = this.pdfStreams.create(buffer) this.sendResult(clientId, { data: '', stream: handle }, client) diff --git a/src/main/browser/installed-browser-cookie-store-limits.test.ts b/src/main/browser/installed-browser-cookie-store-limits.test.ts new file mode 100644 index 00000000000..6dad57cd4d3 --- /dev/null +++ b/src/main/browser/installed-browser-cookie-store-limits.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { + assertInstalledBrowserCookieStoreWithinLimits, + INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES, + INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES, + InstalledBrowserCookieStoreLimitError +} from './installed-browser-cookie-store-limits' + +describe('assertInstalledBrowserCookieStoreWithinLimits', () => { + it('accepts the exact cookie-count and byte boundaries', () => { + expect( + assertInstalledBrowserCookieStoreWithinLimits( + BigInt(INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES), + BigInt(INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES) + ) + ).toBe(INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES) + }) + + it.each([ + { + kind: 'cookies', + cookies: INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES + 1, + bytes: INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES + }, + { + kind: 'bytes', + cookies: INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES, + bytes: INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES + 1 + } + ])('rejects one $kind over its boundary', ({ kind, cookies, bytes }) => { + expect(() => assertInstalledBrowserCookieStoreWithinLimits(cookies, bytes)).toThrow( + expect.objectContaining({ + name: InstalledBrowserCookieStoreLimitError.name, + kind + }) + ) + }) +}) diff --git a/src/main/browser/installed-browser-cookie-store-limits.ts b/src/main/browser/installed-browser-cookie-store-limits.ts new file mode 100644 index 00000000000..be62a7f82fe --- /dev/null +++ b/src/main/browser/installed-browser-cookie-store-limits.ts @@ -0,0 +1,41 @@ +export const INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES = 250_000 +export const INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES = 64 * 1024 * 1024 + +export type InstalledBrowserCookieStoreLimitKind = 'cookies' | 'bytes' + +export class InstalledBrowserCookieStoreLimitError extends Error { + constructor( + readonly kind: InstalledBrowserCookieStoreLimitKind, + readonly observed: number | bigint, + readonly limit: number + ) { + super(`Installed browser cookie store exceeds the ${kind} limit`) + this.name = 'InstalledBrowserCookieStoreLimitError' + } +} + +export function assertInstalledBrowserCookieStoreWithinLimits( + cookieCount: number | bigint, + cookieBytes: number | bigint +): number { + enforceLimit('cookies', cookieCount, INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES) + enforceLimit('bytes', cookieBytes, INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES) + return Number(cookieCount) +} + +export function installedBrowserCookieStoreLimitReason(browserLabel: string): string { + return `${browserLabel} cookie store is too large to import safely (${INSTALLED_BROWSER_COOKIE_STORE_MAX_COOKIES.toLocaleString('en-US')}-cookie and ${INSTALLED_BROWSER_COOKIE_STORE_MAX_BYTES / 1024 / 1024} MiB cookie-data limits).` +} + +function enforceLimit( + kind: InstalledBrowserCookieStoreLimitKind, + observed: number | bigint, + limit: number +): void { + if ( + (typeof observed === 'bigint' && observed > BigInt(limit)) || + (typeof observed === 'number' && (!Number.isSafeInteger(observed) || observed > limit)) + ) { + throw new InstalledBrowserCookieStoreLimitError(kind, observed, limit) + } +} diff --git a/src/main/browser/native-image-screenshot-encoder.ts b/src/main/browser/native-image-screenshot-encoder.ts new file mode 100644 index 00000000000..48a326feb4c --- /dev/null +++ b/src/main/browser/native-image-screenshot-encoder.ts @@ -0,0 +1,79 @@ +import { + assertBrowserScreenshotEncodedBytes, + assertBrowserScreenshotGeometry +} from './browser-screenshot-limits' + +function applyFallbackClip( + image: Electron.NativeImage, + params: Record | undefined +): Electron.NativeImage | null { + if (params?.captureBeyondViewport) { + // Why: capturePage cannot produce pixels outside the currently painted viewport. + return null + } + + const clip = params?.clip + if (!clip || typeof clip !== 'object') { + return image + } + const clipRect = clip as Record + + const x = typeof clipRect.x === 'number' ? clipRect.x : Number.NaN + const y = typeof clipRect.y === 'number' ? clipRect.y : Number.NaN + const width = typeof clipRect.width === 'number' ? clipRect.width : Number.NaN + const height = typeof clipRect.height === 'number' ? clipRect.height : Number.NaN + const scale = + typeof clipRect.scale === 'number' && Number.isFinite(clipRect.scale) && clipRect.scale > 0 + ? clipRect.scale + : 1 + + if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) { + return null + } + + const cropRect = { + x: Math.round(x * scale), + y: Math.round(y * scale), + width: Math.round(width * scale), + height: Math.round(height * scale) + } + assertBrowserScreenshotGeometry(cropRect.width, cropRect.height) + const imageSize = image.getSize() + if ( + cropRect.x < 0 || + cropRect.y < 0 || + cropRect.width <= 0 || + cropRect.height <= 0 || + cropRect.x + cropRect.width > imageSize.width || + cropRect.y + cropRect.height > imageSize.height + ) { + return null + } + + return image.crop(cropRect) +} + +export function encodeNativeImageScreenshot( + image: Electron.NativeImage, + params: Record | undefined +): { data: string } | null { + if (image.isEmpty()) { + return null + } + + const clippedImage = applyFallbackClip(image, params) + if (!clippedImage || clippedImage.isEmpty()) { + return null + } + const clippedSize = clippedImage.getSize() + assertBrowserScreenshotGeometry(clippedSize.width, clippedSize.height) + + const format = params?.format === 'jpeg' ? 'jpeg' : 'png' + const quality = + typeof params?.quality === 'number' && Number.isFinite(params.quality) + ? Math.max(0, Math.min(100, Math.round(params.quality))) + : undefined + const buffer = format === 'jpeg' ? clippedImage.toJPEG(quality ?? 90) : clippedImage.toPNG() + assertBrowserScreenshotEncodedBytes(buffer.length) + return { data: buffer.toString('base64') } +} diff --git a/src/main/browser/safari-cookie-store-decoder.test.ts b/src/main/browser/safari-cookie-store-decoder.test.ts new file mode 100644 index 00000000000..5ec0e09be33 --- /dev/null +++ b/src/main/browser/safari-cookie-store-decoder.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from 'vitest' +import { + decodeSafariCookieStore, + SAFARI_COOKIE_STORE_MAX_COOKIES, + SAFARI_COOKIE_STORE_MAX_FILE_BYTES, + SAFARI_COOKIE_STORE_MAX_PAGES, + SAFARI_COOKIE_STORE_MAX_PARSED_BYTES, + SafariCookieStoreLimitError, + removeExpiredSafariCookiesInPlace, + type SafariCookieDecodeLimits +} from './safari-cookie-store-decoder' + +type TestCookie = { + domain: string + name: string + value: string + path?: string + secure?: boolean + httpOnly?: boolean + expiration?: number +} + +function buildCookie(input: TestCookie): Buffer { + const strings = [input.domain, input.name, input.path ?? '/', input.value] + let cursor = 48 + const offsets = strings.map((text) => { + const offset = cursor + cursor += Buffer.byteLength(text) + 1 + return offset + }) + const cookie = Buffer.alloc(cursor) + cookie.writeUInt32LE(cookie.length, 0) + cookie.writeUInt32LE((input.secure ? 1 : 0) | (input.httpOnly ? 4 : 0), 8) + cookie.writeUInt32LE(offsets[0]!, 16) + cookie.writeUInt32LE(offsets[1]!, 20) + cookie.writeUInt32LE(offsets[2]!, 24) + cookie.writeUInt32LE(offsets[3]!, 28) + cookie.writeDoubleLE(input.expiration ?? 0, 40) + strings.forEach((text, index) => cookie.write(text, offsets[index]!, 'utf8')) + return cookie +} + +function buildPage(inputs: TestCookie[]): Buffer { + const cookies = inputs.map(buildCookie) + const headerBytes = 8 + cookies.length * 4 + const page = Buffer.alloc(headerBytes + cookies.reduce((sum, cookie) => sum + cookie.length, 0)) + page.writeUInt32BE(0x00000100, 0) + page.writeUInt32LE(cookies.length, 4) + let cursor = headerBytes + cookies.forEach((cookie, index) => { + page.writeUInt32LE(cursor, 8 + index * 4) + cookie.copy(page, cursor) + cursor += cookie.length + }) + return page +} + +function buildStore(pages: TestCookie[][]): Buffer { + const encodedPages = pages.map(buildPage) + const headerBytes = 8 + encodedPages.length * 4 + const store = Buffer.alloc(headerBytes + encodedPages.reduce((sum, page) => sum + page.length, 0)) + store.write('cook', 0, 'utf8') + store.writeUInt32BE(encodedPages.length, 4) + let cursor = headerBytes + encodedPages.forEach((page, index) => { + store.writeUInt32BE(page.length, 8 + index * 4) + page.copy(store, cursor) + cursor += page.length + }) + return store +} + +function limits(overrides: Partial = {}): SafariCookieDecodeLimits { + return { + maxPages: 10, + maxCookies: 10, + maxParsedBytes: 1024, + ...overrides + } +} + +function retainedBytes(cookie: TestCookie): number { + const cleanDomain = cookie.domain.startsWith('.') ? cookie.domain.slice(1) : cookie.domain + const url = `${cookie.secure ? 'https' : 'http'}://${cleanDomain}/` + return [url, cookie.domain, cookie.name, cookie.value, cookie.path ?? '/'].reduce( + (sum, value) => sum + Buffer.byteLength(value), + 0 + ) +} + +describe('decodeSafariCookieStore', () => { + it('preserves page order and removes expired cookies without a second retained list', () => { + const first = { + domain: '.first.example.com', + name: 'first', + value: 'one', + secure: true, + httpOnly: true + } + const expired = { + domain: '.expired.example.com', + name: 'expired', + value: 'old', + expiration: 1 + } + const last = { domain: '.last.example.com', name: 'last', value: 'three' } + + const result = decodeSafariCookieStore(buildStore([[first, expired], [last]])) + const originalList = result.cookies + removeExpiredSafariCookiesInPlace(result.cookies, 2_000_000_000) + + expect(result.totalCookies).toBe(3) + expect(result.cookies).toBe(originalList) + expect(result.cookies.map((cookie) => cookie.name)).toEqual(['first', 'last']) + expect(result.cookies[0]).toMatchObject({ + url: 'https://first.example.com/', + domain: '.first.example.com', + value: 'one', + path: '/', + secure: true, + httpOnly: true, + sameSite: 'unspecified' + }) + }) + + it('accepts the exact cookie and parsed-byte boundaries', () => { + const first = { domain: '.a.example.com', name: 'a', value: 'one' } + const second = { domain: '.b.example.com', name: 'b', value: 'two' } + const maxParsedBytes = retainedBytes(first) + retainedBytes(second) + + const result = decodeSafariCookieStore( + buildStore([[first, second]]), + limits({ maxCookies: 2, maxParsedBytes }) + ) + + expect(result.cookies.map((cookie) => cookie.name)).toEqual(['a', 'b']) + }) + + it('rejects one cookie or parsed byte over the configured boundary', () => { + const first = { domain: '.a.example.com', name: 'a', value: 'one' } + const second = { domain: '.b.example.com', name: 'b', value: 'two' } + const store = buildStore([[first, second]]) + + expect(() => decodeSafariCookieStore(store, limits({ maxCookies: 1 }))).toThrow( + expect.objectContaining({ kind: 'cookies', observed: 2, limit: 1 }) + ) + expect(() => + decodeSafariCookieStore( + store, + limits({ maxParsedBytes: retainedBytes(first) + retainedBytes(second) - 1 }) + ) + ).toThrow( + expect.objectContaining({ + kind: 'parsed-bytes', + observed: retainedBytes(first) + retainedBytes(second) + }) + ) + }) + + it('counts malformed declared entries against the parse budget', () => { + const malformedPage = Buffer.alloc(20) + malformedPage.writeUInt32BE(0x00000100, 0) + malformedPage.writeUInt32LE(3, 4) + const header = Buffer.alloc(12) + header.write('cook', 0, 'utf8') + header.writeUInt32BE(1, 4) + header.writeUInt32BE(malformedPage.length, 8) + + expect(() => + decodeSafariCookieStore(Buffer.concat([header, malformedPage]), limits({ maxCookies: 2 })) + ).toThrow(SafariCookieStoreLimitError) + }) + + it('publishes generous production limits', () => { + expect({ + fileBytes: SAFARI_COOKIE_STORE_MAX_FILE_BYTES, + pages: SAFARI_COOKIE_STORE_MAX_PAGES, + cookies: SAFARI_COOKIE_STORE_MAX_COOKIES, + parsedBytes: SAFARI_COOKIE_STORE_MAX_PARSED_BYTES + }).toEqual({ + fileBytes: 64 * 1024 * 1024, + pages: 65_536, + cookies: 250_000, + parsedBytes: 64 * 1024 * 1024 + }) + }) +}) diff --git a/src/main/browser/safari-cookie-store-decoder.ts b/src/main/browser/safari-cookie-store-decoder.ts new file mode 100644 index 00000000000..b6e58fc3b49 --- /dev/null +++ b/src/main/browser/safari-cookie-store-decoder.ts @@ -0,0 +1,212 @@ +const MAC_EPOCH_DELTA_SECONDS = 978_307_200 + +export const SAFARI_COOKIE_STORE_MAX_FILE_BYTES = 64 * 1024 * 1024 +export const SAFARI_COOKIE_STORE_MAX_PAGES = 65_536 +export const SAFARI_COOKIE_STORE_MAX_COOKIES = 250_000 +export const SAFARI_COOKIE_STORE_MAX_PARSED_BYTES = 64 * 1024 * 1024 + +export type SafariCookieStoreLimitKind = 'pages' | 'cookies' | 'parsed-bytes' + +export class SafariCookieStoreLimitError extends Error { + constructor( + readonly kind: SafariCookieStoreLimitKind, + readonly observed: number, + readonly limit: number + ) { + super(`Safari cookie store exceeds the ${kind} limit`) + this.name = 'SafariCookieStoreLimitError' + } +} + +export type SafariDecodedCookie = { + url: string + name: string + value: string + domain: string + path: string + secure: boolean + httpOnly: boolean + sameSite: 'unspecified' + expirationDate: number | undefined +} + +export type SafariCookieDecodeResult = { + cookies: SafariDecodedCookie[] + totalCookies: number +} + +export type SafariCookieDecodeLimits = { + maxPages: number + maxCookies: number + maxParsedBytes: number +} + +const DEFAULT_LIMITS: SafariCookieDecodeLimits = { + maxPages: SAFARI_COOKIE_STORE_MAX_PAGES, + maxCookies: SAFARI_COOKIE_STORE_MAX_COOKIES, + maxParsedBytes: SAFARI_COOKIE_STORE_MAX_PARSED_BYTES +} + +type DecodeState = { + readonly cookies: SafariDecodedCookie[] + readonly limits: SafariCookieDecodeLimits + decodedCookies: number + inspectedEntries: number + parsedBytes: number +} + +export function decodeSafariCookieStore( + buffer: Buffer, + limits: SafariCookieDecodeLimits = DEFAULT_LIMITS +): SafariCookieDecodeResult { + if (buffer.length < 8 || buffer.subarray(0, 4).toString('utf8') !== 'cook') { + return { cookies: [], totalCookies: 0 } + } + + const pageCount = buffer.readUInt32BE(4) + enforceLimit('pages', pageCount, limits.maxPages) + const pagesOffset = 8 + pageCount * 4 + if (pagesOffset > buffer.length) { + return { cookies: [], totalCookies: 0 } + } + + const state: DecodeState = { + cookies: [], + limits, + decodedCookies: 0, + inspectedEntries: 0, + parsedBytes: 0 + } + let pageOffset = pagesOffset + for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) { + const pageSize = buffer.readUInt32BE(8 + pageIndex * 4) + const pageEnd = Math.min(buffer.length, pageOffset + pageSize) + decodeSafariCookiePage(buffer.subarray(pageOffset, pageEnd), state) + pageOffset += pageSize + } + return { cookies: state.cookies, totalCookies: state.decodedCookies } +} + +function decodeSafariCookiePage(page: Buffer, state: DecodeState): void { + if (page.length < 16 || page.readUInt32BE(0) !== 0x00000100) { + return + } + + const cookieCount = page.readUInt32LE(4) + state.inspectedEntries += cookieCount + enforceLimit('cookies', state.inspectedEntries, state.limits.maxCookies) + if (8 + cookieCount * 4 > page.length) { + return + } + + for (let index = 0; index < cookieCount; index += 1) { + const offset = page.readUInt32LE(8 + index * 4) + const cookie = decodeSafariCookie(page.subarray(offset)) + if (!cookie) { + continue + } + state.decodedCookies += 1 + state.parsedBytes += cookieRetainedBytes(cookie) + enforceLimit('parsed-bytes', state.parsedBytes, state.limits.maxParsedBytes) + state.cookies.push(cookie) + } +} + +export function removeExpiredSafariCookiesInPlace( + cookies: SafariDecodedCookie[], + nowSeconds: number +): void { + let retained = 0 + for (const cookie of cookies) { + if (!cookie.expirationDate || cookie.expirationDate > nowSeconds) { + cookies[retained] = cookie + retained += 1 + } + } + cookies.length = retained +} + +function decodeSafariCookie(buffer: Buffer): SafariDecodedCookie | null { + if (buffer.length < 48) { + return null + } + // Why: the declared size is external; clamp string scans to bytes present in this page. + const size = Math.min(buffer.readUInt32LE(0), buffer.length) + if (size < 48) { + return null + } + + const flags = buffer.readUInt32LE(8) + const secure = (flags & 1) !== 0 + const name = readCString(buffer, buffer.readUInt32LE(20), size) + if (!name) { + return null + } + const value = readCString(buffer, buffer.readUInt32LE(28), size) ?? '' + const path = readCString(buffer, buffer.readUInt32LE(24), size) ?? '/' + const rawUrl = readCString(buffer, buffer.readUInt32LE(16), size) ?? '' + const domain = rawUrl.startsWith('.') ? rawUrl : rawUrl || null + if (!domain) { + return null + } + + const url = deriveUrl(domain, secure) + if (!url) { + return null + } + // Why: Safari stores expiration seconds relative to 2001-01-01. + const expiration = buffer.readDoubleLE(40) + const expirationDate = + expiration > 0 ? Math.round(expiration + MAC_EPOCH_DELTA_SECONDS) : undefined + + return { + url, + name, + value, + domain, + path, + secure, + httpOnly: (flags & 4) !== 0, + sameSite: 'unspecified', + expirationDate + } +} + +function readCString(buffer: Buffer, offset: number, end: number): string | null { + if (offset < 0 || offset >= end) { + return null + } + let cursor = offset + while (cursor < end && buffer[cursor] !== 0) { + cursor += 1 + } + return cursor < end ? buffer.toString('utf8', offset, cursor) : null +} + +function deriveUrl(domain: string, secure: boolean): string | null { + const cleanDomain = domain.startsWith('.') ? domain.slice(1) : domain + if (!cleanDomain || cleanDomain.includes(' ')) { + return null + } + try { + return new URL(`${secure ? 'https' : 'http'}://${cleanDomain}/`).toString() + } catch { + return null + } +} + +function cookieRetainedBytes(cookie: SafariDecodedCookie): number { + return ( + Buffer.byteLength(cookie.url) + + Buffer.byteLength(cookie.name) + + Buffer.byteLength(cookie.value) + + Buffer.byteLength(cookie.domain) + + Buffer.byteLength(cookie.path) + ) +} + +function enforceLimit(kind: SafariCookieStoreLimitKind, observed: number, limit: number): void { + if (observed > limit) { + throw new SafariCookieStoreLimitError(kind, observed, limit) + } +} diff --git a/src/main/browser/snapshot-engine.test.ts b/src/main/browser/snapshot-engine.test.ts index 17f5341b220..c06b5d43d21 100644 --- a/src/main/browser/snapshot-engine.test.ts +++ b/src/main/browser/snapshot-engine.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it, vi } from 'vitest' -import { buildSnapshot, type CdpCommandSender } from './snapshot-engine' +import { + buildSnapshot, + SNAPSHOT_MAX_ENTRIES, + SNAPSHOT_MAX_NAME_CODE_UNITS, + SNAPSHOT_MAX_RETAINED_NAME_CODE_UNITS, + type CdpCommandSender +} from './snapshot-engine' type AXNode = { nodeId: string @@ -193,4 +199,46 @@ describe('buildSnapshot', () => { expect(result.snapshot).toContain('[Footer]') expect(result.snapshot).toContain('heading "Dashboard"') }) + + it('bounds retained entries and page-controlled accessible names', async () => { + const childIds = Array.from({ length: SNAPSHOT_MAX_ENTRIES + 100 }, (_, index) => + String(index + 2) + ) + const nodes: AXNode[] = [node('1', 'WebArea', 'page', { childIds })] + for (const childId of childIds) { + nodes.push( + node( + childId, + 'button', + `${childId.padStart(8, '0')}-${'x'.repeat(SNAPSHOT_MAX_NAME_CODE_UNITS + 100)}` + ) + ) + } + + const result = await buildSnapshot(makeSender(nodes)) + + expect(result.refs).toHaveLength( + Math.min( + SNAPSHOT_MAX_ENTRIES, + Math.floor(SNAPSHOT_MAX_RETAINED_NAME_CODE_UNITS / SNAPSHOT_MAX_NAME_CODE_UNITS) + ) + ) + expect(result.refs[0]?.name).toHaveLength(SNAPSHOT_MAX_NAME_CODE_UNITS) + expect(result.refs.reduce((total, ref) => total + ref.name.length, 0)).toBeLessThanOrEqual( + SNAPSHOT_MAX_RETAINED_NAME_CODE_UNITS + ) + }) + + it('walks malformed cyclic trees without recursive growth', async () => { + const nodes: AXNode[] = [ + node('1', 'WebArea', 'page', { childIds: ['2'] }), + node('2', 'generic', '', { childIds: ['3'] }), + node('3', 'generic', '', { childIds: ['2', '4'] }), + node('4', 'button', 'Reachable', { backendDOMNodeId: 4 }) + ] + + const result = await buildSnapshot(makeSender(nodes)) + + expect(result.refs).toEqual([{ ref: '@e1', role: 'button', name: 'Reachable' }]) + }) }) diff --git a/src/main/browser/snapshot-engine.ts b/src/main/browser/snapshot-engine.ts index 141a3d957dc..1d6aac01a19 100644 --- a/src/main/browser/snapshot-engine.ts +++ b/src/main/browser/snapshot-engine.ts @@ -75,6 +75,18 @@ const HEADING_PATTERN = /^heading$/ const SKIP_ROLES = new Set(['none', 'presentation', 'generic']) +export const SNAPSHOT_MAX_AX_NODES = 50_000 +export const SNAPSHOT_MAX_ENTRIES = 4096 +export const SNAPSHOT_MAX_NAME_CODE_UNITS = 1024 +export const SNAPSHOT_MAX_RETAINED_NAME_CODE_UNITS = 1024 * 1024 +const SNAPSHOT_MAX_VISUAL_DEPTH = 64 + +type SnapshotWalkBudget = { + remainingNodes: number + remainingEntries: number + remainingNameCodeUnits: number +} + export async function buildSnapshot( sendCommand: CdpCommandSender, iframeSessions?: Map, @@ -84,37 +96,50 @@ export async function buildSnapshot( const { nodes } = (await sendCommand('Accessibility.getFullAXTree')) as { nodes: AXNode[] } const nodeById = new Map() - for (const node of nodes) { + for (const node of nodes.slice(0, SNAPSHOT_MAX_AX_NODES)) { nodeById.set(node.nodeId, node) } const entries: SnapshotEntry[] = [] let refCounter = 1 + const budget: SnapshotWalkBudget = { + remainingNodes: SNAPSHOT_MAX_AX_NODES, + remainingEntries: SNAPSHOT_MAX_ENTRIES, + remainingNameCodeUnits: SNAPSHOT_MAX_RETAINED_NAME_CODE_UNITS + } const root = nodes[0] if (!root) { return { snapshot: '', refs: [], refMap: new Map() } } - walkTree(root, nodeById, 0, entries, () => refCounter++) + walkTree(root, nodeById, 0, entries, () => refCounter++, budget) // Why: many modern SPAs use styled
s, s, and custom elements as // interactive controls without proper ARIA roles. These elements are invisible // to the accessibility tree walk above but are clearly interactive (cursor:pointer, // onclick, tabindex, contenteditable). This DOM query pass discovers them and // promotes them to interactive refs so the agent can interact with them. - const cursorInteractiveEntries = await findCursorInteractiveElements(sendCommand, entries) + const cursorInteractiveEntries = + budget.remainingEntries > 0 ? await findCursorInteractiveElements(sendCommand, entries) : [] for (const cie of cursorInteractiveEntries) { + const name = reserveSnapshotName(cie.name, budget) + if (name === null || !reserveSnapshotEntry(budget)) { + break + } cie.ref = `@e${refCounter++}` - entries.push(cie) + entries.push({ ...cie, name }) } // Why: cross-origin iframes have their own AX trees accessible only through // their dedicated CDP session. Append their elements after the parent tree // so the agent can see and interact with iframe content. - const iframeRefSessions: { ref: string; sessionId: string }[] = [] + const iframeRefSessions = new Map() if (iframeSessions && makeIframeSender && iframeSessions.size > 0) { for (const [_frameId, sessionId] of iframeSessions) { + if (budget.remainingEntries <= 0 || budget.remainingNodes <= 0) { + break + } try { const iframeSender = makeIframeSender(sessionId) await iframeSender('Accessibility.enable') @@ -125,15 +150,15 @@ export async function buildSnapshot( continue } const iframeNodeById = new Map() - for (const n of iframeNodes) { + for (const n of iframeNodes.slice(0, budget.remainingNodes)) { iframeNodeById.set(n.nodeId, n) } const iframeRoot = iframeNodes[0] if (iframeRoot) { const startRef = refCounter - walkTree(iframeRoot, iframeNodeById, 1, entries, () => refCounter++) + walkTree(iframeRoot, iframeNodeById, 1, entries, () => refCounter++, budget) for (let i = startRef; i < refCounter; i++) { - iframeRefSessions.push({ ref: `@e${i}`, sessionId }) + iframeRefSessions.set(`@e${i}`, sessionId) } } } catch { @@ -171,12 +196,11 @@ export async function buildSnapshot( } lines.push(`${indent}[${entry.ref}] ${entry.role} "${displayName}"`) refs.push({ ref: entry.ref, role: entry.role, name: displayName }) - const iframeSession = iframeRefSessions.find((s) => s.ref === entry.ref) refMap.set(entry.ref, { backendDOMNodeId: entry.backendDOMNodeId, role: entry.role, name: entry.name, - sessionId: iframeSession?.sessionId, + sessionId: iframeRefSessions.get(entry.ref), nth: total > 1 ? nth : undefined }) } else { @@ -188,109 +212,161 @@ export async function buildSnapshot( } function walkTree( - node: AXNode, + root: AXNode, nodeById: Map, depth: number, entries: SnapshotEntry[], - nextRef: () => number + nextRef: () => number, + budget: SnapshotWalkBudget ): void { - if (node.ignored) { - walkChildren(node, nodeById, depth, entries, nextRef) - return - } + const stack: { node: AXNode; depth: number }[] = [{ node: root, depth }] + const visited = new Set() + while (stack.length > 0 && budget.remainingNodes > 0 && budget.remainingEntries > 0) { + const current = stack.pop()! + const node = current.node + if (visited.has(node.nodeId)) { + continue + } + visited.add(node.nodeId) + budget.remainingNodes -= 1 - const role = node.role?.value ?? '' - const name = node.name?.value ?? '' + const role = node.role?.value ?? '' + const rawName = node.name?.value ?? '' + const isInteractive = INTERACTIVE_ROLES.has(role) + const isHeading = HEADING_PATTERN.test(role) + const isLandmark = LANDMARK_ROLES.has(role) + const isStaticText = role === 'staticText' || role === 'StaticText' + const shouldWalkChildren = + node.ignored === true || + SKIP_ROLES.has(role) || + (!isInteractive && !isHeading && !isLandmark && !isStaticText) || + (!rawName && !isLandmark) - if (SKIP_ROLES.has(role)) { - walkChildren(node, nodeById, depth, entries, nextRef) - return - } + if (shouldWalkChildren) { + pushSnapshotChildren(stack, node, nodeById, current.depth, budget.remainingNodes) + continue + } - const isInteractive = INTERACTIVE_ROLES.has(role) - const isHeading = HEADING_PATTERN.test(role) - const isLandmark = LANDMARK_ROLES.has(role) - const isStaticText = role === 'staticText' || role === 'StaticText' + if (isLandmark) { + const name = reserveSnapshotName(rawName || role, budget) + if (name !== null && reserveSnapshotEntry(budget)) { + entries.push({ + ref: '', + role: formatLandmarkRole(role, rawName ? name : ''), + name, + backendDOMNodeId: node.backendDOMNodeId ?? 0, + depth: current.depth + }) + } + pushSnapshotChildren( + stack, + node, + nodeById, + Math.min(current.depth + 1, SNAPSHOT_MAX_VISUAL_DEPTH), + budget.remainingNodes + ) + continue + } - if (!isInteractive && !isHeading && !isLandmark && !isStaticText) { - walkChildren(node, nodeById, depth, entries, nextRef) - return - } + if (isHeading) { + appendSnapshotEntry(entries, budget, { + ref: '', + role: 'heading', + rawName, + backendDOMNodeId: node.backendDOMNodeId ?? 0, + depth: current.depth + }) + continue + } - if (!name && !isLandmark) { - walkChildren(node, nodeById, depth, entries, nextRef) - return - } + if (isStaticText) { + const name = trimSnapshotName(rawName) + if (name) { + appendSnapshotEntry(entries, budget, { + ref: '', + role: 'text', + rawName: name, + backendDOMNodeId: node.backendDOMNodeId ?? 0, + depth: current.depth + }) + } + continue + } - const hasFocusable = isInteractive && isFocusable(node) - - if (isLandmark) { - entries.push({ - ref: '', - role: formatLandmarkRole(role, name), - name: name || role, - backendDOMNodeId: node.backendDOMNodeId ?? 0, - depth - }) - walkChildren(node, nodeById, depth + 1, entries, nextRef) - return - } - - if (isHeading) { - entries.push({ - ref: '', - role: 'heading', - name, - backendDOMNodeId: node.backendDOMNodeId ?? 0, - depth - }) - return - } - - if (isStaticText && name.trim().length > 0) { - entries.push({ - ref: '', - role: 'text', - name: name.trim(), - backendDOMNodeId: node.backendDOMNodeId ?? 0, - depth - }) - return - } - - if (isInteractive && (hasFocusable || node.backendDOMNodeId)) { - const ref = `@e${nextRef()}` - entries.push({ - ref, - role: formatInteractiveRole(role), - name: name || '(unlabeled)', - backendDOMNodeId: node.backendDOMNodeId ?? 0, - depth - }) - return - } - - walkChildren(node, nodeById, depth, entries, nextRef) -} - -function walkChildren( - node: AXNode, - nodeById: Map, - depth: number, - entries: SnapshotEntry[], - nextRef: () => number -): void { - if (!node.childIds) { - return - } - for (const childId of node.childIds) { - const child = nodeById.get(childId) - if (child) { - walkTree(child, nodeById, depth, entries, nextRef) + if (isInteractive && (isFocusable(node) || node.backendDOMNodeId)) { + appendSnapshotEntry(entries, budget, { + ref: `@e${nextRef()}`, + role: formatInteractiveRole(role), + rawName, + backendDOMNodeId: node.backendDOMNodeId ?? 0, + depth: current.depth + }) } } } +function pushSnapshotChildren( + stack: { node: AXNode; depth: number }[], + node: AXNode, + nodeById: Map, + depth: number, + maxChildren: number +): void { + const childCount = Math.min(node.childIds?.length ?? 0, maxChildren) + for (let index = childCount - 1; index >= 0; index--) { + const child = nodeById.get(node.childIds![index]) + if (child) { + stack.push({ node: child, depth }) + } + } +} + +function appendSnapshotEntry( + entries: SnapshotEntry[], + budget: SnapshotWalkBudget, + entry: Omit & { rawName: string } +): void { + const name = reserveSnapshotName(entry.rawName, budget) + if (name === null || !reserveSnapshotEntry(budget)) { + return + } + const { rawName: _rawName, ...rest } = entry + entries.push({ ...rest, name }) +} + +function reserveSnapshotEntry(budget: SnapshotWalkBudget): boolean { + if (budget.remainingEntries <= 0) { + return false + } + budget.remainingEntries -= 1 + return true +} + +function reserveSnapshotName(rawName: string, budget: SnapshotWalkBudget): string | null { + const maxLength = Math.min(SNAPSHOT_MAX_NAME_CODE_UNITS, budget.remainingNameCodeUnits) + if (maxLength <= 0) { + return null + } + const name = rawName.slice(0, maxLength) + budget.remainingNameCodeUnits -= name.length + return name +} + +function trimSnapshotName(rawName: string): string { + let start = 0 + while (start < rawName.length && /\s/.test(rawName[start])) { + start += 1 + } + if (start === rawName.length) { + return '' + } + let end = rawName.length + while (end > start && /\s/.test(rawName[end - 1])) { + end -= 1 + } + return rawName.slice(start, Math.min(end, start + SNAPSHOT_MAX_NAME_CODE_UNITS)) +} + function isFocusable(node: AXNode): boolean { if (!node.properties) { return true diff --git a/src/main/cache-identity-digest.ts b/src/main/cache-identity-digest.ts new file mode 100644 index 00000000000..fbcbad42abe --- /dev/null +++ b/src/main/cache-identity-digest.ts @@ -0,0 +1,11 @@ +import { createHash } from 'node:crypto' + +/** Fixed-size identity for long-lived maps whose source fields can be arbitrarily large. */ +export function cacheIdentityDigest(parts: readonly string[]): string { + const digest = createHash('sha256') + for (const part of parts) { + digest.update(`${part.length}:`) + digest.update(part) + } + return digest.digest('base64url') +} diff --git a/src/main/claude-accounts/managed-auth-path.ts b/src/main/claude-accounts/managed-auth-path.ts index 2914b72e033..ff4303a083b 100644 --- a/src/main/claude-accounts/managed-auth-path.ts +++ b/src/main/claude-accounts/managed-auth-path.ts @@ -1,7 +1,9 @@ -import { existsSync, lstatSync, readFileSync, realpathSync, writeFileSync } from 'node:fs' +import { existsSync, lstatSync, realpathSync, writeFileSync } from 'node:fs' import { join, relative, resolve, sep } from 'node:path' import { app } from 'electron' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' const MANAGED_AUTH_MARKER = '.orca-managed-claude-auth' @@ -65,8 +67,11 @@ export function readClaudeManagedAuthFile( if (!isOwnedChildFile(managedAuthPath, filePath)) { return null } - return readFileSync(filePath, 'utf-8') - } catch { + return readAgentStateFileSync(filePath) + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } return null } } @@ -80,6 +85,9 @@ export function writeClaudeManagedAuthFile( if (existsSync(filePath) && !isOwnedChildFile(managedAuthPath, filePath)) { throw new Error('Managed Claude auth child file is not owned by Orca.') } + if (existsSync(filePath)) { + readAgentStateFileSync(filePath) + } writeFileAtomically(filePath, contents, { mode: 0o600 }) } @@ -92,7 +100,7 @@ function isManagedAuthMarkerValid(markerPath: string, accountId: string): boolea ) { return false } - return readFileSync(markerPath, 'utf-8').trim() === accountId + return readAgentStateFileSync(markerPath).trim() === accountId } catch { return false } diff --git a/src/main/claude-accounts/oauth-refresh.ts b/src/main/claude-accounts/oauth-refresh.ts index cc717ef51ee..a69c12ad04a 100644 --- a/src/main/claude-accounts/oauth-refresh.ts +++ b/src/main/claude-accounts/oauth-refresh.ts @@ -1,5 +1,6 @@ import { net, session } from 'electron' import { ensureElectronProxyFromEnvironment } from '../network/proxy-settings' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' // Why: the OAuth client id and token endpoint are the public Claude Code // values, verified against the installed `claude` binary (2.1.177) and the @@ -158,7 +159,7 @@ export async function refreshClaudeOauthCredentials( console.warn(`[claude-oauth-refresh] token endpoint returned ${res.status}`) return null } - const data = (await res.json()) as TokenEndpointResponse + const data = await readFetchResponseJsonWithinLimit(res) return applyRefreshedToken(credentialsJson, data, now) } catch (error) { console.warn( diff --git a/src/main/claude-accounts/runtime-auth-service.ts b/src/main/claude-accounts/runtime-auth-service.ts index 58328222528..3ef0ec4b64c 100644 --- a/src/main/claude-accounts/runtime-auth-service.ts +++ b/src/main/claude-accounts/runtime-auth-service.ts @@ -1,10 +1,11 @@ /* eslint-disable max-lines -- Why: keeps file/Keychain/snapshot/env-patch auth semantics together so PTY launch and quota-fetch paths can't drift. */ import { execFileSync } from 'node:child_process' -import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs' +import { chmodSync, existsSync, mkdirSync, rmSync } from 'node:fs' import { dirname, join } from 'node:path' import { app } from 'electron' import type { ClaudeManagedAccount } from '../../shared/types' import type { Store } from '../persistence' +import { readAgentStateFileSync, readAgentStateJsonFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' import type { ClaudeEnvPatch } from './environment' import { @@ -35,6 +36,7 @@ import { setSelectedClaudeAccountIdForTarget, type ClaudeAccountSelectionTarget } from './runtime-selection' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' export type ClaudeRuntimeAuthPreparation = { configDir: string @@ -346,7 +348,7 @@ export class ClaudeRuntimeAuthService { if (this.lastSyncedAccountId === null) { const paths = this.pathResolver.getRuntimePaths() const runtimeCredentialsJson = existsSync(paths.credentialsPath) - ? readFileSync(paths.credentialsPath, 'utf-8') + ? readAgentStateFileSync(paths.credentialsPath) : null await this.captureSystemDefaultSnapshotForManagedEntry( runtimeCredentialsJson, @@ -569,7 +571,7 @@ export class ClaudeRuntimeAuthService { ): Promise { const paths = this.pathResolver.getRuntimePaths() const fileCredentials = existsSync(paths.credentialsPath) - ? readFileSync(paths.credentialsPath, 'utf-8') + ? readAgentStateFileSync(paths.credentialsPath) : null const runtimeOauthAccount = this.readRuntimeOauthAccount() const candidates: ClaudeRuntimeCredentialCandidate[] = [] @@ -1162,7 +1164,7 @@ export class ClaudeRuntimeAuthService { options.credentialsJsonOverride !== undefined ? options.credentialsJsonOverride : existsSync(paths.credentialsPath) - ? readFileSync(paths.credentialsPath, 'utf-8') + ? readAgentStateFileSync(paths.credentialsPath) : null const keychainCredentialsJson = await this.readAggregateClaudeKeychainCredentialsBestEffort( paths.configDir @@ -1293,7 +1295,7 @@ export class ClaudeRuntimeAuthService { return null } try { - const parsed = JSON.parse(readFileSync(snapshotPath, 'utf-8')) as unknown + const parsed = readAgentStateJsonFileSync(snapshotPath) if (this.isSystemDefaultSnapshot(parsed)) { return parsed } @@ -1411,7 +1413,7 @@ export class ClaudeRuntimeAuthService { private readRuntimeCredentialsFile(): string | null { const credentialsPath = this.pathResolver.getRuntimePaths().credentialsPath - return existsSync(credentialsPath) ? readFileSync(credentialsPath, 'utf-8') : null + return existsSync(credentialsPath) ? readAgentStateFileSync(credentialsPath) : null } private runtimeCredentialsBelongToAccount( @@ -1452,7 +1454,7 @@ export class ClaudeRuntimeAuthService { } const paths = this.pathResolver.getRuntimePaths() const currentCredentialsJson = existsSync(paths.credentialsPath) - ? readFileSync(paths.credentialsPath, 'utf-8') + ? readAgentStateFileSync(paths.credentialsPath) : null return currentCredentialsJson === previouslyWrittenCredentialsJson } @@ -1461,7 +1463,7 @@ export class ClaudeRuntimeAuthService { const paths = this.pathResolver.getRuntimePaths() try { const currentCredentialsJson = existsSync(paths.credentialsPath) - ? readFileSync(paths.credentialsPath, 'utf-8') + ? readAgentStateFileSync(paths.credentialsPath) : null return ( currentCredentialsJson !== null && @@ -1551,7 +1553,7 @@ export class ClaudeRuntimeAuthService { return null } try { - const parsed = JSON.parse(readFileSync(configPath, 'utf-8')) as unknown + const parsed = readAgentStateJsonFileSync(configPath) const record = this.asRecord(parsed) if (!record) { return RUNTIME_OAUTH_ACCOUNT_PARSE_ERROR @@ -1761,8 +1763,11 @@ export class ClaudeRuntimeAuthService { private fileContentsEqual(targetPath: string, contents: string): boolean { try { - return existsSync(targetPath) && readFileSync(targetPath, 'utf-8') === contents - } catch { + return existsSync(targetPath) && readAgentStateFileSync(targetPath) === contents + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } return false } } @@ -1783,7 +1788,7 @@ export class ClaudeRuntimeAuthService { return {} } try { - const parsed = JSON.parse(readFileSync(targetPath, 'utf-8')) as unknown + const parsed = readAgentStateJsonFileSync(targetPath) if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { return parsed as Record } diff --git a/src/main/claude-accounts/service.ts b/src/main/claude-accounts/service.ts index 4cf9d256d5a..510b9a5fee5 100644 --- a/src/main/claude-accounts/service.ts +++ b/src/main/claude-accounts/service.ts @@ -2,7 +2,7 @@ for login, credential capture, Keychain storage, selection, and rate-limit refresh. */ import { randomUUID } from 'node:crypto' import { execFileSync, spawn } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, relative, resolve, sep } from 'node:path' import type { @@ -12,6 +12,7 @@ import type { } from '../../shared/types' import type { Store } from '../persistence' import type { RateLimitService } from '../rate-limits/service' +import { readAgentStateFileSync, readAgentStateJsonFileSync } from '../agent-state-file-reader' import { resolveClaudeCommand } from '../codex-cli/command' import type { ClaudeRuntimeAuthService } from './runtime-auth-service' import { @@ -611,7 +612,7 @@ export class ClaudeAccountService { } } const credentialsPath = join(configDir, '.credentials.json') - return existsSync(credentialsPath) ? readFileSync(credentialsPath, 'utf-8') : null + return existsSync(credentialsPath) ? readAgentStateFileSync(credentialsPath) : null } private readOauthAccountFromConfigDir(configDir: string): unknown { @@ -620,7 +621,7 @@ export class ClaudeAccountService { continue } try { - const parsed = JSON.parse(readFileSync(configPath, 'utf-8')) as Record + const parsed = readAgentStateJsonFileSync(configPath) as Record if (parsed.oauthAccount) { return parsed.oauthAccount } diff --git a/src/main/claude-usage/scanner-large-directory.test.ts b/src/main/claude-usage/scanner-large-directory.test.ts index 78d3895c58b..8262a54da83 100644 --- a/src/main/claude-usage/scanner-large-directory.test.ts +++ b/src/main/claude-usage/scanner-large-directory.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it, vi } from 'vitest' -import type { Dirent } from 'node:fs' +import type { Dir, Dirent } from 'node:fs' import type * as FsPromises from 'node:fs/promises' import type * as NodeOs from 'node:os' import { join } from 'node:path' -const { homedirMock, readdirMock } = vi.hoisted(() => ({ +const { homedirMock, opendirMock } = vi.hoisted(() => ({ homedirMock: vi.fn<() => string>(), - readdirMock: vi.fn<(dirPath: string) => Promise>() + opendirMock: vi.fn<(dirPath: string) => Promise>() })) vi.mock('os', async () => { @@ -21,7 +21,7 @@ vi.mock('fs/promises', async () => { const actual = await vi.importActual('fs/promises') return { ...actual, - readdir: readdirMock + opendir: opendirMock } }) @@ -43,24 +43,84 @@ const largeTranscriptEntries = Array.from({ length: FILE_COUNT }, (_, index) => dirent(`session-${index}.jsonl`, 'file') ) +function directory(entries: Dirent[]): Dir { + return { + async *[Symbol.asyncIterator]() { + yield* entries + } + } as Dir +} + +function generatedFileDirectory(count: number): Dir { + return { + async *[Symbol.asyncIterator]() { + for (let index = 0; index < count; index++) { + yield dirent(`session-${index}.jsonl`, 'file') + } + } + } as Dir +} + describe('listClaudeTranscriptFiles large directories', () => { it('keeps nested transcript scans past the JavaScript spread-argument limit', async () => { homedirMock.mockReturnValue(FAKE_HOME) - readdirMock.mockImplementation(async (dirPath) => { + opendirMock.mockImplementation(async (dirPath) => { if (dirPath === PROJECTS_ROOT) { - return [dirent('large-project', 'directory')] + return directory([dirent('large-project', 'directory')]) } if (dirPath === PROJECT_DIR) { - return largeTranscriptEntries + return directory(largeTranscriptEntries) } if (dirPath === TRANSCRIPTS_ROOT) { - return [] + return directory([]) } - throw new Error(`Unexpected readdir path: ${dirPath}`) + throw new Error(`Unexpected opendir path: ${dirPath}`) }) const { listClaudeTranscriptFiles } = await import('./scanner') await expect(listClaudeTranscriptFiles()).resolves.toHaveLength(FILE_COUNT) }) + + it('fails closed instead of silently omitting a transcript past capacity', async () => { + opendirMock.mockImplementation(async (dirPath) => { + if (dirPath === PROJECTS_ROOT) { + return generatedFileDirectory(200_001) + } + if (dirPath === TRANSCRIPTS_ROOT) { + return directory([]) + } + throw new Error(`Unexpected opendir path: ${dirPath}`) + }) + + const { listClaudeTranscriptFiles } = await import('./scanner') + + await expect(listClaudeTranscriptFiles()).rejects.toMatchObject({ + name: 'UsageHistoryScanCapacityError', + resource: 'files', + limit: 200_000 + }) + }) + + it('keeps stable transcripts when a nested directory vanishes during discovery', async () => { + const vanishedDirectory = join(PROJECTS_ROOT, 'vanished') + opendirMock.mockImplementation(async (dirPath) => { + if (dirPath === PROJECTS_ROOT) { + return directory([dirent('stable.jsonl', 'file'), dirent('vanished', 'directory')]) + } + if (dirPath === vanishedDirectory) { + throw Object.assign(new Error('directory vanished'), { code: 'ENOENT' }) + } + if (dirPath === TRANSCRIPTS_ROOT) { + return directory([]) + } + throw new Error(`Unexpected opendir path: ${dirPath}`) + }) + + const { listClaudeTranscriptFiles } = await import('./scanner') + + await expect(listClaudeTranscriptFiles()).resolves.toEqual([ + join(PROJECTS_ROOT, 'stable.jsonl') + ]) + }) }) diff --git a/src/main/claude-usage/scanner.test.ts b/src/main/claude-usage/scanner.test.ts index f7de6cef141..c04879d1684 100644 --- a/src/main/claude-usage/scanner.test.ts +++ b/src/main/claude-usage/scanner.test.ts @@ -8,6 +8,7 @@ import { import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' +import { UsageHistoryScanBudget } from '../usage-history-scan-budget' describe('parseClaudeUsageRecord', () => { it('extracts token usage from assistant transcript lines', () => { @@ -102,6 +103,31 @@ describe('parseClaudeUsageRecord', () => { await rm(root, { recursive: true, force: true }) } }) + + it('fails the whole parse when retained usage records exceed capacity', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-claude-capacity-')) + const filePath = join(root, 'session.jsonl') + const record = (uuid: string) => + JSON.stringify({ + type: 'assistant', + sessionId: 'session-1', + uuid, + timestamp: '2026-04-09T10:00:00.000Z', + message: { usage: { input_tokens: 1 } } + }) + try { + await writeFile(filePath, [record('turn-1'), record('turn-2')].join('\n')) + const budget = new UsageHistoryScanBudget({ records: 1 }) + + await expect(parseClaudeUsageFile(filePath, budget)).rejects.toMatchObject({ + name: 'UsageHistoryScanCapacityError', + resource: 'records', + limit: 1 + }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) }) describe('Claude usage aggregation', () => { diff --git a/src/main/claude-usage/scanner.ts b/src/main/claude-usage/scanner.ts index db1c1f21f7c..f797def80ef 100644 --- a/src/main/claude-usage/scanner.ts +++ b/src/main/claude-usage/scanner.ts @@ -1,10 +1,16 @@ /* eslint-disable max-lines -- Why: transcript discovery, parsing, attribution, and aggregation share one data shape pipeline. Keeping them co-located makes it easier to audit correctness when Claude usage numbers look surprising. */ import { homedir } from 'node:os' import { join, basename } from 'node:path' -import { realpath, readdir, stat } from 'node:fs/promises' -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' +import { realpath, stat } from 'node:fs/promises' import type { Repo } from '../../shared/types' +import { walkUsageHistoryJsonlFiles } from '../usage-history-file-discovery' +import { readUsageHistoryJsonlLines } from '../usage-history-jsonl-reader' +import { + MAX_USAGE_HISTORY_FILES, + UsageHistoryScanBudget, + UsageHistoryScanCapacityError, + getUsageHistoryRetainedBytes +} from '../usage-history-scan-budget' import type { ClaudeUsageAttributedTurn, ClaudeUsageDailyAggregate, @@ -128,54 +134,29 @@ async function yieldToEventLoop(): Promise { await new Promise((resolve) => setTimeout(resolve, 0)) } -async function walkJsonlFiles(dirPath: string): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) - const files: string[] = [] - - for (const entry of entries) { - const fullPath = join(dirPath, entry.name) - if (entry.isDirectory()) { - appendDiscoveredFiles(files, await walkJsonlFiles(fullPath)) - continue - } - if (entry.isFile() && entry.name.endsWith('.jsonl')) { - files.push(fullPath) - } - } - - return files -} - -function appendDiscoveredFiles(target: string[], source: readonly string[]): void { - // Why: long-lived transcript directories can exceed V8's argument limit if - // child file arrays are spread into push(). - for (const filePath of source) { - target.push(filePath) - } -} - -export async function listClaudeTranscriptFiles(): Promise { +export async function listClaudeTranscriptFiles( + budget = new UsageHistoryScanBudget() +): Promise { const roots = [CLAUDE_PROJECTS_DIR, CLAUDE_TRANSCRIPTS_DIR] - const files = await Promise.all( - roots.map(async (root) => { - try { - return await walkJsonlFiles(root) - } catch { - return [] + const files: string[] = [] + for (const root of roots) { + try { + for (const filePath of await walkUsageHistoryJsonlFiles(root, budget)) { + files.push(filePath) } - }) - ) - return [...new Set(files.flat())].sort() + } catch (error) { + if (error instanceof UsageHistoryScanCapacityError) { + throw error + } + } + } + return [...new Set(files)].sort() } export async function getProcessedFileInfo(filePath: string): Promise { const fileStat = await stat(filePath) let lineCount = 0 - const lines = createInterface({ - input: createReadStream(filePath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) - for await (const _line of lines) { + for await (const _line of readUsageHistoryJsonlLines(filePath)) { lineCount++ } return { @@ -308,17 +289,17 @@ export function parseClaudeUsageRecord(line: string): ClaudeUsageParsedTurn | nu return parsed ? stripClaudeSourceMetadata(parsed) : null } -export async function parseClaudeUsageFile(filePath: string): Promise { +export async function parseClaudeUsageFile( + filePath: string, + budget = new UsageHistoryScanBudget() +): Promise { const turns: ClaudeUsageParsedSourceTurn[] = [] const fallbackSessionId = basename(filePath, '.jsonl') - const lines = createInterface({ - input: createReadStream(filePath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) - for await (const line of lines) { + for await (const line of readUsageHistoryJsonlLines(filePath)) { const parsed = parseClaudeUsageSourceRecord(line, fallbackSessionId) if (parsed) { + claimClaudeTurn(budget, parsed) turns.push(parsed) } } @@ -326,7 +307,10 @@ export async function parseClaudeUsageFile(filePath: string): Promise { @@ -334,15 +318,12 @@ async function readClaudeUsageScanFile(filePath: string): Promise<{ let lineCount = 0 const turns: ClaudeUsageParsedSourceTurn[] = [] const fallbackSessionId = basename(filePath, '.jsonl') - const lines = createInterface({ - input: createReadStream(filePath, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) - for await (const line of lines) { + for await (const line of readUsageHistoryJsonlLines(filePath)) { lineCount++ const parsed = parseClaudeUsageSourceRecord(line, fallbackSessionId) if (parsed) { + claimClaudeTurn(budget, parsed) turns.push(parsed) } } @@ -358,6 +339,19 @@ async function readClaudeUsageScanFile(filePath: string): Promise<{ } } +function claimClaudeTurn(budget: UsageHistoryScanBudget, turn: ClaudeUsageParsedSourceTurn): void { + budget.claimRecord( + getUsageHistoryRetainedBytes([ + turn.sessionId, + turn.timestamp, + turn.model, + turn.cwd, + turn.gitBranch, + turn.dedupeKey + ]) + ) +} + function localDayFromTimestamp(timestamp: string): string | null { const parsed = new Date(timestamp) if (Number.isNaN(parsed.getTime())) { @@ -492,6 +486,49 @@ function mergeClaudeDailyAggregates( } } +function claimClaudeUsageProjection( + budget: UsageHistoryScanBudget, + sessions: readonly ClaudeUsageSession[], + dailyAggregates: readonly ClaudeUsageDailyAggregate[] +): void { + for (const session of sessions) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + session.sessionId, + session.firstTimestamp, + session.lastTimestamp, + session.model, + session.lastCwd, + session.lastGitBranch, + session.primaryWorktreeId, + session.primaryRepoId + ]) + ) + for (const location of session.locationBreakdown) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + location.locationKey, + location.projectLabel, + location.repoId, + location.worktreeId + ]) + ) + } + } + for (const daily of dailyAggregates) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + daily.day, + daily.model, + daily.projectKey, + daily.projectLabel, + daily.repoId, + daily.worktreeId + ]) + ) + } +} + function finalizeClaudeSessions( sessionsById: Map ): ClaudeUsageSession[] { @@ -626,7 +663,14 @@ export async function scanClaudeUsageFiles( sessions: ClaudeUsageSession[] dailyAggregates: ClaudeUsageDailyAggregate[] }> { - const files = await listClaudeTranscriptFiles() + if (previousProcessedFiles.length > MAX_USAGE_HISTORY_FILES) { + throw new UsageHistoryScanCapacityError('files', MAX_USAGE_HISTORY_FILES) + } + const budget = new UsageHistoryScanBudget() + const files = await listClaudeTranscriptFiles(budget) + for (const previous of previousProcessedFiles) { + budget.claimPath(previous.path) + } const previousByPath = new Map(previousProcessedFiles.map((file) => [file.path, file])) const worktreeLookup = await buildWorktreeLookup(worktrees) @@ -685,7 +729,12 @@ export async function scanClaudeUsageFiles( // turn for exactly one file; cached files keep the claims they persisted. const turnOwnerByDedupeKey = new Map() for (const [filePath, previous] of reusedByPath) { + for (const session of previous.sessions) { + budget.claimRecords(session.turnCount) + } + claimClaudeUsageProjection(budget, previous.sessions, previous.dailyAggregates) for (const dedupeKey of previous.ownedDedupeKeys) { + budget.claimOwnershipKey(dedupeKey) // First cached claim wins so conflicting projections stay deterministic. if (!turnOwnerByDedupeKey.has(dedupeKey)) { turnOwnerByDedupeKey.set(dedupeKey, filePath) @@ -698,7 +747,9 @@ export async function scanClaudeUsageFiles( const batch = pathsToParse.slice(index, index + FILE_SCAN_BATCH_SIZE) // Why: transcript scans run in Electron's main process. Small parallel // batches cut independent file I/O without letting Settings stay blocked. - const reads = await Promise.all(batch.map((filePath) => readClaudeUsageScanFile(filePath))) + const reads = await Promise.all( + batch.map((filePath) => readClaudeUsageScanFile(filePath, budget)) + ) for (const [batchIndex, filePath] of batch.entries()) { const { processedFile, turns } = reads[batchIndex] // Why: ownership claims must be sequential in sorted-path order so @@ -713,15 +764,20 @@ export async function scanClaudeUsageFiles( hasDeferredClaims = true continue } + if (owner === undefined) { + budget.claimOwnershipKey(turn.dedupeKey) + } turnOwnerByDedupeKey.set(turn.dedupeKey, filePath) ownedDedupeKeys.push(turn.dedupeKey) } ownedTurns.push(stripClaudeSourceMetadata(turn)) } const attributed = await attributeClaudeUsageTurns(ownedTurns, worktreeLookup) + const aggregates = aggregateClaudeUsage(attributed) + claimClaudeUsageProjection(budget, aggregates.sessions, aggregates.dailyAggregates) parsedByPath.set(filePath, { ...processedFile, - ...aggregateClaudeUsage(attributed), + ...aggregates, ownedDedupeKeys, hasDeferredClaims }) @@ -744,14 +800,17 @@ export async function scanClaudeUsageFiles( mergeClaudeDailyAggregates(dailyByKey, processed.dailyAggregates) } + const sessions = finalizeClaudeSessions(sessionsById) + const dailyAggregates = [...dailyByKey.values()].sort((left, right) => + left.day === right.day + ? left.projectLabel.localeCompare(right.projectLabel) + : left.day.localeCompare(right.day) + ) + claimClaudeUsageProjection(budget, sessions, dailyAggregates) return { processedFiles, - sessions: finalizeClaudeSessions(sessionsById), - dailyAggregates: [...dailyByKey.values()].sort((left, right) => - left.day === right.day - ? left.projectLabel.localeCompare(right.projectLabel) - : left.day.localeCompare(right.day) - ) + sessions, + dailyAggregates } } diff --git a/src/main/claude-usage/store.ts b/src/main/claude-usage/store.ts index a020186ee90..bea648c6a5e 100644 --- a/src/main/claude-usage/store.ts +++ b/src/main/claude-usage/store.ts @@ -1,7 +1,6 @@ /* eslint-disable max-lines -- Why: this store is the single main-process owner for Claude usage persistence, scan gating, and query semantics. Keeping those policy decisions together avoids split-brain range/scope logic across multiple files. */ import { app } from 'electron' -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { join } from 'node:path' import type { ClaudeUsageBreakdownKind, ClaudeUsageBreakdownRow, @@ -15,6 +14,10 @@ import type { } from '../../shared/claude-usage-types' import type { AutomationRunUsage } from '../../shared/automations-types' import type { Store } from '../persistence' +import { + readUsageProjectionStateFile, + writeUsageProjectionStateFileWithRecovery +} from '../usage-projection-state-file' import { loadKnownUsageWorktreesByRepo, type UsageWorktreeRef } from '../usage-worktree-metadata' import type { ClaudeUsagePersistedState } from './types' import { createWorktreeRefs, getSessionProjectLabel, scanClaudeUsageFiles } from './scanner' @@ -326,10 +329,11 @@ export class ClaudeUsageStore { private load(): ClaudeUsagePersistedState { try { const usageFile = getClaudeUsageFile() - if (!existsSync(usageFile)) { + const raw = readUsageProjectionStateFile(usageFile) + if (raw === null) { return getDefaultState() } - const parsed = JSON.parse(readFileSync(usageFile, 'utf-8')) as ClaudeUsagePersistedState + const parsed = JSON.parse(raw) as ClaudeUsagePersistedState if (parsed.schemaVersion !== SCHEMA_VERSION) { // Why: scanner semantics affect persisted totals, so old Claude caches // must be rebuilt after parser/source changes instead of reused briefly. @@ -363,16 +367,12 @@ export class ClaudeUsageStore { private writeToDisk(): void { const usageFile = getClaudeUsageFile() - const dir = dirname(usageFile) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - // Why: scans can refresh while the app is in active use. Use the same - // atomic temp-file pattern as the main store so a crash or concurrent write - // cannot leave a truncated analytics file as the common failure mode. - const tmpFile = `${usageFile}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp` - writeFileSync(tmpFile, JSON.stringify(this.state, null, 2), 'utf-8') - renameSync(tmpFile, usageFile) + this.state = writeUsageProjectionStateFileWithRecovery(usageFile, this.state, (error) => { + const reset = getDefaultState() + reset.scanState.enabled = this.state.scanState.enabled + reset.scanState.lastScanError = error.message + return reset + }) } async setEnabled(enabled: boolean): Promise { diff --git a/src/main/cli/cli-installer.test.ts b/src/main/cli/cli-installer.test.ts index 9c17fd21116..523d637ab9e 100644 --- a/src/main/cli/cli-installer.test.ts +++ b/src/main/cli/cli-installer.test.ts @@ -7,6 +7,7 @@ import { readFile, readlink, symlink, + truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -27,7 +28,7 @@ vi.mock('node:child_process', () => ({ execFile: execFileMock })) -import { CliInstaller } from './cli-installer' +import { CLI_LAUNCHER_INSPECTION_MAX_BYTES, CliInstaller } from './cli-installer' import { buildAppImageCliWrapper } from './appimage-cli-wrapper' import { WindowsUserPathRegistryReader, @@ -706,6 +707,48 @@ describe('CliInstaller', () => { } ) + it.skipIf(process.platform === 'win32')( + 'preserves an oversized regular launcher even when its prefix resembles an Orca launcher', + async () => { + const fixture = await makeFixture() + const commandDir = join(fixture.root, 'bin') + const installPath = join(commandDir, 'orca') + const resourcesPath = await createPackagedMacLauncher(fixture.root) + await mkdir(commandDir, { recursive: true }) + await writeFile( + installPath, + [ + '#!/usr/bin/env bash', + 'ELECTRON_RUN_AS_NODE=1', + 'ORCA_NODE_OPTIONS=x', + 'NODE_REPL_EXTERNAL_MODULE=x', + `CLI='${join(fixture.root, 'out', 'cli', 'index.js')}'`, + '' + ].join('\n'), + 'utf8' + ) + await truncate(installPath, CLI_LAUNCHER_INSPECTION_MAX_BYTES + 1) + + const installer = new CliInstaller({ + platform: 'darwin', + isPackaged: true, + resourcesPath, + commandPathOverride: installPath, + processPathEnv: commandDir + }) + + await expect(installer.getStatus()).resolves.toMatchObject({ + state: 'conflict', + currentTarget: null + }) + await expect(installer.install()).rejects.toThrow('Refusing to replace non-Orca command') + await expect(lstat(installPath)).resolves.toHaveProperty( + 'size', + CLI_LAUNCHER_INSPECTION_MAX_BYTES + 1 + ) + } + ) + // Why: a dev build can temporarily own the public command on developer // machines; packaged Orca should treat that as stale, not a hard conflict. it.skipIf(process.platform === 'win32')( diff --git a/src/main/cli/cli-installer.ts b/src/main/cli/cli-installer.ts index 18d2fcc8f18..4e1e81c3001 100644 --- a/src/main/cli/cli-installer.ts +++ b/src/main/cli/cli-installer.ts @@ -2,21 +2,16 @@ import { app } from 'electron' import { execFile } from 'node:child_process' import { constants, existsSync } from 'node:fs' -import { - access, - lstat, - mkdir, - readFile, - readlink, - stat, - symlink, - unlink, - writeFile -} from 'node:fs/promises' +import { access, lstat, mkdir, readlink, stat, symlink, unlink, writeFile } from 'node:fs/promises' import { homedir } from 'node:os' import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path' import { promisify } from 'node:util' import type { CliInstallMethod, CliInstallStatus } from '../../shared/cli-install-types' +import { nodeFileContentsEqual } from '../../shared/node-file-content-equality' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' import { buildAppImageCliWrapper } from './appimage-cli-wrapper' import { invalidateWindowsUserPathRegistryCache, @@ -32,6 +27,7 @@ const LINUX_COMMAND_NAME = 'orca-ide' const LEGACY_LINUX_COMMAND_NAME = 'orca' const DEV_LAUNCHER_DIR = ['cli', 'bin'] const WINDOWS_PATH_WRITE_TIMEOUT_MS = 5_000 +export const CLI_LAUNCHER_INSPECTION_MAX_BYTES = 64 * 1024 type CliInstallerOptions = { platform?: NodeJS.Platform @@ -500,19 +496,18 @@ export class CliInstaller { }) } - const currentContent = await readFile(commandPath, 'utf8') const expectedContent = buildAppImageCliWrapper(appImagePath) + const matches = await nodeFileContentsEqual(commandPath, expectedContent) return this.buildStatus({ commandPath, launcherPath: appImagePath, installMethod: 'wrapper', supported: true, - state: currentContent === expectedContent ? 'installed' : 'stale', + state: matches ? 'installed' : 'stale', currentTarget: appImagePath, - detail: - currentContent === expectedContent - ? `Registered at ${commandPath}.` - : `${commandPath} points to a different launcher.` + detail: matches + ? `Registered at ${commandPath}.` + : `${commandPath} points to a different launcher.` }) } catch (error) { if (isMissingError(error)) { @@ -538,8 +533,7 @@ export class CliInstaller { const stats = await lstat(commandPath) if (!stats.isSymbolicLink()) { if (stats.isFile()) { - const currentContent = await readFile(commandPath, 'utf8') - const managedTarget = extractManagedUnixLauncherTarget(currentContent) + const managedTarget = await readManagedUnixLauncherTarget(commandPath) if (managedTarget) { return this.buildStatus({ commandPath, @@ -694,19 +688,18 @@ export class CliInstaller { }) } - const currentContent = await readFile(commandPath, 'utf8') const expectedContent = buildWindowsForwarder(launcherPath) + const matches = await nodeFileContentsEqual(commandPath, expectedContent) return this.buildStatus({ commandPath, launcherPath, installMethod: 'wrapper', supported: true, - state: currentContent === expectedContent ? 'installed' : 'stale', + state: matches ? 'installed' : 'stale', currentTarget: launcherPath, - detail: - currentContent === expectedContent - ? `Registered at ${commandPath}.` - : `${commandPath} points to a different launcher.` + detail: matches + ? `Registered at ${commandPath}.` + : `${commandPath} points to a different launcher.` }) } catch (error) { if (isMissingError(error)) { @@ -1013,6 +1006,20 @@ function extractManagedUnixLauncherTarget(content: string): string | null { : null } +async function readManagedUnixLauncherTarget(filePath: string): Promise { + try { + const contents = ( + await readNodeFileWithinLimit(filePath, CLI_LAUNCHER_INSPECTION_MAX_BYTES) + ).buffer.toString('utf8') + return extractManagedUnixLauncherTarget(contents) + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + return null + } + throw error + } +} + function extractShellAssignment(content: string, name: string): string | null { const match = new RegExp(`^${name}=('([^']*)'|"([^"]*)"|([^\\n]+))$`, 'm').exec(content) if (!match) { diff --git a/src/main/cli/linux-bare-orca-dispatcher.test.ts b/src/main/cli/linux-bare-orca-dispatcher.test.ts index b2bb40e558b..b95b5c87a54 100644 --- a/src/main/cli/linux-bare-orca-dispatcher.test.ts +++ b/src/main/cli/linux-bare-orca-dispatcher.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, stat, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' @@ -7,7 +7,10 @@ vi.mock('electron', () => ({ app: { isPackaged: true } })) -import { installLinuxBareOrcaDispatcher } from './linux-bare-orca-dispatcher' +import { + LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES, + installLinuxBareOrcaDispatcher +} from './linux-bare-orca-dispatcher' const created: string[] = [] @@ -67,6 +70,45 @@ describe('installLinuxBareOrcaDispatcher', () => { expect(second.state).toBe('installed') }) + it('recognizes an owned dispatcher at the exact inspection boundary', async () => { + const { homePath, resourcesPath } = await makeFixture() + const dispatcherPath = join(homePath, '.local', 'bin', 'orca') + const ownedPrefix = '#!/bin/sh\n# orca-serve-bare-orca-dispatcher\n' + await mkdir(join(homePath, '.local', 'bin'), { recursive: true }) + await writeFile( + dispatcherPath, + ownedPrefix + + ' '.repeat(LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES - Buffer.byteLength(ownedPrefix)), + 'utf8' + ) + + const result = await installLinuxBareOrcaDispatcher({ + resourcesPath, + homePath, + appImagePath: null + }) + + expect(result.state).toBe('installed') + expect((await stat(dispatcherPath)).size).toBeLessThan(LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES) + }) + + it('preserves an oversized dispatcher even when its prefix contains the ownership marker', async () => { + const { homePath, resourcesPath } = await makeFixture() + const dispatcherPath = join(homePath, '.local', 'bin', 'orca') + await mkdir(join(homePath, '.local', 'bin'), { recursive: true }) + await writeFile(dispatcherPath, '#!/bin/sh\n# orca-serve-bare-orca-dispatcher\n', 'utf8') + await truncate(dispatcherPath, LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES + 1) + + const result = await installLinuxBareOrcaDispatcher({ + resourcesPath, + homePath, + appImagePath: null + }) + + expect(result.state).toBe('skipped-foreign') + expect((await stat(dispatcherPath)).size).toBe(LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES + 1) + }) + it('quotes a resources path containing spaces so the exec line cannot be split', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-bare-dispatcher-space-')) created.push(root) diff --git a/src/main/cli/linux-bare-orca-dispatcher.ts b/src/main/cli/linux-bare-orca-dispatcher.ts index 3dc8df2906c..c77f296cc5f 100644 --- a/src/main/cli/linux-bare-orca-dispatcher.ts +++ b/src/main/cli/linux-bare-orca-dispatcher.ts @@ -1,13 +1,15 @@ import { existsSync } from 'node:fs' -import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises' +import { chmod, mkdir, writeFile } from 'node:fs/promises' import { homedir } from 'node:os' import { dirname, join } from 'node:path' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' import { buildAppImageCliWrapper, quoteShell } from './appimage-cli-wrapper' import { getBundledLauncherPath } from './cli-installer' // Why: marks a dispatcher this function wrote so repeat serve starts overwrite // our own file idempotently but never clobber a user's own ~/.local/bin/orca. const DISPATCHER_MARKER = '# orca-serve-bare-orca-dispatcher' +export const LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES = 64 * 1024 export type LinuxBareOrcaDispatcherOptions = { /** Packaged app resources root; the bundled `orca-ide` launcher lives under it. */ @@ -106,7 +108,9 @@ function withMarker(script: string): string { async function isOwnedDispatcher(dispatcherPath: string): Promise { try { - return (await readFile(dispatcherPath, 'utf8')).includes(DISPATCHER_MARKER) + return ( + await readNodeFileWithinLimit(dispatcherPath, LINUX_BARE_ORCA_DISPATCHER_MAX_BYTES) + ).buffer.includes(DISPATCHER_MARKER) } catch { return false } diff --git a/src/main/cli/linux-terminal-orca-cli-shim.test.ts b/src/main/cli/linux-terminal-orca-cli-shim.test.ts index 81af17ba779..29f63a472e7 100644 --- a/src/main/cli/linux-terminal-orca-cli-shim.test.ts +++ b/src/main/cli/linux-terminal-orca-cli-shim.test.ts @@ -1,4 +1,4 @@ -import { chmodSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { chmodSync, mkdirSync, readFileSync, statSync, truncateSync, writeFileSync } from 'node:fs' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -8,7 +8,14 @@ vi.mock('electron', () => ({ app: { isPackaged: true } })) -import { ensureLinuxTerminalOrcaCliShimDir } from './linux-terminal-orca-cli-shim' +import { + _getLinuxTerminalShimCacheSizeForTests, + _isLinuxTerminalShimCacheKeyRetainableForTests, + _resetLinuxTerminalShimCacheForTests, + ensureLinuxTerminalOrcaCliShimDir, + LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES, + LINUX_TERMINAL_SHIM_CACHE_MAX_ENTRIES +} from './linux-terminal-orca-cli-shim' const created: string[] = [] @@ -23,6 +30,7 @@ async function makeFixture(): Promise<{ userDataPath: string; resourcesPath: str } afterEach(async () => { + _resetLinuxTerminalShimCacheForTests() await Promise.all(created.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) }) @@ -91,6 +99,25 @@ describe('ensureLinuxTerminalOrcaCliShimDir', () => { expect(content).not.toContain(resourcesPath) }) + it('replaces a large sparse file at the owned shim path without loading it whole', async () => { + const { userDataPath, resourcesPath } = await makeFixture() + const shimDir = join(userDataPath, 'linux-orca-cli-shim') + const shimPath = join(shimDir, 'orca') + mkdirSync(shimDir, { recursive: true }) + writeFileSync(shimPath, 'stale') + truncateSync(shimPath, 256 * 1024 * 1024) + + expect( + ensureLinuxTerminalOrcaCliShimDir({ + userDataPath, + resourcesPath, + appImagePath: null + }) + ).toBe(shimDir) + expect(statSync(shimPath).size).toBeLessThan(64 * 1024) + expect(readFileSync(shimPath, 'utf8')).toContain('orca-ide') + }) + it('returns null (and does not memoize) when the bundled launcher is missing', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-terminal-cli-shim-missing-')) created.push(root) @@ -115,4 +142,31 @@ describe('ensureLinuxTerminalOrcaCliShimDir', () => { }) expect(recovered).toBe(join(userDataPath, 'linux-orca-cli-shim')) }) + + it('bounds successful user-data path memoization', async () => { + const { userDataPath, resourcesPath } = await makeFixture() + for (let index = 0; index <= LINUX_TERMINAL_SHIM_CACHE_MAX_ENTRIES; index += 1) { + expect( + ensureLinuxTerminalOrcaCliShimDir({ + userDataPath: join(userDataPath, String(index)), + resourcesPath, + appImagePath: null + }) + ).not.toBeNull() + } + expect(_getLinuxTerminalShimCacheSizeForTests()).toBe(LINUX_TERMINAL_SHIM_CACHE_MAX_ENTRIES) + }) + + it('measures memoized paths by UTF-8 bytes', () => { + expect( + _isLinuxTerminalShimCacheKeyRetainableForTests( + 'x'.repeat(LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES) + ) + ).toBe(true) + expect( + _isLinuxTerminalShimCacheKeyRetainableForTests( + '😀'.repeat(LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES / 4 + 1) + ) + ).toBe(false) + }) }) diff --git a/src/main/cli/linux-terminal-orca-cli-shim.ts b/src/main/cli/linux-terminal-orca-cli-shim.ts index 56f38df15ce..df682923173 100644 --- a/src/main/cli/linux-terminal-orca-cli-shim.ts +++ b/src/main/cli/linux-terminal-orca-cli-shim.ts @@ -1,5 +1,7 @@ -import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' +import { nodeFileContentsEqualSync } from '../../shared/node-file-content-equality' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' import { buildBareOrcaCliScript } from './linux-bare-orca-dispatcher' const SHIM_DIR_NAME = 'linux-orca-cli-shim' @@ -8,6 +10,37 @@ const SHIM_DIR_NAME = 'linux-orca-cli-shim' // changes with the install itself, so one successful write per process is enough. // Failures are NOT cached so a transient fs error retries on the next spawn. const ensuredShimDirs = new Map() +export const LINUX_TERMINAL_SHIM_CACHE_MAX_ENTRIES = 64 +export const LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES = 64 * 1024 + +function getEnsuredShimDir(userDataPath: string): string | undefined { + const cached = ensuredShimDirs.get(userDataPath) + if (cached === undefined) { + return undefined + } + ensuredShimDirs.delete(userDataPath) + ensuredShimDirs.set(userDataPath, cached) + return cached +} + +function rememberEnsuredShimDir(userDataPath: string, shimDir: string): void { + if ( + measureUtf8ByteLength(userDataPath, { + stopAfterBytes: LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES + }).exceededLimit + ) { + return + } + ensuredShimDirs.delete(userDataPath) + ensuredShimDirs.set(userDataPath, shimDir) + while (ensuredShimDirs.size > LINUX_TERMINAL_SHIM_CACHE_MAX_ENTRIES) { + const oldest = ensuredShimDirs.keys().next().value + if (oldest === undefined) { + return + } + ensuredShimDirs.delete(oldest) + } +} export type LinuxTerminalOrcaCliShimOptions = { userDataPath: string @@ -27,7 +60,7 @@ export type LinuxTerminalOrcaCliShimOptions = { export function ensureLinuxTerminalOrcaCliShimDir( options: LinuxTerminalOrcaCliShimOptions ): string | null { - const cached = ensuredShimDirs.get(options.userDataPath) + const cached = getEnsuredShimDir(options.userDataPath) if (cached !== undefined) { return cached } @@ -47,7 +80,7 @@ export function ensureLinuxTerminalOrcaCliShimDir( const shimDir = join(options.userDataPath, SHIM_DIR_NAME) const shimPath = join(shimDir, 'orca') try { - if (readShim(shimPath) !== resolved.script) { + if (!shimMatches(shimPath, resolved.script)) { mkdirSync(shimDir, { recursive: true }) writeFileSync(shimPath, resolved.script, 'utf8') } @@ -57,14 +90,28 @@ export function ensureLinuxTerminalOrcaCliShimDir( } catch { return null } - ensuredShimDirs.set(options.userDataPath, shimDir) + rememberEnsuredShimDir(options.userDataPath, shimDir) return shimDir } -function readShim(shimPath: string): string | null { +export function _resetLinuxTerminalShimCacheForTests(): void { + ensuredShimDirs.clear() +} + +export function _getLinuxTerminalShimCacheSizeForTests(): number { + return ensuredShimDirs.size +} + +export function _isLinuxTerminalShimCacheKeyRetainableForTests(userDataPath: string): boolean { + return !measureUtf8ByteLength(userDataPath, { + stopAfterBytes: LINUX_TERMINAL_SHIM_CACHE_KEY_MAX_BYTES + }).exceededLimit +} + +function shimMatches(shimPath: string, expected: string): boolean { try { - return readFileSync(shimPath, 'utf8') + return nodeFileContentsEqualSync(shimPath, expected) } catch { - return null + return false } } diff --git a/src/main/cli/wsl-cli-installer-bounds.test.ts b/src/main/cli/wsl-cli-installer-bounds.test.ts new file mode 100644 index 00000000000..61694684913 --- /dev/null +++ b/src/main/cli/wsl-cli-installer-bounds.test.ts @@ -0,0 +1,80 @@ +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, stat, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import type { CliInstallStatus } from '../../shared/cli-install-types' +import { WSL_CLI_INSPECTION_MAX_BYTES, WslCliInstaller, _internals } from './wsl-cli-installer' + +function installedHostStatus(): CliInstallStatus { + const launcherPath = 'C:\\Orca\\resources\\bin\\orca.exe' + return { + platform: 'win32', + commandName: 'orca', + commandPath: launcherPath, + pathDirectory: 'C:\\Orca\\resources\\bin', + pathConfigured: true, + launcherPath, + installMethod: 'wrapper', + supported: true, + state: 'installed', + currentTarget: launcherPath, + unsupportedReason: null, + detail: null + } +} + +describe('WSL CLI launcher inspection bounds', () => { + it('accepts encoded file output at the exact byte ceiling and rejects one byte more', () => { + const prefix = _internals.WSL_BOUNDED_FILE_OUTPUT_PREFIX + const exact = Buffer.alloc(WSL_CLI_INSPECTION_MAX_BYTES, 0x61) + const oversized = Buffer.alloc(WSL_CLI_INSPECTION_MAX_BYTES + 1, 0x61) + + expect( + _internals.parseBoundedWslFileOutput(`${prefix}${exact.toString('base64')}`) + ).toHaveLength(WSL_CLI_INSPECTION_MAX_BYTES) + expect(_internals.parseBoundedWslFileOutput(`${prefix}${oversized.toString('base64')}`)).toBe( + 'not_file' + ) + }) + + it.skipIf(process.platform === 'win32')( + 'fails closed on a sparse oversized launcher without reading it whole', + async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-wsl-cli-bounded-read-')) + const home = join(root, 'home') + const commandPath = join(home, '.local', 'bin', 'orca-ide') + await mkdir(join(home, '.local', 'bin'), { recursive: true }) + await writeFile(commandPath, '# Orca managed WSL CLI launcher\n', 'utf8') + await truncate(commandPath, 256 * 1024 * 1024) + const runner = async (_distro: string, command: string): Promise => { + if (command.includes('printf %s "$HOME"')) { + return home + } + if (command.includes('command -v powershell.exe')) { + return 'yes' + } + if (command.includes('case ":$PATH:"')) { + return 'yes' + } + return execFileSync('bash', ['-c', command], { + encoding: 'utf8', + maxBuffer: 256 * 1024 + }) + } + const installer = new WslCliInstaller({ + platform: 'win32', + distro: 'Ubuntu', + hostInstaller: { getStatus: async () => installedHostStatus() }, + wslRunner: runner + }) + + try { + await expect(installer.getStatus()).resolves.toMatchObject({ state: 'conflict' }) + await expect(stat(commandPath)).resolves.toHaveProperty('size', 256 * 1024 * 1024) + } finally { + await rm(root, { recursive: true, force: true }) + } + } + ) +}) diff --git a/src/main/cli/wsl-cli-installer.test.ts b/src/main/cli/wsl-cli-installer.test.ts index 13bd8088de3..d7d8be6c3fd 100644 --- a/src/main/cli/wsl-cli-installer.test.ts +++ b/src/main/cli/wsl-cli-installer.test.ts @@ -134,7 +134,7 @@ function createWslRunner( } return '' } - if (command.includes('cat ')) { + if (command.includes('head -c ')) { if (command.includes(commandPath)) { return files.get(commandPath) ?? '__ORCA_MISSING__' } @@ -267,7 +267,10 @@ describe('WslCliInstaller', () => { distro: 'Ubuntu', hostInstaller: { getStatus: async () => makeHostStatus('C:\\Orca\\orca.cmd') }, wslRunner: async (distro, command) => { - if (command.includes('cat /home/alice/.local/share/orca/orca-wsl-bridge.ps1')) { + if ( + command.includes('head -c ') && + command.includes('/home/alice/.local/share/orca/orca-wsl-bridge.ps1') + ) { return `${_internals.buildWslBridgeScript()}\n` } return wsl.runner(distro, command) @@ -828,9 +831,6 @@ describe('WslCliInstaller', () => { distro: 'Ubuntu', hostInstaller: { getStatus: async () => makeHostStatus('C:\\Orca\\orca.cmd') }, wslRunner: async (distro, command) => { - if (command.includes('cat /home/alice/.local/share/orca/orca-wsl-bridge.ps1')) { - return 'user bridge' - } if (command.includes('rm -f')) { throw new Error('__ORCA_CONFLICT__') } diff --git a/src/main/cli/wsl-cli-installer.ts b/src/main/cli/wsl-cli-installer.ts index f6a1684e19c..7153329c35f 100644 --- a/src/main/cli/wsl-cli-installer.ts +++ b/src/main/cli/wsl-cli-installer.ts @@ -24,6 +24,8 @@ const BRIDGE_MANAGED_MARKER = getWslBridgeMarker() const WSL_COMMAND_NAME = 'orca-ide' const LEGACY_WSL_COMMAND_NAME = 'orca' const WSL_COMMAND_TIMEOUT_MS = 10_000 +const WSL_BOUNDED_FILE_OUTPUT_PREFIX = '__ORCA_BOUNDED_FILE_BASE64__:' +export const WSL_CLI_INSPECTION_MAX_BYTES = 64 * 1024 function normalizeManagedScriptContent(content: string): string { return content.replace(/\n+$/u, '\n') @@ -383,6 +385,7 @@ export class WslCliInstaller { const output = await this.run( distro, [ + 'set -o pipefail', `if [ -L ${quoteShell(commandPath)} ]; then`, ' printf __ORCA_NOT_FILE__', `elif [ ! -e ${quoteShell(commandPath)} ]; then`, @@ -390,7 +393,8 @@ export class WslCliInstaller { `elif [ ! -f ${quoteShell(commandPath)} ]; then`, ' printf __ORCA_NOT_FILE__', 'else', - ` cat ${quoteShell(commandPath)}`, + ` printf ${quoteShell(WSL_BOUNDED_FILE_OUTPUT_PREFIX)}`, + ` head -c ${WSL_CLI_INSPECTION_MAX_BYTES + 1} -- ${quoteShell(commandPath)} | base64`, 'fi' ].join('\n') ) @@ -400,7 +404,7 @@ export class WslCliInstaller { if (output === '__ORCA_NOT_FILE__') { return 'not_file' } - return output + return parseBoundedWslFileOutput(output) } private buildStatus(args: { @@ -506,10 +510,24 @@ function buildEncodedWslBashCommand(command: string): string { return `set -o pipefail; printf %s ${quoteShell(encoded)} | base64 -d | bash` } +function parseBoundedWslFileOutput(output: string): string | 'not_file' { + if (!output.startsWith(WSL_BOUNDED_FILE_OUTPUT_PREFIX)) { + return output + } + const encoded = output.slice(WSL_BOUNDED_FILE_OUTPUT_PREFIX.length).replace(/\s/gu, '') + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded)) { + return 'not_file' + } + const decoded = Buffer.from(encoded, 'base64') + return decoded.length <= WSL_CLI_INSPECTION_MAX_BYTES ? decoded.toString('utf8') : 'not_file' +} + export const _internals = { buildEncodedWslBashCommand, buildWslBridgeScript, buildWslLauncher, getBridgePathFromCommandPath, - parseManagedLauncherTarget + parseManagedLauncherTarget, + parseBoundedWslFileOutput, + WSL_BOUNDED_FILE_OUTPUT_PREFIX } diff --git a/src/main/cli/wsl-cli-registration-registry.test.ts b/src/main/cli/wsl-cli-registration-registry.test.ts index 95bf554a104..92c60caa85e 100644 --- a/src/main/cli/wsl-cli-registration-registry.test.ts +++ b/src/main/cli/wsl-cli-registration-registry.test.ts @@ -1,11 +1,12 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { getWslCliRegistrationCandidates, recordWslCliRegistrationObservations, - recordWslCliRegistrationRemoved + recordWslCliRegistrationRemoved, + WSL_CLI_REGISTRY_MAX_BYTES } from './wsl-cli-registration-registry' describe('WSL CLI registration registry', () => { @@ -144,6 +145,16 @@ describe('WSL CLI registration registry', () => { ).resolves.toEqual(['Ubuntu', 'Debian']) }) + it('rediscovers available distros without reading an oversized sparse registry', async () => { + const registryPath = join(userDataPath, 'wsl-cli-registrations.json') + await writeFile(registryPath, '{"schemaVersion":2}', 'utf8') + await truncate(registryPath, WSL_CLI_REGISTRY_MAX_BYTES + 1) + + await expect( + getWslCliRegistrationCandidates(userDataPath, ['Ubuntu', 'Debian']) + ).resolves.toEqual(['Ubuntu', 'Debian']) + }) + it('stops reconciling a registration removed through Settings', async () => { await recordWslCliRegistrationObservations(userDataPath, [ { distro: 'Ubuntu', inspected: true, managed: true } diff --git a/src/main/cli/wsl-cli-registration-registry.ts b/src/main/cli/wsl-cli-registration-registry.ts index d1a84a42f52..70dc6934553 100644 --- a/src/main/cli/wsl-cli-registration-registry.ts +++ b/src/main/cli/wsl-cli-registration-registry.ts @@ -1,5 +1,9 @@ -import { mkdir, readFile } from 'node:fs/promises' +import { mkdir } from 'node:fs/promises' import { join } from 'node:path' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' import { getKeyedSerializedQueueTail, runKeyedSerializedOperation } from './keyed-promise-queue' import { normalizeWslDistroKey } from './wsl-cli-registration-operation' @@ -7,6 +11,7 @@ import { normalizeWslDistroKey } from './wsl-cli-registration-operation' const REGISTRY_FILE_NAME = 'wsl-cli-registrations.json' const REGISTRY_SCHEMA_VERSION = 2 const DEFAULT_NEGATIVE_INSPECTION_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +export const WSL_CLI_REGISTRY_MAX_BYTES = 1024 * 1024 // Why: the registry is advisory; cap per-distro bookkeeping so hosts that // cycle many uniquely named distros cannot grow the file without bound. const MAX_INSPECTION_ENTRIES = 64 @@ -126,11 +131,18 @@ function getRegistryPath(userDataPath: string): string { async function readState(userDataPath: string): Promise { try { - return parseState(await readFile(getRegistryPath(userDataPath), 'utf8')) + const { buffer } = await readNodeFileWithinLimit( + getRegistryPath(userDataPath), + WSL_CLI_REGISTRY_MAX_BYTES + ) + return parseState(buffer.toString('utf8')) } catch (error) { if (isMissingError(error)) { return emptyState() } + if (error instanceof NodeFileReadTooLargeError) { + return emptyState() + } throw error } } diff --git a/src/main/codex-accounts/codex-wsl-runtime-home-retention.test.ts b/src/main/codex-accounts/codex-wsl-runtime-home-retention.test.ts new file mode 100644 index 00000000000..adcc9d6372e --- /dev/null +++ b/src/main/codex-accounts/codex-wsl-runtime-home-retention.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { CodexWslRuntimeHomeRetention } from './codex-wsl-runtime-home-retention' + +const BOUNDS = { + maxEntries: 2, + maxKeyBytes: 8, + maxValueBytes: 8, + maxTotalBytes: 18 +} + +describe('CodexWslRuntimeHomeRetention', () => { + it('retains the coordinated state and distinguishes an explicit system account', () => { + const retention = new CodexWslRuntimeHomeRetention(BOUNDS) + retention.setRuntimeHomePath('a', '/home') + retention.setLastWrittenAuthJson('a', 'auth') + retention.setLastSyncedAccountId('a', null) + + expect(retention.getRuntimeHomePath('a')).toBe('/home') + expect(retention.getLastWrittenAuthJson('a')).toBe('auth') + expect(retention.hasLastSyncedAccountId('a')).toBe(true) + expect(retention.getLastSyncedAccountId('a')).toBeNull() + }) + + it('evicts the least-recently-used distro at the entry bound', () => { + const retention = new CodexWslRuntimeHomeRetention(BOUNDS) + retention.setRuntimeHomePath('a', 'one') + retention.setRuntimeHomePath('b', 'two') + retention.getRuntimeHomePath('a') + retention.setRuntimeHomePath('c', 'three') + + expect(retention.getRuntimeHomePath('a')).toBe('one') + expect(retention.getRuntimeHomePath('b')).toBeUndefined() + expect(retention.getRuntimeHomePath('c')).toBe('three') + expect(retention.evidence().entries).toBe(2) + }) + + it('bounds aggregate retained bytes and fails closed for oversized fields', () => { + const retention = new CodexWslRuntimeHomeRetention(BOUNDS) + retention.setRuntimeHomePath('a', '12345678') + retention.setRuntimeHomePath('b', '12345678') + + expect(retention.evidence()).toEqual({ entries: 2, retainedBytes: 18 }) + + retention.setLastWrittenAuthJson('b', '123456789') + + expect(retention.getRuntimeHomePath('b')).toBeUndefined() + expect(retention.evidence()).toEqual({ entries: 1, retainedBytes: 9 }) + }) +}) diff --git a/src/main/codex-accounts/codex-wsl-runtime-home-retention.ts b/src/main/codex-accounts/codex-wsl-runtime-home-retention.ts new file mode 100644 index 00000000000..e15e4709ef7 --- /dev/null +++ b/src/main/codex-accounts/codex-wsl-runtime-home-retention.ts @@ -0,0 +1,140 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export type CodexWslRuntimeHomeRetentionBounds = { + maxEntries: number + maxKeyBytes: number + maxValueBytes: number + maxTotalBytes: number +} + +export const DEFAULT_CODEX_WSL_RUNTIME_HOME_RETENTION_BOUNDS: CodexWslRuntimeHomeRetentionBounds = { + maxEntries: 64, + maxKeyBytes: 1024, + maxValueBytes: 4 * 1024 * 1024, + maxTotalBytes: 8 * 1024 * 1024 +} + +type CodexWslRuntimeHomeEntry = { + runtimeHomePath?: string + lastWrittenAuthJson?: string | null + lastSyncedAccountId?: string | null + retainedBytes: number +} + +export class CodexWslRuntimeHomeRetention { + private readonly entries = new Map() + private retainedBytes = 0 + + constructor( + private readonly bounds: CodexWslRuntimeHomeRetentionBounds = DEFAULT_CODEX_WSL_RUNTIME_HOME_RETENTION_BOUNDS + ) { + for (const value of Object.values(bounds)) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new RangeError('Codex WSL runtime-home retention bounds must be positive integers') + } + } + } + + getRuntimeHomePath(distro: string): string | undefined { + return this.get(distro)?.runtimeHomePath + } + + hasLastSyncedAccountId(distro: string): boolean { + return this.get(distro)?.lastSyncedAccountId !== undefined + } + + getLastSyncedAccountId(distro: string): string | null | undefined { + return this.get(distro)?.lastSyncedAccountId + } + + getLastWrittenAuthJson(distro: string): string | null | undefined { + return this.get(distro)?.lastWrittenAuthJson + } + + setRuntimeHomePath(distro: string, runtimeHomePath: string): void { + this.update(distro, { runtimeHomePath }) + } + + setLastSyncedAccountId(distro: string, accountId: string | null): void { + this.update(distro, { lastSyncedAccountId: accountId }) + } + + setLastWrittenAuthJson(distro: string, authJson: string | null): void { + this.update(distro, { lastWrittenAuthJson: authJson }) + } + + evidence(): { entries: number; retainedBytes: number } { + return { entries: this.entries.size, retainedBytes: this.retainedBytes } + } + + private get(distro: string): CodexWslRuntimeHomeEntry | undefined { + const entry = this.entries.get(distro) + if (entry) { + this.entries.delete(distro) + this.entries.set(distro, entry) + } + return entry + } + + private update( + distro: string, + patch: Omit, 'retainedBytes'> + ): void { + const previous = this.entries.get(distro) + const next = { ...previous, ...patch, retainedBytes: 0 } + const retainedBytes = this.measureEntry(distro, next) + this.delete(distro) + if (retainedBytes === null || retainedBytes > this.bounds.maxTotalBytes) { + return + } + while ( + this.entries.size >= this.bounds.maxEntries || + this.retainedBytes + retainedBytes > this.bounds.maxTotalBytes + ) { + const oldest = this.entries.keys().next().value + if (oldest === undefined) { + return + } + this.delete(oldest) + } + next.retainedBytes = retainedBytes + this.entries.set(distro, next) + this.retainedBytes += retainedBytes + } + + private measureEntry(distro: string, entry: CodexWslRuntimeHomeEntry): number | null { + const keyBytes = measureUtf8ByteLength(distro, { + stopAfterBytes: this.bounds.maxKeyBytes + }) + if (keyBytes.exceededLimit) { + return null + } + let retainedBytes = keyBytes.byteLength + for (const value of [ + entry.runtimeHomePath, + entry.lastWrittenAuthJson, + entry.lastSyncedAccountId + ]) { + if (typeof value !== 'string') { + continue + } + const valueBytes = measureUtf8ByteLength(value, { + stopAfterBytes: this.bounds.maxValueBytes + }) + if (valueBytes.exceededLimit) { + return null + } + retainedBytes += valueBytes.byteLength + } + return retainedBytes + } + + private delete(distro: string): void { + const entry = this.entries.get(distro) + if (!entry) { + return + } + this.entries.delete(distro) + this.retainedBytes -= entry.retainedBytes + } +} diff --git a/src/main/codex-accounts/host-codex-managed-home-ownership.ts b/src/main/codex-accounts/host-codex-managed-home-ownership.ts index 6f0603c73f4..a7d2d4a09c5 100644 --- a/src/main/codex-accounts/host-codex-managed-home-ownership.ts +++ b/src/main/codex-accounts/host-codex-managed-home-ownership.ts @@ -1,5 +1,6 @@ -import { existsSync, lstatSync, readFileSync, realpathSync } from 'node:fs' +import { existsSync, lstatSync, realpathSync } from 'node:fs' import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { readAgentStateFileSync } from '../agent-state-file-reader' type HostCodexManagedHomeOwnershipOptions = { candidatePath: string @@ -87,7 +88,7 @@ export function assertOwnedHostCodexManagedHomePath({ if (!markerIsRegularFile) { throw new Error('Managed Codex home ownership marker is not a regular file.') } - const markerContents = readFileSync(markerPath, 'utf-8') + const markerContents = readAgentStateFileSync(markerPath) if (expectedAccountId !== undefined && markerContents.trim() !== expectedAccountId) { throw new Error('Managed Codex home ownership marker does not match its account ID.') } diff --git a/src/main/codex-accounts/legacy-history-migration.test.ts b/src/main/codex-accounts/legacy-history-migration.test.ts new file mode 100644 index 00000000000..7b47c93d9ca --- /dev/null +++ b/src/main/codex-accounts/legacy-history-migration.test.ts @@ -0,0 +1,109 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + CODEX_LEGACY_HISTORY_READ_CHUNK_BYTES, + mergeCodexLegacyHistorySync +} from './legacy-history-migration' + +describe('Codex legacy history migration', () => { + let root = '' + let legacyHistoryPath = '' + let runtimeHistoryPath = '' + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-codex-legacy-history-')) + legacyHistoryPath = join(root, 'legacy-history.jsonl') + runtimeHistoryPath = join(root, 'runtime-history.jsonl') + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('preserves the legacy merge order and exact normalized output in limit', () => { + const longLegacyLine = `🦀${'x'.repeat(CODEX_LEGACY_HISTORY_READ_CHUNK_BYTES + 8)}` + writeFileSync(runtimeHistoryPath, 'first\n\nsecond\r\nlast') + writeFileSync(legacyHistoryPath, `second\r\nthird\nfirst\n\nthird\n${longLegacyLine}`) + + const result = mergeCodexLegacyHistorySync({ legacyHistoryPath, runtimeHistoryPath }) + + expect(result).toMatchObject({ kind: 'merged', addedLineCount: 2, outputLineCount: 5 }) + expect(readFileSync(runtimeHistoryPath, 'utf8')).toBe( + `first\nsecond\r\nlast\nthird\n${longLegacyLine}\n` + ) + }) + + it('leaves the runtime file byte-for-byte unchanged when output exceeds its limit', () => { + const originalRuntime = 'runtime\n' + writeFileSync(runtimeHistoryPath, originalRuntime) + writeFileSync(legacyHistoryPath, 'new-value\n') + + const result = mergeCodexLegacyHistorySync({ + legacyHistoryPath, + runtimeHistoryPath, + limits: { maxOutputBytes: Buffer.byteLength(originalRuntime) } + }) + + expect(result).toEqual({ + kind: 'skipped', + reason: 'output-bytes', + observed: Buffer.byteLength(originalRuntime) + Buffer.byteLength('new-value\n'), + limit: Buffer.byteLength(originalRuntime) + }) + expect(readFileSync(runtimeHistoryPath, 'utf8')).toBe(originalRuntime) + }) + + it('does not create partial output for an oversized source file or line', () => { + writeFileSync(legacyHistoryPath, '') + truncateSync(legacyHistoryPath, 4096) + + const fileResult = mergeCodexLegacyHistorySync({ + legacyHistoryPath, + runtimeHistoryPath, + limits: { maxFileBytes: 1024 } + }) + + expect(fileResult).toMatchObject({ kind: 'skipped', reason: 'file-bytes', limit: 1024 }) + expect(existsSync(runtimeHistoryPath)).toBe(false) + + writeFileSync(legacyHistoryPath, 'oversized\n') + const lineResult = mergeCodexLegacyHistorySync({ + legacyHistoryPath, + runtimeHistoryPath, + limits: { maxLineBytes: 4 } + }) + + expect(lineResult).toMatchObject({ kind: 'skipped', reason: 'line-bytes', limit: 4 }) + expect(existsSync(runtimeHistoryPath)).toBe(false) + }) + + it('bounds input records even when every record is empty or duplicated', () => { + writeFileSync(legacyHistoryPath, 'same\nsame\n\nsame\n') + + const result = mergeCodexLegacyHistorySync({ + legacyHistoryPath, + runtimeHistoryPath, + limits: { maxInputLines: 3 } + }) + + expect(result).toMatchObject({ + kind: 'skipped', + reason: 'input-lines', + observed: 4, + limit: 3 + }) + expect(existsSync(runtimeHistoryPath)).toBe(false) + }) + + it('leaves an existing empty-only runtime file untouched when no record survives', () => { + writeFileSync(runtimeHistoryPath, '\n\n') + writeFileSync(legacyHistoryPath, '\n') + + expect(mergeCodexLegacyHistorySync({ legacyHistoryPath, runtimeHistoryPath })).toEqual({ + kind: 'empty' + }) + expect(readFileSync(runtimeHistoryPath, 'utf8')).toBe('\n\n') + }) +}) diff --git a/src/main/codex-accounts/legacy-history-migration.ts b/src/main/codex-accounts/legacy-history-migration.ts new file mode 100644 index 00000000000..81dbd40c3ae --- /dev/null +++ b/src/main/codex-accounts/legacy-history-migration.ts @@ -0,0 +1,230 @@ +import { randomUUID } from 'node:crypto' +import { + closeSync, + existsSync, + fstatSync, + mkdirSync, + openSync, + readSync, + rmSync, + writeSync +} from 'node:fs' +import { dirname } from 'node:path' +import { renameFileWithWindowsRetry } from './fs-utils' + +export const CODEX_LEGACY_HISTORY_FILE_MAX_BYTES = 32 * 1024 * 1024 +export const CODEX_LEGACY_HISTORY_LINE_MAX_BYTES = 1024 * 1024 +export const CODEX_LEGACY_HISTORY_INPUT_MAX_LINES = 200_000 +export const CODEX_LEGACY_HISTORY_OUTPUT_MAX_BYTES = 32 * 1024 * 1024 +export const CODEX_LEGACY_HISTORY_READ_CHUNK_BYTES = 64 * 1024 + +export type CodexLegacyHistoryMigrationLimits = { + maxFileBytes: number + maxLineBytes: number + maxInputLines: number + maxOutputBytes: number +} + +export const DEFAULT_CODEX_LEGACY_HISTORY_MIGRATION_LIMITS: CodexLegacyHistoryMigrationLimits = { + maxFileBytes: CODEX_LEGACY_HISTORY_FILE_MAX_BYTES, + maxLineBytes: CODEX_LEGACY_HISTORY_LINE_MAX_BYTES, + maxInputLines: CODEX_LEGACY_HISTORY_INPUT_MAX_LINES, + maxOutputBytes: CODEX_LEGACY_HISTORY_OUTPUT_MAX_BYTES +} + +export type CodexLegacyHistorySkipReason = + | 'file-bytes' + | 'line-bytes' + | 'input-lines' + | 'output-bytes' + +export type CodexLegacyHistoryMigrationResult = + | { + kind: 'merged' + addedLineCount: number + outputBytes: number + outputLineCount: number + } + | { kind: 'empty' } + | { + kind: 'skipped' + reason: CodexLegacyHistorySkipReason + observed: number + limit: number + } + +class CodexLegacyHistoryCapacityError extends Error { + constructor( + readonly reason: CodexLegacyHistorySkipReason, + readonly observed: number, + readonly limit: number + ) { + super(`Codex legacy history exceeded ${reason} limit (${observed} > ${limit})`) + this.name = 'CodexLegacyHistoryCapacityError' + } +} + +export function mergeCodexLegacyHistorySync(options: { + legacyHistoryPath: string + runtimeHistoryPath: string + limits?: Partial +}): CodexLegacyHistoryMigrationResult { + const limits = resolveLimits(options.limits) + const temporaryPath = `${options.runtimeHistoryPath}.${process.pid}.${randomUUID()}.migration.tmp` + let outputDescriptor: number | null = null + + try { + mkdirSync(dirname(options.runtimeHistoryPath), { recursive: true }) + outputDescriptor = openSync(temporaryPath, 'wx') + const seenLines = new Set() + let inputLineCount = 0 + let outputBytes = 0 + let addedLineCount = 0 + + const retainLine = (line: string, fromLegacy: boolean): void => { + inputLineCount += 1 + assertWithinCapacity('input-lines', inputLineCount, limits.maxInputLines) + if (!line || seenLines.has(line)) { + return + } + + const encodedLine = Buffer.from(`${line}\n`, 'utf8') + assertWithinCapacity('output-bytes', outputBytes + encodedLine.length, limits.maxOutputBytes) + seenLines.add(line) + writeAll(outputDescriptor!, encodedLine) + outputBytes += encodedLine.length + if (fromLegacy) { + addedLineCount += 1 + } + } + + if (existsSync(options.runtimeHistoryPath)) { + readHistoryLinesSync(options.runtimeHistoryPath, limits, (line) => retainLine(line, false)) + } + readHistoryLinesSync(options.legacyHistoryPath, limits, (line) => retainLine(line, true)) + + closeSync(outputDescriptor) + outputDescriptor = null + if (seenLines.size === 0) { + rmSync(temporaryPath, { force: true }) + return { kind: 'empty' } + } + + renameFileWithWindowsRetry(temporaryPath, options.runtimeHistoryPath) + return { + kind: 'merged', + addedLineCount, + outputBytes, + outputLineCount: seenLines.size + } + } catch (error) { + closeIgnoringErrors(outputDescriptor) + rmSync(temporaryPath, { force: true }) + if (error instanceof CodexLegacyHistoryCapacityError) { + return { + kind: 'skipped', + reason: error.reason, + observed: error.observed, + limit: error.limit + } + } + throw error + } +} + +function readHistoryLinesSync( + filePath: string, + limits: CodexLegacyHistoryMigrationLimits, + onLine: (line: string) => void +): void { + const descriptor = openSync(filePath, 'r') + try { + const initialBytes = fstatSync(descriptor).size + assertWithinCapacity('file-bytes', initialBytes, limits.maxFileBytes) + const readBuffer = Buffer.allocUnsafe(CODEX_LEGACY_HISTORY_READ_CHUNK_BYTES) + let sourceBytes = 0 + let fragments: Buffer[] = [] + let fragmentBytes = 0 + + while (true) { + const bytesRead = readSync(descriptor, readBuffer, 0, readBuffer.length, null) + if (bytesRead === 0) { + break + } + sourceBytes += bytesRead + assertWithinCapacity('file-bytes', sourceBytes, limits.maxFileBytes) + + let offset = 0 + while (offset < bytesRead) { + const newlineIndex = readBuffer.indexOf(0x0a, offset) + const end = newlineIndex === -1 || newlineIndex >= bytesRead ? bytesRead : newlineIndex + const nextFragmentBytes = end - offset + assertWithinCapacity('line-bytes', fragmentBytes + nextFragmentBytes, limits.maxLineBytes) + if (nextFragmentBytes > 0) { + fragments.push(Buffer.from(readBuffer.subarray(offset, end))) + fragmentBytes += nextFragmentBytes + } + if (newlineIndex === -1 || newlineIndex >= bytesRead) { + break + } + onLine(decodeLine(fragments, fragmentBytes)) + fragments = [] + fragmentBytes = 0 + offset = newlineIndex + 1 + } + } + + if (fragmentBytes > 0) { + onLine(decodeLine(fragments, fragmentBytes)) + } + } finally { + closeSync(descriptor) + } +} + +function decodeLine(fragments: Buffer[], bytes: number): string { + if (fragments.length === 1) { + return fragments[0]!.toString('utf8') + } + return Buffer.concat(fragments, bytes).toString('utf8') +} + +function writeAll(descriptor: number, buffer: Buffer): void { + let offset = 0 + while (offset < buffer.length) { + offset += writeSync(descriptor, buffer, offset, buffer.length - offset) + } +} + +function assertWithinCapacity( + reason: CodexLegacyHistorySkipReason, + observed: number, + limit: number +): void { + if (observed > limit) { + throw new CodexLegacyHistoryCapacityError(reason, observed, limit) + } +} + +function resolveLimits( + overrides: Partial | undefined +): CodexLegacyHistoryMigrationLimits { + const limits = { ...DEFAULT_CODEX_LEGACY_HISTORY_MIGRATION_LIMITS, ...overrides } + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + return limits +} + +function closeIgnoringErrors(descriptor: number | null): void { + if (descriptor === null) { + return + } + try { + closeSync(descriptor) + } catch { + // The original migration error is more actionable than cleanup failure. + } +} diff --git a/src/main/codex-accounts/legacy-managed-state-migration.test.ts b/src/main/codex-accounts/legacy-managed-state-migration.test.ts new file mode 100644 index 00000000000..e43b7d8f6d9 --- /dev/null +++ b/src/main/codex-accounts/legacy-managed-state-migration.test.ts @@ -0,0 +1,163 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { migrateLegacyManagedCodexStateSync } from './legacy-managed-state-migration' + +describe('Codex legacy managed-state migration', () => { + let root = '' + let managedAccountsRoot = '' + let metadataDir = '' + let runtimeHomePath = '' + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-codex-managed-state-')) + managedAccountsRoot = join(root, 'codex-accounts') + metadataDir = join(root, 'metadata') + runtimeHomePath = join(root, 'runtime') + mkdirSync(managedAccountsRoot) + mkdirSync(metadataDir) + mkdirSync(runtimeHomePath) + }) + + afterEach(() => { + vi.restoreAllMocks() + rmSync(root, { recursive: true, force: true }) + }) + + function createManagedHome(accountId: string): string { + const homePath = join(managedAccountsRoot, accountId, 'home') + mkdirSync(homePath, { recursive: true }) + writeFileSync(join(homePath, '.orca-managed-home'), '') + return homePath + } + + function migrate( + limits?: Parameters[0]['limits'] + ): ReturnType { + return migrateLegacyManagedCodexStateSync({ + managedAccountsRoot, + metadataDir, + runtimeHomePath, + limits + }) + } + + it('preserves ordinary history, session, diagnostic, and one-shot marker behavior', () => { + const homePath = createManagedHome('account-1') + writeFileSync(join(runtimeHomePath, 'history.jsonl'), '{"id":"shared"}\n') + writeFileSync(join(homePath, 'history.jsonl'), '{"id":"shared"}\n{"id":"managed"}\n') + mkdirSync(join(homePath, 'sessions')) + mkdirSync(join(runtimeHomePath, 'sessions')) + writeFileSync(join(homePath, 'sessions', 'conflict.jsonl'), 'legacy\n') + writeFileSync(join(runtimeHomePath, 'sessions', 'conflict.jsonl'), 'runtime\n') + + const summary = migrate() + + expect(summary).toMatchObject({ + diagnosticRecordsOmitted: 0, + diagnosticRecordsWritten: 1, + historySkippedHomeCount: 0, + managedHomeDiscoverySkipped: false, + migratedHomeCount: 1, + sessionSkippedHomeCount: 0 + }) + expect(readFileSync(join(runtimeHomePath, 'history.jsonl'), 'utf8')).toBe( + '{"id":"shared"}\n{"id":"managed"}\n' + ) + expect( + readFileSync( + join(runtimeHomePath, 'sessions', 'conflict.orca-legacy-account-1.jsonl'), + 'utf8' + ) + ).toBe('legacy\n') + const diagnostic = readFileSync(join(metadataDir, 'migration-diagnostics.jsonl'), 'utf8') + expect(diagnostic).toContain('"type":"session-conflict"') + + const marker = JSON.parse(readFileSync(join(metadataDir, 'migration-v1.json'), 'utf8')) + expect(Object.keys(marker).sort()).toEqual(['completedAt', 'migratedHomeCount']) + expect(marker.migratedHomeCount).toBe(1) + expect(migrate()).toBeNull() + }) + + it('skips all homes deterministically when managed-home discovery exceeds its cap', () => { + const firstHome = createManagedHome('account-1') + createManagedHome('account-2') + writeFileSync(join(firstHome, 'history.jsonl'), '{"id":"must-not-import"}\n') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + const summary = migrate({ maxAccountEntries: 1 }) + + expect(summary).toMatchObject({ + managedHomeDiscoverySkipped: true, + migratedHomeCount: 0 + }) + expect(existsSync(join(runtimeHomePath, 'history.jsonl'))).toBe(false) + const marker = JSON.parse(readFileSync(join(metadataDir, 'migration-v1.json'), 'utf8')) + expect(marker).toMatchObject({ + managedHomeDiscoverySkipped: true, + migratedHomeCount: 0 + }) + expect(readFileSync(join(metadataDir, 'migration-diagnostics.jsonl'), 'utf8')).toContain( + '"type":"managed-home-discovery-skipped"' + ) + expect(warnSpy).toHaveBeenCalledTimes(1) + }) + + it('records history and session capacity skips without partial output', () => { + const homePath = createManagedHome('account-1') + writeFileSync(join(homePath, 'history.jsonl'), 'oversized\n') + mkdirSync(join(homePath, 'sessions')) + writeFileSync(join(homePath, 'sessions', 'a.jsonl'), 'a') + writeFileSync(join(homePath, 'sessions', 'b.jsonl'), 'b') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + const summary = migrate({ + history: { maxFileBytes: 4 }, + sessions: { maxEntries: 1 } + }) + + expect(summary).toMatchObject({ + historySkippedHomeCount: 1, + sessionSkippedHomeCount: 1 + }) + expect(existsSync(join(runtimeHomePath, 'history.jsonl'))).toBe(false) + expect(existsSync(join(runtimeHomePath, 'sessions'))).toBe(false) + const marker = JSON.parse(readFileSync(join(metadataDir, 'migration-v1.json'), 'utf8')) + expect(marker).toMatchObject({ + historySkippedHomeCount: 1, + sessionSkippedHomeCount: 1 + }) + const diagnostics = readFileSync(join(metadataDir, 'migration-diagnostics.jsonl'), 'utf8') + expect(diagnostics).toContain('"type":"history-skipped"') + expect(diagnostics).toContain('"type":"sessions-skipped"') + expect(warnSpy).toHaveBeenCalledTimes(1) + }) + + it('caps diagnostic output while still preserving every conflicting session', () => { + const homePath = createManagedHome('account-1') + mkdirSync(join(homePath, 'sessions')) + mkdirSync(join(runtimeHomePath, 'sessions')) + for (const name of ['a.jsonl', 'b.jsonl']) { + writeFileSync(join(homePath, 'sessions', name), `legacy-${name}`) + writeFileSync(join(runtimeHomePath, 'sessions', name), `runtime-${name}`) + } + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + const summary = migrate({ maxDiagnosticRecords: 1 }) + + expect(summary).toMatchObject({ + diagnosticRecordsOmitted: 1, + diagnosticRecordsWritten: 1 + }) + expect(existsSync(join(runtimeHomePath, 'sessions', 'a.orca-legacy-account-1.jsonl'))).toBe( + true + ) + expect(existsSync(join(runtimeHomePath, 'sessions', 'b.orca-legacy-account-1.jsonl'))).toBe( + true + ) + const marker = JSON.parse(readFileSync(join(metadataDir, 'migration-v1.json'), 'utf8')) + expect(marker.diagnosticRecordsOmitted).toBe(1) + expect(warnSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/codex-accounts/legacy-managed-state-migration.ts b/src/main/codex-accounts/legacy-managed-state-migration.ts new file mode 100644 index 00000000000..9da9cbf10e5 --- /dev/null +++ b/src/main/codex-accounts/legacy-managed-state-migration.ts @@ -0,0 +1,317 @@ +import { appendFileSync, existsSync, opendirSync } from 'node:fs' +import { join } from 'node:path' +import { writeFileAtomically } from './fs-utils' +import { + mergeCodexLegacyHistorySync, + type CodexLegacyHistoryMigrationLimits +} from './legacy-history-migration' +import { + migrateCodexLegacySessionsSync, + type CodexLegacySessionMigrationLimits +} from './legacy-session-migration' + +export const CODEX_LEGACY_MANAGED_ACCOUNT_MAX_ENTRIES = 4096 +export const CODEX_LEGACY_MANAGED_HOME_MAX_COUNT = 256 +export const CODEX_LEGACY_MIGRATION_DIAGNOSTIC_MAX_RECORDS = 2048 +export const CODEX_LEGACY_MIGRATION_DIAGNOSTIC_MAX_BYTES = 1024 * 1024 + +type LegacyManagedHome = { + accountId: string + homePath: string +} + +type ManagedHomeDiscoveryResult = + | { kind: 'complete'; homes: LegacyManagedHome[] } + | { + kind: 'skipped' + reason: 'account-entries' | 'managed-homes' + observed: number + limit: number + } + +type MigrationDiagnostic = Record + +export type CodexLegacyManagedStateMigrationLimits = { + maxAccountEntries: number + maxManagedHomes: number + maxDiagnosticRecords: number + maxDiagnosticBytes: number + history?: Partial + sessions?: Partial +} + +export type CodexLegacyManagedStateMigrationSummary = { + diagnosticRecordsOmitted: number + diagnosticRecordsWritten: number + historySkippedHomeCount: number + managedHomeDiscoverySkipped: boolean + migratedHomeCount: number + sessionSkippedHomeCount: number +} + +const DEFAULT_LIMITS: CodexLegacyManagedStateMigrationLimits = { + maxAccountEntries: CODEX_LEGACY_MANAGED_ACCOUNT_MAX_ENTRIES, + maxManagedHomes: CODEX_LEGACY_MANAGED_HOME_MAX_COUNT, + maxDiagnosticRecords: CODEX_LEGACY_MIGRATION_DIAGNOSTIC_MAX_RECORDS, + maxDiagnosticBytes: CODEX_LEGACY_MIGRATION_DIAGNOSTIC_MAX_BYTES +} + +export function migrateLegacyManagedCodexStateSync(options: { + managedAccountsRoot: string + metadataDir: string + runtimeHomePath: string + limits?: Partial +}): CodexLegacyManagedStateMigrationSummary | null { + const markerPath = join(options.metadataDir, 'migration-v1.json') + if (existsSync(markerPath)) { + return null + } + + const limits = resolveLimits(options.limits) + const diagnostics = new MigrationDiagnosticWriter( + join(options.metadataDir, 'migration-diagnostics.jsonl'), + limits.maxDiagnosticRecords, + limits.maxDiagnosticBytes + ) + const discovery = discoverLegacyManagedHomesSync(options.managedAccountsRoot, limits) + let historySkippedHomeCount = 0 + let sessionSkippedHomeCount = 0 + + if (discovery.kind === 'skipped') { + diagnostics.append({ + type: 'managed-home-discovery-skipped', + reason: discovery.reason, + observed: discovery.observed, + limit: discovery.limit + }) + } else { + for (const home of discovery.homes) { + const legacyHistoryPath = join(home.homePath, 'history.jsonl') + if (existsSync(legacyHistoryPath)) { + const historyResult = mergeCodexLegacyHistorySync({ + legacyHistoryPath, + runtimeHistoryPath: join(options.runtimeHomePath, 'history.jsonl'), + limits: limits.history + }) + if (historyResult.kind === 'skipped') { + historySkippedHomeCount += 1 + diagnostics.append({ + type: 'history-skipped', + accountId: home.accountId, + legacyHistoryPath, + reason: historyResult.reason, + observed: historyResult.observed, + limit: historyResult.limit + }) + } + } + + const legacySessionsRoot = join(home.homePath, 'sessions') + if (!existsSync(legacySessionsRoot)) { + continue + } + const sessionResult = migrateCodexLegacySessionsSync({ + accountId: home.accountId, + legacySessionsRoot, + runtimeSessionsRoot: join(options.runtimeHomePath, 'sessions'), + limits: limits.sessions, + onConflict: ({ runtimeFilePath, preservedPath }) => { + diagnostics.append({ + type: 'session-conflict', + accountId: home.accountId, + runtimeFilePath, + preservedPath + }) + } + }) + if (sessionResult.kind === 'skipped') { + sessionSkippedHomeCount += 1 + diagnostics.append({ + type: 'sessions-skipped', + accountId: home.accountId, + legacySessionsRoot, + reason: sessionResult.reason, + observed: sessionResult.observed, + limit: sessionResult.limit + }) + } + } + } + + const summary: CodexLegacyManagedStateMigrationSummary = { + diagnosticRecordsOmitted: diagnostics.omittedCount, + diagnosticRecordsWritten: diagnostics.writtenCount, + historySkippedHomeCount, + managedHomeDiscoverySkipped: discovery.kind === 'skipped', + migratedHomeCount: discovery.kind === 'complete' ? discovery.homes.length : 0, + sessionSkippedHomeCount + } + writeMigrationMarker(markerPath, summary) + warnAboutCapacitySkips(summary, markerPath) + return summary +} + +function discoverLegacyManagedHomesSync( + managedAccountsRoot: string, + limits: CodexLegacyManagedStateMigrationLimits +): ManagedHomeDiscoveryResult { + if (!existsSync(managedAccountsRoot)) { + return { kind: 'complete', homes: [] } + } + + const homes: LegacyManagedHome[] = [] + let entryCount = 0 + const directory = opendirSync(managedAccountsRoot) + try { + while (true) { + const entry = directory.readSync() + if (entry === null) { + break + } + entryCount += 1 + if (entryCount > limits.maxAccountEntries) { + return { + kind: 'skipped', + reason: 'account-entries', + observed: entryCount, + limit: limits.maxAccountEntries + } + } + if (!entry.isDirectory()) { + continue + } + const homePath = join(managedAccountsRoot, entry.name, 'home') + if (!existsSync(join(homePath, '.orca-managed-home'))) { + continue + } + homes.push({ accountId: entry.name, homePath }) + if (homes.length > limits.maxManagedHomes) { + return { + kind: 'skipped', + reason: 'managed-homes', + observed: homes.length, + limit: limits.maxManagedHomes + } + } + } + } finally { + closeDirectoryIgnoringAlreadyClosed(directory) + } + return { + kind: 'complete', + homes: homes.sort(compareManagedHomePaths) + } +} + +function compareManagedHomePaths(left: LegacyManagedHome, right: LegacyManagedHome): number { + if (left.homePath < right.homePath) { + return -1 + } + return left.homePath > right.homePath ? 1 : 0 +} + +function writeMigrationMarker( + markerPath: string, + summary: CodexLegacyManagedStateMigrationSummary +): void { + const capacityDetails = { + ...(summary.managedHomeDiscoverySkipped ? { managedHomeDiscoverySkipped: true } : {}), + ...(summary.historySkippedHomeCount > 0 + ? { historySkippedHomeCount: summary.historySkippedHomeCount } + : {}), + ...(summary.sessionSkippedHomeCount > 0 + ? { sessionSkippedHomeCount: summary.sessionSkippedHomeCount } + : {}), + ...(summary.diagnosticRecordsOmitted > 0 + ? { diagnosticRecordsOmitted: summary.diagnosticRecordsOmitted } + : {}) + } + writeFileAtomically( + markerPath, + `${JSON.stringify({ + completedAt: Date.now(), + migratedHomeCount: summary.migratedHomeCount, + ...capacityDetails + })}\n` + ) +} + +function warnAboutCapacitySkips( + summary: CodexLegacyManagedStateMigrationSummary, + markerPath: string +): void { + const skippedCount = + Number(summary.managedHomeDiscoverySkipped) + + summary.historySkippedHomeCount + + summary.sessionSkippedHomeCount + if (skippedCount === 0 && summary.diagnosticRecordsOmitted === 0) { + return + } + console.warn('[codex-runtime-home] Legacy state migration completed with bounded skips:', { + skippedCount, + diagnosticRecordsOmitted: summary.diagnosticRecordsOmitted, + markerPath + }) +} + +class MigrationDiagnosticWriter { + private bytesWritten = 0 + private failed = false + omittedCount = 0 + writtenCount = 0 + + constructor( + private readonly path: string, + private readonly maxRecords: number, + private readonly maxBytes: number + ) {} + + append(record: MigrationDiagnostic): void { + const line = `${JSON.stringify(record)}\n` + const lineBytes = Buffer.byteLength(line) + if ( + this.failed || + this.writtenCount >= this.maxRecords || + lineBytes > this.maxBytes - this.bytesWritten + ) { + this.omittedCount += 1 + return + } + try { + appendFileSync(this.path, line, { encoding: 'utf8' }) + this.writtenCount += 1 + this.bytesWritten += lineBytes + } catch (error) { + this.failed = true + this.omittedCount += 1 + console.warn('[codex-runtime-home] Failed to append migration diagnostic:', error) + } + } +} + +function resolveLimits( + overrides: Partial | undefined +): CodexLegacyManagedStateMigrationLimits { + const limits = { ...DEFAULT_LIMITS, ...overrides } + for (const [name, value] of Object.entries({ + maxAccountEntries: limits.maxAccountEntries, + maxManagedHomes: limits.maxManagedHomes, + maxDiagnosticRecords: limits.maxDiagnosticRecords, + maxDiagnosticBytes: limits.maxDiagnosticBytes + })) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + return limits +} + +function closeDirectoryIgnoringAlreadyClosed(directory: ReturnType): void { + try { + directory.closeSync() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ERR_DIR_CLOSED') { + throw error + } + } +} diff --git a/src/main/codex-accounts/legacy-session-migration.test.ts b/src/main/codex-accounts/legacy-session-migration.test.ts new file mode 100644 index 00000000000..7ba50dd7302 --- /dev/null +++ b/src/main/codex-accounts/legacy-session-migration.test.ts @@ -0,0 +1,174 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + CODEX_LEGACY_SESSION_COMPARE_CHUNK_BYTES, + codexLegacySessionFilesEqualSync, + migrateCodexLegacySessionsSync +} from './legacy-session-migration' + +describe('Codex legacy session migration', () => { + let root = '' + let legacySessionsRoot = '' + let runtimeSessionsRoot = '' + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-codex-legacy-sessions-')) + legacySessionsRoot = join(root, 'legacy') + runtimeSessionsRoot = join(root, 'runtime') + mkdirSync(legacySessionsRoot) + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('preserves creation of the runtime sessions directory for an empty legacy tree', () => { + const result = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot + }) + + expect(result).toMatchObject({ + kind: 'migrated', + copiedFileCount: 0, + discoveredEntryCount: 0, + discoveredFileCount: 0 + }) + expect(existsSync(runtimeSessionsRoot)).toBe(true) + }) + + it('copies and conflict-preserves ordinary files in the historical sorted order', () => { + mkdirSync(join(legacySessionsRoot, 'nested')) + mkdirSync(join(runtimeSessionsRoot, 'nested'), { recursive: true }) + writeFileSync(join(legacySessionsRoot, 'z-copy.jsonl'), 'copy\n') + writeFileSync(join(legacySessionsRoot, 'a-conflict.jsonl'), 'legacy\n') + writeFileSync(join(runtimeSessionsRoot, 'a-conflict.jsonl'), 'runtime\n') + writeFileSync(join(legacySessionsRoot, 'nested', 'same.jsonl'), 'same\n') + writeFileSync(join(runtimeSessionsRoot, 'nested', 'same.jsonl'), 'same\n') + const conflicts: string[] = [] + + const result = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot, + onConflict: ({ runtimeFilePath }) => conflicts.push(runtimeFilePath) + }) + + expect(result).toMatchObject({ + kind: 'migrated', + conflictCount: 1, + copiedFileCount: 2, + discoveredEntryCount: 4, + discoveredFileCount: 3 + }) + expect(conflicts).toEqual([join(runtimeSessionsRoot, 'a-conflict.jsonl')]) + expect(readFileSync(join(runtimeSessionsRoot, 'z-copy.jsonl'), 'utf8')).toBe('copy\n') + expect( + readFileSync(join(runtimeSessionsRoot, 'a-conflict.orca-legacy-account-1.jsonl'), 'utf8') + ).toBe('legacy\n') + expect(readFileSync(join(runtimeSessionsRoot, 'a-conflict.jsonl'), 'utf8')).toBe('runtime\n') + }) + + it('compares sparse multi-megabyte collisions with fixed-size buffers', () => { + mkdirSync(runtimeSessionsRoot) + const legacyPath = join(legacySessionsRoot, 'large.jsonl') + const runtimePath = join(runtimeSessionsRoot, 'large.jsonl') + const sparseBytes = CODEX_LEGACY_SESSION_COMPARE_CHUNK_BYTES * 128 + for (const path of [legacyPath, runtimePath]) { + writeFileSync(path, 'same-prefix') + truncateSync(path, sparseBytes) + } + + expect(codexLegacySessionFilesEqualSync(legacyPath, runtimePath)).toBe(true) + const result = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot + }) + + expect(result).toMatchObject({ kind: 'migrated', conflictCount: 0, copiedFileCount: 0 }) + expect(existsSync(join(runtimeSessionsRoot, 'large.orca-legacy-account-1.jsonl'))).toBe(false) + }) + + it('preflights the entry cap before copying any file', () => { + writeFileSync(join(legacySessionsRoot, 'a.jsonl'), 'a') + writeFileSync(join(legacySessionsRoot, 'b.jsonl'), 'b') + + const result = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot, + limits: { maxEntries: 1 } + }) + + expect(result).toMatchObject({ + kind: 'skipped', + reason: 'entries', + observed: 2, + limit: 1 + }) + expect(existsSync(runtimeSessionsRoot)).toBe(false) + }) + + it('preflights depth and aggregate file bytes before copying', () => { + mkdirSync(join(legacySessionsRoot, 'one', 'two'), { recursive: true }) + writeFileSync(join(legacySessionsRoot, 'one', 'two', 'deep.jsonl'), 'deep') + + const depthResult = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot, + limits: { maxDepth: 1 } + }) + + expect(depthResult).toMatchObject({ kind: 'skipped', reason: 'depth', observed: 2, limit: 1 }) + expect(existsSync(runtimeSessionsRoot)).toBe(false) + + rmSync(join(legacySessionsRoot, 'one'), { recursive: true }) + writeFileSync(join(legacySessionsRoot, 'a.jsonl'), '1234') + writeFileSync(join(legacySessionsRoot, 'b.jsonl'), '5678') + const byteResult = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot, + limits: { maxTotalFileBytes: 7 } + }) + + expect(byteResult).toMatchObject({ + kind: 'skipped', + reason: 'total-file-bytes', + observed: 8, + limit: 7 + }) + expect(existsSync(runtimeSessionsRoot)).toBe(false) + }) + + it('bounds retained path text independently of entry count', () => { + writeFileSync(join(legacySessionsRoot, 'long-name.jsonl'), 'content') + + const result = migrateCodexLegacySessionsSync({ + accountId: 'account-1', + legacySessionsRoot, + runtimeSessionsRoot, + limits: { maxPathCodeUnits: legacySessionsRoot.length } + }) + + expect(result).toMatchObject({ + kind: 'skipped', + reason: 'path-code-units', + limit: legacySessionsRoot.length + }) + expect(existsSync(runtimeSessionsRoot)).toBe(false) + }) +}) diff --git a/src/main/codex-accounts/legacy-session-migration.ts b/src/main/codex-accounts/legacy-session-migration.ts new file mode 100644 index 00000000000..b36b84f16c6 --- /dev/null +++ b/src/main/codex-accounts/legacy-session-migration.ts @@ -0,0 +1,305 @@ +import { randomUUID } from 'node:crypto' +import { + closeSync, + existsSync, + fstatSync, + mkdirSync, + openSync, + opendirSync, + readSync, + rmSync, + statSync +} from 'node:fs' +import { dirname, extname, join, relative } from 'node:path' +import { copyFileWithWindowsRetry, renameFileWithWindowsRetry } from './fs-utils' + +export const CODEX_LEGACY_SESSION_MAX_DEPTH = 32 +export const CODEX_LEGACY_SESSION_MAX_ENTRIES = 50_000 +export const CODEX_LEGACY_SESSION_MAX_TOTAL_FILE_BYTES = 4 * 1024 * 1024 * 1024 +export const CODEX_LEGACY_SESSION_MAX_PATH_CODE_UNITS = 4 * 1024 * 1024 +export const CODEX_LEGACY_SESSION_COMPARE_CHUNK_BYTES = 64 * 1024 + +export type CodexLegacySessionMigrationLimits = { + maxDepth: number + maxEntries: number + maxTotalFileBytes: number + maxPathCodeUnits: number +} + +export const DEFAULT_CODEX_LEGACY_SESSION_MIGRATION_LIMITS: CodexLegacySessionMigrationLimits = { + maxDepth: CODEX_LEGACY_SESSION_MAX_DEPTH, + maxEntries: CODEX_LEGACY_SESSION_MAX_ENTRIES, + maxTotalFileBytes: CODEX_LEGACY_SESSION_MAX_TOTAL_FILE_BYTES, + maxPathCodeUnits: CODEX_LEGACY_SESSION_MAX_PATH_CODE_UNITS +} + +export type CodexLegacySessionSkipReason = + | 'depth' + | 'entries' + | 'total-file-bytes' + | 'path-code-units' + +export type CodexLegacySessionMigrationResult = + | { + kind: 'migrated' + conflictCount: number + copiedFileCount: number + discoveredEntryCount: number + discoveredFileBytes: number + discoveredFileCount: number + } + | { + kind: 'skipped' + reason: CodexLegacySessionSkipReason + observed: number + limit: number + visitedEntryCount: number + } + +type TraversalResult = + | { + kind: 'complete' + entryCount: number + fileBytes: number + filePaths: string[] + } + | { + kind: 'skipped' + reason: CodexLegacySessionSkipReason + observed: number + limit: number + visitedEntryCount: number + } + +class CodexLegacySessionCapacityError extends Error { + constructor( + readonly reason: CodexLegacySessionSkipReason, + readonly observed: number, + readonly limit: number, + readonly visitedEntryCount: number + ) { + super(`Codex legacy sessions exceeded ${reason} limit (${observed} > ${limit})`) + this.name = 'CodexLegacySessionCapacityError' + } +} + +export function migrateCodexLegacySessionsSync(options: { + accountId: string + legacySessionsRoot: string + runtimeSessionsRoot: string + limits?: Partial + onConflict?: (conflict: { runtimeFilePath: string; preservedPath: string }) => void +}): CodexLegacySessionMigrationResult { + const limits = resolveLimits(options.limits) + const traversal = collectLegacySessionFilesSync(options.legacySessionsRoot, limits) + if (traversal.kind === 'skipped') { + return traversal + } + + mkdirSync(options.runtimeSessionsRoot, { recursive: true }) + let conflictCount = 0 + let copiedFileCount = 0 + for (const legacyFilePath of traversal.filePaths) { + const relativePath = relative(options.legacySessionsRoot, legacyFilePath) + const runtimeFilePath = join(options.runtimeSessionsRoot, relativePath) + mkdirSync(dirname(runtimeFilePath), { recursive: true }) + if (!existsSync(runtimeFilePath)) { + copyFileAtomicallySync(legacyFilePath, runtimeFilePath) + copiedFileCount += 1 + continue + } + if (codexLegacySessionFilesEqualSync(legacyFilePath, runtimeFilePath)) { + continue + } + + const preservedPath = getPreservedLegacySessionPath(runtimeFilePath, options.accountId) + copyFileAtomicallySync(legacyFilePath, preservedPath) + conflictCount += 1 + copiedFileCount += 1 + options.onConflict?.({ runtimeFilePath, preservedPath }) + } + + return { + kind: 'migrated', + conflictCount, + copiedFileCount, + discoveredEntryCount: traversal.entryCount, + discoveredFileBytes: traversal.fileBytes, + discoveredFileCount: traversal.filePaths.length + } +} + +function collectLegacySessionFilesSync( + rootPath: string, + limits: CodexLegacySessionMigrationLimits +): TraversalResult { + let entryCount = 0 + let fileBytes = 0 + let pathCodeUnits = rootPath.length + const filePaths: string[] = [] + const rootStats = statSync(rootPath) + if (rootStats.isFile()) { + assertWithinCapacity('total-file-bytes', rootStats.size, limits.maxTotalFileBytes, 0) + assertWithinCapacity('path-code-units', pathCodeUnits, limits.maxPathCodeUnits, 0) + return { kind: 'complete', entryCount: 0, fileBytes: rootStats.size, filePaths: [rootPath] } + } + + const pendingDirectories = [{ depth: 0, path: rootPath }] + try { + while (pendingDirectories.length > 0) { + const current = pendingDirectories.pop()! + const directory = opendirSync(current.path) + try { + while (true) { + const entry = directory.readSync() + if (entry === null) { + break + } + entryCount += 1 + assertWithinCapacity('entries', entryCount, limits.maxEntries, entryCount) + const childPath = join(current.path, entry.name) + pathCodeUnits += childPath.length + assertWithinCapacity( + 'path-code-units', + pathCodeUnits, + limits.maxPathCodeUnits, + entryCount + ) + + if (entry.isDirectory()) { + const childDepth = current.depth + 1 + assertWithinCapacity('depth', childDepth, limits.maxDepth, entryCount) + pendingDirectories.push({ depth: childDepth, path: childPath }) + continue + } + if (!entry.isFile()) { + continue + } + + const size = statSync(childPath).size + fileBytes += size + assertWithinCapacity('total-file-bytes', fileBytes, limits.maxTotalFileBytes, entryCount) + filePaths.push(childPath) + } + } finally { + closeDirectoryIgnoringAlreadyClosed(directory) + } + } + } catch (error) { + if (error instanceof CodexLegacySessionCapacityError) { + return { + kind: 'skipped', + reason: error.reason, + observed: error.observed, + limit: error.limit, + visitedEntryCount: error.visitedEntryCount + } + } + throw error + } + + return { kind: 'complete', entryCount, fileBytes, filePaths: filePaths.sort() } +} + +export function codexLegacySessionFilesEqualSync(leftPath: string, rightPath: string): boolean { + let leftDescriptor: number | null = null + let rightDescriptor: number | null = null + try { + leftDescriptor = openSync(leftPath, 'r') + rightDescriptor = openSync(rightPath, 'r') + if (fstatSync(leftDescriptor).size !== fstatSync(rightDescriptor).size) { + return false + } + + const leftBuffer = Buffer.allocUnsafe(CODEX_LEGACY_SESSION_COMPARE_CHUNK_BYTES) + const rightBuffer = Buffer.allocUnsafe(CODEX_LEGACY_SESSION_COMPARE_CHUNK_BYTES) + while (true) { + const leftBytes = readFullChunkSync(leftDescriptor, leftBuffer) + const rightBytes = readFullChunkSync(rightDescriptor, rightBuffer) + if (leftBytes !== rightBytes) { + return false + } + if (leftBytes === 0) { + return true + } + if (!leftBuffer.subarray(0, leftBytes).equals(rightBuffer.subarray(0, rightBytes))) { + return false + } + } + } finally { + closeIgnoringErrors(leftDescriptor) + closeIgnoringErrors(rightDescriptor) + } +} + +function readFullChunkSync(descriptor: number, buffer: Buffer): number { + let offset = 0 + while (offset < buffer.length) { + const bytesRead = readSync(descriptor, buffer, offset, buffer.length - offset, null) + if (bytesRead === 0) { + break + } + offset += bytesRead + } + return offset +} + +function copyFileAtomicallySync(sourcePath: string, targetPath: string): void { + const temporaryPath = `${targetPath}.${process.pid}.${randomUUID()}.migration.tmp` + try { + copyFileWithWindowsRetry(sourcePath, temporaryPath) + renameFileWithWindowsRetry(temporaryPath, targetPath) + } finally { + rmSync(temporaryPath, { force: true }) + } +} + +function getPreservedLegacySessionPath(runtimeFilePath: string, accountId: string): string { + const extension = extname(runtimeFilePath) + const basename = runtimeFilePath.slice(0, runtimeFilePath.length - extension.length) + return `${basename}.orca-legacy-${accountId}${extension}` +} + +function assertWithinCapacity( + reason: CodexLegacySessionSkipReason, + observed: number, + limit: number, + visitedEntryCount: number +): void { + if (!Number.isSafeInteger(observed) || observed > limit) { + throw new CodexLegacySessionCapacityError(reason, observed, limit, visitedEntryCount) + } +} + +function resolveLimits( + overrides: Partial | undefined +): CodexLegacySessionMigrationLimits { + const limits = { ...DEFAULT_CODEX_LEGACY_SESSION_MIGRATION_LIMITS, ...overrides } + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + return limits +} + +function closeIgnoringErrors(descriptor: number | null): void { + if (descriptor === null) { + return + } + try { + closeSync(descriptor) + } catch { + // Preserve the comparison result or original read error. + } +} + +function closeDirectoryIgnoringAlreadyClosed(directory: ReturnType): void { + try { + directory.closeSync() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ERR_DIR_CLOSED') { + throw error + } + } +} diff --git a/src/main/codex-accounts/legacy-shared-auth-migration.ts b/src/main/codex-accounts/legacy-shared-auth-migration.ts index b21d858cf04..4f1405ba96e 100644 --- a/src/main/codex-accounts/legacy-shared-auth-migration.ts +++ b/src/main/codex-accounts/legacy-shared-auth-migration.ts @@ -1,6 +1,7 @@ -import { lstatSync, readFileSync } from 'node:fs' +import { lstatSync } from 'node:fs' import { join } from 'node:path' import type { CodexManagedAccount } from '../../shared/types' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from './fs-utils' import { assertOwnedHostCodexManagedHomePath } from './host-codex-managed-home-ownership' import { codexAuthMatchesManagedAccount, compareCodexAuthFreshness } from './codex-auth-identity' @@ -174,7 +175,7 @@ function migrateSharedMcpCredentials( function readRegularFile(filePath: string): string | null { const state = regularFileState(filePath) - return state === 'missing' ? null : readFileSync(filePath, 'utf-8') + return state === 'missing' ? null : readAgentStateFileSync(filePath) } function regularFileState(filePath: string): 'missing' | 'present' { diff --git a/src/main/codex-accounts/runtime-home-service.ts b/src/main/codex-accounts/runtime-home-service.ts index dfa49c74cd1..db4e55b7929 100644 --- a/src/main/codex-accounts/runtime-home-service.ts +++ b/src/main/codex-accounts/runtime-home-service.ts @@ -1,35 +1,22 @@ /* eslint-disable max-lines -- Why: keeps Codex's whole runtime-home contract in one place so account-switch semantics don't drift across launch/login/quota paths. */ import { - appendFileSync, - copyFileSync, existsSync, chmodSync, lstatSync, mkdirSync, readlinkSync, - readdirSync, - readFileSync, renameSync, rmdirSync, rmSync, - statSync, symlinkSync, unlinkSync } from 'node:fs' import { execFileSync } from 'node:child_process' -import { - dirname, - extname, - isAbsolute, - join, - parse, - relative, - resolve, - win32 as pathWin32 -} from 'node:path' +import { dirname, isAbsolute, join, resolve, win32 as pathWin32 } from 'node:path' import { app } from 'electron' import type { CodexManagedAccount } from '../../shared/types' import type { Store } from '../persistence' +import { readAgentStateFileSync, readAgentStateJsonFileSync } from '../agent-state-file-reader' import { WSL_CODEX_RUNTIME_HOME_SEGMENTS } from '../pty/codex-home-wsl-env' import { writeFileAtomically } from './fs-utils' import { @@ -69,6 +56,9 @@ import { codexAuthMatchesSystemDefaultIdentity } from './codex-auth-identity' import { migrateLegacySharedAuthToPerAccountHome } from './legacy-shared-auth-migration' +import { migrateLegacyManagedCodexStateSync } from './legacy-managed-state-migration' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { CodexWslRuntimeHomeRetention } from './codex-wsl-runtime-home-retention' type CodexSystemDefaultSnapshot = { authJson: string | null @@ -114,9 +104,7 @@ export class CodexRuntimeHomeService { // Last auth.json Orca wrote to the runtime home; a later diff signals an out-of-band change (Codex token refresh, or external login to adopt). private lastWrittenAuthJson: string | null = null // Why: WSL terminals have per-distro runtime homes; sharing the host baseline can make stale WSL auth look newer than managed storage. - private readonly lastWrittenWslAuthJsonByDistro = new Map() - private readonly lastSyncedWslAccountIdByDistro = new Map() - private readonly wslRuntimeHomePathByDistro = new Map() + private readonly wslRuntimeHomeRetention = new CodexWslRuntimeHomeRetention() private skipNextReadBackForAccountId: string | null = null // Why: a flag-ON host account refreshes auth in its own home. Remember that // provenance so a later deselect/rollback never adopts stale shared bytes. @@ -640,7 +628,7 @@ export class CodexRuntimeHomeService { this.skipNextReadBackForAccountId = null } this.lastSyncedAccountId = activeAccount.id - this.writeRuntimeAuth(readFileSync(activeAuthPath, 'utf-8')) + this.writeRuntimeAuth(readAgentStateFileSync(activeAuthPath)) } // Why: re-auth/add-account write fresh managed tokens, so skip the next read-back to avoid clobbering them with stale runtime tokens. @@ -691,7 +679,7 @@ export class CodexRuntimeHomeService { options.lastWrittenAuthJson === undefined ? this.lastWrittenAuthJson : options.lastWrittenAuthJson - const runtimeContents = readFileSync(runtimeAuthPath, 'utf-8') + const runtimeContents = readAgentStateFileSync(runtimeAuthPath) if (lastWrittenAuthJson !== null && runtimeContents === lastWrittenAuthJson) { return 'unchanged' } @@ -746,7 +734,7 @@ export class CodexRuntimeHomeService { if (!existsSync(runtimeAuthPath) || this.lastWrittenAuthJson === null) { return 'rejected' } - const runtimeContents = readFileSync(runtimeAuthPath, 'utf-8') + const runtimeContents = readAgentStateFileSync(runtimeAuthPath) if (runtimeContents === this.lastWrittenAuthJson) { return 'unchanged' } @@ -803,11 +791,11 @@ export class CodexRuntimeHomeService { // Why: the system-default account changes outside Orca, so read its real home directly to avoid a stale cached runtime copy. return this.getWslSystemCodexHomePath(target) } - const cachedRuntimeHomePath = this.wslRuntimeHomePathByDistro.get(distro) + const cachedRuntimeHomePath = this.wslRuntimeHomeRetention.getRuntimeHomePath(distro) if ( cachedRuntimeHomePath && - this.lastSyncedWslAccountIdByDistro.has(distro) && - this.lastSyncedWslAccountIdByDistro.get(distro) === selectedAccountId + this.wslRuntimeHomeRetention.hasLastSyncedAccountId(distro) && + this.wslRuntimeHomeRetention.getLastSyncedAccountId(distro) === selectedAccountId ) { // Why: RateLimitService resolves provenance twice per poll; stay path-only so it doesn't block main on UNC reads and a wsl.exe probe. return cachedRuntimeHomePath @@ -836,14 +824,14 @@ export class CodexRuntimeHomeService { if (!runtimeHomePath) { return null } - this.wslRuntimeHomePathByDistro.set(distro, runtimeHomePath) + this.wslRuntimeHomeRetention.setRuntimeHomePath(distro, runtimeHomePath) mkdirSync(runtimeHomePath, { recursive: true }) this.safeMigrateLegacyWslActiveHomePointer(distro, runtimeHomePath) this.seedWslRuntimeHome(runtimeHomePath, activeAccount, distro) const runtimeAuthPath = join(runtimeHomePath, 'auth.json') - const previousWslAccountId = this.lastSyncedWslAccountIdByDistro.get(distro) ?? null + const previousWslAccountId = this.wslRuntimeHomeRetention.getLastSyncedAccountId(distro) ?? null if (previousWslAccountId) { if (this.skipNextReadBackForAccountId === previousWslAccountId) { this.skipNextReadBackForAccountId = null @@ -855,9 +843,10 @@ export class CodexRuntimeHomeService { if (previousWslAccount) { this.readBackRefreshedTokensFromPath(runtimeAuthPath, { updateLastWrittenAuthJson: true, - lastWrittenAuthJson: this.lastWrittenWslAuthJsonByDistro.get(distro) ?? null, + lastWrittenAuthJson: + this.wslRuntimeHomeRetention.getLastWrittenAuthJson(distro) ?? null, setLastWrittenAuthJson: (contents) => { - this.lastWrittenWslAuthJsonByDistro.set(distro, contents) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, contents) }, expectedAccountId: previousWslAccount.id }) @@ -867,10 +856,10 @@ export class CodexRuntimeHomeService { const activeAuthPath = activeAccount ? join(activeAccount.managedHomePath, 'auth.json') : null if (activeAccount && activeAuthPath && existsSync(activeAuthPath)) { - const activeAuth = readFileSync(activeAuthPath, 'utf-8') + const activeAuth = readAgentStateFileSync(activeAuthPath) this.writeRuntimeAuthAtPath(runtimeAuthPath, activeAuth) - this.lastWrittenWslAuthJsonByDistro.set(distro, activeAuth) - this.lastSyncedWslAccountIdByDistro.set(distro, activeAccount.id) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, activeAuth) + this.wslRuntimeHomeRetention.setLastSyncedAccountId(distro, activeAccount.id) return runtimeHomePath } if (activeAccount && activeAuthPath) { @@ -889,10 +878,11 @@ export class CodexRuntimeHomeService { const systemAuthPath = this.getWslSystemCodexAuthPath({ runtime: 'wsl', wslDistro: distro }) if (systemAuthPath && existsSync(systemAuthPath)) { - const systemAuth = readFileSync(systemAuthPath, 'utf-8') - const mirroredSystemDefaultAuth = this.lastWrittenWslAuthJsonByDistro.get(distro) ?? null + const systemAuth = readAgentStateFileSync(systemAuthPath) + const mirroredSystemDefaultAuth = + this.wslRuntimeHomeRetention.getLastWrittenAuthJson(distro) ?? null const runtimeAuth = existsSync(runtimeAuthPath) - ? readFileSync(runtimeAuthPath, 'utf-8') + ? readAgentStateFileSync(runtimeAuthPath) : null if ( runtimeAuth !== null && @@ -904,19 +894,19 @@ export class CodexRuntimeHomeService { ) { // Why: WSL baselines are lost on restart, so a same-identity fresher runtime auth is a token refresh; copy it back before mirroring ~/.codex. this.writeRuntimeAuthAtPath(systemAuthPath, runtimeAuth) - this.lastWrittenWslAuthJsonByDistro.set(distro, runtimeAuth) - this.lastSyncedWslAccountIdByDistro.set(distro, null) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, runtimeAuth) + this.wslRuntimeHomeRetention.setLastSyncedAccountId(distro, null) return runtimeHomePath } this.writeRuntimeAuthAtPath(runtimeAuthPath, systemAuth) - this.lastWrittenWslAuthJsonByDistro.set(distro, systemAuth) - this.lastSyncedWslAccountIdByDistro.set(distro, null) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, systemAuth) + this.wslRuntimeHomeRetention.setLastSyncedAccountId(distro, null) return runtimeHomePath } rmSync(runtimeAuthPath, { force: true }) - this.lastWrittenWslAuthJsonByDistro.set(distro, null) - this.lastSyncedWslAccountIdByDistro.set(distro, null) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, null) + this.wslRuntimeHomeRetention.setLastSyncedAccountId(distro, null) return runtimeHomePath } @@ -948,16 +938,16 @@ export class CodexRuntimeHomeService { return } - const runtimeHomePath = this.wslRuntimeHomePathByDistro.get(distro) + const runtimeHomePath = this.wslRuntimeHomeRetention.getRuntimeHomePath(distro) if (!runtimeHomePath) { return } this.readBackRefreshedTokensFromPath(join(runtimeHomePath, 'auth.json'), { updateLastWrittenAuthJson: true, - lastWrittenAuthJson: this.lastWrittenWslAuthJsonByDistro.get(distro) ?? null, + lastWrittenAuthJson: this.wslRuntimeHomeRetention.getLastWrittenAuthJson(distro) ?? null, setLastWrittenAuthJson: (contents) => { - this.lastWrittenWslAuthJsonByDistro.set(distro, contents) + this.wslRuntimeHomeRetention.setLastWrittenAuthJson(distro, contents) }, expectedAccountId: account.id }) @@ -1056,7 +1046,7 @@ export class CodexRuntimeHomeService { if (existsSync(configPath)) { writeFileAtomically( runtimeConfigPath, - prepareWslRuntimeSeedConfig(readFileSync(configPath, 'utf-8'), homePath) + prepareWslRuntimeSeedConfig(readAgentStateFileSync(configPath), homePath) ) return } @@ -1080,7 +1070,7 @@ export class CodexRuntimeHomeService { if (!existsSync(managedAuthPath)) { continue } - const managedAuthContents = readFileSync(managedAuthPath, 'utf-8') + const managedAuthContents = readAgentStateFileSync(managedAuthPath) if (codexAuthMatchesManagedAccount(runtimeAuthContents, account, managedAuthContents)) { matches.push({ account, managedAuthPath, managedAuthContents }) } @@ -1172,14 +1162,6 @@ export class CodexRuntimeHomeService { return join(this.getRuntimeMetadataDir(), 'active', 'host', 'home') } - private getMigrationMarkerPath(): string { - return join(this.getRuntimeMetadataDir(), 'migration-v1.json') - } - - private getMigrationDiagnosticsPath(): string { - return join(this.getRuntimeMetadataDir(), 'migration-diagnostics.jsonl') - } - private getManagedAccountsRoot(): string { return join(app.getPath('userData'), 'codex-accounts') } @@ -1284,150 +1266,11 @@ export class CodexRuntimeHomeService { } private migrateLegacyManagedStateIfNeeded(): void { - if (existsSync(this.getMigrationMarkerPath())) { - return - } - - const managedHomes = this.getLegacyManagedHomes() - for (const managedHomePath of managedHomes) { - const accountId = parse(relative(this.getManagedAccountsRoot(), managedHomePath)).dir.split( - /[\\/]/ - )[0] - if (!accountId) { - continue - } - this.migrateLegacyHistory(managedHomePath) - this.migrateLegacySessions(managedHomePath, accountId) - } - - // Why: migration is one-shot; re-importing every startup would replay stale managed-home state into the shared runtime. - writeFileAtomically( - this.getMigrationMarkerPath(), - `${JSON.stringify({ completedAt: Date.now(), migratedHomeCount: managedHomes.length })}\n` - ) - } - - private getLegacyManagedHomes(): string[] { - const managedAccountsRoot = this.getManagedAccountsRoot() - if (!existsSync(managedAccountsRoot)) { - return [] - } - - const accountEntries = readdirSync(managedAccountsRoot, { withFileTypes: true }) - const managedHomes: string[] = [] - for (const entry of accountEntries) { - if (!entry.isDirectory()) { - continue - } - const managedHomePath = join(managedAccountsRoot, entry.name, 'home') - if (existsSync(join(managedHomePath, '.orca-managed-home'))) { - managedHomes.push(managedHomePath) - } - } - return managedHomes.sort() - } - - private migrateLegacyHistory(managedHomePath: string): void { - const legacyHistoryPath = join(managedHomePath, 'history.jsonl') - if (!existsSync(legacyHistoryPath)) { - return - } - - const runtimeHistoryPath = join(this.getRuntimeHomePath(), 'history.jsonl') - const existingLines = existsSync(runtimeHistoryPath) - ? readFileSync(runtimeHistoryPath, 'utf-8').split('\n').filter(Boolean) - : [] - const mergedLines = [...existingLines] - const seenLines = new Set(existingLines) - for (const line of readFileSync(legacyHistoryPath, 'utf-8').split('\n')) { - if (!line || seenLines.has(line)) { - continue - } - seenLines.add(line) - mergedLines.push(line) - } - - if (mergedLines.length === 0) { - return - } - writeFileAtomically(runtimeHistoryPath, `${mergedLines.join('\n')}\n`) - } - - private migrateLegacySessions(managedHomePath: string, accountId: string): void { - const legacySessionsRoot = join(managedHomePath, 'sessions') - if (!existsSync(legacySessionsRoot)) { - return - } - - const runtimeSessionsRoot = join(this.getRuntimeHomePath(), 'sessions') - mkdirSync(runtimeSessionsRoot, { recursive: true }) - for (const legacyFilePath of this.listFilesRecursively(legacySessionsRoot)) { - const relativePath = relative(legacySessionsRoot, legacyFilePath) - const runtimeFilePath = join(runtimeSessionsRoot, relativePath) - mkdirSync(dirname(runtimeFilePath), { recursive: true }) - if (!existsSync(runtimeFilePath)) { - copyFileSync(legacyFilePath, runtimeFilePath) - continue - } - - const legacyContents = readFileSync(legacyFilePath) - const runtimeContents = readFileSync(runtimeFilePath) - if (runtimeContents.equals(legacyContents)) { - continue - } - - const preservedPath = this.getPreservedLegacySessionPath(runtimeFilePath, accountId) - copyFileSync(legacyFilePath, preservedPath) - this.appendMigrationDiagnostic({ - type: 'session-conflict', - accountId, - runtimeFilePath, - preservedPath - }) - } - } - - private listFilesRecursively(rootPath: string): string[] { - const stat = statSync(rootPath) - if (!stat.isDirectory()) { - return [rootPath] - } - - const files: string[] = [] - for (const entry of readdirSync(rootPath, { withFileTypes: true })) { - const childPath = join(rootPath, entry.name) - if (entry.isDirectory()) { - this.appendListedFiles(files, this.listFilesRecursively(childPath)) - continue - } - if (entry.isFile()) { - files.push(childPath) - } - } - return files.sort() - } - - private appendListedFiles(target: string[], source: readonly string[]): void { - // Why: tolerate directories larger than V8's argument limit for spread calls. - for (const filePath of source) { - target.push(filePath) - } - } - - private getPreservedLegacySessionPath(runtimeFilePath: string, accountId: string): string { - const extension = extname(runtimeFilePath) - const basename = runtimeFilePath.slice(0, runtimeFilePath.length - extension.length) - return `${basename}.orca-legacy-${accountId}${extension}` - } - - private appendMigrationDiagnostic(record: Record): void { - const diagnosticsPath = this.getMigrationDiagnosticsPath() - try { - appendFileSync(diagnosticsPath, `${JSON.stringify(record)}\n`, { encoding: 'utf-8' }) - } catch (error) { - // Why: diagnostics must not fail the one-shot migration after the session file is already preserved. - console.warn('[codex-runtime-home] Failed to append migration diagnostic:', error) - } + migrateLegacyManagedCodexStateSync({ + managedAccountsRoot: this.getManagedAccountsRoot(), + metadataDir: this.getRuntimeMetadataDir(), + runtimeHomePath: this.getRuntimeHomePath() + }) } private captureSystemDefaultSnapshot(options: { force: boolean }): void { @@ -1438,7 +1281,7 @@ export class CodexRuntimeHomeService { const runtimeAuthPath = join(getSystemCodexHomePath(), 'auth.json') const snapshot: CodexSystemDefaultSnapshot = { - authJson: existsSync(runtimeAuthPath) ? readFileSync(runtimeAuthPath, 'utf-8') : null + authJson: existsSync(runtimeAuthPath) ? readAgentStateFileSync(runtimeAuthPath) : null } writeFileAtomically(snapshotPath, `${JSON.stringify(snapshot, null, 2)}\n`, { mode: 0o600 }) } @@ -1451,7 +1294,7 @@ export class CodexRuntimeHomeService { } try { - const runtimeAuth = readFileSync(runtimeAuthPath, 'utf-8') + const runtimeAuth = readAgentStateFileSync(runtimeAuthPath) if (!existsSync(systemDefaultAuthPath)) { const snapshot = this.readSystemDefaultSnapshot(this.getSystemDefaultSnapshotPath()) const mirroredSystemDefaultAuth = this.lastWrittenAuthJson ?? snapshot?.authJson ?? null @@ -1467,7 +1310,7 @@ export class CodexRuntimeHomeService { } return } - const systemDefaultAuth = readFileSync(systemDefaultAuthPath, 'utf-8') + const systemDefaultAuth = readAgentStateFileSync(systemDefaultAuthPath) if (runtimeAuth !== systemDefaultAuth) { const snapshot = this.readSystemDefaultSnapshot(this.getSystemDefaultSnapshotPath()) const mirroredSystemDefaultAuth = this.lastWrittenAuthJson ?? snapshot?.authJson ?? null @@ -1496,7 +1339,7 @@ export class CodexRuntimeHomeService { const runtimeAuthPath = this.getRuntimeAuthPath() const systemDefaultAuthPath = join(getSystemCodexHomePath(), 'auth.json') if (existsSync(systemDefaultAuthPath)) { - const systemDefaultAuth = readFileSync(systemDefaultAuthPath, 'utf-8') + const systemDefaultAuth = readAgentStateFileSync(systemDefaultAuthPath) this.captureSystemDefaultSnapshot({ force: true }) this.writeRuntimeAuth(systemDefaultAuth) return @@ -1566,7 +1409,7 @@ export class CodexRuntimeHomeService { private readSystemDefaultAuth(): string | null { const systemDefaultAuthPath = join(getSystemCodexHomePath(), 'auth.json') - return existsSync(systemDefaultAuthPath) ? readFileSync(systemDefaultAuthPath, 'utf-8') : null + return existsSync(systemDefaultAuthPath) ? readAgentStateFileSync(systemDefaultAuthPath) : null } private writeRuntimeAuth(contents: string): void { @@ -1592,8 +1435,11 @@ export class CodexRuntimeHomeService { private fileContentsEqual(targetPath: string, contents: string): boolean { try { - return existsSync(targetPath) && readFileSync(targetPath, 'utf-8') === contents - } catch { + return existsSync(targetPath) && readAgentStateFileSync(targetPath) === contents + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } return false } } @@ -1635,7 +1481,7 @@ export class CodexRuntimeHomeService { private readRuntimeLogoutMarker(): CodexRuntimeLogoutMarker | null { let parsed: unknown try { - parsed = JSON.parse(readFileSync(this.getRuntimeLogoutMarkerPath(), 'utf-8')) as unknown + parsed = readAgentStateJsonFileSync(this.getRuntimeLogoutMarkerPath()) } catch { return null } @@ -1665,7 +1511,7 @@ export class CodexRuntimeHomeService { private readSystemDefaultSnapshot(snapshotPath: string): CodexSystemDefaultSnapshot | null { let rawContents: string try { - rawContents = readFileSync(snapshotPath, 'utf-8') + rawContents = readAgentStateFileSync(snapshotPath) } catch { return null } diff --git a/src/main/codex-accounts/service.ts b/src/main/codex-accounts/service.ts index 54b676aa9c1..a4733767740 100644 --- a/src/main/codex-accounts/service.ts +++ b/src/main/codex-accounts/service.ts @@ -1,7 +1,7 @@ /* eslint-disable max-lines -- Why: keeps Codex account lifecycle, path safety, login, and identity parsing in one audited main-process module. */ import { randomUUID } from 'node:crypto' import { execFileSync, spawn, type ChildProcess } from 'node:child_process' -import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs' import { dirname, join, resolve, sep } from 'node:path' import { homedir } from 'node:os' import { app } from 'electron' @@ -44,6 +44,8 @@ import { type CodexAccountSelectionTarget } from './runtime-selection' import { assertOwnedHostCodexManagedHomePath } from './host-codex-managed-home-ownership' +import { readAgentStateFileSync } from '../agent-state-file-reader' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' const LOGIN_TIMEOUT_MS = 120_000 const MAX_LOGIN_OUTPUT_CHARS = 4_000 @@ -133,7 +135,7 @@ function killLoginProcessTree(child: ChildProcess): void { function readLoginAuthSnapshot(authJsonPath: string): string | null | undefined { try { - return readFileSync(authJsonPath, 'utf-8') + return readAgentStateFileSync(authJsonPath) } catch (error) { const code = (error as NodeJS.ErrnoException).code if (code === 'ENOENT' || code === 'ENOTDIR') { @@ -412,7 +414,7 @@ export class CodexAccountService { try { // Why: a single read avoids an exists/read race and halves filesystem // probes whenever an accounts snapshot resolves this live identity. - contents = readFileSync(authFilePath, 'utf-8') + contents = readAgentStateFileSync(authFilePath) } catch (error) { const code = (error as NodeJS.ErrnoException | null)?.code if (code === 'ENOENT' || code === 'ENOTDIR') { @@ -708,7 +710,7 @@ export class CodexAccountService { try { return { - contents: readFileSync(primaryConfigPath, 'utf-8'), + contents: readAgentStateFileSync(primaryConfigPath), sourceHomePath, sourceHooksPath: join(sourceHomePath, 'hooks.json') } @@ -739,7 +741,7 @@ export class CodexAccountService { // Why: the config is read over UNC but consumed by Codex inside WSL, so // path rewrites must anchor to the Linux-side ~/.codex, not the UNC path. return { - contents: readFileSync(configPath, 'utf-8'), + contents: readAgentStateFileSync(configPath), sourceHomePath: `${wslHome}/.codex`, sourceHooksPath: `${wslHome}/.codex/hooks.json` } @@ -767,10 +769,13 @@ export class CodexAccountService { private writeManagedConfig(managedHomePath: string, contents: string): void { const configPath = join(managedHomePath, 'config.toml') try { - if (existsSync(configPath) && readFileSync(configPath, 'utf-8') === contents) { + if (existsSync(configPath) && readAgentStateFileSync(configPath) === contents) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // Why: a read error must not make a stale config look current; atomic write owns ACL repair and error surfacing. } writeFileAtomically(configPath, contents) @@ -939,7 +944,7 @@ export class CodexAccountService { } if ( expectedAccountId !== undefined && - readFileSync(join(candidatePath, '.orca-managed-home'), 'utf-8').trim() !== + readAgentStateFileSync(join(candidatePath, '.orca-managed-home')).trim() !== expectedAccountId ) { throw new Error('Managed WSL Codex home ownership marker does not match its account ID.') @@ -1247,7 +1252,7 @@ export class CodexAccountService { this.assertManagedHomePath(managedHomePath, expectedAccountId), 'auth.json' ) - const authFileContents = readFileSync(authFilePath, 'utf-8') + const authFileContents = readAgentStateFileSync(authFilePath) let parsed: Record try { parsed = JSON.parse(authFileContents) as Record diff --git a/src/main/codex-cli/command.ts b/src/main/codex-cli/command.ts index 177b7e0ea9c..bf4a3410915 100644 --- a/src/main/codex-cli/command.ts +++ b/src/main/codex-cli/command.ts @@ -1,6 +1,7 @@ -import { accessSync, constants, existsSync, readdirSync, statSync } from 'node:fs' +import { accessSync, constants, statSync } from 'node:fs' import { homedir } from 'node:os' import { delimiter, dirname, join } from 'node:path' +import { discoverNvmVersionBinDirectories } from './nvm-version-directory-discovery' type ResolveCommandOptions = { pathEnv?: string | null @@ -27,29 +28,6 @@ function splitPath(pathEnv: string | null | undefined): string[] { .filter(Boolean) } -function parseVersionSegment(raw: string): number[] { - return raw - .replace(/^v/i, '') - .split('.') - .map((segment) => Number.parseInt(segment, 10)) - .map((segment) => (Number.isFinite(segment) ? segment : 0)) -} - -function compareVersionDesc(left: string, right: string): number { - const leftParts = parseVersionSegment(left) - const rightParts = parseVersionSegment(right) - const length = Math.max(leftParts.length, rightParts.length) - - for (let index = 0; index < length; index += 1) { - const delta = (rightParts[index] ?? 0) - (leftParts[index] ?? 0) - if (delta !== 0) { - return delta - } - } - - return right.localeCompare(left) -} - function findFirstExecutable( platform: NodeJS.Platform, directories: string[], @@ -123,19 +101,6 @@ function getBaseVersionManagerDirectories(platform: NodeJS.Platform, homePath: s return directories } -function getNvmVersionDirectories(homePath: string): string[] { - const nvmVersionsDir = join(homePath, '.nvm', 'versions', 'node') - if (!existsSync(nvmVersionsDir)) { - return [] - } - - return readdirSync(nvmVersionsDir, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name) - .sort(compareVersionDesc) - .map((entry) => join(nvmVersionsDir, entry, 'bin')) -} - function getVersionManagerDirectories( platform: NodeJS.Platform, homePath: string, @@ -147,7 +112,7 @@ function getVersionManagerDirectories( // command resolution probes the newest installed Node versions explicitly. const firstNvmMatch = findFirstExecutable( platform, - getNvmVersionDirectories(homePath), + discoverNvmVersionBinDirectories(homePath), executableNames ) if (firstNvmMatch) { @@ -189,7 +154,7 @@ export function resolveCliCommands( // Why: agent detection probes many CLIs at once; compute expensive install // directories, especially nvm versions, once per detection pass. const installDirectories = [ - ...getNvmVersionDirectories(homePath), + ...discoverNvmVersionBinDirectories(homePath), ...getBaseVersionManagerDirectories(platform, homePath) ] const resolved = new Map() diff --git a/src/main/codex-cli/nvm-version-directory-discovery.test.ts b/src/main/codex-cli/nvm-version-directory-discovery.test.ts new file mode 100644 index 00000000000..c2d725dec1c --- /dev/null +++ b/src/main/codex-cli/nvm-version-directory-discovery.test.ts @@ -0,0 +1,93 @@ +import { join } from 'node:path' +import type * as NodeFs from 'node:fs' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const opendirSyncMock = vi.hoisted(() => vi.fn()) + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), + opendirSync: opendirSyncMock +})) + +import { discoverNvmVersionBinDirectories } from './nvm-version-directory-discovery' + +type FakeEntry = { name: string; directory?: boolean } + +function useEntries(entries: FakeEntry[]): { closeSync: ReturnType } { + let index = 0 + const closeSync = vi.fn() + opendirSyncMock.mockReturnValue({ + closeSync, + readSync: vi.fn(() => { + const entry = entries[index] + index += 1 + return entry + ? { + name: entry.name, + isDirectory: () => entry.directory !== false + } + : null + }) + }) + return { closeSync } +} + +describe('nvm version directory discovery', () => { + beforeEach(() => { + opendirSyncMock.mockReset() + }) + + it('preserves newest-version ordering for an ordinary listing', () => { + useEntries([ + { name: 'v20.18.0' }, + { name: 'README', directory: false }, + { name: 'v24.2.0' }, + { name: 'v22.14.0' } + ]) + + expect(discoverNvmVersionBinDirectories('/home/alice')).toEqual([ + join('/home/alice', '.nvm', 'versions', 'node', 'v24.2.0', 'bin'), + join('/home/alice', '.nvm', 'versions', 'node', 'v22.14.0', 'bin'), + join('/home/alice', '.nvm', 'versions', 'node', 'v20.18.0', 'bin') + ]) + }) + + it('accepts the exact entry and retained-name limits', () => { + const { closeSync } = useEntries([{ name: 'v20' }, { name: 'v22' }]) + + expect( + discoverNvmVersionBinDirectories('/home/alice', { + maxEntries: 2, + maxRetainedNameBytes: 6 + }) + ).toHaveLength(2) + expect(closeSync).toHaveBeenCalledOnce() + }) + + it('fails closed and closes the stream on the first entry over the count limit', () => { + const { closeSync } = useEntries([ + { name: 'v20' }, + { name: 'v22' }, + { name: 'v24' }, + { name: 'v26' } + ]) + + expect( + discoverNvmVersionBinDirectories('/home/alice', { + maxEntries: 2 + }) + ).toEqual([]) + expect(closeSync).toHaveBeenCalledOnce() + }) + + it('fails closed when retained directory names exceed the byte limit', () => { + const { closeSync } = useEntries([{ name: 'v20' }, { name: 'v22' }]) + + expect( + discoverNvmVersionBinDirectories('/home/alice', { + maxRetainedNameBytes: 5 + }) + ).toEqual([]) + expect(closeSync).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/codex-cli/nvm-version-directory-discovery.ts b/src/main/codex-cli/nvm-version-directory-discovery.ts new file mode 100644 index 00000000000..c537eef95b0 --- /dev/null +++ b/src/main/codex-cli/nvm-version-directory-discovery.ts @@ -0,0 +1,100 @@ +import { opendirSync } from 'node:fs' +import { join } from 'node:path' + +export const NVM_VERSION_DISCOVERY_MAX_ENTRIES = 4_096 +export const NVM_VERSION_DISCOVERY_MAX_RETAINED_NAME_BYTES = 1024 * 1024 + +export type NvmVersionDiscoveryLimits = { + maxEntries: number + maxRetainedNameBytes: number +} + +function parseVersionSegment(raw: string): number[] { + return raw + .replace(/^v/i, '') + .split('.') + .map((segment) => Number.parseInt(segment, 10)) + .map((segment) => (Number.isFinite(segment) ? segment : 0)) +} + +function compareVersionDesc(left: string, right: string): number { + const leftParts = parseVersionSegment(left) + const rightParts = parseVersionSegment(right) + const length = Math.max(leftParts.length, rightParts.length) + + for (let index = 0; index < length; index += 1) { + const delta = (rightParts[index] ?? 0) - (leftParts[index] ?? 0) + if (delta !== 0) { + return delta + } + } + + return right.localeCompare(left) +} + +function resolveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} + +function closeNvmVersionsDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch { + // The OS may already have closed a fully consumed directory stream. + } +} + +export function discoverNvmVersionBinDirectories( + homePath: string, + requestedLimits: Partial = {} +): string[] { + const maxEntries = resolveLimit( + requestedLimits.maxEntries, + NVM_VERSION_DISCOVERY_MAX_ENTRIES, + 'maxEntries' + ) + const maxRetainedNameBytes = resolveLimit( + requestedLimits.maxRetainedNameBytes, + NVM_VERSION_DISCOVERY_MAX_RETAINED_NAME_BYTES, + 'maxRetainedNameBytes' + ) + const versionsDir = join(homePath, '.nvm', 'versions', 'node') + let directory: ReturnType + try { + directory = opendirSync(versionsDir, { bufferSize: 32 }) + } catch { + return [] + } + + const versionNames: string[] = [] + let scannedEntries = 0 + let retainedNameBytes = 0 + try { + for (let entry = directory.readSync(); entry !== null; entry = directory.readSync()) { + scannedEntries += 1 + if (scannedEntries > maxEntries) { + return [] + } + if (!entry.isDirectory()) { + continue + } + retainedNameBytes += Buffer.byteLength(entry.name, 'utf8') + if (retainedNameBytes > maxRetainedNameBytes) { + return [] + } + versionNames.push(entry.name) + } + } finally { + closeNvmVersionsDirectory(directory) + } + + return versionNames + .sort(compareVersionDesc) + .map((versionName) => join(versionsDir, versionName, 'bin')) +} diff --git a/src/main/codex-usage/scanner-large-directory.test.ts b/src/main/codex-usage/scanner-large-directory.test.ts index 1ef57ff9d03..6a7e4aa5099 100644 --- a/src/main/codex-usage/scanner-large-directory.test.ts +++ b/src/main/codex-usage/scanner-large-directory.test.ts @@ -1,12 +1,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { Dirent, Stats } from 'node:fs' +import type { Dir, Dirent, Stats } from 'node:fs' import type * as FsPromises from 'node:fs/promises' import { join } from 'node:path' -const { getLegacyCopiedCodexSessionBridgeScanPreferenceMock, readdirMock, statMock } = vi.hoisted( +const { getLegacyCopiedCodexSessionBridgeScanPreferenceMock, opendirMock, statMock } = vi.hoisted( () => ({ getLegacyCopiedCodexSessionBridgeScanPreferenceMock: vi.fn(), - readdirMock: vi.fn<(dirPath: string) => Promise>(), + opendirMock: vi.fn<(dirPath: string) => Promise>(), statMock: vi.fn<(filePath: string) => Promise>() }) ) @@ -15,7 +15,7 @@ vi.mock('fs/promises', async () => { const actual = await vi.importActual('fs/promises') return { ...actual, - readdir: readdirMock, + opendir: opendirMock, stat: statMock } }) @@ -49,26 +49,44 @@ const largeSessionEntries = Array.from({ length: FILE_COUNT }, (_, index) => dirent(`session-${index}.jsonl`, 'file') ) +function directory(entries: Dirent[]): Dir { + return { + async *[Symbol.asyncIterator]() { + yield* entries + } + } as Dir +} + +function generatedFileDirectory(count: number): Dir { + return { + async *[Symbol.asyncIterator]() { + for (let index = 0; index < count; index++) { + yield dirent(`session-${index}.jsonl`, 'file') + } + } + } as Dir +} + describe('listCodexSessionFiles large directories', () => { beforeEach(() => { getLegacyCopiedCodexSessionBridgeScanPreferenceMock.mockReset() getLegacyCopiedCodexSessionBridgeScanPreferenceMock.mockReturnValue(null) - readdirMock.mockReset() + opendirMock.mockReset() statMock.mockReset() }) it('keeps nested session scans past the JavaScript spread-argument limit', async () => { - readdirMock.mockImplementation(async (dirPath) => { + opendirMock.mockImplementation(async (dirPath) => { if (dirPath === RUNTIME_SESSIONS_ROOT) { - return [dirent('bulk', 'directory')] + return directory([dirent('bulk', 'directory')]) } if (dirPath === RUNTIME_BULK_DIR) { - return largeSessionEntries + return directory(largeSessionEntries) } if (dirPath === SYSTEM_SESSIONS_ROOT) { - return [] + return directory([]) } - throw new Error(`Unexpected readdir path: ${dirPath}`) + throw new Error(`Unexpected opendir path: ${dirPath}`) }) statMock.mockImplementation(async (filePath) => { const match = /session-(\d+)\.jsonl$/.exec(filePath.replaceAll('\\', '/')) @@ -83,4 +101,24 @@ describe('listCodexSessionFiles large directories', () => { await expect(listCodexSessionFiles()).resolves.toHaveLength(FILE_COUNT) expect(getLegacyCopiedCodexSessionBridgeScanPreferenceMock).not.toHaveBeenCalled() }) + + it('fails closed instead of silently omitting a rollout past capacity', async () => { + opendirMock.mockImplementation(async (dirPath) => { + if (dirPath === RUNTIME_SESSIONS_ROOT) { + return generatedFileDirectory(200_001) + } + if (dirPath === SYSTEM_SESSIONS_ROOT) { + return directory([]) + } + throw new Error(`Unexpected opendir path: ${dirPath}`) + }) + + const { listCodexSessionFiles } = await import('./scanner') + + await expect(listCodexSessionFiles()).rejects.toMatchObject({ + name: 'UsageHistoryScanCapacityError', + resource: 'files', + limit: 200_000 + }) + }) }) diff --git a/src/main/codex-usage/scanner.test.ts b/src/main/codex-usage/scanner.test.ts index 6426222461f..e31a5b4f476 100644 --- a/src/main/codex-usage/scanner.test.ts +++ b/src/main/codex-usage/scanner.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it, vi } from 'vitest' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' const { getPathMock } = vi.hoisted(() => ({ getPathMock: vi.fn<(name: string) => string>() @@ -10,7 +13,8 @@ vi.mock('electron', () => ({ } })) -import { attributeCodexUsageEvent, parseCodexUsageRecord } from './scanner' +import { attributeCodexUsageEvent, parseCodexUsageFile, parseCodexUsageRecord } from './scanner' +import { UsageHistoryScanBudget } from '../usage-history-scan-budget' describe('parseCodexUsageRecord', () => { it('uses token totals only as a duplicate baseline', () => { @@ -321,3 +325,37 @@ describe('attributeCodexUsageEvent', () => { expect(attributed?.worktreeId).toBeNull() }) }) + +describe('parseCodexUsageFile capacity', () => { + it('fails the whole parse when retained usage events exceed capacity', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-codex-capacity-')) + const filePath = join(root, 'rollout.jsonl') + const event = (timestamp: string, total: number) => + JSON.stringify({ + timestamp, + type: 'event_msg', + payload: { + type: 'token_count', + info: { + total_token_usage: { input_tokens: total, total_tokens: total }, + last_token_usage: { input_tokens: 1, total_tokens: 1 } + } + } + }) + try { + await writeFile( + filePath, + [event('2026-04-09T10:00:00.000Z', 1), event('2026-04-09T10:00:01.000Z', 2)].join('\n') + ) + const budget = new UsageHistoryScanBudget({ records: 1 }) + + await expect(parseCodexUsageFile(filePath, [], { budget })).rejects.toMatchObject({ + name: 'UsageHistoryScanCapacityError', + resource: 'records', + limit: 1 + }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) diff --git a/src/main/codex-usage/scanner.ts b/src/main/codex-usage/scanner.ts index eb5fbcaab14..a837b8df676 100644 --- a/src/main/codex-usage/scanner.ts +++ b/src/main/codex-usage/scanner.ts @@ -1,14 +1,21 @@ /* eslint-disable max-lines -- Why: Codex discovery, incremental parsing, attribution, and aggregation all depend on the same event-normalization rules. Keeping them together makes the duplicate-snapshot logic easier to audit when usage totals look wrong. */ import { basename, join, win32, posix } from 'node:path' -import { createReadStream, existsSync } from 'node:fs' -import { realpath, readdir, stat } from 'node:fs/promises' -import { createInterface } from 'node:readline' +import { existsSync } from 'node:fs' +import { realpath, stat } from 'node:fs/promises' import type { Repo } from '../../shared/types' import { areWorktreePathsEqual } from '../ipc/worktree-logic' import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from '../codex/codex-home-paths' import { getCodexAccountHomeSessionDirectories } from '../codex/codex-account-home-discovery' import { getLegacyCopiedCodexSessionBridgeScanPreference } from '../codex/codex-session-bridge' import { canonicalizeUsageWorktreePaths } from '../usage-worktree-canonicalizer' +import { walkUsageHistoryJsonlFiles } from '../usage-history-file-discovery' +import { readUsageHistoryJsonlLines } from '../usage-history-jsonl-reader' +import { + MAX_USAGE_HISTORY_FILES, + UsageHistoryScanBudget, + UsageHistoryScanCapacityError, + getUsageHistoryRetainedBytes +} from '../usage-history-scan-budget' import type { CodexUsageAttributedEvent, CodexUsageDailyAggregate, @@ -93,39 +100,6 @@ async function yieldToEventLoop(): Promise { await new Promise((resolve) => setImmediate(resolve)) } -async function walkJsonlFiles( - dirPath: string, - progress: { entriesVisited: number } = { entriesVisited: 0 } -): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) - const files: string[] = [] - - for (const entry of entries) { - progress.entriesVisited += 1 - if (progress.entriesVisited % YIELD_EVERY_DISCOVERY_ENTRIES === 0) { - await yieldToEventLoop() - } - const fullPath = join(dirPath, entry.name) - if (entry.isDirectory()) { - appendDiscoveredFiles(files, await walkJsonlFiles(fullPath, progress)) - continue - } - if (entry.isFile() && entry.name.endsWith('.jsonl')) { - files.push(fullPath) - } - } - - return files -} - -function appendDiscoveredFiles(target: string[], source: readonly string[]): void { - // Why: large session directories can exceed V8's argument limit if child - // file arrays are spread into push(). - for (const filePath of source) { - target.push(filePath) - } -} - export function getCodexSessionsDirectory(): string { // Why: Orca-launched Codex processes receive an Orca-owned CODEX_HOME, so // callers that need the primary runtime path should not consult ambient @@ -148,12 +122,19 @@ function hasLegacyCopiedSessionBridgeMarkers(): boolean { return existsSync(join(getOrcaManagedCodexHomePath(), '.orca-session-copies')) } -export async function listCodexSessionFiles(): Promise { +export async function listCodexSessionFiles( + budget = new UsageHistoryScanBudget() +): Promise { const files: string[] = [] for (const dirPath of getCodexSessionDirectories()) { try { - appendDiscoveredFiles(files, await walkJsonlFiles(dirPath)) - } catch { + for (const filePath of await walkUsageHistoryJsonlFiles(dirPath, budget)) { + files.push(filePath) + } + } catch (error) { + if (error instanceof UsageHistoryScanCapacityError) { + throw error + } // Missing or unreadable history in one home should not hide the other. } } @@ -894,6 +875,62 @@ function mergeDailyAggregates( } } +function claimCodexUsageProjection( + budget: UsageHistoryScanBudget, + sessions: readonly CodexUsageSession[], + dailyAggregates: readonly CodexUsageDailyAggregate[] +): void { + for (const session of sessions) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + session.sessionId, + session.firstTimestamp, + session.lastTimestamp, + session.primaryModel, + session.primaryProjectLabel, + session.primaryWorktreeId, + session.primaryRepoId + ]) + ) + for (const location of session.locationBreakdown) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + location.locationKey, + location.projectLabel, + location.repoId, + location.worktreeId + ]) + ) + } + for (const model of session.modelBreakdown) { + budget.claimProjection(getUsageHistoryRetainedBytes([model.modelKey, model.modelLabel])) + } + for (const locationModel of session.locationModelBreakdown) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + locationModel.locationKey, + locationModel.modelKey, + locationModel.modelLabel, + locationModel.repoId, + locationModel.worktreeId + ]) + ) + } + } + for (const daily of dailyAggregates) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + daily.day, + daily.model, + daily.projectKey, + daily.projectLabel, + daily.repoId, + daily.worktreeId + ]) + ) + } +} + export function parseCodexUsageRecord( line: string, context: CodexUsageParseContext @@ -997,16 +1034,14 @@ export function parseCodexUsageRecord( export async function parseCodexUsageFile( filePath: string, worktrees: (CodexUsageWorktreeRef & { canonicalPath: string })[], - options: { skipInitialBytes?: number; claimEventKey?: (eventKey: string) => boolean } = {} + options: { + skipInitialBytes?: number + claimEventKey?: (eventKey: string) => boolean + budget?: UsageHistoryScanBudget + } = {} ): Promise { + const budget = options.budget ?? new UsageHistoryScanBudget() const processedFile = await getProcessedFileInfo(filePath) - const lines = createInterface({ - input: createReadStream(filePath, { - encoding: 'utf-8', - start: options.skipInitialBytes ?? 0 - }), - crlfDelay: Infinity - }) const events: CodexUsageAttributedEvent[] = [] const context: CodexUsageParseContext = { sessionId: basename(filePath, '.jsonl'), @@ -1021,7 +1056,9 @@ export async function parseCodexUsageFile( const ownedEventKeys = new Set() let hasDeferredClaims = false - for await (const line of lines) { + for await (const line of readUsageHistoryJsonlLines(filePath, { + start: options.skipInitialBytes ?? 0 + })) { const parsed = parseCodexUsageRecord(line, context) if (!parsed) { continue @@ -1033,6 +1070,18 @@ export async function parseCodexUsageFile( hasDeferredClaims = true continue } + budget.claimRecord( + getUsageHistoryRetainedBytes([ + parsed.sessionId, + parsed.timestamp, + parsed.eventKey, + parsed.cwd, + parsed.model + ]) + ) + if (!ownedEventKeys.has(parsed.eventKey)) { + budget.claimOwnershipKey(parsed.eventKey) + } ownedEventKeys.add(parsed.eventKey) const attributed = await attributeCodexUsageEvent(parsed, worktrees) if (attributed) { @@ -1040,9 +1089,11 @@ export async function parseCodexUsageFile( } } + const aggregates = aggregateCodexUsage(events) + claimCodexUsageProjection(budget, aggregates.sessions, aggregates.dailyAggregates) return { ...processedFile, - ...aggregateCodexUsage(events), + ...aggregates, ownedEventKeys: [...ownedEventKeys], hasDeferredClaims } @@ -1056,7 +1107,14 @@ export async function scanCodexUsageFiles( sessions: CodexUsageSession[] dailyAggregates: CodexUsageDailyAggregate[] }> { - const files = await listCodexSessionFiles() + if (previousProcessedFiles.length > MAX_USAGE_HISTORY_FILES) { + throw new UsageHistoryScanCapacityError('files', MAX_USAGE_HISTORY_FILES) + } + const budget = new UsageHistoryScanBudget() + const files = await listCodexSessionFiles(budget) + for (const previous of previousProcessedFiles) { + budget.claimPath(previous.path) + } const previousByPath = new Map(previousProcessedFiles.map((file) => [file.path, file])) const worktreesWithCanonicalPaths = await buildWorktreesWithCanonicalPaths(worktrees) const legacySourceSkipBytesByPath = getLegacySourceSkipBytesByPath(files) @@ -1107,7 +1165,12 @@ export async function scanCodexUsageFiles( // deterministic. const eventOwnerByKey = new Map() for (const [filePath, previous] of reusedByPath) { + for (const session of previous.sessions) { + budget.claimRecords(session.eventCount) + } + claimCodexUsageProjection(budget, previous.sessions, previous.dailyAggregates) for (const eventKey of previous.ownedEventKeys) { + budget.claimOwnershipKey(eventKey) // First cached claim wins so conflicting projections stay deterministic. if (!eventOwnerByKey.has(eventKey)) { eventOwnerByKey.set(eventKey, filePath) @@ -1119,6 +1182,7 @@ export async function scanCodexUsageFiles( for (const [index, filePath] of pathsToParse.entries()) { const processed = await parseCodexUsageFile(filePath, worktreesWithCanonicalPaths, { skipInitialBytes: legacySourceSkipBytesByPath.get(filePath) ?? 0, + budget, claimEventKey: (eventKey) => { const owner = eventOwnerByKey.get(eventKey) if (owner !== undefined && owner !== filePath) { @@ -1151,14 +1215,17 @@ export async function scanCodexUsageFiles( mergeDailyAggregates(dailyByKey, processed.dailyAggregates) } + const sessions = finalizeSessions(sessionsById) + const dailyAggregates = [...dailyByKey.values()].sort((left, right) => + left.day === right.day + ? left.projectLabel.localeCompare(right.projectLabel) + : left.day.localeCompare(right.day) + ) + claimCodexUsageProjection(budget, sessions, dailyAggregates) return { processedFiles, - sessions: finalizeSessions(sessionsById), - dailyAggregates: [...dailyByKey.values()].sort((left, right) => - left.day === right.day - ? left.projectLabel.localeCompare(right.projectLabel) - : left.day.localeCompare(right.day) - ) + sessions, + dailyAggregates } } diff --git a/src/main/codex-usage/store.ts b/src/main/codex-usage/store.ts index ca42ba34061..b873d9ae9eb 100644 --- a/src/main/codex-usage/store.ts +++ b/src/main/codex-usage/store.ts @@ -1,7 +1,6 @@ /* eslint-disable max-lines -- Why: this store owns Codex analytics persistence, scan policy, and renderer query semantics. Keeping them together prevents the Codex range/scope rules from drifting away from the scanner’s event model. */ import { app } from 'electron' -import { dirname, join } from 'node:path' -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' import type { CodexUsageBreakdownKind, CodexUsageBreakdownRow, @@ -15,6 +14,10 @@ import type { } from '../../shared/codex-usage-types' import type { AutomationRunUsage } from '../../shared/automations-types' import type { Store } from '../persistence' +import { + readUsageProjectionStateFile, + writeUsageProjectionStateFileWithRecovery +} from '../usage-projection-state-file' import { loadKnownUsageWorktreesByRepo, type UsageWorktreeRef } from '../usage-worktree-metadata' import type { CodexUsagePersistedState } from './types' import { createWorktreeRefs, scanCodexUsageFiles } from './scanner' @@ -336,10 +339,11 @@ export class CodexUsageStore { private load(): CodexUsagePersistedState { try { const usageFile = getCodexUsageFile() - if (!existsSync(usageFile)) { + const raw = readUsageProjectionStateFile(usageFile) + if (raw === null) { return getDefaultState() } - const parsed = JSON.parse(readFileSync(usageFile, 'utf-8')) as CodexUsagePersistedState + const parsed = JSON.parse(raw) as CodexUsagePersistedState return normalizePersistedState({ ...getDefaultState(), ...parsed, @@ -356,13 +360,12 @@ export class CodexUsageStore { private writeToDisk(): void { const usageFile = getCodexUsageFile() - const dir = dirname(usageFile) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - const tmpFile = `${usageFile}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp` - writeFileSync(tmpFile, JSON.stringify(this.state), 'utf-8') - renameSync(tmpFile, usageFile) + this.state = writeUsageProjectionStateFileWithRecovery(usageFile, this.state, (error) => { + const reset = getDefaultState() + reset.scanState.enabled = this.state.scanState.enabled + reset.scanState.lastScanError = error.message + return reset + }) } async setEnabled(enabled: boolean): Promise { diff --git a/src/main/codex/codex-account-home-discovery.test.ts b/src/main/codex/codex-account-home-discovery.test.ts new file mode 100644 index 00000000000..5e973a4e386 --- /dev/null +++ b/src/main/codex/codex-account-home-discovery.test.ts @@ -0,0 +1,96 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { pathState, ownershipMock } = vi.hoisted(() => ({ + pathState: { userData: '' }, + ownershipMock: vi.fn() +})) + +vi.mock('./codex-home-paths', () => ({ + getOrcaUserDataPath: () => pathState.userData, + getSystemCodexHomePath: () => join(pathState.userData, 'system-codex') +})) + +vi.mock('../codex-accounts/host-codex-managed-home-ownership', () => ({ + assertOwnedHostCodexManagedHomePath: ownershipMock +})) + +import { getCodexAccountHomeSessionDirectories } from './codex-account-home-discovery' + +let root = '' +let accountsRoot = '' + +async function createAccount(accountId: string): Promise { + const sessionsPath = join(accountsRoot, accountId, 'home', 'sessions') + await mkdir(sessionsPath, { recursive: true }) + return sessionsPath +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-codex-account-discovery-')) + pathState.userData = root + accountsRoot = join(root, 'codex-accounts') + await mkdir(accountsRoot) + ownershipMock.mockReset() +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) + +describe('Codex account-home discovery limits', () => { + it('preserves sorted valid account homes below every limit', async () => { + const later = await createAccount('z-account') + const earlier = await createAccount('a-account') + await writeFile(join(accountsRoot, 'not-an-account'), 'x') + + expect(getCodexAccountHomeSessionDirectories()).toEqual([earlier, later]) + }) + + it('accepts the exact entry limit and fails closed on the next entry', async () => { + const sessionsPath = await createAccount('account-1') + await writeFile(join(accountsRoot, 'ignored'), 'x') + + expect(getCodexAccountHomeSessionDirectories({ maxEntries: 2 })).toEqual([sessionsPath]) + await writeFile(join(accountsRoot, 'overflow'), 'x') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + expect(getCodexAccountHomeSessionDirectories({ maxEntries: 2 })).toEqual([]) + expect(warnSpy).toHaveBeenCalledWith( + '[codex-usage] Account-home discovery exceeded 2 entries; skipping homes' + ) + }) + + it('accepts the exact home limit and fails closed on the next valid home', async () => { + await createAccount('account-1') + await createAccount('account-2') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + expect(getCodexAccountHomeSessionDirectories({ maxHomes: 2 })).toHaveLength(2) + expect(getCodexAccountHomeSessionDirectories({ maxHomes: 1 })).toEqual([]) + expect(warnSpy).toHaveBeenCalledWith( + '[codex-usage] Account-home discovery exceeded 1 homes; skipping homes' + ) + }) + + it('bounds retained account and session path strings at the exact code-unit limit', async () => { + const accountId = 'account-1' + const sessionsPath = await createAccount(accountId) + const accountHome = join(accountsRoot, accountId, 'home') + const exactPathCodeUnits = accountId.length + accountHome.length + sessionsPath.length + + expect(getCodexAccountHomeSessionDirectories({ maxPathCodeUnits: exactPathCodeUnits })).toEqual( + [sessionsPath] + ) + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + expect( + getCodexAccountHomeSessionDirectories({ maxPathCodeUnits: exactPathCodeUnits - 1 }) + ).toEqual([]) + expect(warnSpy).toHaveBeenCalledWith( + `[codex-usage] Account-home discovery exceeded ${exactPathCodeUnits - 1} path code units; skipping homes` + ) + }) +}) diff --git a/src/main/codex/codex-account-home-discovery.ts b/src/main/codex/codex-account-home-discovery.ts index d8d30bcd922..72f6d6ee504 100644 --- a/src/main/codex/codex-account-home-discovery.ts +++ b/src/main/codex/codex-account-home-discovery.ts @@ -1,34 +1,102 @@ -import { lstatSync, readdirSync } from 'node:fs' +import { lstatSync, opendirSync } from 'node:fs' import { join } from 'node:path' import { getOrcaUserDataPath, getSystemCodexHomePath } from './codex-home-paths' import { assertOwnedHostCodexManagedHomePath } from '../codex-accounts/host-codex-managed-home-ownership' +export const CODEX_ACCOUNT_HOME_DISCOVERY_MAX_ENTRIES = 4096 +export const CODEX_ACCOUNT_HOME_DISCOVERY_MAX_HOMES = 256 +export const CODEX_ACCOUNT_HOME_DISCOVERY_MAX_PATH_CODE_UNITS = 1024 * 1024 + +export type CodexAccountHomeDiscoveryLimits = { + maxEntries: number + maxHomes: number + maxPathCodeUnits: number +} + /** Session roots of per-account self-contained host Codex homes present on disk. * Why disk-enumerated, not settings-driven: rollouts retained after an account * change must still be counted, and CLI callers have no settings store. WSL * account homes live inside their distro and are scanned by their own lane. */ -export function getCodexAccountHomeSessionDirectories(): string[] { +export function getCodexAccountHomeSessionDirectories( + limitOverrides: Partial = {} +): string[] { + const limits: CodexAccountHomeDiscoveryLimits = { + maxEntries: CODEX_ACCOUNT_HOME_DISCOVERY_MAX_ENTRIES, + maxHomes: CODEX_ACCOUNT_HOME_DISCOVERY_MAX_HOMES, + maxPathCodeUnits: CODEX_ACCOUNT_HOME_DISCOVERY_MAX_PATH_CODE_UNITS, + ...limitOverrides + } + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + const accountsRoot = join(getOrcaUserDataPath(), 'codex-accounts') try { - return readdirSync(accountsRoot, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .flatMap((entry) => { - const accountHome = join(accountsRoot, entry.name, 'home') - try { - assertOwnedHostCodexManagedHomePath({ - candidatePath: accountHome, - managedAccountsRoot: accountsRoot, - systemCodexHomePath: getSystemCodexHomePath(), - expectedAccountId: entry.name - }) - const sessionsPath = join(accountHome, 'sessions') - // Why: a redirected sessions root could make usage scan unrelated, unbounded trees. - return lstatSync(sessionsPath).isDirectory() ? [sessionsPath] : [] - } catch { - return [] + const accountIds: string[] = [] + let entryCount = 0 + let pathCodeUnits = 0 + const directory = opendirSync(accountsRoot) + try { + for (let entry = directory.readSync(); entry !== null; entry = directory.readSync()) { + entryCount += 1 + if (entryCount > limits.maxEntries) { + return failCodexAccountHomeDiscovery('entries', limits.maxEntries) } - }) + pathCodeUnits += entry.name.length + if (pathCodeUnits > limits.maxPathCodeUnits) { + return failCodexAccountHomeDiscovery('path code units', limits.maxPathCodeUnits) + } + if (entry.isDirectory()) { + accountIds.push(entry.name) + } + } + } finally { + closeCodexAccountDirectory(directory) + } + + const sessionDirectories: string[] = [] + for (const accountId of accountIds.sort()) { + const accountHome = join(accountsRoot, accountId, 'home') + const sessionsPath = join(accountHome, 'sessions') + pathCodeUnits += accountHome.length + sessionsPath.length + if (pathCodeUnits > limits.maxPathCodeUnits) { + return failCodexAccountHomeDiscovery('path code units', limits.maxPathCodeUnits) + } + try { + assertOwnedHostCodexManagedHomePath({ + candidatePath: accountHome, + managedAccountsRoot: accountsRoot, + systemCodexHomePath: getSystemCodexHomePath(), + expectedAccountId: accountId + }) + // Why: a redirected sessions root could make usage scan unrelated, unbounded trees. + if (lstatSync(sessionsPath).isDirectory()) { + sessionDirectories.push(sessionsPath) + if (sessionDirectories.length > limits.maxHomes) { + return failCodexAccountHomeDiscovery('homes', limits.maxHomes) + } + } + } catch { + // A missing, redirected, or invalid account home is not a usage source. + } + } + return sessionDirectories } catch { return [] } } + +function closeCodexAccountDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch { + // The OS may have already closed a failed directory stream. + } +} + +function failCodexAccountHomeDiscovery(resource: string, limit: number): [] { + console.warn(`[codex-usage] Account-home discovery exceeded ${limit} ${resource}; skipping homes`) + return [] +} diff --git a/src/main/codex/codex-app-server-capability-cache.test.ts b/src/main/codex/codex-app-server-capability-cache.test.ts index c6dea6e0031..632fa1b8e91 100644 --- a/src/main/codex/codex-app-server-capability-cache.test.ts +++ b/src/main/codex/codex-app-server-capability-cache.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it, vi } from 'vitest' import { CODEX_APP_SERVER_CAPABILITY_RETRY_INTERVAL_MS, CodexAppServerCapabilityCache, - getCodexAppServerHostKey + getCodexAppServerHostKey, + MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS, + MAX_CODEX_APP_SERVER_HOST_KEY_CODE_UNITS, + type CodexAppServerHostKey } from './codex-app-server-capability-cache' const unsupportedError = new Error('unsupported') @@ -114,4 +117,48 @@ describe('CodexAppServerCapabilityCache', () => { expect(getCodexAppServerHostKey({ kind: 'wsl', distro: 'Ubuntu' })).toBe('wsl:Ubuntu') expect(getCodexAppServerHostKey({ kind: 'wsl', distro: 'Debian' })).toBe('wsl:Debian') }) + + it('retains exactly the host limit and LRU-evicts on the next host', () => { + const cache = new CodexAppServerCapabilityCache() + const hostKeys = Array.from( + { length: MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS }, + (_, index): CodexAppServerHostKey => `wsl:distro-${index}` + ) + hostKeys.forEach((hostKey) => cache.rememberUnsupported(hostKey, 0)) + + expect(cache.shouldTry(hostKeys[0], 1)).toBe(false) + cache.rememberUnsupported('wsl:overflow', 0) + + expect(cache.shouldTry(hostKeys[0], 1)).toBe(false) + expect(cache.shouldTry(hostKeys[1], 1)).toBe(true) + expect(cache.shouldTry('wsl:overflow', 1)).toBe(false) + }) + + it('bounds oversized generated and direct host keys', () => { + const exactDistro = 'x'.repeat(MAX_CODEX_APP_SERVER_HOST_KEY_CODE_UNITS - 'wsl:'.length) + const oversizedDistro = `${exactDistro}x` + + expect(getCodexAppServerHostKey({ kind: 'wsl', distro: exactDistro })).toBe( + `wsl:${exactDistro}` + ) + const firstDigest = getCodexAppServerHostKey({ kind: 'wsl', distro: oversizedDistro }) + expect(firstDigest).toMatch(/^wsl:sha256:[a-f0-9]{64}$/) + expect(getCodexAppServerHostKey({ kind: 'wsl', distro: oversizedDistro })).toBe(firstDigest) + + const directOversized = `wsl:${oversizedDistro}` as CodexAppServerHostKey + const cache = new CodexAppServerCapabilityCache() + cache.rememberUnsupported(directOversized, 0) + expect(cache.shouldTry(directOversized, 1)).toBe(false) + }) + + it('clear forgets supported and unsupported host state', () => { + const cache = new CodexAppServerCapabilityCache() + cache.rememberSupported('native') + cache.rememberUnsupported('wsl:Ubuntu', 0) + + cache.clear() + + expect(cache.isKnownSupported('native')).toBe(false) + expect(cache.shouldTry('wsl:Ubuntu', 1)).toBe(true) + }) }) diff --git a/src/main/codex/codex-app-server-capability-cache.ts b/src/main/codex/codex-app-server-capability-cache.ts index 85816c9cf0c..1decbbf29da 100644 --- a/src/main/codex/codex-app-server-capability-cache.ts +++ b/src/main/codex/codex-app-server-capability-cache.ts @@ -1,18 +1,36 @@ +import { createHash } from 'node:crypto' + // Why: suppress a known-missing RPC surface without pinning it forever — an // in-place codex upgrade during a long Orca session self-heals after the // interval, mirroring GitCapabilityCache's rationale. export const CODEX_APP_SERVER_CAPABILITY_RETRY_INTERVAL_MS = 30 * 60_000 +export const MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS = 64 +export const MAX_CODEX_APP_SERVER_HOST_KEY_CODE_UNITS = 4 * 1024 /** Execution host that runs the codex binary. WSL distros are isolated from * the native host and from each other — each can carry a different codex. */ export type CodexAppServerHostKey = 'native' | `wsl:${string}` +function boundHostKey(hostKey: CodexAppServerHostKey): CodexAppServerHostKey { + if (hostKey.length <= MAX_CODEX_APP_SERVER_HOST_KEY_CODE_UNITS) { + return hostKey + } + return `wsl:sha256:${createHash('sha256').update(hostKey.slice('wsl:'.length)).digest('hex')}` +} + export function getCodexAppServerHostKey( host: { kind: 'native' } | { kind: 'wsl'; distro: string } ): CodexAppServerHostKey { - return host.kind === 'wsl' ? `wsl:${host.distro}` : 'native' + if (host.kind === 'native') { + return 'native' + } + return boundHostKey(`wsl:${host.distro}`) } +type CodexAppServerCapabilityState = + | { kind: 'supported' } + | { kind: 'unsupported'; retryAfterMs: number } + /** * Capability cache for the codex app-server trust-grant RPC pair, modeled on * GitCapabilityCache but with a synchronous runner: the grant client blocks @@ -20,33 +38,56 @@ export function getCodexAppServerHostKey( * unsupported mark alone is what keeps later installs off the dead probe. */ export class CodexAppServerCapabilityCache { - private readonly retryAfterByHost = new Map() - private readonly supportedHosts = new Set() + private readonly stateByHost = new Map() + + private getState(hostKey: CodexAppServerHostKey): CodexAppServerCapabilityState | undefined { + const retainedHostKey = boundHostKey(hostKey) + const state = this.stateByHost.get(retainedHostKey) + if (state) { + this.stateByHost.delete(retainedHostKey) + this.stateByHost.set(retainedHostKey, state) + } + return state + } + + private remember(hostKey: CodexAppServerHostKey, state: CodexAppServerCapabilityState): void { + const retainedHostKey = boundHostKey(hostKey) + this.stateByHost.delete(retainedHostKey) + this.stateByHost.set(retainedHostKey, state) + while (this.stateByHost.size > MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS) { + const oldestHost = this.stateByHost.keys().next().value + if (oldestHost === undefined) { + return + } + this.stateByHost.delete(oldestHost) + } + } shouldTry(hostKey: CodexAppServerHostKey, nowMs = Date.now()): boolean { - const retryAfterMs = this.retryAfterByHost.get(hostKey) - if (retryAfterMs === undefined) { + const state = this.getState(hostKey) + if (!state || state.kind === 'supported') { return true } - if (nowMs < retryAfterMs) { + if (nowMs < state.retryAfterMs) { return false } - this.retryAfterByHost.delete(hostKey) + this.stateByHost.delete(boundHostKey(hostKey)) return true } isKnownSupported(hostKey: CodexAppServerHostKey): boolean { - return this.supportedHosts.has(hostKey) + return this.getState(hostKey)?.kind === 'supported' } rememberUnsupported(hostKey: CodexAppServerHostKey, nowMs = Date.now()): void { - this.supportedHosts.delete(hostKey) - this.retryAfterByHost.set(hostKey, nowMs + CODEX_APP_SERVER_CAPABILITY_RETRY_INTERVAL_MS) + this.remember(hostKey, { + kind: 'unsupported', + retryAfterMs: nowMs + CODEX_APP_SERVER_CAPABILITY_RETRY_INTERVAL_MS + }) } rememberSupported(hostKey: CodexAppServerHostKey): void { - this.retryAfterByHost.delete(hostKey) - this.supportedHosts.add(hostKey) + this.remember(hostKey, { kind: 'supported' }) } runWithFallbackSync( @@ -56,7 +97,7 @@ export class CodexAppServerCapabilityCache { isUnsupportedError: (error: unknown) => boolean, nowMs = Date.now() ): T { - if (!this.supportedHosts.has(hostKey) && !this.shouldTry(hostKey, nowMs)) { + if (!this.shouldTry(hostKey, nowMs)) { return runFallback() } try { @@ -76,8 +117,7 @@ export class CodexAppServerCapabilityCache { } clear(): void { - this.retryAfterByHost.clear() - this.supportedHosts.clear() + this.stateByHost.clear() } } diff --git a/src/main/codex/codex-app-server-fragment-memory.test.ts b/src/main/codex/codex-app-server-fragment-memory.test.ts new file mode 100644 index 00000000000..d4336d704df --- /dev/null +++ b/src/main/codex/codex-app-server-fragment-memory.test.ts @@ -0,0 +1,45 @@ +import type { ChildProcess, ChildProcessWithoutNullStreams, spawn } from 'node:child_process' +import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { describe, expect, it, vi } from 'vitest' +import { runCodexAppServerSession } from './codex-app-server-session' + +describe('runCodexAppServerSession fragmented output', () => { + it('parses a response delivered as 100,000 one-byte fragments', async () => { + const child = new EventEmitter() as ChildProcessWithoutNullStreams + child.stdin = new PassThrough() + child.stdout = new PassThrough() + child.stderr = new PassThrough() + child.kill = vi.fn(() => true) as ChildProcess['kill'] + child.stdin.on('data', (bytes: Buffer) => { + const request = JSON.parse(bytes.toString()) as { id?: number; method: string } + if (request.id === undefined) { + return + } + const response = Buffer.from( + `${request.method === 'fragment/get' ? ' '.repeat(99_950) : ''}${JSON.stringify({ + id: request.id, + result: request.method === 'fragment/get' ? { value: 'complete' } : {} + })}\n` + ) + for (let index = 0; index < response.byteLength; index += 1) { + child.stdout.emit('data', response.subarray(index, index + 1)) + } + }) + child.stdin.once('finish', () => { + queueMicrotask(() => { + child.emit('exit', 0, null) + child.emit('close', 0, null) + }) + }) + const spawnImpl = vi.fn(() => child) as unknown as typeof spawn + + const result = await runCodexAppServerSession( + { command: 'codex', args: ['app-server'], timeoutMs: 5_000 }, + ({ request }) => request('fragment/get'), + spawnImpl + ) + + expect(result).toEqual({ value: 'complete' }) + }) +}) diff --git a/src/main/codex/codex-app-server-grant-bridge.ts b/src/main/codex/codex-app-server-grant-bridge.ts index c85c4a1c479..c20d7090462 100644 --- a/src/main/codex/codex-app-server-grant-bridge.ts +++ b/src/main/codex/codex-app-server-grant-bridge.ts @@ -16,6 +16,10 @@ import type { CodexUserHookTrustRebaseRequest, CodexUserHookTrustRebaseResult } from './codex-user-hook-trust-rebase-client' +import { + findLastNonEmptyCodexAppServerGrantLine, + parseCodexAppServerGrantJson +} from './codex-app-server-grant-json' // Why: hook install/refresh is synchronous launch prep — a Codex pane must // not start before its trust is settled — but a stdio JSON-RPC session needs @@ -114,12 +118,11 @@ function runCodexAppServerEntrySync( `codex trust-grant entry killed by ${spawned.signal} after ${request.invocation.timeoutMs}ms deadline` ) } - const lines = (spawned.stdout ?? '').split('\n').filter((line) => line.trim().length > 0) - const lastLine = lines.at(-1) + const lastLine = findLastNonEmptyCodexAppServerGrantLine(spawned.stdout ?? '') let envelope: GrantEntryEnvelope | null = null if (lastLine) { try { - envelope = JSON.parse(lastLine) as GrantEntryEnvelope + envelope = parseCodexAppServerGrantJson(lastLine) } catch { envelope = null } diff --git a/src/main/codex/codex-app-server-grant-entry.ts b/src/main/codex/codex-app-server-grant-entry.ts index 8e16892d279..14c465391ce 100644 --- a/src/main/codex/codex-app-server-grant-entry.ts +++ b/src/main/codex/codex-app-server-grant-entry.ts @@ -11,22 +11,21 @@ import { import { writeSync } from 'node:fs' import { runCodexHookTrustGrantSession } from './codex-app-server-client' import { runCodexUserHookTrustRebaseSession } from './codex-user-hook-trust-rebase-client' +import { readNodeReadableTextWithinLimit } from '../../shared/node-readable-text' +import { parseCodexAppServerGrantJson } from './codex-app-server-grant-json' const HARD_EXIT_MARGIN_MS = 2_000 +const GRANT_ENTRY_MAX_INPUT_BYTES = 16 * 1024 * 1024 async function readStdin(): Promise { - const chunks: Buffer[] = [] - for await (const chunk of process.stdin) { - chunks.push(chunk as Buffer) - } - return Buffer.concat(chunks).toString('utf8') + return await readNodeReadableTextWithinLimit(process.stdin, GRANT_ENTRY_MAX_INPUT_BYTES) } async function main(): Promise { const raw = await readStdin() let request: CodexAppServerEntryRequest try { - request = JSON.parse(raw) as CodexAppServerEntryRequest + request = parseCodexAppServerGrantJson(raw) } catch (error) { process.stdout.write( `${JSON.stringify({ diff --git a/src/main/codex/codex-app-server-grant-json.test.ts b/src/main/codex/codex-app-server-grant-json.test.ts new file mode 100644 index 00000000000..8a2178b799b --- /dev/null +++ b/src/main/codex/codex-app-server-grant-json.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it, vi } from 'vitest' +import { + CODEX_APP_SERVER_GRANT_JSON_MAX_STRUCTURAL_TOKENS, + findLastNonEmptyCodexAppServerGrantLine, + parseCodexAppServerGrantJson +} from './codex-app-server-grant-json' + +describe('Codex app-server grant JSON admission', () => { + it('preserves the last non-empty output line without splitting all output', () => { + expect(findLastNonEmptyCodexAppServerGrantLine('diagnostic\n\n{"ok":true}\n \t\r')).toBe( + '{"ok":true}' + ) + expect(findLastNonEmptyCodexAppServerGrantLine('\n \t\r')).toBeNull() + }) + + it('rejects structurally amplified grant JSON before parsing', () => { + const text = `{"values":[${'0,'.repeat(CODEX_APP_SERVER_GRANT_JSON_MAX_STRUCTURAL_TOKENS)}0]}` + const parseSpy = vi.spyOn(JSON, 'parse') + try { + expect(() => parseCodexAppServerGrantJson(text)).toThrow(/JSON structure exceeds/) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) +}) diff --git a/src/main/codex/codex-app-server-grant-json.ts b/src/main/codex/codex-app-server-grant-json.ts new file mode 100644 index 00000000000..47cb25d3f4b --- /dev/null +++ b/src/main/codex/codex-app-server-grant-json.ts @@ -0,0 +1,51 @@ +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' + +export const CODEX_APP_SERVER_GRANT_JSON_MAX_STRUCTURAL_TOKENS = 1_000_000 +export const CODEX_APP_SERVER_GRANT_JSON_MAX_NESTING_DEPTH = 128 + +export function parseCodexAppServerGrantJson(text: string): T { + assertJsonTextStructureWithinLimits(text, { + structuralTokens: CODEX_APP_SERVER_GRANT_JSON_MAX_STRUCTURAL_TOKENS, + nestingDepth: CODEX_APP_SERVER_GRANT_JSON_MAX_NESTING_DEPTH + }) + return JSON.parse(text) as T +} + +export function findLastNonEmptyCodexAppServerGrantLine(output: string): string | null { + let end = output.length + while (end > 0) { + const newline = output.lastIndexOf('\n', end - 1) + const start = newline + 1 + const line = output.slice(start, end) + if (hasNonWhitespace(line)) { + return line + } + end = newline === -1 ? 0 : newline + } + return null +} + +function hasNonWhitespace(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + if (!isEcmaWhitespace(value.charCodeAt(index))) { + return true + } + } + return false +} + +function isEcmaWhitespace(code: number): boolean { + return ( + code === 0x20 || + (code >= 0x09 && code <= 0x0d) || + code === 0xa0 || + code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || + code === 0x2028 || + code === 0x2029 || + code === 0x202f || + code === 0x205f || + code === 0x3000 || + code === 0xfeff + ) +} diff --git a/src/main/codex/codex-app-server-session.ts b/src/main/codex/codex-app-server-session.ts index 04748e6eaa2..eea85544eb9 100644 --- a/src/main/codex/codex-app-server-session.ts +++ b/src/main/codex/codex-app-server-session.ts @@ -1,4 +1,5 @@ import { spawn, type ChildProcess, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import { waitForProcessExitUntil } from './codex-process-exit-deadline' import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal' @@ -117,7 +118,7 @@ export async function runCodexAppServerSession( windowsHide: true }) as ChildProcessWithoutNullStreams - let stderrTail = '' + const stderrTail = new GrowingByteBuffer() let exited = false let nextRequestId = 1 let timedOut = false @@ -145,10 +146,15 @@ export async function runCodexAppServerSession( child.on('close', () => { failPending(buildEarlyExitError()) }) - // Why: JSONL can contain non-ASCII hook paths. Stream decoding must retain a - // multibyte character split across pipe chunks or the response becomes invalid JSON. - child.stderr.setEncoding('utf8').on('data', (chunk: string) => { - stderrTail = (stderrTail + chunk).slice(-STDERR_TAIL_MAX_BYTES) + // Why: retain raw bytes so split UTF-8 code points decode only after a full line arrives. + child.stderr.on('data', (chunk: Buffer) => { + if (chunk.byteLength >= STDERR_TAIL_MAX_BYTES) { + stderrTail.clear() + stderrTail.append(chunk.subarray(chunk.byteLength - STDERR_TAIL_MAX_BYTES)) + return + } + stderrTail.append(chunk) + stderrTail.retainSuffix(STDERR_TAIL_MAX_BYTES) }) // Why: a child can exit between the liveness check and stdin.write(); an // EPIPE must reject the RPC instead of becoming an unhandled stream error. @@ -156,21 +162,25 @@ export async function runCodexAppServerSession( failPending(error) }) - let stdoutBuffer = '' - child.stdout.setEncoding('utf8').on('data', (chunk: string) => { - stdoutBuffer += chunk - if (Buffer.byteLength(stdoutBuffer) > STDOUT_LINE_MAX_BYTES) { + const stdoutBuffer = new GrowingByteBuffer() + child.stdout.on('data', (chunk: Buffer) => { + stdoutBuffer.append(chunk) + if (stdoutBuffer.byteLength > STDOUT_LINE_MAX_BYTES) { // Why: Windows process-tree termination is asynchronous; stop buffered // chunks from spawning another taskkill for the same oversized response. child.stdout.destroy() + stdoutBuffer.clear() killCodexAppServerProcessTree(child) failPending(new Error('codex app-server emitted an oversized JSONL response')) return } - let newlineIndex - while ((newlineIndex = stdoutBuffer.indexOf('\n')) !== -1) { - const line = stdoutBuffer.slice(0, newlineIndex).trim() - stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1) + while (true) { + const newlineIndex = stdoutBuffer.indexOfByte(0x0a) + if (newlineIndex === -1) { + break + } + const line = stdoutBuffer.takePrefixString(newlineIndex).trim() + stdoutBuffer.discardPrefix(1) if (!line) { continue } @@ -263,13 +273,14 @@ export async function runCodexAppServerSession( } function buildEarlyExitError(): Error { - if (stderrIndicatesMissingAppServer(stderrTail)) { + const stderr = stderrTail.toString() + if (stderrIndicatesMissingAppServer(stderr)) { return new CodexAppServerUnsupportedError( - `codex CLI does not support the app-server subcommand: ${stderrTail.trim().slice(0, 400)}` + `codex CLI does not support the app-server subcommand: ${stderr.trim().slice(0, 400)}` ) } return new Error( - `codex app-server exited before completing the session${stderrTail ? `: ${stderrTail.trim().slice(0, 400)}` : ''}` + `codex app-server exited before completing the session${stderr ? `: ${stderr.trim().slice(0, 400)}` : ''}` ) } @@ -289,10 +300,11 @@ export async function runCodexAppServerSession( error instanceof Error && !(error instanceof CodexAppServerUnsupportedError) && !(error instanceof CodexAppServerTimeoutError) && - stderrIndicatesMissingAppServer(stderrTail) + stderrIndicatesMissingAppServer(stderrTail.toString()) ) { + const stderr = stderrTail.toString() throw new CodexAppServerUnsupportedError( - `codex CLI does not support the app-server subcommand: ${stderrTail.trim().slice(0, 400)}` + `codex CLI does not support the app-server subcommand: ${stderr.trim().slice(0, 400)}` ) } throw error @@ -312,5 +324,7 @@ export async function runCodexAppServerSession( } } clearTimeout(deadline) + stdoutBuffer.clear() + stderrTail.clear() } } diff --git a/src/main/codex/codex-config-mirror.ts b/src/main/codex/codex-config-mirror.ts index 26933c1ba02..5869d362397 100644 --- a/src/main/codex/codex-config-mirror.ts +++ b/src/main/codex/codex-config-mirror.ts @@ -1,5 +1,6 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { dirname, join } from 'node:path' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from './codex-home-paths' import { rewriteRelativePathConfigValues } from './codex-config-path-reference-rewrite' @@ -58,7 +59,7 @@ function syncSystemConfigIntoManagedCodexHomeUnsafe({ return } - const rawSystemConfig = systemConfigExists ? readFileSync(systemConfigPath, 'utf-8') : '' + const rawSystemConfig = systemConfigExists ? readAgentStateFileSync(systemConfigPath) : '' const sourceConfigDir = resolveCodexConfigMirrorSourceDirectory(systemHomePath) if (!runtimeConfigExists) { writeFileAtomically( @@ -69,7 +70,7 @@ function syncSystemConfigIntoManagedCodexHomeUnsafe({ } const systemConfig = prepareSystemConfigForRuntimeMirror(rawSystemConfig, sourceConfigDir) - const runtimeConfig = readFileSync(runtimeConfigPath, 'utf-8') + const runtimeConfig = readAgentStateFileSync(runtimeConfigPath) const mergedConfig = mergeSystemCodexConfigIntoRuntime(runtimeConfig, systemConfig) if (mergedConfig !== runtimeConfig) { writeFileAtomically(runtimeConfigPath, mergedConfig) diff --git a/src/main/codex/codex-config-settings-upsert.ts b/src/main/codex/codex-config-settings-upsert.ts deleted file mode 100644 index 963d46d239a..00000000000 --- a/src/main/codex/codex-config-settings-upsert.ts +++ /dev/null @@ -1,322 +0,0 @@ -import { - createTomlLineScanState, - getTomlTableHeader, - isTomlStructuralLine, - updateTomlLineScanState -} from './config-toml-line-scan' -import { parseTomlKeyPath, parseTomlTableHeaderPath } from './config-toml-key-path' - -const TUI_STRUCTURED_PREFIX = 'tui.' - -// Why: promoted [tui] settings are keyed by structured path (tui.) so their -// baseline/update entries can never collide with a top-level key of the same name. -export function tuiStructuredKey(key: string): string { - return `${TUI_STRUCTURED_PREFIX}${key}` -} - -export function isTuiStructuredKey(structuredKey: string): boolean { - return structuredKey.startsWith(TUI_STRUCTURED_PREFIX) -} - -export function tuiKeyFromStructuredKey(structuredKey: string): string { - return structuredKey.slice(TUI_STRUCTURED_PREFIX.length) -} - -// Why: promoted updates arrive keyed by structured path; the preamble and [tui] -// regions are disjoint, so a mixed batch (e.g. /model + a status-line change) -// composes in one rewrite — top-level keys land in the preamble, tui. -// entries wherever the [tui] placement rule puts them. -export function upsertPromotedSettingsInContent( - content: string, - updates: Map -): string { - const topLevelUpdates = new Map() - const tuiUpdates = new Map() - for (const [key, raw] of updates) { - if (isTuiStructuredKey(key)) { - tuiUpdates.set(tuiKeyFromStructuredKey(key), raw) - } else { - topLevelUpdates.set(key, raw) - } - } - let result = content - if (topLevelUpdates.size > 0) { - result = upsertTopLevelSettingsInContent(result, topLevelUpdates) - } - if (tuiUpdates.size > 0) { - result = upsertTuiSettingsInContent(result, tuiUpdates) - } - return result -} - -export function upsertTopLevelSettingsInContent( - content: string, - updates: Map -): string { - const lines = content.split('\n') - let state = createTomlLineScanState() - let preambleEnd = lines.length - const keyLineIndexes = new Map() - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index] ?? '' - if (isTomlStructuralLine(state)) { - if (getTomlTableHeader(line)) { - preambleEnd = index - break - } - const parsed = parseTomlKeyPath(line) - const key = parsed?.segments.length === 1 ? parsed.segments[0] : null - if (parsed && line[parsed.end] === '=' && key && updates.has(key)) { - keyLineIndexes.set(key, index) - } - } - state = updateTomlLineScanState(state, line) - } - - // Why: CRLF configs keep a trailing \r after the split; new lines must use - // the file's existing endings or a Windows-owned config becomes mixed-EOL. - const usesCrlf = content.includes('\r\n') - const insertions: string[] = [] - for (const [key, raw] of updates) { - const existingIndex = keyLineIndexes.get(key) - const rendered = `${key} = ${raw}` - if (existingIndex !== undefined) { - lines[existingIndex] = lines[existingIndex]?.endsWith('\r') ? `${rendered}\r` : rendered - } else { - insertions.push(usesCrlf ? `${rendered}\r` : rendered) - } - } - if (insertions.length > 0) { - let insertAt = preambleEnd - while (insertAt > 0 && (lines[insertAt - 1] ?? '').trim() === '') { - insertAt -= 1 - } - if (insertAt === preambleEnd && preambleEnd < lines.length) { - insertions.push(usesCrlf ? '\r' : '') - } - lines.splice(insertAt, 0, ...insertions) - } - return joinPreservingTrailingNewline(lines, usesCrlf) -} - -type TuiPlacementScan = { - bareKeyIndexes: Map - dottedKeyIndexes: Map - hasBareTuiTable: boolean - hasDottedTuiKey: boolean - blocksNewTuiTable: boolean - blockedAbsentKeys: Set - bareBodyInsertIndex: number - lastDottedTuiIndex: number -} - -/** - * Upserts promoted `[tui]` keys (keyed by bare name) into the system config, - * placing each per the design's total placement rule: replace an existing key - * in place keeping its form; else insert bare into the first `[tui]` body; else - * dotted in the preamble beside existing dotted `tui.*` keys; else create one - * `[tui]` table at EOF for every key that reaches that branch. Rendering follows - * the destination — bare inside a table, dotted in the preamble — so no `tui` - * table is ever defined twice. - */ -export function upsertTuiSettingsInContent(content: string, updates: Map): string { - const lines = content.split('\n') - const scan = scanTuiPlacement(lines, updates) - const usesCrlf = content.includes('\r\n') - const bareBodyInserts: string[] = [] - const dottedPreambleInserts: string[] = [] - const newTableKeys: string[] = [] - - for (const [key, raw] of updates) { - const dottedIndex = scan.dottedKeyIndexes.get(key) - if (dottedIndex !== undefined) { - lines[dottedIndex] = withTrailingCr(lines[dottedIndex]!, `${tuiStructuredKey(key)} = ${raw}`) - continue - } - const bareIndex = scan.bareKeyIndexes.get(key) - if (bareIndex !== undefined) { - lines[bareIndex] = withTrailingCr(lines[bareIndex]!, `${key} = ${raw}`) - continue - } - // Why: adding a scalar beside an existing tui. descendant would turn valid TOML invalid. - if (scan.blockedAbsentKeys.has(key)) { - continue - } - if (scan.hasBareTuiTable) { - bareBodyInserts.push(`${key} = ${raw}`) - } else if (scan.hasDottedTuiKey) { - dottedPreambleInserts.push(`${tuiStructuredKey(key)} = ${raw}`) - } else if (!scan.blocksNewTuiTable) { - // Why: inline/array tui definitions block this branch because adding a - // plain [tui] beside either would make the config invalid. - newTableKeys.push(`${key} = ${raw}`) - } - } - - // Why: the config shape routes every absent key to the same branch, so at most - // one insert group is non-empty; still apply EOF→body→preamble so a splice - // never shifts a lower index a later splice depends on. - if (newTableKeys.length > 0) { - appendNewTuiTable(lines, newTableKeys, usesCrlf) - } - if (bareBodyInserts.length > 0) { - lines.splice( - scan.bareBodyInsertIndex, - 0, - ...bareBodyInserts.map((line) => withCrLine(line, usesCrlf)) - ) - } - if (dottedPreambleInserts.length > 0) { - lines.splice( - scan.lastDottedTuiIndex + 1, - 0, - ...dottedPreambleInserts.map((line) => withCrLine(line, usesCrlf)) - ) - } - return joinPreservingTrailingNewline(lines, usesCrlf) -} - -function scanTuiPlacement(lines: string[], updates: Map): TuiPlacementScan { - let state = createTomlLineScanState() - let inPreamble = true - let tuiTableSeen = false - let tuiBodyActive = false - let tuiBodyHeaderIndex = -1 - let tuiBodyEndIndex = -1 - let hasDottedTuiKey = false - let blocksNewTuiTable = false - let lastDottedTuiIndex = -1 - const bareKeyIndexes = new Map() - const dottedKeyIndexes = new Map() - const blockedAbsentKeys = new Set() - - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index] ?? '' - if (isTomlStructuralLine(state)) { - const header = getTomlTableHeader(line) - if (header) { - if (tuiBodyActive) { - tuiBodyEndIndex = index - tuiBodyActive = false - } - const table = parseTomlTableHeaderPath(header) - if ( - table && - !table.isArray && - table.segments.length === 1 && - table.segments[0] === 'tui' && - !tuiTableSeen - ) { - tuiTableSeen = true - tuiBodyActive = true - tuiBodyHeaderIndex = index - } - // Why: a root [[tui]] is already an array, so appending [tui] would - // redefine it and make an otherwise valid config unparseable. - if (table?.isArray && table.segments.length === 1 && table.segments[0] === 'tui') { - blocksNewTuiTable = true - } - const descendantKey = - table?.segments[0] === 'tui' && table.segments.length > 1 ? table.segments[1] : null - if (descendantKey && updates.has(descendantKey)) { - blockedAbsentKeys.add(descendantKey) - } - inPreamble = false - state = updateTomlLineScanState(state, line) - continue - } - if (inPreamble) { - // Why: any dotted `tui.*` key (allowlisted or not) already defines the - // implicit tui table, so a new `[tui]` table at EOF would duplicate it. - const parsed = parseTomlKeyPath(line) - const isAssignment = parsed && line[parsed.end] === '=' - if (isAssignment && parsed.segments[0] === 'tui' && parsed.segments.length > 1) { - hasDottedTuiKey = true - lastDottedTuiIndex = index - const promotedKey = parsed.segments.length === 2 ? parsed.segments[1] : null - if (promotedKey && updates.has(promotedKey)) { - dottedKeyIndexes.set(promotedKey, index) - } - const descendantKey = parsed.segments.length > 2 ? parsed.segments[1] : null - if (descendantKey && updates.has(descendantKey)) { - blockedAbsentKeys.add(descendantKey) - } - } else if (isAssignment && parsed.segments.length === 1 && parsed.segments[0] === 'tui') { - blocksNewTuiTable = true - } - } else if (tuiBodyActive) { - const parsed = parseTomlKeyPath(line) - const key = parsed?.segments.length === 1 ? parsed.segments[0] : null - if (parsed && line[parsed.end] === '=' && key && updates.has(key)) { - bareKeyIndexes.set(key, index) - } - const descendantKey = parsed && parsed.segments.length > 1 ? parsed.segments[0] : null - if (descendantKey && updates.has(descendantKey)) { - blockedAbsentKeys.add(descendantKey) - } - } - } - state = updateTomlLineScanState(state, line) - } - if (tuiBodyActive) { - tuiBodyEndIndex = lines.length - } - - return { - bareKeyIndexes, - dottedKeyIndexes, - hasBareTuiTable: tuiTableSeen, - hasDottedTuiKey, - blocksNewTuiTable, - blockedAbsentKeys, - bareBodyInsertIndex: computeBareBodyInsertIndex(lines, tuiBodyHeaderIndex, tuiBodyEndIndex), - lastDottedTuiIndex - } -} - -// Why: TOML forbids adding bare keys to `[tui]` after a `[tui.*]` subtable opens, -// so absent keys land at the body's end — before trailing blanks and before the -// next header — which is the only valid spot. -function computeBareBodyInsertIndex( - lines: string[], - headerIndex: number, - endIndex: number -): number { - if (headerIndex === -1) { - return -1 - } - let insertAt = endIndex - while (insertAt > headerIndex + 1 && (lines[insertAt - 1] ?? '').trim() === '') { - insertAt -= 1 - } - return insertAt -} - -function appendNewTuiTable(lines: string[], keyRenders: string[], usesCrlf: boolean): void { - let appendAt = lines.length - while (appendAt > 0 && (lines[appendAt - 1] ?? '').trim() === '') { - appendAt -= 1 - } - // Why: separate the new table from prior content with a blank line, unless the - // file was empty/blank, where a leading blank would be spurious. - const block = appendAt > 0 ? ['', '[tui]', ...keyRenders] : ['[tui]', ...keyRenders] - lines.splice(appendAt, 0, ...block.map((line) => withCrLine(line, usesCrlf))) -} - -function withTrailingCr(originalLine: string, rendered: string): string { - return originalLine.endsWith('\r') ? `${rendered}\r` : rendered -} - -function withCrLine(rendered: string, usesCrlf: boolean): string { - return usesCrlf ? `${rendered}\r` : rendered -} - -// Why: a missing trailing newline is restored in the file's own EOL so a -// preamble-only or table-appended rewrite matches the source's newline behavior. -function joinPreservingTrailingNewline(lines: string[], usesCrlf: boolean): string { - const result = lines.join('\n') - if (result.endsWith('\n') || result.length === 0) { - return result - } - return result.endsWith('\r') ? `${result}\n` : `${result}${usesCrlf ? '\r\n' : '\n'}` -} diff --git a/src/main/codex/codex-home-paths.ts b/src/main/codex/codex-home-paths.ts index 61e2eba1eca..0f2ee9cfb13 100644 --- a/src/main/codex/codex-home-paths.ts +++ b/src/main/codex/codex-home-paths.ts @@ -3,7 +3,6 @@ import { existsSync, lstatSync, mkdirSync, - readFileSync, readlinkSync, rmdirSync, rmSync, @@ -14,8 +13,11 @@ import { } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' +import { nodeSourceAndCopyContentsEqualSync } from '../../shared/node-source-copy-content-equality' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' const CODEX_GLOBAL_INSTRUCTIONS_ENTRY = 'AGENTS.md' +const CODEX_RESOURCE_COPY_MARKER_MAX_BYTES = 64 * 1024 const CODEX_SYSTEM_RESOURCE_ENTRIES = [ 'skills', @@ -124,7 +126,7 @@ function linkSystemCodexResource( // rewriting an unchanged file across the UNC boundary on every launch. if ( entryName === CODEX_GLOBAL_INSTRUCTIONS_ENTRY && - copiedFileContentsMatch(sourcePath, targetPath) + nodeSourceAndCopyContentsEqualSync(sourcePath, targetPath) ) { return } @@ -213,19 +215,6 @@ function pathEntryExists(entryPath: string): boolean { } } -function copiedFileContentsMatch(sourcePath: string, targetPath: string): boolean { - try { - // Why: reading a FIFO or device synchronously can block Codex launch. - // Follow source symlinks, but only compare two regular files. - if (!statSync(sourcePath).isFile() || !lstatSync(targetPath).isFile()) { - return false - } - return readFileSync(sourcePath).equals(readFileSync(targetPath)) - } catch { - return false - } -} - function targetAlreadyPointsToSource(targetPath: string, sourcePath: string): boolean { try { return ( @@ -264,7 +253,10 @@ function markCopiedResource(managedHomePath: string, entryName: string, sourcePa function readCopiedResourceSourcePath(managedHomePath: string, entryName: string): string | null { try { const parsed: unknown = JSON.parse( - readFileSync(getResourceCopyMarkerPath(managedHomePath, entryName), 'utf-8') + readNodeFileSyncWithinLimit( + getResourceCopyMarkerPath(managedHomePath, entryName), + CODEX_RESOURCE_COPY_MARKER_MAX_BYTES + ).buffer.toString('utf8') ) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return null diff --git a/src/main/codex/codex-hook-trust-grant.ts b/src/main/codex/codex-hook-trust-grant.ts index 94eb3117b45..b6fc569034b 100644 --- a/src/main/codex/codex-hook-trust-grant.ts +++ b/src/main/codex/codex-hook-trust-grant.ts @@ -15,6 +15,7 @@ import { codexAppServerCapabilityCache, getCodexAppServerHostKey } from './codex-app-server-capability-cache' +import { CodexHostRetryDeadlines } from './codex-host-retry-deadlines' import { writeCodexTrustGrantLedgerHome, type CodexTrustGrantBinaryStamp, @@ -72,7 +73,7 @@ const diagnostics = { lastFallbackReason: null as CodexTrustGrantFallbackReason | null } export type CodexTrustGrantDiagnostics = typeof diagnostics -const transientRetryAfterByHost = new Map() +const transientRetryAfterByHost = new CodexHostRetryDeadlines() export function getCodexTrustGrantDiagnostics(): CodexTrustGrantDiagnostics { return { ...diagnostics } diff --git a/src/main/codex/codex-host-retry-deadlines.test.ts b/src/main/codex/codex-host-retry-deadlines.test.ts new file mode 100644 index 00000000000..978bb7e1ff5 --- /dev/null +++ b/src/main/codex/codex-host-retry-deadlines.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS, + type CodexAppServerHostKey +} from './codex-app-server-capability-cache' +import { CodexHostRetryDeadlines } from './codex-host-retry-deadlines' + +function host(index: number): CodexAppServerHostKey { + return `wsl:distro-${index}` +} + +describe('CodexHostRetryDeadlines', () => { + it('preserves every deadline through the exact host limit', () => { + const deadlines = new CodexHostRetryDeadlines() + for (let index = 0; index < MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS; index += 1) { + deadlines.set(host(index), index + 1) + } + + expect(deadlines.sizeForTest()).toBe(MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS) + for (let index = 0; index < MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS; index += 1) { + expect(deadlines.get(host(index))).toBe(index + 1) + } + }) + + it('evicts the least recently used host above the limit', () => { + const deadlines = new CodexHostRetryDeadlines() + for (let index = 0; index < MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS; index += 1) { + deadlines.set(host(index), index + 1) + } + expect(deadlines.get(host(0))).toBe(1) + + deadlines.set(host(MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS), 999) + + expect(deadlines.sizeForTest()).toBe(MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS) + expect(deadlines.get(host(0))).toBe(1) + expect(deadlines.get(host(1))).toBeUndefined() + expect(deadlines.get(host(MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS))).toBe(999) + }) +}) diff --git a/src/main/codex/codex-host-retry-deadlines.ts b/src/main/codex/codex-host-retry-deadlines.ts new file mode 100644 index 00000000000..20b2f9df41a --- /dev/null +++ b/src/main/codex/codex-host-retry-deadlines.ts @@ -0,0 +1,41 @@ +import { + MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS, + type CodexAppServerHostKey +} from './codex-app-server-capability-cache' + +export class CodexHostRetryDeadlines { + private readonly deadlineByHost = new Map() + + get(hostKey: CodexAppServerHostKey): number | undefined { + const deadline = this.deadlineByHost.get(hostKey) + if (deadline !== undefined) { + this.deadlineByHost.delete(hostKey) + this.deadlineByHost.set(hostKey, deadline) + } + return deadline + } + + set(hostKey: CodexAppServerHostKey, deadline: number): void { + this.deadlineByHost.delete(hostKey) + this.deadlineByHost.set(hostKey, deadline) + while (this.deadlineByHost.size > MAX_CODEX_APP_SERVER_CAPABILITY_HOSTS) { + const oldestHost = this.deadlineByHost.keys().next().value + if (oldestHost === undefined) { + return + } + this.deadlineByHost.delete(oldestHost) + } + } + + delete(hostKey: CodexAppServerHostKey): void { + this.deadlineByHost.delete(hostKey) + } + + clear(): void { + this.deadlineByHost.clear() + } + + sizeForTest(): number { + return this.deadlineByHost.size + } +} diff --git a/src/main/codex/codex-real-home-hook-install.ts b/src/main/codex/codex-real-home-hook-install.ts index 1d59c17ac14..769d99ec21d 100644 --- a/src/main/codex/codex-real-home-hook-install.ts +++ b/src/main/codex/codex-real-home-hook-install.ts @@ -1,10 +1,11 @@ -import { existsSync, mkdirSync, readFileSync, statSync, unlinkSync } from 'node:fs' +import { existsSync, mkdirSync, statSync, unlinkSync } from 'node:fs' import { join } from 'node:path' import { writeFileAtomically } from '../codex-accounts/fs-utils' import { buildManagedCommandHook, createManagedCommandMatcher, MANAGED_HOOK_TIMEOUT_SECONDS, + readHooksJsonRawForGenerationCheck, readHooksJsonWithRaw, removeManagedCommands, writeHooksJson, @@ -74,7 +75,9 @@ function assertHooksJsonGeneration( hooksWritePath: string, expectedRaw: string | null ): void { - const currentRaw = existsSync(hooksJsonPath) ? readFileSync(hooksJsonPath, 'utf-8') : null + const currentRaw = existsSync(hooksJsonPath) + ? readHooksJsonRawForGenerationCheck(hooksJsonPath) + : null if (currentRaw !== expectedRaw || resolveHooksJsonWritePath(hooksJsonPath) !== hooksWritePath) { // Why: the pre-mutation RPC can overlap a user's editor save. Abort rather // than atomically replacing a newer file with the stale parsed snapshot. diff --git a/src/main/codex/codex-session-backfill-marker.ts b/src/main/codex/codex-session-backfill-marker.ts index d3d5a07e5f7..f03fbbccbd6 100644 --- a/src/main/codex/codex-session-backfill-marker.ts +++ b/src/main/codex/codex-session-backfill-marker.ts @@ -1,5 +1,6 @@ -import { mkdirSync, readFileSync, rmSync } from 'node:fs' +import { mkdirSync, rmSync } from 'node:fs' import { dirname } from 'node:path' +import { readAgentStateJsonFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' import type { CodexSessionBackfillSummary } from './codex-session-backfill-types' @@ -12,7 +13,7 @@ export function hasCompletedCodexSessionBackfillMarker( systemSessionsRoot: string ): boolean { try { - const parsed: unknown = JSON.parse(readFileSync(markerPath, 'utf-8')) + const parsed = readAgentStateJsonFileSync(markerPath) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return false } diff --git a/src/main/codex/codex-session-bridge.ts b/src/main/codex/codex-session-bridge.ts index e546ce9d8f8..6681f64407b 100644 --- a/src/main/codex/codex-session-bridge.ts +++ b/src/main/codex/codex-session-bridge.ts @@ -3,13 +3,13 @@ import { linkSync, lstatSync, mkdirSync, - readFileSync, readlinkSync, renameSync, rmSync, symlinkSync } from 'node:fs' import { dirname, isAbsolute, join, relative, sep } from 'node:path' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from './codex-home-paths' import { listCodexSessionJsonlFiles, @@ -334,7 +334,7 @@ function getLegacySessionCopyMarkerPath(relativePath: string): string { function readLegacyCopiedSessionMarker(relativePath: string): LegacyCopiedSessionMarker | null { try { const parsed: unknown = JSON.parse( - readFileSync(getLegacySessionCopyMarkerPath(relativePath), 'utf-8') + readAgentStateFileSync(getLegacySessionCopyMarkerPath(relativePath)) ) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return null diff --git a/src/main/codex/codex-session-file-listing.test.ts b/src/main/codex/codex-session-file-listing.test.ts new file mode 100644 index 00000000000..59ea9f9f14e --- /dev/null +++ b/src/main/codex/codex-session-file-listing.test.ts @@ -0,0 +1,112 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + CodexSessionListingCapacityError, + listCodexSessionJsonlFilesIncrementally, + listCodexSessionJsonlFilesWithinLimits +} from './codex-session-file-listing' + +const tempRoots: string[] = [] + +async function makeRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-codex-session-listing-')) + tempRoots.push(root) + return root +} + +async function collect(files: AsyncIterable): Promise { + const collected: string[] = [] + for await (const file of files) { + collected.push(file) + } + return collected +} + +afterEach(async () => { + vi.restoreAllMocks() + await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('Codex session file-listing limits', () => { + it('preserves sorted JSONL discovery below every limit', async () => { + const root = await makeRoot() + await mkdir(join(root, '2026', '01', '02'), { recursive: true }) + const later = join(root, '2026', '01', '02', 'z.jsonl') + const earlier = join(root, '2026', '01', '02', 'a.jsonl') + await writeFile(later, '{}') + await writeFile(earlier, '{}') + await writeFile(join(root, '2026', '01', '02', 'ignored.txt'), 'x') + + expect(listCodexSessionJsonlFilesWithinLimits(root)).toEqual([earlier, later]) + }) + + it('accepts exact entry and file limits, then rejects the next file', async () => { + const root = await makeRoot() + await writeFile(join(root, 'a.jsonl'), '{}') + await writeFile(join(root, 'b.jsonl'), '{}') + + expect( + listCodexSessionJsonlFilesWithinLimits(root, { maxEntries: 2, maxFiles: 2 }) + ).toHaveLength(2) + await writeFile(join(root, 'c.jsonl'), '{}') + expect(() => + listCodexSessionJsonlFilesWithinLimits(root, { maxEntries: 3, maxFiles: 2 }) + ).toThrow('Codex session listing exceeded 2 files') + }) + + it('bounds a zero-file directory tree by entries and depth', async () => { + const root = await makeRoot() + await mkdir(join(root, 'a', 'b'), { recursive: true }) + + expect(listCodexSessionJsonlFilesWithinLimits(root, { maxEntries: 2, maxDepth: 2 })).toEqual([]) + expect(() => + listCodexSessionJsonlFilesWithinLimits(root, { maxEntries: 1, maxDepth: 2 }) + ).toThrow('Codex session listing exceeded 1 entries') + expect(() => + listCodexSessionJsonlFilesWithinLimits(root, { maxEntries: 2, maxDepth: 1 }) + ).toThrow('Codex session listing exceeded 1 depth') + }) + + it('accepts the exact aggregate path limit and rejects one code unit less', async () => { + const root = await makeRoot() + const filePath = join(root, 'a.jsonl') + await writeFile(filePath, '{}') + const exactPathCodeUnits = root.length + filePath.length + + expect( + listCodexSessionJsonlFilesWithinLimits(root, { + maxPathCodeUnits: exactPathCodeUnits + }) + ).toEqual([filePath]) + expect(() => + listCodexSessionJsonlFilesWithinLimits(root, { + maxPathCodeUnits: exactPathCodeUnits - 1 + }) + ).toThrow(`Codex session listing exceeded ${exactPathCodeUnits - 1} path code units`) + }) + + it('stops an incremental scan at capacity and reports it as a failed directory', async () => { + const root = await makeRoot() + await writeFile(join(root, 'a.jsonl'), '{}') + await writeFile(join(root, 'b.jsonl'), '{}') + const onDirectoryError = vi.fn() + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + const files = await collect( + listCodexSessionJsonlFilesIncrementally( + root, + { batchSize: 1, limits: { maxFiles: 1 }, yieldMs: 0 }, + onDirectoryError + ) + ) + + expect(files).toHaveLength(1) + expect(onDirectoryError).toHaveBeenCalledWith( + root, + expect.any(CodexSessionListingCapacityError) + ) + expect(warnSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/codex/codex-session-file-listing.ts b/src/main/codex/codex-session-file-listing.ts index 8b0a8dd6f2e..cfdf1459001 100644 --- a/src/main/codex/codex-session-file-listing.ts +++ b/src/main/codex/codex-session-file-listing.ts @@ -1,10 +1,25 @@ -import { readdirSync } from 'node:fs' +import { opendirSync } from 'node:fs' import { opendir } from 'node:fs/promises' import { join } from 'node:path' +import { + CodexSessionListingBudget, + CodexSessionListingCapacityError, + type CodexSessionListingLimits +} from './codex-session-listing-budget' + +export { + CODEX_SESSION_LISTING_MAX_DEPTH, + CODEX_SESSION_LISTING_MAX_ENTRIES, + CODEX_SESSION_LISTING_MAX_FILES, + CODEX_SESSION_LISTING_MAX_PATH_CODE_UNITS, + CodexSessionListingCapacityError +} from './codex-session-listing-budget' export type CodexSessionBridgeIncrementalOptions = { /** Directory entries to process before yielding back to the event loop. */ batchSize?: number + /** Optional lower limits for tests or constrained callers. */ + limits?: Partial /** Delay after each processed batch; zero still yields on a timer turn. */ yieldMs?: number } @@ -19,32 +34,59 @@ const INCREMENTAL_BRIDGE_YIELD_MS = 10 * that run outside the CLI launch path. */ export function listCodexSessionJsonlFiles(rootPath: string): string[] { + return listCodexSessionJsonlFilesWithinLimits(rootPath) +} + +export function listCodexSessionJsonlFilesWithinLimits( + rootPath: string, + limits: Partial = {} +): string[] { + const budget = new CodexSessionListingBudget(limits) + budget.claimDepth(0) + budget.claimPath(rootPath) const files: string[] = [] - try { - for (const entry of readdirSync(rootPath, { withFileTypes: true })) { - const childPath = join(rootPath, entry.name) - if (entry.isDirectory()) { - appendSessionFilePaths(files, listCodexSessionJsonlFiles(childPath)) - continue + const pendingDirectories = [{ depth: 0, path: rootPath }] + + while (pendingDirectories.length > 0) { + const current = pendingDirectories.pop()! + let directory: ReturnType + try { + directory = opendirSync(current.path) + } catch (error) { + warnAboutCodexSessionListingError(error) + continue + } + try { + for (let entry = directory.readSync(); entry !== null; entry = directory.readSync()) { + budget.claimEntry() + const childPath = join(current.path, entry.name) + budget.claimPath(childPath) + if (entry.isDirectory()) { + const depth = current.depth + 1 + budget.claimDepth(depth) + pendingDirectories.push({ depth, path: childPath }) + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + budget.claimFile() + files.push(childPath) + } } - if (entry.isFile() && entry.name.endsWith('.jsonl')) { - files.push(childPath) + } catch (error) { + if (error instanceof CodexSessionListingCapacityError) { + throw error } + warnAboutCodexSessionListingError(error) + } finally { + closeCodexSessionDirectory(directory) } - } catch (error) { - console.warn('[codex-session-bridge] Failed to list system Codex sessions:', error) } return files.sort() } -/** - * Appends session paths without spreading large arrays into a single call. - */ -function appendSessionFilePaths(target: string[], source: readonly string[]): void { - // Why: existing Codex homes can accumulate enough nested sessions to exceed - // V8's argument limit if child arrays are spread into push(). - for (const filePath of source) { - target.push(filePath) +function closeCodexSessionDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch { + // The OS may have already closed a failed directory stream. } } @@ -89,21 +131,26 @@ async function* listCodexSessionFilesIncrementally( ): AsyncGenerator { const batchSize = Math.max(1, options.batchSize ?? INCREMENTAL_BRIDGE_BATCH_SIZE) const yieldMs = Math.max(0, options.yieldMs ?? INCREMENTAL_BRIDGE_YIELD_MS) - const pendingDirectories = [rootPath] + const budget = new CodexSessionListingBudget(options.limits) + budget.claimDepth(0) + budget.claimPath(rootPath) + const pendingDirectories = [{ depth: 0, path: rootPath }] let entriesSinceYield = 0 while (pendingDirectories.length > 0) { - const currentDirectory = pendingDirectories.pop() - if (!currentDirectory) { - continue - } + const currentDirectory = pendingDirectories.pop()! try { - const directory = await opendir(currentDirectory) + const directory = await opendir(currentDirectory.path) for await (const entry of directory) { - const childPath = join(currentDirectory, entry.name) + budget.claimEntry() + const childPath = join(currentDirectory.path, entry.name) + budget.claimPath(childPath) if (entry.isDirectory()) { - pendingDirectories.push(childPath) + const depth = currentDirectory.depth + 1 + budget.claimDepth(depth) + pendingDirectories.push({ depth, path: childPath }) } else if (entry.isFile() && isSessionFile(entry.name)) { + budget.claimFile() yield childPath } entriesSinceYield += 1 @@ -113,12 +160,19 @@ async function* listCodexSessionFilesIncrementally( } } } catch (error) { - await onDirectoryError?.(currentDirectory, error) - console.warn('[codex-session-bridge] Failed to list system Codex sessions:', error) + await onDirectoryError?.(currentDirectory.path, error) + warnAboutCodexSessionListingError(error) + if (error instanceof CodexSessionListingCapacityError) { + return + } } } } +function warnAboutCodexSessionListingError(error: unknown): void { + console.warn('[codex-session-bridge] Failed to list system Codex sessions:', error) +} + /** * Defers incremental bridge work to a later timer turn. */ diff --git a/src/main/codex/codex-session-index-heal-jsonl.test.ts b/src/main/codex/codex-session-index-heal-jsonl.test.ts new file mode 100644 index 00000000000..4a9729a335e --- /dev/null +++ b/src/main/codex/codex-session-index-heal-jsonl.test.ts @@ -0,0 +1,45 @@ +import { appendFileSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES, + readCodexSessionIndexHealJsonlRecords +} from './codex-session-index-heal-jsonl' + +describe('Codex session index heal JSONL reader', () => { + const roots: string[] = [] + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + }) + + function makePath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-heal-jsonl-')) + roots.push(root) + return join(root, 'ledger.jsonl') + } + + it('accepts a valid record exactly at the per-line byte limit', () => { + const path = makePath() + const prefix = '{"padding":"' + const suffix = '"}' + const padding = 'x'.repeat( + MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES - prefix.length - suffix.length + ) + writeFileSync(path, `${prefix}${padding}${suffix}\n`) + + expect([...readCodexSessionIndexHealJsonlRecords(path)]).toEqual([{ padding }]) + }) + + it('skips an oversized sparse line and resumes at the next record', () => { + const path = makePath() + writeFileSync(path, '{"ignored":"') + truncateSync(path, MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES + 8 * 1024 * 1024) + appendFileSync(path, '\n{"kept":true}\n') + + expect([...readCodexSessionIndexHealJsonlRecords(path)]).toEqual([{ kept: true }]) + }) +}) diff --git a/src/main/codex/codex-session-index-heal-jsonl.ts b/src/main/codex/codex-session-index-heal-jsonl.ts new file mode 100644 index 00000000000..cb67e9a8d80 --- /dev/null +++ b/src/main/codex/codex-session-index-heal-jsonl.ts @@ -0,0 +1,102 @@ +import { closeSync, openSync, readSync } from 'node:fs' + +export const MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES = 1024 * 1024 +const HEAL_JSONL_READ_CHUNK_BYTES = 64 * 1024 + +function isNotFoundError(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT' +} + +function parseJsonlRecord(raw: string): Record | null { + if (!raw.trim()) { + return null + } + try { + const parsed: unknown = JSON.parse(raw) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : null + } catch { + return null + } +} + +export function* readCodexSessionIndexHealJsonlRecords( + filePath: string +): Generator> { + let descriptor: number + try { + descriptor = openSync(filePath, 'r') + } catch (error) { + if (isNotFoundError(error)) { + return + } + throw error + } + + const readBuffer = Buffer.allocUnsafe(HEAL_JSONL_READ_CHUNK_BYTES) + let lineParts: Buffer[] = [] + let lineBytes = 0 + let skippingOversizedLine = false + + const resetLine = (): void => { + lineParts = [] + lineBytes = 0 + skippingOversizedLine = false + } + const appendSegment = (start: number, end: number): void => { + if (skippingOversizedLine || start === end) { + return + } + const nextBytes = lineBytes + end - start + if (nextBytes > MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES) { + // Why: a corrupt line may span an arbitrarily large sparse file; discard until its newline. + lineParts = [] + lineBytes = 0 + skippingOversizedLine = true + return + } + lineParts.push(Buffer.from(readBuffer.subarray(start, end))) + lineBytes = nextBytes + } + const takeRecord = (): Record | null => { + const raw = + lineParts.length === 1 + ? lineParts[0].toString('utf8') + : Buffer.concat(lineParts, lineBytes).toString('utf8') + return parseJsonlRecord(raw) + } + + try { + while (true) { + const bytesRead = readSync(descriptor, readBuffer, 0, readBuffer.length, null) + if (bytesRead === 0) { + break + } + let start = 0 + for (let index = 0; index < bytesRead; index += 1) { + if (readBuffer[index] !== 0x0a) { + continue + } + appendSegment(start, index) + if (!skippingOversizedLine) { + const record = takeRecord() + if (record) { + yield record + } + } + resetLine() + start = index + 1 + } + appendSegment(start, bytesRead) + } + if (!skippingOversizedLine && lineBytes > 0) { + const record = takeRecord() + if (record) { + yield record + } + } + } finally { + closeSync(descriptor) + } +} diff --git a/src/main/codex/codex-session-index-heal-state-memory.test.ts b/src/main/codex/codex-session-index-heal-state-memory.test.ts new file mode 100644 index 00000000000..86717e717ed --- /dev/null +++ b/src/main/codex/codex-session-index-heal-state-memory.test.ts @@ -0,0 +1,81 @@ +import { + closeSync, + mkdtempSync, + openSync, + rmSync, + truncateSync, + writeFileSync, + writeSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + CODEX_SESSION_INDEX_HEAL_VERSION, + CodexSessionIndexHealCapacityError, + collectPendingHealThreads, + isHealMarkerCurrent, + MAX_CODEX_SESSION_INDEX_HEAL_MARKER_FILE_BYTES, + MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS, + type CodexSessionIndexHealPaths +} from './codex-session-index-heal-state' + +describe('Codex session index heal state memory bounds', () => { + const roots: string[] = [] + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + }) + + function makePaths(): CodexSessionIndexHealPaths { + const root = mkdtempSync(join(tmpdir(), 'orca-heal-state-memory-')) + roots.push(root) + return { + auditLogPath: join(root, 'audit.jsonl'), + systemSessionsRoot: '/sessions', + healLedgerPath: join(root, 'heal.jsonl'), + healMarkerPath: join(root, 'marker.json') + } + } + + it('fails closed when processed state exceeds the retained-thread ceiling', () => { + const paths = makePaths() + const descriptor = openSync(paths.healLedgerPath, 'w') + try { + for (let start = 0; start <= MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS; start += 1000) { + const count = Math.min(1000, MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS + 1 - start) + const lines = Array.from({ length: count }, (_, offset) => { + const suffix = (start + offset).toString(16).padStart(12, '0') + return JSON.stringify({ + v: CODEX_SESSION_INDEX_HEAL_VERSION, + systemSessionsRoot: paths.systemSessionsRoot, + threadId: `00000000-0000-0000-0000-${suffix}`, + outcome: 'healed' + }) + }) + writeSync(descriptor, `${lines.join('\n')}\n`) + } + } finally { + closeSync(descriptor) + } + + expect(() => collectPendingHealThreads(paths)).toThrow(CodexSessionIndexHealCapacityError) + }) + + it('rejects an oversized sparse completion marker', () => { + const paths = makePaths() + writeFileSync( + paths.healMarkerPath, + JSON.stringify({ + version: CODEX_SESSION_INDEX_HEAL_VERSION, + systemSessionsRoot: paths.systemSessionsRoot, + auditBytes: 0 + }) + ) + truncateSync(paths.healMarkerPath, MAX_CODEX_SESSION_INDEX_HEAL_MARKER_FILE_BYTES + 1) + + expect(isHealMarkerCurrent(paths, 0)).toBe(false) + }) +}) diff --git a/src/main/codex/codex-session-index-heal-state.ts b/src/main/codex/codex-session-index-heal-state.ts index 4efc124316e..c546ba2faa4 100644 --- a/src/main/codex/codex-session-index-heal-state.ts +++ b/src/main/codex/codex-session-index-heal-state.ts @@ -1,10 +1,17 @@ -import { appendFileSync, mkdirSync, readFileSync, statSync } from 'node:fs' +import { appendFileSync, mkdirSync, statSync } from 'node:fs' import { dirname } from 'node:path' import { isPathInsideOrEqual, normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' +import { + MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES, + readCodexSessionIndexHealJsonlRecords +} from './codex-session-index-heal-jsonl' + +export { MAX_CODEX_SESSION_INDEX_HEAL_JSONL_LINE_BYTES } // State files for the session index heal: which backfilled rollouts exist // (the backfill audit ledger), which thread ids this pass already processed @@ -14,6 +21,22 @@ import { writeFileAtomically } from '../codex-accounts/fs-utils' // Bump to re-drive the heal for every host after a semantics change; already // processed thread ids are re-read because ledger lines are version-scoped. export const CODEX_SESSION_INDEX_HEAL_VERSION = 3 +export const MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS = 100_000 +export const MAX_CODEX_SESSION_INDEX_HEAL_MARKER_FILE_BYTES = 64 * 1024 + +const MAX_AUDIT_TARGET_LENGTH = 64 * 1024 +const MAX_AUDIT_RECORD_ID_LENGTH = 128 +const MAX_ROLLOUT_STAMP_LENGTH = 512 +const CODEX_THREAD_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export class CodexSessionIndexHealCapacityError extends Error { + constructor(kind: 'pending' | 'processed') { + super( + `Codex session index heal ${kind} state exceeds ${MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS} entries` + ) + this.name = 'CodexSessionIndexHealCapacityError' + } +} // Why: an unsupported CLI stays unsupported until upgraded; re-probing once a // day is enough to notice an upgrade without a per-startup spawn. @@ -53,11 +76,11 @@ export type HealMarkerSummary = { export function collectPendingHealThreads(paths: CodexSessionIndexHealPaths): PendingHealThread[] { const processed = readProcessedHealThreads(paths) const pendingByThreadId = new Map() - for (const line of readJsonlLines(paths.auditLogPath, true)) { + for (const line of readCodexSessionIndexHealJsonlRecords(paths.auditLogPath)) { if (line.action !== 'hardlink' && line.action !== 'copy' && line.action !== 'existing') { continue } - if (typeof line.target !== 'string') { + if (typeof line.target !== 'string' || line.target.length > MAX_AUDIT_TARGET_LENGTH) { continue } // Why: the append-only audit can contain runs for several custom Codex @@ -66,10 +89,13 @@ export function collectPendingHealThreads(paths: CodexSessionIndexHealPaths): Pe continue } const match = CODEX_ROLLOUT_THREAD_ID_PATTERN.exec(lastPathSegment(line.target)) - if (!match) { + if (!match || match[1].length > MAX_ROLLOUT_STAMP_LENGTH) { continue } const threadId = match[2].toLowerCase() + if (typeof line.recordId === 'string' && line.recordId.length > MAX_AUDIT_RECORD_ID_LENGTH) { + continue + } const auditRecordId = typeof line.recordId === 'string' ? line.recordId : null if ( processed.healedThreadIds.has(threadId) || @@ -82,6 +108,12 @@ export function collectPendingHealThreads(paths: CodexSessionIndexHealPaths): Pe pendingByThreadId.delete(threadId) continue } + if ( + !pendingByThreadId.has(threadId) && + pendingByThreadId.size >= MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS + ) { + throw new CodexSessionIndexHealCapacityError('pending') + } pendingByThreadId.set(threadId, { threadId, rolloutStamp: match[1], auditRecordId }) } return [...pendingByThreadId.values()].sort((left, right) => @@ -102,23 +134,49 @@ function readProcessedHealThreads(paths: CodexSessionIndexHealPaths): { const missingAuditRecords = new Set() const legacyMissingThreadIds = new Set() const expectedRoot = normalizeRuntimePathForComparison(paths.systemSessionsRoot) - for (const line of readJsonlLines(paths.healLedgerPath)) { - if ( - line.v === CODEX_SESSION_INDEX_HEAL_VERSION && - typeof line.threadId === 'string' && - typeof line.systemSessionsRoot === 'string' && - (line.outcome === 'healed' || line.outcome === 'missing') && - normalizeRuntimePathForComparison(line.systemSessionsRoot) === expectedRoot - ) { - const threadId = line.threadId.toLowerCase() - if (line.outcome === 'healed') { - healedThreadIds.add(threadId) - } else if (typeof line.auditRecordId === 'string') { - missingAuditRecords.add(`${threadId}\0${line.auditRecordId}`) - } else { - legacyMissingThreadIds.add(threadId) + let trackedEntries = 0 + const addTrackedEntry = (entries: Set, key: string): void => { + if (entries.has(key)) { + return + } + if (trackedEntries >= MAX_CODEX_SESSION_INDEX_HEAL_TRACKED_THREADS) { + throw new CodexSessionIndexHealCapacityError('processed') + } + entries.add(key) + trackedEntries += 1 + } + try { + for (const line of readCodexSessionIndexHealJsonlRecords(paths.healLedgerPath)) { + if ( + line.v === CODEX_SESSION_INDEX_HEAL_VERSION && + typeof line.threadId === 'string' && + CODEX_THREAD_ID_PATTERN.test(line.threadId) && + typeof line.systemSessionsRoot === 'string' && + (line.outcome === 'healed' || line.outcome === 'missing') && + normalizeRuntimePathForComparison(line.systemSessionsRoot) === expectedRoot + ) { + const threadId = line.threadId.toLowerCase() + if (line.outcome === 'healed') { + addTrackedEntry(healedThreadIds, threadId) + } else if ( + typeof line.auditRecordId === 'string' && + line.auditRecordId.length <= MAX_AUDIT_RECORD_ID_LENGTH + ) { + addTrackedEntry(missingAuditRecords, `${threadId}\0${line.auditRecordId}`) + } else if (line.auditRecordId === undefined) { + addTrackedEntry(legacyMissingThreadIds, threadId) + } } } + } catch (error) { + if (error instanceof CodexSessionIndexHealCapacityError) { + throw error + } + return { + healedThreadIds: new Set(), + missingAuditRecords: new Set(), + legacyMissingThreadIds: new Set() + } } return { healedThreadIds, missingAuditRecords, legacyMissingThreadIds } } @@ -153,35 +211,6 @@ export function appendHealLedgerRecord( } } -function readJsonlLines(filePath: string, throwOnReadFailure = false): Record[] { - let contents: string - try { - contents = readFileSync(filePath, 'utf-8') - } catch (error) { - if (throwOnReadFailure && !isNotFoundError(error)) { - // Why: the audit is the heal work queue. Treating EACCES/EIO as empty - // would write a completion marker that permanently skips every session. - throw error - } - return [] - } - const lines: Record[] = [] - for (const raw of contents.split('\n')) { - if (!raw.trim()) { - continue - } - try { - const parsed: unknown = JSON.parse(raw) - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - lines.push(parsed as Record) - } - } catch { - // Skip torn/corrupt lines; both ledgers are append-only diagnostics. - } - } - return lines -} - export function readAuditLogSize(auditLogPath: string): number { try { return statSync(auditLogPath).size @@ -202,7 +231,12 @@ export function isHealMarkerCurrent( auditBytes: number ): boolean { try { - const parsed: unknown = JSON.parse(readFileSync(paths.healMarkerPath, 'utf-8')) + const parsed: unknown = JSON.parse( + readNodeFileSyncWithinLimit( + paths.healMarkerPath, + MAX_CODEX_SESSION_INDEX_HEAL_MARKER_FILE_BYTES + ).buffer.toString('utf8') + ) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return false } diff --git a/src/main/codex/codex-session-listing-budget.ts b/src/main/codex/codex-session-listing-budget.ts new file mode 100644 index 00000000000..bcb355eb618 --- /dev/null +++ b/src/main/codex/codex-session-listing-budget.ts @@ -0,0 +1,81 @@ +export const CODEX_SESSION_LISTING_MAX_DEPTH = 32 +export const CODEX_SESSION_LISTING_MAX_ENTRIES = 200_000 +export const CODEX_SESSION_LISTING_MAX_FILES = 100_000 +export const CODEX_SESSION_LISTING_MAX_PATH_CODE_UNITS = 32 * 1024 * 1024 + +export type CodexSessionListingLimits = { + maxDepth: number + maxEntries: number + maxFiles: number + maxPathCodeUnits: number +} + +const DEFAULT_LIMITS: CodexSessionListingLimits = { + maxDepth: CODEX_SESSION_LISTING_MAX_DEPTH, + maxEntries: CODEX_SESSION_LISTING_MAX_ENTRIES, + maxFiles: CODEX_SESSION_LISTING_MAX_FILES, + maxPathCodeUnits: CODEX_SESSION_LISTING_MAX_PATH_CODE_UNITS +} + +export type CodexSessionListingCapacityResource = 'depth' | 'entries' | 'files' | 'path code units' + +export class CodexSessionListingCapacityError extends Error { + constructor( + readonly resource: CodexSessionListingCapacityResource, + readonly observed: number, + readonly limit: number + ) { + super(`Codex session listing exceeded ${limit} ${resource}`) + this.name = 'CodexSessionListingCapacityError' + } +} + +export class CodexSessionListingBudget { + readonly limits: CodexSessionListingLimits + private entries = 0 + private files = 0 + private pathCodeUnits = 0 + + constructor(limits: Partial = {}) { + this.limits = { ...DEFAULT_LIMITS, ...limits } + for (const [name, value] of Object.entries(this.limits)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + } + + claimDepth(depth: number): void { + if (depth > this.limits.maxDepth) { + throw new CodexSessionListingCapacityError('depth', depth, this.limits.maxDepth) + } + } + + claimEntry(): void { + this.entries += 1 + if (this.entries > this.limits.maxEntries) { + throw new CodexSessionListingCapacityError('entries', this.entries, this.limits.maxEntries) + } + } + + claimFile(): void { + this.files += 1 + if (this.files > this.limits.maxFiles) { + throw new CodexSessionListingCapacityError('files', this.files, this.limits.maxFiles) + } + } + + claimPath(path: string): void { + this.pathCodeUnits += path.length + if ( + !Number.isSafeInteger(this.pathCodeUnits) || + this.pathCodeUnits > this.limits.maxPathCodeUnits + ) { + throw new CodexSessionListingCapacityError( + 'path code units', + this.pathCodeUnits, + this.limits.maxPathCodeUnits + ) + } + } +} diff --git a/src/main/codex/codex-trust-config-rollback.test.ts b/src/main/codex/codex-trust-config-rollback.test.ts index 0eae14ff986..554470fe19f 100644 --- a/src/main/codex/codex-trust-config-rollback.test.ts +++ b/src/main/codex/codex-trust-config-rollback.test.ts @@ -8,10 +8,13 @@ import { rmSync, statSync, symlinkSync, + truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { MAX_AGENT_STATE_FILE_BYTES } from '../agent-state-file-reader' import { captureCodexTrustConfig, restoreCodexTrustConfig } from './codex-trust-config-rollback' const roots: string[] = [] @@ -29,6 +32,15 @@ function tempConfigPath(): string { } describe('Codex trust config rollback', () => { + it('rejects a sparse config above 4 MiB without changing it', () => { + const configPath = tempConfigPath() + writeFileSync(configPath, '') + truncateSync(configPath, MAX_AGENT_STATE_FILE_BYTES + 1) + + expect(() => captureCodexTrustConfig(configPath)).toThrow(NodeFileReadTooLargeError) + expect(statSync(configPath).size).toBe(MAX_AGENT_STATE_FILE_BYTES + 1) + }) + it('treats a missing config as absent and tolerates it remaining absent', () => { const configPath = tempConfigPath() const snapshot = captureCodexTrustConfig(configPath) diff --git a/src/main/codex/codex-trust-config-rollback.ts b/src/main/codex/codex-trust-config-rollback.ts index 0642b834483..df3ad72c228 100644 --- a/src/main/codex/codex-trust-config-rollback.ts +++ b/src/main/codex/codex-trust-config-rollback.ts @@ -1,10 +1,6 @@ import { chmodSync, - closeSync, - fstatSync, lstatSync, - openSync, - readFileSync, readlinkSync, realpathSync, unlinkSync, @@ -12,6 +8,8 @@ import { } from 'node:fs' import { randomUUID } from 'node:crypto' import { dirname, resolve } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { MAX_AGENT_STATE_FILE_BYTES } from '../agent-state-file-reader' import { renameFileWithWindowsRetry } from '../codex-accounts/fs-utils' export type CodexTrustConfigSnapshot = @@ -43,27 +41,22 @@ function resolveConfigRestorePath(tomlPath: string): string { export function captureCodexTrustConfig(tomlPath: string): CodexTrustConfigSnapshot { const restorePath = resolveConfigRestorePath(tomlPath) - let descriptor: number try { - descriptor = openSync(restorePath, 'r') + // Why: contents and mode come from one descriptor, so a path replacement + // cannot pair one file's bytes with another file's permissions. + const { buffer, stats } = readNodeFileSyncWithinLimit(restorePath, MAX_AGENT_STATE_FILE_BYTES) + return { + existed: true, + contents: buffer, + mode: stats.mode, + restorePath + } } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return restorePath === tomlPath ? { existed: false } : { existed: false, restorePath } } throw error } - try { - // Why: read and stat the same open file so replacement between two path - // lookups cannot pair one file's contents with another file's mode. - return { - existed: true, - contents: readFileSync(descriptor), - mode: fstatSync(descriptor).mode, - restorePath - } - } finally { - closeSync(descriptor) - } } export function restoreCodexTrustConfig( @@ -82,7 +75,11 @@ export function restoreCodexTrustConfig( } const { restorePath } = snapshot try { - if (readFileSync(restorePath).equals(snapshot.contents)) { + if ( + readNodeFileSyncWithinLimit(restorePath, MAX_AGENT_STATE_FILE_BYTES).buffer.equals( + snapshot.contents + ) + ) { // Why: the RPC may change permissions without changing bytes; rollback // restores the complete captured file state, not only its contents. chmodSync(restorePath, snapshot.mode) diff --git a/src/main/codex/codex-trust-grant-ledger.test.ts b/src/main/codex/codex-trust-grant-ledger.test.ts index cb4c5450c6d..2eeb651343f 100644 --- a/src/main/codex/codex-trust-grant-ledger.test.ts +++ b/src/main/codex/codex-trust-grant-ledger.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { @@ -9,6 +9,7 @@ import { removeCodexTrustGrantLedgerHome, writeCodexTrustGrantLedgerHome } from './codex-trust-grant-ledger' +import { MAX_AGENT_STATE_FILE_BYTES } from '../agent-state-file-reader' let userDataDir: string let previousUserDataPath: string | undefined @@ -81,6 +82,26 @@ describe('codex trust grant ledger', () => { expect(readCodexTrustGrantLedgerHome(home)).not.toBeNull() }) + it('preserves the prior ledger when a grant exceeds its read ceiling', () => { + const home = join(userDataDir, 'codex-runtime-home', 'home') + const ledgerPath = getCodexTrustGrantLedgerPath() + writeCodexTrustGrantLedgerHome(home, { binary: null, entries: {} }) + const before = readFileSync(ledgerPath, 'utf8') + + expect(() => + writeCodexTrustGrantLedgerHome(home, { + binary: null, + entries: { + oversized: { + signature: 'x'.repeat(MAX_AGENT_STATE_FILE_BYTES), + trustedHash: 'sha256:test' + } + } + }) + ).toThrow('JSON output exceeds') + expect(readFileSync(ledgerPath, 'utf8')).toBe(before) + }) + it('matches binary stamps only on identical identity', () => { const stamp = { kind: 'native' as const, path: '/bin/codex', size: 1, mtimeMs: 2 } const wslStamp = { diff --git a/src/main/codex/codex-trust-grant-ledger.ts b/src/main/codex/codex-trust-grant-ledger.ts index b1d20421bce..6a28a6831c1 100644 --- a/src/main/codex/codex-trust-grant-ledger.ts +++ b/src/main/codex/codex-trust-grant-ledger.ts @@ -1,5 +1,7 @@ -import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs' import { dirname, join } from 'node:path' +import { MAX_AGENT_STATE_FILE_BYTES, readAgentStateJsonFileSync } from '../agent-state-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' import { getOrcaManagedCodexHomePath } from './codex-home-paths' import { normalizeCodexProjectPathForLookup } from './config-toml-trust' @@ -46,7 +48,7 @@ function readLedgerFile(ledgerPath: string): CodexTrustGrantLedgerFile { return empty } try { - const parsed: unknown = JSON.parse(readFileSync(ledgerPath, 'utf-8')) + const parsed = readAgentStateJsonFileSync(ledgerPath) if ( !parsed || typeof parsed !== 'object' || @@ -69,7 +71,8 @@ function readLedgerFile(ledgerPath: string): CodexTrustGrantLedgerFile { function persistLedgerFile(ledgerPath: string, file: CodexTrustGrantLedgerFile): void { mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }) - writeFileSync(ledgerPath, `${JSON.stringify(file, null, 2)}\n`, { + const { serialized } = stringifyJsonWithinByteLimit(file, MAX_AGENT_STATE_FILE_BYTES - 1, 2) + writeFileSync(ledgerPath, `${serialized}\n`, { encoding: 'utf-8', mode: 0o600 }) diff --git a/src/main/codex/codex-user-hook-trust-rebase.ts b/src/main/codex/codex-user-hook-trust-rebase.ts index 0020e106c2f..9085ba7dece 100644 --- a/src/main/codex/codex-user-hook-trust-rebase.ts +++ b/src/main/codex/codex-user-hook-trust-rebase.ts @@ -7,6 +7,7 @@ import { import { runCodexUserHookTrustRebaseSessionSync } from './codex-app-server-grant-bridge' import { isCodexAppServerUnsupportedError } from './codex-app-server-session' import { CODEX_TRUST_GRANT_TRANSIENT_RETRY_INTERVAL_MS } from './codex-hook-trust-grant' +import { CodexHostRetryDeadlines } from './codex-host-retry-deadlines' import { createCodexHookTrustEntry } from './codex-hook-identity' import { resolveCodexTrustGrantHost } from './codex-trust-grant-host' import { @@ -32,7 +33,7 @@ let runSessionSync: RebaseSessionRunnerSync = runCodexUserHookTrustRebaseSession // Why: launch prep re-runs the callers on every pane spawn. A host stuck // without a usable rebase lane (old CLI, unmatched keys) must not pay a codex // session each time — bound retries like the grant lane does. -const rebaseRetryAfterByHost = new Map() +const rebaseRetryAfterByHost = new CodexHostRetryDeadlines() function rememberRebaseSessionFailure(hostKey: CodexAppServerHostKey, error: unknown): void { if (isCodexAppServerUnsupportedError(error)) { diff --git a/src/main/codex/codex-wsl-hook-install-plan.test.ts b/src/main/codex/codex-wsl-hook-install-plan.test.ts index 83ab0141a34..89f47334d90 100644 --- a/src/main/codex/codex-wsl-hook-install-plan.test.ts +++ b/src/main/codex/codex-wsl-hook-install-plan.test.ts @@ -6,7 +6,12 @@ vi.mock('node:child_process', () => ({ execFile: execFileMock })) -import { _internals, createCodexWslRuntimeHookInstallPlan } from './codex-wsl-hook-install-plan' +import { + _internals, + createCodexWslRuntimeHookInstallPlan, + WSL_CANONICAL_PATH_CACHE_MAX_ENTRIES, + WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES +} from './codex-wsl-hook-install-plan' const originalPlatform = process.platform @@ -125,6 +130,17 @@ describe('canonicalizeWslLinuxPath', () => { expect(execFileMock).toHaveBeenCalledTimes(2) }) + it('clears in-flight ownership when spawning wsl.exe throws synchronously', () => { + setPlatform('win32') + execFileMock.mockImplementationOnce(() => { + throw new Error('invalid WSL argv') + }) + + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/alias')).toBeNull() + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/alias')).toBeNull() + expect(execFileMock).toHaveBeenCalledTimes(2) + }) + it('keeps the last known-good cache when revalidation later fails', () => { setPlatform('win32') _internals.canonicalizeWslLinuxPath('Ubuntu', '/home/alias') @@ -208,4 +224,48 @@ describe('canonicalizeWslLinuxPath', () => { expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/alias')).toBeNull() }) + + it('LRU-evicts canonical identities beyond the entry cap', () => { + setPlatform('win32') + for (let index = 0; index <= WSL_CANONICAL_PATH_CACHE_MAX_ENTRIES; index += 1) { + const logicalPath = `/home/runtime-${index}` + _internals.canonicalizeWslLinuxPath('Ubuntu', logicalPath) + const callback = execFileMock.mock.calls[index][3] as ( + error: Error | null, + stdout: string + ) => void + callback(null, `/canonical/runtime-${index}\n`) + } + + expect(_internals.getWslCanonicalPathCacheSize()).toBe(WSL_CANONICAL_PATH_CACHE_MAX_ENTRIES) + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/runtime-0')).toBeNull() + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/runtime-1')).toBe( + '/canonical/runtime-1' + ) + }) + + it('does not retain a canonical identity larger than the aggregate byte budget', () => { + setPlatform('win32') + _internals.canonicalizeWslLinuxPath('Ubuntu', '/home/oversized') + const callback = execFileMock.mock.calls[0][3] as (error: Error | null, stdout: string) => void + callback(null, `/${'x'.repeat(WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES)}\n`) + + expect(_internals.getWslCanonicalPathCacheSize()).toBe(0) + expect(_internals.getWslCanonicalPathCacheBytes()).toBe(0) + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', '/home/oversized')).toBeNull() + }) + + it('retains a canonical identity at the exact aggregate byte budget', () => { + setPlatform('win32') + const logicalPath = '/home/exact-limit' + const keyBytes = Buffer.byteLength(`Ubuntu\0${logicalPath}`, 'utf8') + const canonicalPath = `/${'x'.repeat(WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES - keyBytes - 1)}` + _internals.canonicalizeWslLinuxPath('Ubuntu', logicalPath) + const callback = execFileMock.mock.calls[0][3] as (error: Error | null, stdout: string) => void + callback(null, canonicalPath) + + expect(_internals.getWslCanonicalPathCacheSize()).toBe(1) + expect(_internals.getWslCanonicalPathCacheBytes()).toBe(WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES) + expect(_internals.canonicalizeWslLinuxPath('Ubuntu', logicalPath)).toBe(canonicalPath) + }) }) diff --git a/src/main/codex/codex-wsl-hook-install-plan.ts b/src/main/codex/codex-wsl-hook-install-plan.ts index 4db2bdea93f..31c92729771 100644 --- a/src/main/codex/codex-wsl-hook-install-plan.ts +++ b/src/main/codex/codex-wsl-hook-install-plan.ts @@ -48,17 +48,81 @@ function toDefaultWslLinuxPath(windowsPath: string): string { const WSL_CANONICALIZE_TIMEOUT_MS = 5000 const WSL_PATH_MISSING_OUTPUT = '__ORCA_WSL_PATH_MISSING__' +export const WSL_CANONICAL_PATH_CACHE_MAX_ENTRIES = 256 +export const WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES = 2 * 1024 * 1024 // Why: `readlink -f` over wsl.exe stalls up to the timeout on a cold or wedged // distro. Running it synchronously on the Electron main process froze the UI on // every Codex WSL launch, so resolve it off-thread and cache the latest result. -const canonicalWslPathCache = new Map() +const canonicalWslPathCache = new Map() +let canonicalWslPathCacheBytes = 0 const inFlightWslCanonicalizations = new Map>() function wslCanonicalizeCacheKey(distro: string, linuxPath: string): string { return `${distro}\x00${linuxPath}` } +function getCachedWslCanonicalPath(key: string): string | null { + const cached = canonicalWslPathCache.get(key) + if (!cached) { + return null + } + canonicalWslPathCache.delete(key) + canonicalWslPathCache.set(key, cached) + return cached.canonicalPath +} + +function deleteCachedWslCanonicalPath(key: string): void { + const cached = canonicalWslPathCache.get(key) + if (!cached) { + return + } + canonicalWslPathCache.delete(key) + canonicalWslPathCacheBytes -= cached.retainedBytes +} + +function cacheWslCanonicalPath(key: string, canonicalPath: string): void { + deleteCachedWslCanonicalPath(key) + const retainedBytes = Buffer.byteLength(key, 'utf8') + Buffer.byteLength(canonicalPath, 'utf8') + if (retainedBytes > WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES) { + return + } + while ( + canonicalWslPathCache.size >= WSL_CANONICAL_PATH_CACHE_MAX_ENTRIES || + canonicalWslPathCacheBytes + retainedBytes > WSL_CANONICAL_PATH_CACHE_MAX_UTF8_BYTES + ) { + const oldestKey = canonicalWslPathCache.keys().next().value as string | undefined + if (oldestKey === undefined) { + break + } + deleteCachedWslCanonicalPath(oldestKey) + } + canonicalWslPathCache.set(key, { canonicalPath, retainedBytes }) + canonicalWslPathCacheBytes += retainedBytes +} + +function settleWslCanonicalization(key: string, settlement: WslCanonicalPathSettlement): void { + if (settlement.status === 'resolved') { + cacheWslCanonicalPath(key, settlement.canonicalPath) + } else if (settlement.status === 'missing') { + // Why: a successful directory probe is stronger than a transport error; + // clear the identity so stale trust can be revoked and later rediscovered. + deleteCachedWslCanonicalPath(key) + } + // Why: keep the last known-good cache on timeout/transient WSL failures. + // Dropping it forces the next launch onto the logical `/mnt/...` guess, + // which is wrong under custom automount roots and rewrites trust keys. + const settledListeners = inFlightWslCanonicalizations.get(key) ?? new Set() + inFlightWslCanonicalizations.delete(key) + for (const listener of settledListeners) { + try { + listener(settlement) + } catch (listenerError) { + console.warn('[codex-wsl-hook-path] failed to reconcile canonical path', listenerError) + } + } +} + function scheduleWslLinuxPathCanonicalization( distro: string, linuxPath: string, @@ -102,40 +166,28 @@ function scheduleWslLinuxPathCanonicalization( 'sh', linuxPath ] - execFile( - 'wsl.exe', - args, - { encoding: 'utf-8', timeout: WSL_CANONICALIZE_TIMEOUT_MS, windowsHide: true }, - (error, stdout) => { - const canonicalPath = stdout.trim() - const resolvedPath = !error && canonicalPath.startsWith('/') ? canonicalPath : null - const pathMissing = !error && canonicalPath === WSL_PATH_MISSING_OUTPUT - const settlement: WslCanonicalPathSettlement = resolvedPath - ? { status: 'resolved', canonicalPath: resolvedPath } - : pathMissing - ? { status: 'missing' } - : { status: 'unavailable' } - if (settlement.status === 'resolved') { - canonicalWslPathCache.set(key, canonicalPath) - } else if (settlement.status === 'missing') { - // Why: a successful directory probe is stronger than a transport error; - // clear the identity so stale trust can be revoked and later rediscovered. - canonicalWslPathCache.delete(key) + try { + execFile( + 'wsl.exe', + args, + { encoding: 'utf-8', timeout: WSL_CANONICALIZE_TIMEOUT_MS, windowsHide: true }, + (error, stdout) => { + const canonicalPath = stdout.trim() + const resolvedPath = !error && canonicalPath.startsWith('/') ? canonicalPath : null + const pathMissing = !error && canonicalPath === WSL_PATH_MISSING_OUTPUT + settleWslCanonicalization( + key, + resolvedPath + ? { status: 'resolved', canonicalPath: resolvedPath } + : pathMissing + ? { status: 'missing' } + : { status: 'unavailable' } + ) } - // Why: keep the last known-good cache on timeout/transient WSL failures. - // Dropping it forces the next launch onto the logical `/mnt/...` guess, - // which is wrong under custom automount roots and rewrites trust keys. - const settledListeners = inFlightWslCanonicalizations.get(key) ?? new Set() - inFlightWslCanonicalizations.delete(key) - for (const listener of settledListeners) { - try { - listener(settlement) - } catch (listenerError) { - console.warn('[codex-wsl-hook-path] failed to reconcile canonical path', listenerError) - } - } - } - ) + ) + } catch { + settleWslCanonicalization(key, { status: 'unavailable' }) + } } function canonicalizeWslLinuxPath( @@ -147,7 +199,7 @@ function canonicalizeWslLinuxPath( if (process.platform !== 'win32') { return linuxPath } - const cached = canonicalWslPathCache.get(wslCanonicalizeCacheKey(distro, linuxPath)) + const cached = getCachedWslCanonicalPath(wslCanonicalizeCacheKey(distro, linuxPath)) // Why: every launch revalidates asynchronously. Returning the cache keeps // launch prep synchronous while settlement repairs or revokes trust in-place. scheduleWslLinuxPathCanonicalization(distro, linuxPath, windowsPath, onSettled) @@ -199,6 +251,13 @@ export const _internals = { canonicalizeWslLinuxPath, resetWslCanonicalPathCache(): void { canonicalWslPathCache.clear() + canonicalWslPathCacheBytes = 0 inFlightWslCanonicalizations.clear() + }, + getWslCanonicalPathCacheSize(): number { + return canonicalWslPathCache.size + }, + getWslCanonicalPathCacheBytes(): number { + return canonicalWslPathCacheBytes } } diff --git a/src/main/codex/codex-wsl-reconciliation-generations.test.ts b/src/main/codex/codex-wsl-reconciliation-generations.test.ts new file mode 100644 index 00000000000..7d7166d6344 --- /dev/null +++ b/src/main/codex/codex-wsl-reconciliation-generations.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import { CodexWslReconciliationGenerations } from './codex-wsl-reconciliation-generations' + +const BOUNDS = { + maxEntries: 2, + maxKeyBytes: 4, + maxTotalKeyBytes: 6 +} + +describe('CodexWslReconciliationGenerations', () => { + it('invalidates an older reconciliation for the same runtime home', () => { + const generations = new CodexWslReconciliationGenerations(BOUNDS) + const stale = generations.advance('a') + const current = generations.advance('a') + + expect(generations.isCurrent('a', stale)).toBe(false) + expect(generations.isCurrent('a', current)).toBe(true) + }) + + it('bounds retained homes and fails closed for an evicted callback', () => { + const generations = new CodexWslReconciliationGenerations(BOUNDS) + const evicted = generations.advance('a') + generations.advance('bb') + generations.advance('ccc') + + expect(generations.isCurrent('a', evicted)).toBe(false) + expect(generations.evidence()).toEqual({ entries: 2, keyBytes: 5 }) + }) + + it('does not retain an oversized home key', () => { + const generations = new CodexWslReconciliationGenerations(BOUNDS) + const generation = generations.advance('oversized') + + expect(generations.isCurrent('oversized', generation)).toBe(false) + expect(generations.evidence()).toEqual({ entries: 0, keyBytes: 0 }) + }) +}) diff --git a/src/main/codex/codex-wsl-reconciliation-generations.ts b/src/main/codex/codex-wsl-reconciliation-generations.ts new file mode 100644 index 00000000000..caf2cfeb1bb --- /dev/null +++ b/src/main/codex/codex-wsl-reconciliation-generations.ts @@ -0,0 +1,76 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export type CodexWslReconciliationGenerationBounds = { + maxEntries: number + maxKeyBytes: number + maxTotalKeyBytes: number +} + +export const DEFAULT_CODEX_WSL_RECONCILIATION_GENERATION_BOUNDS: CodexWslReconciliationGenerationBounds = + { + maxEntries: 256, + maxKeyBytes: 64 * 1024, + maxTotalKeyBytes: 1024 * 1024 + } + +type RetainedGeneration = { + generation: number + keyBytes: number +} + +export class CodexWslReconciliationGenerations { + private readonly generations = new Map() + private nextGeneration = 0 + private retainedKeyBytes = 0 + + constructor( + private readonly bounds: CodexWslReconciliationGenerationBounds = DEFAULT_CODEX_WSL_RECONCILIATION_GENERATION_BOUNDS + ) { + for (const value of Object.values(bounds)) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new RangeError('Codex WSL reconciliation bounds must be positive integers') + } + } + } + + advance(key: string): number { + const generation = ++this.nextGeneration + const keyBytes = measureUtf8ByteLength(key, { + stopAfterBytes: this.bounds.maxKeyBytes + }) + this.delete(key) + if (keyBytes.exceededLimit || keyBytes.byteLength > this.bounds.maxTotalKeyBytes) { + return generation + } + while ( + this.generations.size >= this.bounds.maxEntries || + this.retainedKeyBytes + keyBytes.byteLength > this.bounds.maxTotalKeyBytes + ) { + const oldest = this.generations.keys().next().value + if (oldest === undefined) { + return generation + } + this.delete(oldest) + } + this.generations.set(key, { generation, keyBytes: keyBytes.byteLength }) + this.retainedKeyBytes += keyBytes.byteLength + return generation + } + + isCurrent(key: string, generation: number): boolean { + return this.generations.get(key)?.generation === generation + } + + evidence(): { entries: number; keyBytes: number } { + return { entries: this.generations.size, keyBytes: this.retainedKeyBytes } + } + + private delete(key: string): void { + const retained = this.generations.get(key) + if (!retained) { + return + } + this.generations.delete(key) + this.retainedKeyBytes -= retained.keyBytes + } +} diff --git a/src/main/codex/config-settings-promotion.test.ts b/src/main/codex/config-settings-promotion.test.ts index 1f372fa7c84..f6ceabc362f 100644 --- a/src/main/codex/config-settings-promotion.test.ts +++ b/src/main/codex/config-settings-promotion.test.ts @@ -19,7 +19,7 @@ import type * as CodexFsUtils from '../codex-accounts/fs-utils' const { homedirMock, promotionTestState } = vi.hoisted(() => ({ homedirMock: vi.fn<() => string>(), - promotionTestState: { failAtomicWrite: false, atomicWritePaths: [] as string[] } + promotionTestState: { failAtomicWrite: false } })) vi.mock('node:os', async (importOriginal) => { @@ -35,7 +35,6 @@ vi.mock('../codex-accounts/fs-utils', async (importOriginal) => { return { ...actual, writeFileAtomically: (...args: Parameters) => { - promotionTestState.atomicWritePaths.push(args[0]) if (promotionTestState.failAtomicWrite) { throw new Error('injected atomic write failure') } @@ -45,15 +44,7 @@ vi.mock('../codex-accounts/fs-utils', async (importOriginal) => { }) import { syncSystemConfigIntoManagedCodexHome } from './codex-config-mirror' -import { - upsertPromotedSettingsInContent, - upsertTopLevelSettingsInContent -} from './codex-config-settings-upsert' - -// The exact [tui] block codex 0.144.6 writes via config/batchWrite (all four -// promoted keys single-line, theme a string). -const CODEX_TUI_BLOCK = - '[tui]\nstatus_line = ["model-with-reasoning", "task-progress"]\nstatus_line_use_colors = true\nterminal_title = ["model"]\ntheme = "dark-photon"\n' +import { upsertTopLevelSettingsInContent } from './config-settings-promotion' let tmpHome: string let userDataDir: string @@ -66,7 +57,6 @@ beforeEach(() => { process.env.ORCA_USER_DATA_PATH = userDataDir homedirMock.mockReturnValue(tmpHome) promotionTestState.failAtomicWrite = false - promotionTestState.atomicWritePaths.length = 0 // Why: promotion writes into homedir()/.codex — if the mock ever fails to // intercept, these tests would rewrite the developer's real Codex config. if (homedir() !== tmpHome) { @@ -127,13 +117,6 @@ function simulateCodexSettingWrite(key: string, rawValue: string): void { writeFileSync(runtimeConfigPath(), next, 'utf-8') } -// Codex reads then rewrites the whole runtime config; simulate that by writing -// a known runtime config directly (its EOL is normalized by the mirror anyway). -function setRuntimeConfig(content: string): void { - mkdirSync(runtimeHomeDir(), { recursive: true }) - writeFileSync(runtimeConfigPath(), content, 'utf-8') -} - function simulateCodexSettingRemoval(key: string): void { const existing = readFileSync(runtimeConfigPath(), 'utf-8') const linePattern = new RegExp(`^${key}[ \\t]*=.*\\n?`, 'm') @@ -419,363 +402,6 @@ describe('codex settings write-back promotion', () => { }) }) -describe('codex [tui] settings write-back promotion', () => { - it('promotes a runtime [tui] block (codex 0.144.6 shape) into ~/.codex and survives the remirror', () => { - writeSystemConfig('model = "gpt-5"\n') - syncSystemConfigIntoManagedCodexHome() - - // The user customizes the status line/theme inside Orca-launched Codex. - writeFileSync(runtimeConfigPath(), `${readRuntimeConfig()}\n${CODEX_TUI_BLOCK}`, 'utf-8') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe(`model = "gpt-5"\n\n${CODEX_TUI_BLOCK}`) - const runtime = readRuntimeConfig() - expect(runtime).toContain('status_line = ["model-with-reasoning", "task-progress"]') - expect(runtime).toContain('status_line_use_colors = true') - expect(runtime).toContain('terminal_title = ["model"]') - expect(runtime).toContain('theme = "dark-photon"') - - const settledSystem = readSystemConfig() - const settledRuntime = readRuntimeConfig() - syncSystemConfigIntoManagedCodexHome() - expect(readSystemConfig()).toBe(settledSystem) - expect(readRuntimeConfig()).toBe(settledRuntime) - }) - - it('replaces a promoted key in an existing [tui] table, leaving non-promoted neighbors untouched', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui]\nanimations = true\ntheme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui]\nanimations = true\ntheme = "light"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "gpt-5"\n\n[tui]\nanimations = true\ntheme = "light"\n' - ) - }) - - it('promotes a changed status_line array value', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui]\nstatus_line = ["model"]\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig( - 'model = "gpt-5"\n\n[tui]\nstatus_line = ["model-with-reasoning", "task-progress"]\n' - ) - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "gpt-5"\n\n[tui]\nstatus_line = ["model-with-reasoning", "task-progress"]\n' - ) - }) - - it('promotes a model change and a status-line change in one pass into their regions', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark-photon"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "o4"\n\n[tui]\ntheme = "dark-photon"\nstatus_line = ["model"]\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "o4"\n\n[tui]\ntheme = "dark-photon"\nstatus_line = ["model"]\n' - ) - }) - - it('detects and replaces a dotted-form system tui key without creating a [tui] table', () => { - writeSystemConfig('model = "gpt-5"\ntui.theme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - // toml_edit preserves the dotted form when codex rewrites the value. - setRuntimeConfig('model = "gpt-5"\ntui.theme = "light"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe('model = "gpt-5"\ntui.theme = "light"\n') - expect(readSystemConfig()).not.toContain('[tui]') - }) - - it('promotes through a quoted tui table without creating a duplicate table', () => { - writeSystemConfig('model = "gpt-5"\n\n["tui"]\ntheme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n["tui"]\ntheme = "light"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe('model = "gpt-5"\n\n["tui"]\ntheme = "light"\n') - expect(readSystemConfig()).not.toContain('\n[tui]\n') - }) - - it('inserts a second dotted tui key beside an existing dotted-only tui config', () => { - writeSystemConfig('model = "gpt-5"\ntui.theme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\ntui.theme = "dark"\ntui.status_line = ["model"]\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "gpt-5"\ntui.theme = "dark"\ntui.status_line = ["model"]\n' - ) - expect(readSystemConfig()).not.toContain('[tui]') - }) - - it('inserts dotted beside a non-promoted dotted tui key instead of creating a [tui] table', () => { - // Why: any dotted tui.* key already defines the implicit tui table, so a - // fresh [tui] table at EOF would be a duplicate-definition parse error. - writeSystemConfig('model = "gpt-5"\ntui.pet = "cat"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\ntui.pet = "cat"\ntui.theme = "dark-photon"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe('model = "gpt-5"\ntui.pet = "cat"\ntui.theme = "dark-photon"\n') - expect(readSystemConfig()).not.toContain('[tui]') - }) - - it('creates a [tui] table at EOF when the only tui presence is a subtable', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui.notifications]\nenabled = true\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig( - 'model = "gpt-5"\n\n[tui.notifications]\nenabled = true\n\n[tui]\nstatus_line = ["model"]\n' - ) - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "gpt-5"\n\n[tui.notifications]\nenabled = true\n\n[tui]\nstatus_line = ["model"]\n' - ) - }) - - it('creates exactly one [tui] table for two keys promoted in one pass', () => { - writeSystemConfig('model = "gpt-5"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark-photon"\nstatus_line = ["model"]\n') - syncSystemConfigIntoManagedCodexHome() - - const system = readSystemConfig() - expect(system.match(/^\[tui\]$/gm)?.length).toBe(1) - expect(system).toBe( - 'model = "gpt-5"\n\n[tui]\nstatus_line = ["model"]\ntheme = "dark-photon"\n' - ) - }) - - it('lets an outside ~/.codex [tui] edit win over a conflicting in-Codex tui change', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui]\ntheme = "in-codex"\n') - writeSystemConfig('model = "gpt-5"\n\n[tui]\ntheme = "outside-edit"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe('model = "gpt-5"\n\n[tui]\ntheme = "outside-edit"\n') - expect(readRuntimeConfig()).toContain('theme = "outside-edit"') - }) - - it('does not promote a [tui] key deletion', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui]\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toContain('theme = "dark"') - }) - - it('inserts a promoted key into a CRLF system [tui] table preserving CRLF', () => { - writeSystemConfig('model = "gpt-5"\r\n\r\n[tui]\r\ntheme = "dark"\r\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark"\nstatus_line = ["model"]\n') - syncSystemConfigIntoManagedCodexHome() - - const system = readSystemConfig() - expect(system).toContain('status_line = ["model"]\r\n') - expect(system).toBe( - 'model = "gpt-5"\r\n\r\n[tui]\r\ntheme = "dark"\r\nstatus_line = ["model"]\r\n' - ) - }) - - it('never appends a [tui] table when the system config defines tui inline', () => { - writeSystemConfig('model = "gpt-5"\n') - syncSystemConfigIntoManagedCodexHome() - - // In-Codex tui change racing an outside edit that adds an inline tui table: - // appending [tui] would make the system config unparseable, so the change - // is dropped instead. - setRuntimeConfig('model = "gpt-5"\n\n[tui]\ntheme = "dark-photon"\n') - writeSystemConfig('model = "gpt-5"\ntui = { animations = false }\n') - promotionTestState.atomicWritePaths.length = 0 - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe('model = "gpt-5"\ntui = { animations = false }\n') - expect(promotionTestState.atomicWritePaths).not.toContain(systemConfigPath()) - }) - - it('ignores an allowlisted key nested under a [tui.*] subtable', () => { - writeSystemConfig('model = "gpt-5"\n\n[tui.notifications]\ntheme = "should-not-promote"\n') - syncSystemConfigIntoManagedCodexHome() - - setRuntimeConfig('model = "gpt-5"\n\n[tui.notifications]\ntheme = "changed-in-subtable"\n') - syncSystemConfigIntoManagedCodexHome() - - expect(readSystemConfig()).toBe( - 'model = "gpt-5"\n\n[tui.notifications]\ntheme = "should-not-promote"\n' - ) - }) -}) - -describe('upsertPromotedSettingsInContent', () => { - it('replaces a bare key in place inside the [tui] table', () => { - expect( - upsertPromotedSettingsInContent( - '[tui]\ntheme = "dark"\n', - new Map([['tui.theme', '"light"']]) - ) - ).toBe('[tui]\ntheme = "light"\n') - }) - - it('inserts a bare key at the end of the [tui] body, before a subtable', () => { - expect( - upsertPromotedSettingsInContent( - '[tui]\ntheme = "dark"\n\n[tui.notifications]\nenabled = true\n', - new Map([['tui.status_line', '["model"]']]) - ) - ).toBe( - '[tui]\ntheme = "dark"\nstatus_line = ["model"]\n\n[tui.notifications]\nenabled = true\n' - ) - }) - - it('replaces a dotted preamble tui key in place, keeping the dotted form', () => { - expect( - upsertPromotedSettingsInContent('tui.theme = "dark"\n', new Map([['tui.theme', '"light"']])) - ).toBe('tui.theme = "light"\n') - }) - - it('inserts a dotted tui key beside an existing dotted tui key', () => { - expect( - upsertPromotedSettingsInContent( - 'tui.theme = "dark"\n\n[features]\nx = 1\n', - new Map([['tui.status_line', '["model"]']]) - ) - ).toBe('tui.theme = "dark"\ntui.status_line = ["model"]\n\n[features]\nx = 1\n') - }) - - it('creates a [tui] table from empty content', () => { - expect(upsertPromotedSettingsInContent('', new Map([['tui.theme', '"dark"']]))).toBe( - '[tui]\ntheme = "dark"\n' - ) - }) - - it('drops an absent key instead of appending [tui] beside an inline tui table', () => { - expect( - upsertPromotedSettingsInContent( - 'tui = { animations = false }\n', - new Map([['tui.theme', '"dark"']]) - ) - ).toBe('tui = { animations = false }\n') - }) - - it('drops an absent key beside a quoted inline tui table', () => { - expect( - upsertPromotedSettingsInContent( - '"tui" = { animations = false }\n', - new Map([['tui.theme', '"dark"']]) - ) - ).toBe('"tui" = { animations = false }\n') - }) - - it('inserts beside a quoted dotted tui key instead of appending a table', () => { - expect( - upsertPromotedSettingsInContent('"tui" . "pet" = "cat"\n', new Map([['tui.theme', '"dark"']])) - ).toBe('"tui" . "pet" = "cat"\ntui.theme = "dark"\n') - }) - - it('creates a [tui] super-table at EOF after a [tui.*] subtable', () => { - expect( - upsertPromotedSettingsInContent( - '[tui.notifications]\nenabled = true\n', - new Map([['tui.theme', '"dark"']]) - ) - ).toBe('[tui.notifications]\nenabled = true\n\n[tui]\ntheme = "dark"\n') - }) - - it('drops an absent scalar that would redefine an existing tui key table', () => { - expect( - upsertPromotedSettingsInContent( - '[tui."theme"]\nvariant = "dark"\n', - new Map([['tui.theme', '"light"']]) - ) - ).toBe('[tui."theme"]\nvariant = "dark"\n') - }) - - it('drops an absent scalar that would redefine a dotted tui key table', () => { - expect( - upsertPromotedSettingsInContent( - 'tui.theme.variant = "dark"\n', - new Map([['tui.theme', '"light"']]) - ) - ).toBe('tui.theme.variant = "dark"\n') - }) - - it('does not mistake a dotted tui key inside an array table for a root key', () => { - expect( - upsertPromotedSettingsInContent( - '[[profiles]]\ntui.theme = "profile-theme"\n', - new Map([['tui.theme', '"root-theme"']]) - ) - ).toBe('[[profiles]]\ntui.theme = "profile-theme"\n\n[tui]\ntheme = "root-theme"\n') - }) - - it('does not append a table beside a root tui array-of-tables', () => { - expect( - upsertPromotedSettingsInContent( - '[[tui]]\ntheme = "array-theme"\n', - new Map([['tui.theme', '"root-theme"']]) - ) - ).toBe('[[tui]]\ntheme = "array-theme"\n') - }) - - it('does not append a table beside a quoted root tui array-of-tables', () => { - expect( - upsertPromotedSettingsInContent( - '[["tui"]]\ntheme = "array-theme"\n', - new Map([['tui.theme', '"root-theme"']]) - ) - ).toBe('[["tui"]]\ntheme = "array-theme"\n') - }) - - it('creates one [tui] table for multiple keys reaching the new-table branch', () => { - expect( - upsertPromotedSettingsInContent( - '', - new Map([ - ['tui.status_line', '["model"]'], - ['tui.theme', '"dark"'] - ]) - ) - ).toBe('[tui]\nstatus_line = ["model"]\ntheme = "dark"\n') - }) - - it('routes a mixed top-level + tui batch to its two regions in one rewrite', () => { - expect( - upsertPromotedSettingsInContent( - 'model = "gpt-5"\n\n[tui]\ntheme = "dark"\n', - new Map([ - ['model', '"o4"'], - ['tui.theme', '"light"'] - ]) - ) - ).toBe('model = "o4"\n\n[tui]\ntheme = "light"\n') - }) - - it('inserts into a CRLF [tui] table with CRLF endings', () => { - expect( - upsertPromotedSettingsInContent( - '[tui]\r\ntheme = "dark"\r\n', - new Map([['tui.status_line', '["model"]']]) - ) - ).toBe('[tui]\r\ntheme = "dark"\r\nstatus_line = ["model"]\r\n') - }) -}) - describe('upsertTopLevelSettingsInContent', () => { it('writes into empty content', () => { expect(upsertTopLevelSettingsInContent('', new Map([['model', '"x"']]))).toBe('model = "x"\n') @@ -802,12 +428,6 @@ describe('upsertTopLevelSettingsInContent', () => { ).toBe('# keep\nmodel = "new"\n\n[t]\nk = 1\n') }) - it('replaces a quoted top-level key instead of adding its bare equivalent', () => { - expect( - upsertTopLevelSettingsInContent('"model" = "old"\n', new Map([['model', '"new"']])) - ).toBe('model = "new"\n') - }) - it('inserts with CRLF endings into CRLF content', () => { expect( upsertTopLevelSettingsInContent('[features]\r\nhooks = true\r\n', new Map([['model', '"x"']])) diff --git a/src/main/codex/config-settings-promotion.ts b/src/main/codex/config-settings-promotion.ts index 8a194fef99e..372f1806c19 100644 --- a/src/main/codex/config-settings-promotion.ts +++ b/src/main/codex/config-settings-promotion.ts @@ -2,13 +2,13 @@ import { existsSync, lstatSync, mkdirSync, - readFileSync, readlinkSync, realpathSync, statSync, writeFileSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' +import { readAgentStateFileSync, readAgentStateJsonFileSync } from '../agent-state-file-reader' import { writeFileAtomically } from '../codex-accounts/fs-utils' import { parseWslUncPath } from '../../shared/wsl-paths' import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from './codex-home-paths' @@ -18,8 +18,6 @@ import { isTomlStructuralLine, updateTomlLineScanState } from './config-toml-line-scan' -import { parseTomlKeyPath, parseTomlTableHeaderPath } from './config-toml-key-path' -import { tuiStructuredKey, upsertPromotedSettingsInContent } from './codex-config-settings-upsert' // Why: the mirror reverts in-Codex config changes each launch; promotion salvages them by diffing the last baseline. @@ -31,45 +29,6 @@ export const PROMOTED_CODEX_SETTING_KEYS = [ 'sandbox_mode' ] as const -// Why: the [tui] keys the Codex TUI's user-facing pickers persist (status line, -// terminal title, theme). Like the top-level list, every key here gets written -// into the user's real ~/.codex/config.toml on promotion — grow it deliberately. -export const PROMOTED_CODEX_TUI_SETTING_KEYS = [ - 'status_line', - 'status_line_use_colors', - 'terminal_title', - 'theme' -] as const - -// Why: promotion diffs and upserts operate on structured keys — top-level keys -// keep their bare name, [tui] keys are namespaced tui. so their baseline -// entries cannot collide with a top-level key of the same name. -const PROMOTED_STRUCTURED_KEYS: readonly string[] = [ - ...PROMOTED_CODEX_SETTING_KEYS, - ...PROMOTED_CODEX_TUI_SETTING_KEYS.map(tuiStructuredKey) -] - -function isPromotedTuiKey(key: string): boolean { - return (PROMOTED_CODEX_TUI_SETTING_KEYS as readonly string[]).includes(key) -} - -// Returns the structured tui key a scanned line's key represents, or null. In -// the preamble it recognizes the dotted `tui.` form a user may hand-author; -// inside the first `[tui]` table body it recognizes the bare `` form Codex -// writes. Both map to the same structured key so either config shape promotes. -function matchTuiStructuredKey( - keyPath: string[], - inPreamble: boolean, - tuiBodyActive: boolean -): string | null { - if (inPreamble) { - const tuiKey = keyPath.length === 2 && keyPath[0] === 'tui' ? keyPath[1] : null - return tuiKey && isPromotedTuiKey(tuiKey) ? tuiStructuredKey(tuiKey) : null - } - const tuiKey = keyPath.length === 1 ? keyPath[0] : null - return tuiBodyActive && tuiKey && isPromotedTuiKey(tuiKey) ? tuiStructuredKey(tuiKey) : null -} - type TopLevelSettingValue = { raw: string // Why: a multiline string/array value can't be replaced line-by-line, so it's excluded from promotion. @@ -91,7 +50,7 @@ function readSettingsBaseline(runtimeHomePath: string): Map | nu return null } try { - const parsed: unknown = JSON.parse(readFileSync(baselinePath, 'utf-8')) + const parsed = readAgentStateJsonFileSync(baselinePath) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return null } @@ -111,67 +70,24 @@ function readSettingsBaseline(runtimeHomePath: string): Map | nu } } -function matchPromotedStructuredKey( - line: string, - inPreamble: boolean, - tuiBodyActive: boolean -): { structuredKey: string; raw: string } | null { - const parsed = parseTomlKeyPath(line) - if (!parsed || line[parsed.end] !== '=') { - return null - } - const raw = line.slice(parsed.end + 1).trim() - const topLevelKey = parsed.segments.length === 1 ? parsed.segments[0] : null - if ( - inPreamble && - topLevelKey && - (PROMOTED_CODEX_SETTING_KEYS as readonly string[]).includes(topLevelKey) - ) { - return { structuredKey: topLevelKey, raw } - } - const tuiKey = matchTuiStructuredKey(parsed.segments, inPreamble, tuiBodyActive) - return tuiKey ? { structuredKey: tuiKey, raw } : null -} - -// Why: top-level preamble scalars keep the historical behavior; [tui] keys are -// collected from the first bare [tui] table body or the dotted preamble form, -// keyed by structured path. Any table header (including [tui.*] subtables) ends -// the [tui] body, and [profiles.*]/other tables are still ignored. -function readPromotedSettingValues(configPath: string): Map { +// Why: only top-level preamble keys are scanned; rewriting nested [profiles.*] tables isn't worth the risk here. +function readTopLevelSettingValues(configPath: string): Map { const result = new Map() if (!existsSync(configPath)) { return result } - const lines = readFileSync(configPath, 'utf-8').split('\n') + const lines = readAgentStateFileSync(configPath).split('\n') let state = createTomlLineScanState() - let inPreamble = true - let tuiTableSeen = false - let tuiBodyActive = false for (const line of lines) { if (isTomlStructuralLine(state)) { - const header = getTomlTableHeader(line) - if (header) { - const table = parseTomlTableHeaderPath(header) - tuiBodyActive = - table !== null && - !table.isArray && - table.segments.length === 1 && - table.segments[0] === 'tui' && - !tuiTableSeen - if (tuiBodyActive) { - tuiTableSeen = true - } - inPreamble = false - state = updateTomlLineScanState(state, line) - continue + if (getTomlTableHeader(line)) { + break } - const matched = matchPromotedStructuredKey(line, inPreamble, tuiBodyActive) - if (matched) { + const match = /^[ \t]*([A-Za-z0-9_-]+)[ \t]*=[ \t]*(.*?)[ \t\r]*$/.exec(line) + const key = match?.[1] + if (key && (PROMOTED_CODEX_SETTING_KEYS as readonly string[]).includes(key)) { const nextState = updateTomlLineScanState(state, line) - result.set(matched.structuredKey, { - raw: matched.raw, - multiline: !isTomlStructuralLine(nextState) - }) + result.set(key, { raw: match?.[2] ?? '', multiline: !isTomlStructuralLine(nextState) }) state = nextState continue } @@ -193,7 +109,7 @@ export function snapshotCodexRuntimeSettingsBaseline( const runtimeTomlPath = join(runtimeHomePath, 'config.toml') // Why: record an empty baseline even for a missing runtime config, so Codex's first write still diffs and promotes. const settings: Record = {} - for (const [key, value] of readPromotedSettingValues(runtimeTomlPath)) { + for (const [key, value] of readTopLevelSettingValues(runtimeTomlPath)) { if (!value.multiline) { settings[key] = value.raw } @@ -202,7 +118,7 @@ export function snapshotCodexRuntimeSettingsBaseline( const baselinePath = getSettingsBaselinePath(runtimeHomePath) const serialized = `${JSON.stringify(file, null, 2)}\n` // Why: launch prep runs repeatedly; skip byte-identical rewrites to avoid needless disk writes. - if (existsSync(baselinePath) && readFileSync(baselinePath, 'utf-8') === serialized) { + if (existsSync(baselinePath) && readAgentStateFileSync(baselinePath) === serialized) { return } writeFileSync(baselinePath, serialized, { @@ -257,9 +173,10 @@ function promoteCodexRuntimeSettingsToSystemUnsafe(homes: CodexSettingsPromotion if (!baseline) { return } - const runtimeValues = readPromotedSettingValues(runtimeTomlPath) - const changedRuntimeValues = new Map() - for (const key of PROMOTED_STRUCTURED_KEYS) { + const runtimeValues = readTopLevelSettingValues(runtimeTomlPath) + const systemValues = readTopLevelSettingValues(systemTomlPath) + const updates = new Map() + for (const key of PROMOTED_CODEX_SETTING_KEYS) { const runtime = runtimeValues.get(key) if (!runtime || runtime.multiline) { continue @@ -268,14 +185,6 @@ function promoteCodexRuntimeSettingsToSystemUnsafe(homes: CodexSettingsPromotion // Orca mirrored this value and nothing touched it since — not a change. continue } - changedRuntimeValues.set(key, runtime.raw) - } - if (changedRuntimeValues.size === 0) { - return - } - const systemValues = readPromotedSettingValues(systemTomlPath) - const updates = new Map() - for (const [key, runtimeRaw] of changedRuntimeValues) { const system = systemValues.get(key) if (system?.multiline) { continue @@ -284,7 +193,7 @@ function promoteCodexRuntimeSettingsToSystemUnsafe(homes: CodexSettingsPromotion if (system?.raw !== baseline.get(key)) { continue } - updates.set(key, runtimeRaw) + updates.set(key, runtime.raw) } if (updates.size === 0) { return @@ -295,11 +204,8 @@ function promoteCodexRuntimeSettingsToSystemUnsafe(homes: CodexSettingsPromotion // Why: a dangling symlink may target an unmade dir tree; create its real parent so the atomic temp write has a home. mkdirSync(dirname(writeTarget.path), { recursive: true, mode: 0o700 }) const targetExists = existsSync(writeTarget.path) - const systemContent = targetExists ? readFileSync(writeTarget.path, 'utf-8') : '' - const nextContent = upsertPromotedSettingsInContent(systemContent, updates) - if (nextContent === systemContent) { - return - } + const systemContent = targetExists ? readAgentStateFileSync(writeTarget.path) : '' + const nextContent = upsertTopLevelSettingsInContent(systemContent, updates) if (targetExists && parseWslUncPath(writeTarget.path)) { // Why: \\wsl$ 9P symlink metadata is unreliable; write through the existing file to preserve the WSL-side inode. writeFileSync(writeTarget.path, nextContent, 'utf-8') @@ -346,3 +252,55 @@ function resolveDanglingSymlinkTarget(linkPath: string): string { // Why: replacing any link in a cycle would destroy dotfile-manager state; abort instead. throw new Error(`Codex config symlink cycle at ${linkPath}`) } + +export function upsertTopLevelSettingsInContent( + content: string, + updates: Map +): string { + const lines = content.split('\n') + let state = createTomlLineScanState() + let preambleEnd = lines.length + const keyLineIndexes = new Map() + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? '' + if (isTomlStructuralLine(state)) { + if (getTomlTableHeader(line)) { + preambleEnd = index + break + } + const match = /^[ \t]*([A-Za-z0-9_-]+)[ \t]*=/.exec(line) + if (match?.[1] && updates.has(match[1])) { + keyLineIndexes.set(match[1], index) + } + } + state = updateTomlLineScanState(state, line) + } + + // Why: match the file's existing EOL (CRLF split leaves a trailing \r) so a Windows config doesn't go mixed-EOL. + const usesCrlf = content.includes('\r\n') + const insertions: string[] = [] + for (const [key, raw] of updates) { + const existingIndex = keyLineIndexes.get(key) + const rendered = `${key} = ${raw}` + if (existingIndex !== undefined) { + lines[existingIndex] = lines[existingIndex]?.endsWith('\r') ? `${rendered}\r` : rendered + } else { + insertions.push(usesCrlf ? `${rendered}\r` : rendered) + } + } + if (insertions.length > 0) { + let insertAt = preambleEnd + while (insertAt > 0 && (lines[insertAt - 1] ?? '').trim() === '') { + insertAt -= 1 + } + if (insertAt === preambleEnd && preambleEnd < lines.length) { + insertions.push(usesCrlf ? '\r' : '') + } + lines.splice(insertAt, 0, ...insertions) + } + const result = lines.join('\n') + if (result.endsWith('\n') || result.length === 0) { + return result + } + return result.endsWith('\r') ? `${result}\n` : `${result}${usesCrlf ? '\r\n' : '\n'}` +} diff --git a/src/main/codex/config-toml-key-path.ts b/src/main/codex/config-toml-key-path.ts deleted file mode 100644 index 25eaa4c98c6..00000000000 --- a/src/main/codex/config-toml-key-path.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { parseTomlSingleLineStringValue } from './config-toml-line-scan' - -export type ParsedTomlKeyPath = { - segments: string[] - end: number -} - -export type ParsedTomlTableHeaderPath = ParsedTomlKeyPath & { - isArray: boolean -} - -export function parseTomlTableHeaderPath(header: string): ParsedTomlTableHeaderPath | null { - const trimmed = header.trim() - let source: string - let isArray: boolean - if (trimmed.startsWith('[[')) { - if (!trimmed.endsWith(']]')) { - return null - } - source = trimmed.slice(2, -2) - isArray = true - } else { - if (!trimmed.startsWith('[') || !trimmed.endsWith(']') || trimmed.endsWith(']]')) { - return null - } - source = trimmed.slice(1, -1) - isArray = false - } - const parsed = parseTomlKeyPath(source) - if (!parsed || parsed.end !== source.length) { - return null - } - return { ...parsed, isArray } -} - -export function parseTomlKeyPath(source: string, offset = 0): ParsedTomlKeyPath | null { - const segments: string[] = [] - let index = skipTomlKeyWhitespace(source, offset) - while (index < source.length) { - const quoted = parseTomlSingleLineStringValue(source, index) - if (quoted) { - segments.push(quoted.value) - index = quoted.end - } else { - const bare = /^[A-Za-z0-9_-]+/.exec(source.slice(index)) - if (!bare) { - return null - } - segments.push(bare[0]) - index += bare[0].length - } - index = skipTomlKeyWhitespace(source, index) - if (source[index] !== '.') { - return { segments, end: index } - } - index = skipTomlKeyWhitespace(source, index + 1) - } - return null -} - -function skipTomlKeyWhitespace(source: string, offset: number): number { - let index = offset - while (source[index] === ' ' || source[index] === '\t') { - index += 1 - } - return index -} diff --git a/src/main/codex/config-toml-trust.ts b/src/main/codex/config-toml-trust.ts index a4768730c5b..f1f5c991e68 100644 --- a/src/main/codex/config-toml-trust.ts +++ b/src/main/codex/config-toml-trust.ts @@ -3,7 +3,6 @@ import { existsSync, lstatSync, mkdirSync, - readFileSync, realpathSync, statSync, unlinkSync, @@ -12,6 +11,7 @@ import { import { basename, dirname, join, posix as pathPosix, win32 as pathWin32 } from 'node:path' import { createHash, randomUUID } from 'node:crypto' import { renameFileWithWindowsRetry } from '../codex-accounts/fs-utils' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { foldWslUncPathCaseInsensitiveParts } from '../../shared/wsl-paths' import { writeRollingFileBackup } from '../rolling-file-backup' import { @@ -327,7 +327,7 @@ function isCodexEventLabel(value: string): value is CodexEventLabel { // Why: strip a leading BOM (some Windows editors write one) so header regexes anchored at `^[ \t]*\[` still match. function readTomlFile(configPath: string): string { - const raw = readFileSync(configPath, 'utf-8') + const raw = readAgentStateFileSync(configPath) return raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw } diff --git a/src/main/codex/hook-service.ts b/src/main/codex/hook-service.ts index 1c4052cefca..e42bf1620e7 100644 --- a/src/main/codex/hook-service.ts +++ b/src/main/codex/hook-service.ts @@ -1,8 +1,9 @@ /* eslint-disable max-lines -- Why: getStatus + install + remove all share the managed-command and trust-key derivation. Splitting would hide that the three operations must agree on group index, event label, and command bytes. */ -import { existsSync, readFileSync, statSync, unlinkSync } from 'node:fs' +import { existsSync, statSync, unlinkSync } from 'node:fs' import { join, win32 as pathWin32 } from 'node:path' import type { SFTPWrapper } from 'ssh2' import type { AgentHookInstallState, AgentHookInstallStatus } from '../../shared/agent-hook-types' +import { readAgentStateFileSync } from '../agent-state-file-reader' import { buildManagedCommandHook, createManagedCommandMatcher, @@ -11,6 +12,7 @@ import { hookDefinitionHasManagedCommand, MANAGED_HOOK_TIMEOUT_SECONDS, readHooksJson, + readHooksJsonRawForGenerationCheck, readHooksJsonWithRaw, removeManagedCommands, wrapPosixHookCommand, @@ -81,6 +83,7 @@ import { } from './codex-managed-trust-reconciliation' import type { CodexTrustGrantLedgerHome } from './codex-trust-grant-ledger' import { mutateRealHomeHooksPreservingUserTrust } from './codex-user-hook-trust-rebase' +import { CodexWslReconciliationGenerations } from './codex-wsl-reconciliation-generations' // Why: Pre/PostToolUse feed the live in-flight-tool readout; PermissionRequest exits with no decision so Codex still shows its approval UI while Orca flips the pane to waiting. const CODEX_EVENTS = [ @@ -541,7 +544,7 @@ function applyMirroredRuntimeUserHookTrustStates( return } - const existing = readFileSync(tomlPath, 'utf-8') + const existing = readAgentStateFileSync(tomlPath) let updated = existing for (const { entry, enabled } of entries) { const headerKeyPattern = buildHookTrustHeaderKeyPattern(computeTrustKey(entry)) @@ -645,7 +648,7 @@ function cleanupLegacySystemManagedHooks(): void { afterHooks: nextHooks, writeHooks: () => { if ( - readFileSync(legacyConfigPath, 'utf-8') !== previousRaw || + readHooksJsonRawForGenerationCheck(legacyConfigPath) !== previousRaw || resolveHooksJsonWritePath(legacyConfigPath) !== hooksWritePath ) { // Why: the pre-mutation RPC may overlap a user save; downgrade must @@ -691,7 +694,7 @@ function cleanupLegacyCodexProfileHooks(): void { return } - const existing = readFileSync(profilePath, 'utf-8') + const existing = readAgentStateFileSync(profilePath) const next = stripLegacyManagedProfileBlock(existing) if (next === existing) { return @@ -1021,16 +1024,14 @@ function getWslReconciliationKey(runtimeHomePath: string): string { } export class CodexHookService { - private readonly wslReconciliationGeneration = new Map() + private readonly wslReconciliationGenerations = new CodexWslReconciliationGenerations() private supersedeWslReconciliation(runtimeHomePath: string | null | undefined): number { if (!runtimeHomePath) { return 0 } const key = getWslReconciliationKey(runtimeHomePath) - const generation = (this.wslReconciliationGeneration.get(key) ?? 0) + 1 - this.wslReconciliationGeneration.set(key, generation) - return generation + return this.wslReconciliationGenerations.advance(key) } installForRuntimeHome( @@ -1058,7 +1059,7 @@ export class CodexHookService { : null const action = getWslHookReconciliationAction({ settlement, - isCurrentGeneration: this.wslReconciliationGeneration.get(key) === generation, + isCurrentGeneration: this.wslReconciliationGenerations.isCurrent(key, generation), installedTrustConfigPath, resolvedTrustConfigPath: resolvedPlan?.trustConfigPath ?? null, installSucceeded diff --git a/src/main/codex/hook-trust-promotion.ts b/src/main/codex/hook-trust-promotion.ts index 16b19ceeae6..76276add0fb 100644 --- a/src/main/codex/hook-trust-promotion.ts +++ b/src/main/codex/hook-trust-promotion.ts @@ -1,5 +1,7 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { existsSync, writeFileSync } from 'node:fs' import { join } from 'node:path' +import { MAX_AGENT_STATE_FILE_BYTES, readAgentStateJsonFileSync } from '../agent-state-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' import { createManagedCommandMatcher, readHooksJson, @@ -51,7 +53,7 @@ function readHookTrustProvenance( return null } try { - const parsed: unknown = JSON.parse(readFileSync(provenancePath, 'utf-8')) + const parsed = readAgentStateJsonFileSync(provenancePath) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return null } @@ -97,7 +99,8 @@ export function snapshotCodexRuntimeHookTrustProvenance( } } const file: HookTrustProvenanceFile = { version: 1, entries } - writeFileSync(getProvenancePath(runtimeHomePath), `${JSON.stringify(file, null, 2)}\n`, { + const { serialized } = stringifyJsonWithinByteLimit(file, MAX_AGENT_STATE_FILE_BYTES - 1, 2) + writeFileSync(getProvenancePath(runtimeHomePath), `${serialized}\n`, { encoding: 'utf-8', mode: 0o600 }) diff --git a/src/main/computer/macos-computer-use-permission-status.test.ts b/src/main/computer/macos-computer-use-permission-status.test.ts index cfa511b11ce..09323ccc85f 100644 --- a/src/main/computer/macos-computer-use-permission-status.test.ts +++ b/src/main/computer/macos-computer-use-permission-status.test.ts @@ -1,12 +1,17 @@ import { execFileSync, spawn, spawnSync } from 'node:child_process' -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { mkdtemp, rm, stat } from 'node:fs/promises' import { join } from 'node:path' +import { EventEmitter } from 'node:events' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as nodeBoundedFileReader from '../../shared/node-bounded-file-reader' const resolveHelperAppPathMock = vi.hoisted(() => vi.fn()) const resolveHelperExecutablePathMock = vi.hoisted(() => vi.fn()) const permissionStatusTempDir = '/tmp/orca-computer-use-permissions-test' const permissionStatusPath = join(permissionStatusTempDir, 'status.json') +const { readNodeFileWithinLimitMock } = vi.hoisted(() => ({ + readNodeFileWithinLimitMock: vi.fn() +})) vi.mock('child_process', () => ({ execFileSync: vi.fn(), @@ -30,11 +35,15 @@ vi.mock('child_process', () => ({ vi.mock('fs/promises', () => ({ mkdtemp: vi.fn(), - readFile: vi.fn(), rm: vi.fn(), stat: vi.fn() })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, readNodeFileWithinLimit: readNodeFileWithinLimitMock } +}) + vi.mock('./macos-native-provider-paths', () => ({ resolveMacOSComputerUseAppPath: resolveHelperAppPathMock, resolveMacOSComputerUseExecutablePath: resolveHelperExecutablePathMock @@ -48,7 +57,7 @@ describe('getComputerUsePermissionStatus', () => { vi.mocked(spawnSync).mockClear() vi.mocked(execFileSync).mockReset() vi.mocked(mkdtemp).mockReset() - vi.mocked(readFile).mockReset() + readNodeFileWithinLimitMock.mockReset() vi.mocked(rm).mockReset() vi.mocked(stat).mockReset() resolveHelperAppPathMock.mockReset() @@ -162,6 +171,56 @@ describe('getComputerUsePermissionStatus', () => { }) }) + it('kills the helper and rejects oversized launch output', async () => { + const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions') + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter & { setEncoding: ReturnType } + stderr: EventEmitter & { setEncoding: ReturnType } + kill: ReturnType + } + child.stdout = Object.assign(new EventEmitter(), { setEncoding: vi.fn() }) + child.stderr = Object.assign(new EventEmitter(), { setEncoding: vi.fn() }) + child.kill = vi.fn() + vi.mocked(spawn).mockImplementationOnce(() => child as unknown as ReturnType) + + const statusPromise = getComputerUsePermissionStatus() + await Promise.resolve() + child.stdout.emit('data', 'x'.repeat(64 * 1024 + 1)) + + await expect(statusPromise).rejects.toMatchObject({ + name: 'RuntimeClientError', + code: 'accessibility_error', + message: 'Permission helper returned too much launch output' + }) + expect(child.kill).toHaveBeenCalledTimes(1) + }) + + it('preserves helper diagnostics delivered as 50,000 one-byte fragments', async () => { + const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions') + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter & { setEncoding: ReturnType } + stderr: EventEmitter & { setEncoding: ReturnType } + kill: ReturnType + } + child.stdout = Object.assign(new EventEmitter(), { setEncoding: vi.fn() }) + child.stderr = Object.assign(new EventEmitter(), { setEncoding: vi.fn() }) + child.kill = vi.fn() + vi.mocked(spawn).mockImplementationOnce(() => child as unknown as ReturnType) + const statusPromise = getComputerUsePermissionStatus() + await Promise.resolve() + + for (let index = 0; index < 50_000; index += 1) { + child.stderr.emit('data', ' ') + } + child.stderr.emit('data', 'permission denied') + child.emit('close', 1) + + await expect(statusPromise).rejects.toMatchObject({ + code: 'accessibility_error', + message: 'Could not check permissions: permission denied' + }) + }) + it('reads permission status through the helper app identity', async () => { const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions') mockPermissionStatus('{"accessibility":"granted","screenshots":"not-granted"}') @@ -187,13 +246,26 @@ describe('getComputerUsePermissionStatus', () => { { stdio: ['ignore', 'pipe', 'pipe'] } ) expect(spawnSync).not.toHaveBeenCalled() - expect(readFile).toHaveBeenCalledWith(permissionStatusPath, 'utf8') + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith(permissionStatusPath, 64 * 1024) expect(rm).toHaveBeenCalledWith(permissionStatusTempDir, { recursive: true, force: true }) }) + it('maps an oversized permission status file to a stable runtime error', async () => { + const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions') + readNodeFileWithinLimitMock.mockRejectedValueOnce( + new nodeBoundedFileReader.NodeFileReadTooLargeError(64 * 1024 + 1, 64 * 1024) + ) + + await expect(getComputerUsePermissionStatus()).rejects.toMatchObject({ + name: 'RuntimeClientError', + code: 'accessibility_error', + message: 'Permission helper returned too much status data' + }) + }) + it('returns unavailable permission status when the helper app is missing on macOS', async () => { const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions') resolveHelperAppPathMock.mockReturnValue(null) @@ -213,7 +285,10 @@ describe('getComputerUsePermissionStatus', () => { function mockPermissionStatus(json: string): void { vi.mocked(spawnSync).mockReturnValue({ status: 0 } as ReturnType) - vi.mocked(readFile).mockResolvedValue(json) + readNodeFileWithinLimitMock.mockResolvedValue({ + buffer: Buffer.from(json), + stats: { size: Buffer.byteLength(json) } + }) } function setPlatform(platform: NodeJS.Platform): void { diff --git a/src/main/computer/macos-computer-use-permission-status.ts b/src/main/computer/macos-computer-use-permission-status.ts index b23c3e9ab96..431ea47b08f 100644 --- a/src/main/computer/macos-computer-use-permission-status.ts +++ b/src/main/computer/macos-computer-use-permission-status.ts @@ -1,5 +1,5 @@ import { spawn } from 'node:child_process' -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { mkdtemp, rm, stat } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { setTimeout as delay } from 'node:timers/promises' @@ -13,8 +13,14 @@ import { resolveMacOSComputerUseExecutablePath } from './macos-native-provider-paths' import { RuntimeClientError } from './runtime-client-error' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' const PERMISSION_STATUS_HELPER_LAUNCH_TIMEOUT_MS = 5_000 +export const PERMISSION_STATUS_MAX_BYTES = 64 * 1024 export function getComputerUsePermissionStatus(): Promise { return getComputerUsePermissionStatusAsync() @@ -86,10 +92,22 @@ async function readPermissionStatusFromHelperApp( for (let attempt = 0; attempt < 50; attempt++) { if (await fileExists(statusPath)) { - const output = await readFile(statusPath, 'utf8') - return JSON.parse(output) as Partial< - Record - > + try { + const output = ( + await readNodeFileWithinLimit(statusPath, PERMISSION_STATUS_MAX_BYTES) + ).buffer.toString('utf8') + return JSON.parse(output) as Partial< + Record + > + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw new RuntimeClientError( + 'accessibility_error', + 'Permission helper returned too much status data' + ) + } + throw error + } } await delay(100) } @@ -108,16 +126,41 @@ function launchPermissionStatusHelper(helperAppPath: string, statusPath: string) stdio: ['ignore', 'pipe', 'pipe'] } ) - let stdout = '' - let stderr = '' + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() + let outputBytes = 0 launch.stdout?.setEncoding('utf8') launch.stderr?.setEncoding('utf8') const onStdoutData = (chunk: string): void => { - stdout += chunk + const bytes = Buffer.from(chunk) + outputBytes += bytes.byteLength + if (outputBytes > PERMISSION_STATUS_MAX_BYTES) { + launch.kill() + settleReject( + new RuntimeClientError( + 'accessibility_error', + 'Permission helper returned too much launch output' + ) + ) + return + } + stdout.append(bytes) } const onStderrData = (chunk: string): void => { - stderr += chunk + const bytes = Buffer.from(chunk) + outputBytes += bytes.byteLength + if (outputBytes > PERMISSION_STATUS_MAX_BYTES) { + launch.kill() + settleReject( + new RuntimeClientError( + 'accessibility_error', + 'Permission helper returned too much launch output' + ) + ) + return + } + stderr.append(bytes) } let settled = false let launchTimeout: ReturnType | null = null @@ -137,6 +180,8 @@ function launchPermissionStatusHelper(helperAppPath: string, statusPath: string) } settled = true removeListeners() + stdout.clear() + stderr.clear() resolve() } const settleReject = (error: Error): void => { @@ -145,6 +190,8 @@ function launchPermissionStatusHelper(helperAppPath: string, statusPath: string) } settled = true removeListeners() + stdout.clear() + stderr.clear() reject(error) } const onError = (): void => { @@ -160,7 +207,8 @@ function launchPermissionStatusHelper(helperAppPath: string, statusPath: string) settleResolve() return } - const detail = stderr.trim() || stdout.trim() || `exit ${status ?? 'unknown'}` + const detail = + stderr.toString().trim() || stdout.toString().trim() || `exit ${status ?? 'unknown'}` settleReject( new RuntimeClientError('accessibility_error', `Could not check permissions: ${detail}`) ) diff --git a/src/main/computer/macos-computer-use-permissions.test.ts b/src/main/computer/macos-computer-use-permissions.test.ts index a8b6442b719..974c7ab8bcc 100644 --- a/src/main/computer/macos-computer-use-permissions.test.ts +++ b/src/main/computer/macos-computer-use-permissions.test.ts @@ -1,7 +1,8 @@ import { execFileSync, spawn, spawnSync } from 'node:child_process' -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { mkdtemp, rm, stat } from 'node:fs/promises' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as nodeBoundedFileReader from '../../shared/node-bounded-file-reader' import { openComputerUsePermissions, resetComputerUsePermissions @@ -12,6 +13,9 @@ const resolveHelperExecutablePathMock = vi.hoisted(() => vi.fn()) const permissionStatusTempDir = '/tmp/orca-computer-use-permissions-test' const helperAppPath = '/Applications/Orca Computer Use.app' const helperInfoPlistPath = join(helperAppPath, 'Contents', 'Info.plist') +const { readNodeFileWithinLimitMock } = vi.hoisted(() => ({ + readNodeFileWithinLimitMock: vi.fn() +})) vi.mock('child_process', () => ({ execFileSync: vi.fn(), @@ -35,11 +39,15 @@ vi.mock('child_process', () => ({ vi.mock('fs/promises', () => ({ mkdtemp: vi.fn(), - readFile: vi.fn(), rm: vi.fn(), stat: vi.fn() })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, readNodeFileWithinLimit: readNodeFileWithinLimitMock } +}) + vi.mock('./macos-native-provider-paths', () => ({ resolveMacOSComputerUseAppPath: resolveHelperAppPathMock, resolveMacOSComputerUseExecutablePath: resolveHelperExecutablePathMock @@ -53,7 +61,7 @@ describe('openComputerUsePermissions', () => { vi.mocked(spawnSync).mockClear() vi.mocked(execFileSync).mockReset() vi.mocked(mkdtemp).mockReset() - vi.mocked(readFile).mockReset() + readNodeFileWithinLimitMock.mockReset() vi.mocked(rm).mockReset() vi.mocked(stat).mockReset() resolveHelperAppPathMock.mockReset() @@ -210,9 +218,13 @@ describe('openComputerUsePermissions', () => { it('resets stale macOS TCC grants for the helper bundle id', async () => { resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app') - vi.mocked(readFile) - .mockResolvedValueOnce('{"accessibility":"granted","screenshots":"granted"}') - .mockResolvedValueOnce('{"accessibility":"not-granted","screenshots":"not-granted"}') + readNodeFileWithinLimitMock + .mockResolvedValueOnce( + boundedPermissionStatus('{"accessibility":"granted","screenshots":"granted"}') + ) + .mockResolvedValueOnce( + boundedPermissionStatus('{"accessibility":"not-granted","screenshots":"not-granted"}') + ) vi.mocked(execFileSync).mockReturnValueOnce('com.example.orca.computer-use\n') vi.mocked(spawnSync).mockReturnValue({ status: 0 } as ReturnType) @@ -246,7 +258,14 @@ describe('openComputerUsePermissions', () => { function mockPermissionStatus(json: string): void { vi.mocked(spawnSync).mockReturnValue({ status: 0 } as ReturnType) - vi.mocked(readFile).mockResolvedValue(json) + readNodeFileWithinLimitMock.mockResolvedValue(boundedPermissionStatus(json)) +} + +function boundedPermissionStatus(json: string) { + return { + buffer: Buffer.from(json), + stats: { size: Buffer.byteLength(json) } + } } function setPlatform(platform: NodeJS.Platform): void { diff --git a/src/main/computer/macos-native-provider-client.ts b/src/main/computer/macos-native-provider-client.ts index 0b917037685..faf71f955d2 100644 --- a/src/main/computer/macos-native-provider-client.ts +++ b/src/main/computer/macos-native-provider-client.ts @@ -20,7 +20,7 @@ import { import { resolveMacOSComputerUseExecutablePath } from './macos-native-provider-paths' import { attachMacOSNativeProviderSocketListeners, - consumeNativeProviderLines, + MacOSNativeProviderLineBuffer, startMacOSNativeProviderSocket } from './macos-native-provider-transport' import { validateComputerProviderActionParams } from './computer-provider-action-validation' @@ -37,7 +37,7 @@ export class MacOSNativeProviderClient { private socketToken: string | null = null private nextId = 1 private pending = new Map() - private socketBuffer = '' + private readonly socketBuffer = new MacOSNativeProviderLineBuffer() private providerCapabilities: ComputerProviderCapabilities | null = null private socketListenerCleanup: (() => void) | null = null private socketStartGeneration = 0 @@ -70,20 +70,16 @@ export class MacOSNativeProviderClient { this.socketStartPromise = null this.socketStartGeneration++ this.providerCapabilities = null - this.socketBuffer = '' + this.socketBuffer.clear() this.cleanupActiveSocketListeners() if (socket && !socket.destroyed) { const id = this.nextId++ socket.write(`${JSON.stringify({ id, method: 'terminate', params: {}, token })}\n`) socket.end() } - for (const [id, pending] of this.pending) { - clearTimeout(pending.timer) - pending.reject( - new RuntimeClientError('accessibility_error', 'native macOS provider shut down') - ) - this.pending.delete(id) - } + this.rejectPending( + new RuntimeClientError('accessibility_error', 'native macOS provider shut down') + ) this.cleanupSocketDirectory() } private async call(method: NativeMethod, params: unknown): Promise { @@ -199,7 +195,7 @@ export class MacOSNativeProviderClient { this.socketToken = started.socketToken const socket = started.socket socket.setEncoding('utf8') - this.socketBuffer = '' + this.socketBuffer.clear() this.socketListenerCleanup = attachMacOSNativeProviderSocketListeners(socket, { data: (chunk) => this.handleSocketData(socket, chunk), close: () => this.handleSocketClose(socket), @@ -214,10 +210,11 @@ export class MacOSNativeProviderClient { if (this.socket !== socket) { return } - this.socketBuffer += chunk - this.socketBuffer = consumeNativeProviderLines(this.socketBuffer, (line) => - this.handleLine(line) - ) + try { + this.socketBuffer.feed(chunk, (line) => this.handleLine(line)) + } catch (error) { + this.handleTransportError(socket, error instanceof Error ? error : new Error(String(error))) + } } private handleLine(line: string): void { let response: NativeResponse @@ -245,7 +242,7 @@ export class MacOSNativeProviderClient { } this.cleanupActiveSocketListeners() this.socket = null - this.socketBuffer = '' + this.socketBuffer.clear() this.cleanupSocketDirectory() this.rejectPending( new RuntimeClientError('accessibility_error', 'native macOS helper app connection closed') @@ -259,7 +256,7 @@ export class MacOSNativeProviderClient { this.cleanupActiveSocketListeners() // Why: an active transport error makes the helper socket unreliable for the next request. this.socket = null - this.socketBuffer = '' + this.socketBuffer.clear() if (!socket.destroyed) { socket.destroy() } @@ -275,7 +272,7 @@ export class MacOSNativeProviderClient { } this.cleanupActiveSocketListeners() this.socket = null - this.socketBuffer = '' + this.socketBuffer.clear() if (!socket.destroyed) { socket.destroy() } diff --git a/src/main/computer/macos-native-provider-transport.test.ts b/src/main/computer/macos-native-provider-transport.test.ts new file mode 100644 index 00000000000..e3e17b30a72 --- /dev/null +++ b/src/main/computer/macos-native-provider-transport.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from 'vitest' +import { + MacOSNativeProviderLineBuffer, + MacOSNativeProviderResponseTooLargeError, + consumeNativeProviderLines +} from './macos-native-provider-transport' + +describe('consumeNativeProviderLines', () => { + it('preserves complete lines and returns the partial tail', () => { + const handleLine = vi.fn() + + const remaining = consumeNativeProviderLines('one\ntwo\npartial', handleLine, 16) + + expect(handleLine.mock.calls.map(([line]) => line)).toEqual(['one', 'two']) + expect(remaining).toBe('partial') + }) + + it('rejects a complete oversized response before dispatching it', () => { + const handleLine = vi.fn() + + expect(() => consumeNativeProviderLines('12345\n', handleLine, 4)).toThrow( + MacOSNativeProviderResponseTooLargeError + ) + expect(handleLine).not.toHaveBeenCalled() + }) + + it('rejects a newline-free response once its retained tail crosses the limit', () => { + expect(() => consumeNativeProviderLines('12345', vi.fn(), 4)).toThrow( + 'native macOS provider response exceeded' + ) + }) +}) + +describe('MacOSNativeProviderLineBuffer', () => { + it('parses a response delivered in more than 100,000 one-character fragments', () => { + const buffer = new MacOSNativeProviderLineBuffer() + const handleLine = vi.fn() + const line = JSON.stringify({ id: 1, ok: true, result: 'x'.repeat(100_000) }) + + for (const character of line) { + buffer.feed(character, handleLine) + } + buffer.feed('\n', handleLine) + + expect(handleLine).toHaveBeenCalledWith(line) + }) + + it('clears an oversized fragmented line and accepts a later response', () => { + const buffer = new MacOSNativeProviderLineBuffer(4) + const handleLine = vi.fn() + buffer.feed('12', handleLine) + expect(() => buffer.feed('345', handleLine)).toThrow(MacOSNativeProviderResponseTooLargeError) + + buffer.feed('ok\n', handleLine) + + expect(handleLine).toHaveBeenCalledWith('ok') + }) +}) diff --git a/src/main/computer/macos-native-provider-transport.ts b/src/main/computer/macos-native-provider-transport.ts index 54c4cf773aa..262e6760589 100644 --- a/src/main/computer/macos-native-provider-transport.ts +++ b/src/main/computer/macos-native-provider-transport.ts @@ -6,8 +6,20 @@ import { join } from 'node:path' import { randomUUID } from 'node:crypto' import { connectMacOSProviderSocket } from './macos-native-provider-socket' import { RuntimeClientError } from './runtime-client-error' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' const HELPER_CONNECT_TIMEOUT_MS = 10_000 +export const MACOS_NATIVE_PROVIDER_MAX_RESPONSE_LINE_BYTES = 64 * 1024 * 1024 + +export class MacOSNativeProviderResponseTooLargeError extends Error { + constructor( + readonly observedBytes: number, + readonly maxBytes: number + ) { + super(`native macOS provider response exceeded ${maxBytes} byte line limit`) + this.name = 'MacOSNativeProviderResponseTooLargeError' + } +} export type StartedMacOSProviderSocket = { socket: net.Socket @@ -16,6 +28,40 @@ export type StartedMacOSProviderSocket = { socketToken: string } +export class MacOSNativeProviderLineBuffer { + private readonly pending = new GrowingByteBuffer() + + constructor(private readonly maxLineBytes = MACOS_NATIVE_PROVIDER_MAX_RESPONSE_LINE_BYTES) {} + + feed(chunk: string, handleLine: (line: string) => void): void { + let remaining = chunk + while (remaining.length > 0) { + const newline = remaining.indexOf('\n') + const hasNewline = newline >= 0 + const segment = hasNewline ? remaining.slice(0, newline) : remaining + remaining = hasNewline ? remaining.slice(newline + 1) : '' + const segmentBytes = Buffer.byteLength(segment) + const observedBytes = this.pending.byteLength + segmentBytes + if (observedBytes > this.maxLineBytes) { + this.pending.clear() + throw new MacOSNativeProviderResponseTooLargeError(observedBytes, this.maxLineBytes) + } + this.pending.append(Buffer.from(segment)) + if (!hasNewline) { + return + } + const line = this.pending.takeString('utf8') + if (line.trim()) { + handleLine(line) + } + } + } + + clear(): void { + this.pending.clear() + } +} + export function isMacOS14OrNewer(): boolean { const darwinMajor = Number.parseInt(release().split('.')[0] ?? '', 10) return Number.isFinite(darwinMajor) && darwinMajor >= 23 @@ -47,16 +93,25 @@ export function attachMacOSNativeProviderSocketListeners( export function consumeNativeProviderLines( buffer: string, - handleLine: (line: string) => void + handleLine: (line: string) => void, + maxLineBytes = MACOS_NATIVE_PROVIDER_MAX_RESPONSE_LINE_BYTES ): string { let remaining = buffer while (true) { const newline = remaining.indexOf('\n') if (newline < 0) { + const remainingBytes = Buffer.byteLength(remaining) + if (remainingBytes > maxLineBytes) { + throw new MacOSNativeProviderResponseTooLargeError(remainingBytes, maxLineBytes) + } return remaining } const line = remaining.slice(0, newline) remaining = remaining.slice(newline + 1) + const lineBytes = Buffer.byteLength(line) + if (lineBytes > maxLineBytes) { + throw new MacOSNativeProviderResponseTooLargeError(lineBytes, maxLineBytes) + } if (line.trim()) { handleLine(line) } diff --git a/src/main/computer/sidecar-client.test.ts b/src/main/computer/sidecar-client.test.ts index fb3b28bf444..61c0b5e8426 100644 --- a/src/main/computer/sidecar-client.test.ts +++ b/src/main/computer/sidecar-client.test.ts @@ -8,6 +8,7 @@ import { import { callComputerSidecarAction, callComputerSidecarCapabilities, + COMPUTER_SIDECAR_MAX_QUEUED_CALLS, resetComputerSidecarForTest } from './sidecar-client' @@ -166,6 +167,44 @@ describe('computer sidecar client', () => { await expect(secondCall).resolves.toEqual({ provider: 'second' }) }) + it('fails closed when the serialized call queue reaches its cap', async () => { + const admitted = Array.from({ length: COMPUTER_SIDECAR_MAX_QUEUED_CALLS }, () => + callComputerSidecarCapabilities() + ) + const admittedSettlements = Promise.allSettled(admitted) + const child = children[0]! + + await expect(callComputerSidecarCapabilities()).rejects.toThrow( + 'computer sidecar queue limit reached' + ) + expect(child.sent).toHaveLength(1) + + child.emit('error', new Error('stop saturated sidecar')) + await admittedSettlements + }) + + it('releases queue admission after a call settles', async () => { + const firstCall = callComputerSidecarCapabilities() + const firstChild = children[0]! + const firstRequest = firstChild.sent[0]! + firstChild.emit('message', { + id: firstRequest.id, + ok: true, + result: { provider: 'first' } + }) + await expect(firstCall).resolves.toEqual({ provider: 'first' }) + + const secondCall = callComputerSidecarCapabilities() + const secondRequest = firstChild.sent[1]! + firstChild.emit('message', { + id: secondRequest.id, + ok: true, + result: { provider: 'second' } + }) + + await expect(secondCall).resolves.toEqual({ provider: 'second' }) + }) + it('marks synthetic action results without provider verification as unverified', async () => { const call = callComputerSidecarAction('click', { app: 'Finder', elementIndex: 0 }) const child = children[0]! diff --git a/src/main/computer/sidecar-client.ts b/src/main/computer/sidecar-client.ts index d0cfa9dfb2c..731c6066eca 100644 --- a/src/main/computer/sidecar-client.ts +++ b/src/main/computer/sidecar-client.ts @@ -43,6 +43,7 @@ type PendingRequest = { } const REQUEST_TIMEOUT_MS = 60_000 +export const COMPUTER_SIDECAR_MAX_QUEUED_CALLS = 16 let sidecar: ComputerSidecarProcess | null = null // Why: Node treats unhandled child 'error' events as process exceptions, so @@ -122,10 +123,20 @@ class ComputerSidecarProcess { private pending = new Map() private queueTail: Promise | null = null private queueGeneration = 0 + private queuedCalls = 0 constructor(private readonly entryPath: string) {} call(method: ComputerSidecarMethod, params: unknown): Promise { + if (this.queuedCalls >= COMPUTER_SIDECAR_MAX_QUEUED_CALLS) { + return Promise.reject( + new RuntimeClientError( + 'accessibility_error', + 'computer sidecar queue limit reached; retry the computer-use request' + ) + ) + } + this.queuedCalls += 1 const generation = this.queueGeneration const run = () => { if (generation !== this.queueGeneration) { @@ -136,8 +147,17 @@ class ComputerSidecarProcess { } return this.send(method, params) } - const result = this.queueTail ? this.queueTail.then(run, run) : run() - const tail = result.then( + let result: Promise + try { + result = this.queueTail ? this.queueTail.then(run, run) : run() + } catch (error) { + this.queuedCalls -= 1 + throw error + } + const trackedResult = result.finally(() => { + this.queuedCalls -= 1 + }) + const tail = trackedResult.then( () => undefined, () => undefined ) @@ -147,7 +167,7 @@ class ComputerSidecarProcess { this.queueTail = null } }) - return result + return trackedResult } private send(method: ComputerSidecarMethod, params: unknown): Promise { diff --git a/src/main/crash-reporting/crash-breadcrumb-store.test.ts b/src/main/crash-reporting/crash-breadcrumb-store.test.ts index 944109a195c..4668032e12f 100644 --- a/src/main/crash-reporting/crash-breadcrumb-store.test.ts +++ b/src/main/crash-reporting/crash-breadcrumb-store.test.ts @@ -24,6 +24,44 @@ describe('crash breadcrumb store', () => { expect(snapshot[29].name).toBe('event_31') }) + it('retains bounded renderer high-water profiles across later activity', () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-07-22T12:00:00.000Z')) + recordCrashBreadcrumb('renderer_memory_highwater', { + rendererSurface: 'main', + thresholdPct: 80, + 'store.agentStatusByPaneKey': 500 + }) + for (let index = 0; index < 32; index += 1) { + vi.advanceTimersByTime(60_000) + recordCrashBreadcrumb('renderer_memory', { index }) + } + + const snapshot = getCrashBreadcrumbSnapshot() + + expect(snapshot).toHaveLength(30) + expect(snapshot[0]).toEqual( + expect.objectContaining({ + name: 'renderer_memory_highwater', + data: expect.objectContaining({ thresholdPct: 80 }) + }) + ) + expect(snapshot.at(-1)?.data).toEqual({ index: 31 }) + }) + + it('caps retained high-water profiles', () => { + for (let index = 0; index < 5; index += 1) { + recordCrashBreadcrumb('renderer_memory_highwater', { + rendererSurface: `surface-${index}`, + thresholdPct: 80 + }) + } + + expect( + getCrashBreadcrumbSnapshot().map((breadcrumb) => breadcrumb.data?.rendererSurface) + ).toEqual(['surface-1', 'surface-2', 'surface-3', 'surface-4']) + }) + it('redacts sensitive breadcrumb fields before they can be snapshotted', () => { recordCrashBreadcrumb('workspace_opened', { path: '/Users/alice/project', diff --git a/src/main/crash-reporting/crash-breadcrumb-store.ts b/src/main/crash-reporting/crash-breadcrumb-store.ts index 76cf875619f..ca31e2068d7 100644 --- a/src/main/crash-reporting/crash-breadcrumb-store.ts +++ b/src/main/crash-reporting/crash-breadcrumb-store.ts @@ -5,14 +5,26 @@ import { } from '../../shared/crash-reporting' const MAX_BREADCRUMBS = 30 +// Why: retain two thresholds for each renderer surface without growing the ring. +const MAX_RETAINED_BREADCRUMBS = 4 // Why: coalesceKey embeds an open-string agentType (length-trimmed only, never // enum-checked), so the key space is unbounded over a long multi-agent/SSH session. // Bound the coalesce map the same way ProcessGoneDedupe bounds its key map. const MAX_COALESCE_KEYS = 128 let breadcrumbs: CrashReportBreadcrumb[] = [] +let retainedBreadcrumbs = new Map() let coalescedBreadcrumbs = new Map() +function retainedBreadcrumbKey(breadcrumb: CrashReportBreadcrumb): string | null { + if (breadcrumb.name !== 'renderer_memory_highwater') { + return null + } + const surface = breadcrumb.data?.rendererSurface + const threshold = breadcrumb.data?.thresholdPct + return `${breadcrumb.name}:${String(surface)}:${String(threshold)}` +} + export function recordCrashBreadcrumb(name: string, data?: CrashReportBreadcrumbData): void { const sanitized = sanitizeCrashReportBreadcrumbs([ { @@ -25,6 +37,19 @@ export function recordCrashBreadcrumb(name: string, data?: CrashReportBreadcrumb if (!breadcrumb) { return } + const retainedKey = retainedBreadcrumbKey(breadcrumb) + if (retainedKey) { + retainedBreadcrumbs.delete(retainedKey) + retainedBreadcrumbs.set(retainedKey, breadcrumb) + while (retainedBreadcrumbs.size > MAX_RETAINED_BREADCRUMBS) { + const oldestKey = retainedBreadcrumbs.keys().next() + if (oldestKey.done) { + break + } + retainedBreadcrumbs.delete(oldestKey.value) + } + return + } breadcrumbs.push(breadcrumb) if (breadcrumbs.length > MAX_BREADCRUMBS) { breadcrumbs.shift() @@ -72,14 +97,20 @@ export function recordCoalescedCrashBreadcrumb({ } export function getCrashBreadcrumbSnapshot(): CrashReportBreadcrumb[] { - return breadcrumbs.map((breadcrumb) => ({ - ...breadcrumb, - ...(breadcrumb.data ? { data: { ...breadcrumb.data } } : {}) - })) + // Why: long sessions must retain threshold profiles without growing the 30-entry budget. + const retained = [...retainedBreadcrumbs.values()] + const recent = breadcrumbs.slice(-(MAX_BREADCRUMBS - retained.length)) + return [...retained, ...recent] + .sort((left, right) => left.createdAt.localeCompare(right.createdAt)) + .map((breadcrumb) => ({ + ...breadcrumb, + ...(breadcrumb.data ? { data: { ...breadcrumb.data } } : {}) + })) } export function clearCrashBreadcrumbsForTest(): void { breadcrumbs = [] + retainedBreadcrumbs = new Map() coalescedBreadcrumbs = new Map() } diff --git a/src/main/crash-reporting/crash-report-store.test.ts b/src/main/crash-reporting/crash-report-store.test.ts index 1a14a9fa2e6..14485a136cd 100644 --- a/src/main/crash-reporting/crash-report-store.test.ts +++ b/src/main/crash-reporting/crash-report-store.test.ts @@ -12,8 +12,13 @@ vi.mock('../win32-utils', async (importOriginal) => { return { ...actual, grantDirAclAsync: grantDirAclAsyncMock } }) -import { CrashReportStore } from './crash-report-store' +import { + CrashReportStore, + MAX_CRASH_REPORT_FILE_BYTES, + MAX_CRASH_REPORT_JSON_STRUCTURAL_TOKENS +} from './crash-report-store' import type { CrashReportCreateInput } from '../../shared/crash-reporting' +import * as boundedFileReader from '../../shared/node-bounded-file-reader' const tempDirs: string[] = [] @@ -92,6 +97,46 @@ describe('CrashReportStore', () => { await expect(store.listRecent()).resolves.toEqual([]) }) + it('rejects an oversized sparse report file and recovers with the next report', async () => { + const { store, filePath } = await createStore() + await fs.writeFile(filePath, 'x') + await fs.truncate(filePath, MAX_CRASH_REPORT_FILE_BYTES + 1) + + await expect(store.listRecent()).resolves.toEqual([]) + await expect(store.record(input('after-oversize'))).resolves.toMatchObject({ + reason: 'after-oversize' + }) + await expect(store.listRecent()).resolves.toHaveLength(1) + }) + + it('rejects structurally amplified report JSON before parsing', async () => { + const { store, filePath } = await createStore() + await fs.writeFile( + filePath, + `{"reports":[${'0,'.repeat(MAX_CRASH_REPORT_JSON_STRUCTURAL_TOKENS)}0]}`, + 'utf8' + ) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const parseSpy = vi.spyOn(JSON, 'parse') + + await expect(store.listRecent()).resolves.toEqual([]) + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('preserves prior reports when the next report exceeds the write ceiling', async () => { + const { store, filePath } = await createStore() + await store.record(input('preserved')) + const before = await fs.readFile(filePath, 'utf8') + const writeSpy = vi.spyOn(fs, 'writeFile') + + await expect(store.record(input('x'.repeat(MAX_CRASH_REPORT_FILE_BYTES)))).rejects.toThrow( + 'JSON output exceeds' + ) + + expect(writeSpy).not.toHaveBeenCalled() + await expect(fs.readFile(filePath, 'utf8')).resolves.toBe(before) + }) + it('allows a pending report to reach one terminal status only', async () => { const { store } = await createStore() const report = await store.record(input()) @@ -179,11 +224,13 @@ describe('CrashReportStore', () => { const reloaded = new CrashReportStore(filePath) vi.spyOn(process, 'platform', 'get').mockReturnValue('win32') const readError = Object.assign(new Error('temporary read failure'), { code }) - const readFileSpy = vi.spyOn(fs, 'readFile').mockRejectedValueOnce(readError) + const readSpy = vi + .spyOn(boundedFileReader, 'readNodeFileWithinLimit') + .mockRejectedValueOnce(readError) await expect(reloaded.getLatestPending()).resolves.toMatchObject({ id: report.id }) - expect(readFileSpy).toHaveBeenCalledTimes(2) + expect(readSpy).toHaveBeenCalledTimes(2) if (code === 'EBUSY') { expect(grantDirAclAsyncMock).not.toHaveBeenCalled() } else { diff --git a/src/main/crash-reporting/crash-report-store.ts b/src/main/crash-reporting/crash-report-store.ts index 35f9eb5cb48..e0299dc1e4a 100644 --- a/src/main/crash-reporting/crash-report-store.ts +++ b/src/main/crash-reporting/crash-report-store.ts @@ -12,8 +12,14 @@ import { type CrashReportRecord, type CrashReportStatus } from '../../shared/crash-reporting' +import * as boundedFileReader from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' const MAX_REPORTS = 5 +export const MAX_CRASH_REPORT_FILE_BYTES = 4 * 1024 * 1024 +export const MAX_CRASH_REPORT_JSON_STRUCTURAL_TOKENS = 1_000_000 +export const MAX_CRASH_REPORT_JSON_NESTING_DEPTH = 128 const RELATED_CRASH_WINDOW_MS = 5_000 const WINDOWS_FILE_OPERATION_RETRY_DELAYS_MS = [50, 100, 150, 200, 250] @@ -202,8 +208,18 @@ export class CrashReportStore { try { const raw = await runCrashReportFileOperationWithWindowsRecovery( path.dirname(this.filePath), - () => fs.readFile(this.filePath, 'utf8') + async () => + ( + await boundedFileReader.readNodeFileWithinLimit( + this.filePath, + MAX_CRASH_REPORT_FILE_BYTES + ) + ).buffer.toString('utf8') ) + assertJsonTextStructureWithinLimits(raw, { + structuralTokens: MAX_CRASH_REPORT_JSON_STRUCTURAL_TOKENS, + nestingDepth: MAX_CRASH_REPORT_JSON_NESTING_DEPTH + }) const parsed = JSON.parse(raw) as Partial return Array.isArray(parsed.reports) ? parsed.reports.slice(0, MAX_REPORTS) : [] } catch (error) { @@ -219,8 +235,13 @@ export class CrashReportStore { const tmpPath = `${this.filePath}.${process.pid}.${Date.now()}.${crypto.randomUUID()}.tmp` try { await runCrashReportFileOperationWithWindowsRecovery(directory, async () => { + const { serialized } = stringifyJsonWithinByteLimit( + { reports }, + MAX_CRASH_REPORT_FILE_BYTES - Buffer.byteLength(os.EOL), + 2 + ) await fs.mkdir(directory, { recursive: true }) - await fs.writeFile(tmpPath, `${JSON.stringify({ reports }, null, 2)}${os.EOL}`, 'utf8') + await fs.writeFile(tmpPath, `${serialized}${os.EOL}`, 'utf8') await fs.rename(tmpPath, this.filePath) }) } finally { diff --git a/src/main/daemon/binary-frame.test.ts b/src/main/daemon/binary-frame.test.ts index 81148f18a59..468c34a0dab 100644 --- a/src/main/daemon/binary-frame.test.ts +++ b/src/main/daemon/binary-frame.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { encodeFrame, createFrameParser, FrameType, FRAME_HEADER_SIZE } from './binary-frame' +import { FRAME_MAX_PAYLOAD } from './types' describe('encodeFrame', () => { it('encodes a data frame with correct header', () => { @@ -130,6 +131,22 @@ describe('createFrameParser', () => { expect(onFrame.mock.calls[0][1].length).toBe(0) }) + it('rejects an oversized declared payload from the header alone', () => { + const onFrame = vi.fn() + const parser = createFrameParser(onFrame) + const header = Buffer.alloc(FRAME_HEADER_SIZE) + header[0] = FrameType.Data + header.writeUInt32BE(FRAME_MAX_PAYLOAD + 1, 1) + + expect(() => parser.feed(header)).toThrow( + `Frame payload ${FRAME_MAX_PAYLOAD + 1} exceeds max ${FRAME_MAX_PAYLOAD}` + ) + expect(onFrame).not.toHaveBeenCalled() + + parser.feed(encodeFrame(FrameType.Data, Buffer.from('fresh'))) + expect(onFrame).toHaveBeenCalledOnce() + }) + it('parses different frame types correctly', () => { const onFrame = vi.fn() const parser = createFrameParser(onFrame) diff --git a/src/main/daemon/binary-frame.ts b/src/main/daemon/binary-frame.ts index e8c4fd6ae90..c43a488cbe9 100644 --- a/src/main/daemon/binary-frame.ts +++ b/src/main/daemon/binary-frame.ts @@ -29,6 +29,10 @@ export function createFrameParser( function parse(): void { while (buffer.length >= FRAME_HEADER_SIZE) { const payloadLength = buffer.readUInt32BE(1) + if (payloadLength > FRAME_MAX_PAYLOAD) { + buffer = Buffer.alloc(0) + throw new Error(`Frame payload ${payloadLength} exceeds max ${FRAME_MAX_PAYLOAD}`) + } const totalLength = FRAME_HEADER_SIZE + payloadLength if (buffer.length < totalLength) { diff --git a/src/main/daemon/client.test.ts b/src/main/daemon/client.test.ts index a3d6d9f610d..eb9dbb8de90 100644 --- a/src/main/daemon/client.test.ts +++ b/src/main/daemon/client.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { DaemonClient } from './client' -import { encodeNdjson } from './ndjson' +import { DAEMON_HANDSHAKE_MAX_LINE_BYTES, encodeNdjson } from './ndjson' import type { HelloMessage, DaemonRequest, DaemonEvent } from './types' import { getDaemonSocketPath } from './daemon-spawner' @@ -310,6 +310,31 @@ describe('DaemonClient', () => { expect(socket.listenerCount('close')).toBe(0) }) + it('rejects a newline-free oversized hello response before the timeout', async () => { + client = new DaemonClient({ socketPath, tokenPath }) + const socket = new EventEmitter() as Socket + socket.write = vi.fn(() => true) as unknown as Socket['write'] + socket.destroy = vi.fn() as unknown as Socket['destroy'] + const sendHello = ( + client as unknown as { + sendHello( + socket: Socket, + token: string, + role: 'control' | 'stream', + timeoutMs: number + ): Promise + } + ).sendHello.bind(client) + + const promise = sendHello(socket, 'test-token-123', 'control', 60_000) + socket.emit('data', Buffer.alloc(DAEMON_HANDSHAKE_MAX_LINE_BYTES + 1, 0x78)) + + await expect(promise).rejects.toThrow('Invalid hello response') + expect(socket.listenerCount('data')).toBe(0) + expect(socket.listenerCount('error')).toBe(0) + expect(socket.listenerCount('close')).toBe(0) + }) + it('rejects when the daemon closes before hello completes', async () => { await startMockDaemon({ closeOnHello: true }) diff --git a/src/main/daemon/client.ts b/src/main/daemon/client.ts index d0dd2453471..5f9650ad6cf 100644 --- a/src/main/daemon/client.ts +++ b/src/main/daemon/client.ts @@ -1,9 +1,18 @@ /* eslint-disable max-lines -- Why: daemon handshake, RPC, stream events, and reconnect cleanup share one socket lifecycle. */ import { connect, type Socket } from 'node:net' -import { readFileSync } from 'node:fs' import { randomUUID } from 'node:crypto' import { StringDecoder } from 'node:string_decoder' -import { encodeNdjson, createNdjsonParser } from './ndjson' +import { + DAEMON_HANDSHAKE_MAX_LINE_BYTES, + encodeBoundedNdjson, + encodeNdjson, + createNdjsonParser +} from './ndjson' +import { + DAEMON_CONTROL_SOCKET_MAX_BUFFERED_BYTES, + DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT, + DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT +} from './daemon-admission-limits' import { CLEAN_DISCONNECT_PROTOCOL_VERSION, PROTOCOL_VERSION, @@ -18,10 +27,14 @@ import type { DaemonEvent } from './types' import { addNodePtyRecoveryHint } from './node-pty-error-hints' +import { readDaemonControlFileText } from './daemon-control-file-reader' const CONNECT_TIMEOUT_MS = 5000 const CONNECTION_ATTEMPT_WAIT_MS = CONNECT_TIMEOUT_MS * 4 const REQUEST_TIMEOUT_MS = 30000 +export const DAEMON_CLIENT_MAX_PENDING_REQUESTS = DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT +export const DAEMON_CLIENT_MAX_CONTROL_BUFFERED_BYTES = DAEMON_CONTROL_SOCKET_MAX_BUFFERED_BYTES +export const DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES = DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT export type DaemonClientOptions = { socketPath: string @@ -118,7 +131,7 @@ export class DaemonClient { attemptGeneration: number, sharedBudget: boolean ): Promise { - const token = readFileSync(this.tokenPath, 'utf-8').trim() + const token = readDaemonControlFileText(this.tokenPath).trim() const deadlineMs = Date.now() + timeoutMs const remainingMs = (): number => sharedBudget ? Math.max(1, deadlineMs - Date.now()) : timeoutMs @@ -196,9 +209,18 @@ export class DaemonClient { if (!this.connected || !this.controlSocket) { throw new DaemonProtocolError('Not connected') } + if (this.pendingRequests.size >= DAEMON_CLIENT_MAX_PENDING_REQUESTS) { + throw new DaemonProtocolError('Daemon client pending request limit reached') + } const id = `req-${++this.requestCounter}` const msg = { id, type, ...(payload !== undefined ? { payload } : {}) } + const encoded = encodeBoundedNdjson(msg, DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES + 1) + const socket = this.controlSocket + if (!this.canBufferControlMessage(socket, encoded)) { + socket.destroy(new DaemonProtocolError('Daemon client control buffer limit reached')) + throw new DaemonProtocolError('Daemon client control buffer limit reached') + } return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -212,7 +234,13 @@ export class DaemonClient { timer }) - this.controlSocket!.write(encodeNdjson(msg)) + try { + socket.write(encoded) + } catch (error) { + this.pendingRequests.delete(id) + clearTimeout(timer) + reject(error instanceof Error ? error : new DaemonProtocolError(String(error))) + } }) } @@ -223,7 +251,14 @@ export class DaemonClient { const id = `${NOTIFY_PREFIX}${++this.requestCounter}` const msg = { id, type, ...(payload !== undefined ? { payload } : {}) } - this.controlSocket.write(encodeNdjson(msg)) + const encoded = encodeBoundedNdjson(msg, DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES + 1) + if (!this.canBufferControlMessage(this.controlSocket, encoded)) { + this.controlSocket.destroy( + new DaemonProtocolError('Daemon client control buffer limit reached') + ) + return + } + this.controlSocket.write(encoded) } onEvent(listener: (event: unknown) => void): () => void { @@ -333,7 +368,6 @@ export class DaemonClient { role } - let buffer = '' let settled = false let timer: ReturnType | null = null const cleanup = (): void => { @@ -360,16 +394,12 @@ export class DaemonClient { // Why: daemon socket chunks can split emoji/box-drawing UTF-8 bytes. // Decoding each Buffer independently would permanently inject U+FFFD. const decoder = new StringDecoder('utf8') - const onData = (chunk: Buffer): void => { - buffer += decoder.write(chunk) - const newlineIdx = buffer.indexOf('\n') - if (newlineIdx === -1) { - return - } - - const line = buffer.slice(0, newlineIdx) - try { - const response = JSON.parse(line) as HelloResponse + const parser = createNdjsonParser( + (message) => { + if (settled) { + return + } + const response = message as HelloResponse if (response.ok) { const identity = parseDaemonEndpointIdentity(response.daemonIdentity) if ( @@ -385,9 +415,15 @@ export class DaemonClient { new DaemonProtocolError(addNodePtyRecoveryHint(response.error ?? 'Hello rejected')) ) } - } catch { - finish(new DaemonProtocolError('Invalid hello response')) + }, + () => finish(new DaemonProtocolError('Invalid hello response')), + { maxLineBytes: DAEMON_HANDSHAKE_MAX_LINE_BYTES } + ) + const onData = (chunk: Buffer): void => { + if (settled) { + return } + parser.feed(decoder.write(chunk)) } const onError = (error: Error): void => finish(error) const onClose = (): void => @@ -455,6 +491,11 @@ export class DaemonClient { return () => socket.off('data', onData) } + private canBufferControlMessage(socket: Socket, encoded: string): boolean { + const messageBytes = Buffer.byteLength(encoded, 'utf8') + return messageBytes <= DAEMON_CLIENT_MAX_CONTROL_BUFFERED_BYTES - socket.writableLength + } + private handleDisconnect(generation: number): void { if (!this.disconnectArmed || generation !== this.connectionGeneration) { return diff --git a/src/main/daemon/daemon-admission-limits.test.ts b/src/main/daemon/daemon-admission-limits.test.ts new file mode 100644 index 00000000000..daa7c00f914 --- /dev/null +++ b/src/main/daemon/daemon-admission-limits.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from 'vitest' +import { + DAEMON_CLIENT_ID_MAX_BYTES, + DAEMON_PTY_COMMAND_MAX_BYTES, + DAEMON_PTY_CWD_MAX_BYTES, + DAEMON_PTY_ENV_DELETE_MAX_ENTRIES, + DAEMON_PTY_ENV_MAX_BYTES, + DAEMON_PTY_ENV_MAX_ENTRIES, + DAEMON_PTY_ENV_VALUE_MAX_BYTES, + DAEMON_PTY_HISTORY_SEED_MAX_BYTES, + DAEMON_REQUEST_ID_MAX_BYTES, + DAEMON_SESSION_ID_MAX_BYTES, + daemonHelloAdmissionError, + daemonRequestAdmissionError +} from './daemon-admission-limits' +import { MAX_TERMINAL_COLS, MAX_TERMINAL_ROWS } from '../../shared/terminal-size-limits' + +function createRequest(payload: Record): Record { + return { id: 'req-1', type: 'createOrAttach', payload } +} + +describe('daemon admission limits', () => { + it('preserves ordinary cross-platform create fields without rewriting them', () => { + const request = createRequest({ + sessionId: 'repo::C:\\projects\\orca@@12345678', + cols: 120, + rows: 40, + cwd: '\\\\wsl.localhost\\Ubuntu\\home\\orca', + command: 'printf "ready"', + env: { PATH: 'C:\\Windows\\System32', ORCA_TERMINAL_HANDLE: 'term_123' }, + envToDelete: ['CODEX_HOME'], + terminalWindowsWslDistro: null, + historySeed: '\u001b[32mrestored\u001b[0m' + }) + + expect(daemonRequestAdmissionError(request)).toBeNull() + expect(request).toEqual( + expect.objectContaining({ + payload: expect.objectContaining({ + cwd: '\\\\wsl.localhost\\Ubuntu\\home\\orca', + terminalWindowsWslDistro: null + }) + }) + ) + }) + + it('measures hello and request identifiers by UTF-8 bytes', () => { + const exactClientId = 'é'.repeat(DAEMON_CLIENT_ID_MAX_BYTES / 2) + expect( + daemonHelloAdmissionError({ type: 'hello', role: 'control', clientId: exactClientId }) + ).toBeNull() + expect( + daemonHelloAdmissionError({ + type: 'hello', + role: 'control', + clientId: `${exactClientId}é` + }) + ).toContain(`${DAEMON_CLIENT_ID_MAX_BYTES} bytes`) + + const exactRequestId = 'é'.repeat(DAEMON_REQUEST_ID_MAX_BYTES / 2) + expect(daemonRequestAdmissionError({ id: exactRequestId, type: 'ping' })).toBeNull() + expect(daemonRequestAdmissionError({ id: `${exactRequestId}é`, type: 'ping' })).toContain( + `${DAEMON_REQUEST_ID_MAX_BYTES} bytes` + ) + }) + + it('rejects unbounded retained identifiers and PTY strings', () => { + expect( + daemonRequestAdmissionError({ + id: 'req', + type: 'getCwd', + payload: { sessionId: 's'.repeat(DAEMON_SESSION_ID_MAX_BYTES + 1) } + }) + ).toContain(`${DAEMON_SESSION_ID_MAX_BYTES} bytes`) + expect( + daemonRequestAdmissionError( + createRequest({ + sessionId: 's', + cwd: 'c'.repeat(DAEMON_PTY_CWD_MAX_BYTES + 1) + }) + ) + ).toContain(`${DAEMON_PTY_CWD_MAX_BYTES} bytes`) + expect( + daemonRequestAdmissionError( + createRequest({ + sessionId: 's', + command: 'c'.repeat(DAEMON_PTY_COMMAND_MAX_BYTES + 1) + }) + ) + ).toContain(`${DAEMON_PTY_COMMAND_MAX_BYTES} bytes`) + expect( + daemonRequestAdmissionError( + createRequest({ + sessionId: 's', + historySeed: 'h'.repeat(DAEMON_PTY_HISTORY_SEED_MAX_BYTES + 1) + }) + ) + ).toContain(`${DAEMON_PTY_HISTORY_SEED_MAX_BYTES} bytes`) + }) + + it('caps environment entry counts, individual values, and aggregate bytes', () => { + const tooManyEntries = Object.fromEntries( + Array.from({ length: DAEMON_PTY_ENV_MAX_ENTRIES + 1 }, (_, index) => [`K${index}`, 'v']) + ) + expect( + daemonRequestAdmissionError(createRequest({ sessionId: 's', env: tooManyEntries })) + ).toContain(`${DAEMON_PTY_ENV_MAX_ENTRIES} entries`) + + expect( + daemonRequestAdmissionError( + createRequest({ + sessionId: 's', + env: { VALUE: 'v'.repeat(DAEMON_PTY_ENV_VALUE_MAX_BYTES + 1) } + }) + ) + ).toContain(`${DAEMON_PTY_ENV_VALUE_MAX_BYTES} bytes`) + + const aggregate = Object.fromEntries( + Array.from({ length: 5 }, (_, index) => [ + `K${index}`, + 'v'.repeat(Math.floor(DAEMON_PTY_ENV_MAX_BYTES / 5)) + ]) + ) + expect( + daemonRequestAdmissionError(createRequest({ sessionId: 's', env: aggregate })) + ).toContain(`${DAEMON_PTY_ENV_MAX_BYTES} bytes`) + }) + + it('caps environment deletion lists and rejects malformed retained fields', () => { + expect( + daemonRequestAdmissionError( + createRequest({ + sessionId: 's', + envToDelete: Array.from( + { length: DAEMON_PTY_ENV_DELETE_MAX_ENTRIES + 1 }, + (_, index) => `K${index}` + ) + }) + ) + ).toContain(`${DAEMON_PTY_ENV_DELETE_MAX_ENTRIES} entries`) + expect( + daemonRequestAdmissionError(createRequest({ sessionId: 's', env: ['not-a-record'] })) + ).toBe('createOrAttach payload.env must be a string record') + expect(daemonRequestAdmissionError({ id: 'req', type: 'getCwd', payload: {} })).toBe( + 'getCwd payload.sessionId must be a string' + ) + }) + + it('rejects PTY dimensions that could force pathological native allocations', () => { + expect( + daemonRequestAdmissionError( + createRequest({ sessionId: 's', cols: MAX_TERMINAL_COLS + 1, rows: 24 }) + ) + ).toContain(`1 through ${MAX_TERMINAL_COLS}`) + expect( + daemonRequestAdmissionError({ + id: 'req', + type: 'resize', + payload: { sessionId: 's', cols: 80, rows: MAX_TERMINAL_ROWS + 1 } + }) + ).toContain(`1 through ${MAX_TERMINAL_ROWS}`) + }) +}) diff --git a/src/main/daemon/daemon-admission-limits.ts b/src/main/daemon/daemon-admission-limits.ts new file mode 100644 index 00000000000..babf2933550 --- /dev/null +++ b/src/main/daemon/daemon-admission-limits.ts @@ -0,0 +1,293 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' +import { terminalSizeAdmissionError } from '../../shared/terminal-size-limits' + +export const DAEMON_MAX_TRANSPORT_SOCKETS = 128 +export const DAEMON_MAX_CONTROL_CLIENTS = 32 +export const DAEMON_MAX_STREAM_ATTACHMENTS = 24 +export const DAEMON_HANDSHAKE_TIMEOUT_MS = 10_000 + +export const DAEMON_MAX_ACTIVE_REQUESTS = 256 +export const DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT = 128 +export const DAEMON_MAX_ACTIVE_REQUEST_BYTES = 32 * 1024 * 1024 +export const DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT = 16 * 1024 * 1024 +export const DAEMON_CONTROL_SOCKET_MAX_BUFFERED_BYTES = 32 * 1024 * 1024 + +export const DAEMON_CLIENT_ID_MAX_BYTES = 1024 +export const DAEMON_REQUEST_ID_MAX_BYTES = 1024 +export const DAEMON_REQUEST_TYPE_MAX_BYTES = 256 +export const DAEMON_SESSION_ID_MAX_BYTES = 4 * 1024 +export const DAEMON_PTY_CWD_MAX_BYTES = 256 * 1024 +export const DAEMON_PTY_COMMAND_MAX_BYTES = 4 * 1024 * 1024 +export const DAEMON_PTY_HISTORY_SEED_MAX_BYTES = 12 * 1024 * 1024 +export const DAEMON_PTY_ENV_MAX_ENTRIES = 4096 +export const DAEMON_PTY_ENV_NAME_MAX_BYTES = 32 * 1024 +export const DAEMON_PTY_ENV_VALUE_MAX_BYTES = 1024 * 1024 +export const DAEMON_PTY_ENV_MAX_BYTES = 4 * 1024 * 1024 +export const DAEMON_PTY_ENV_DELETE_MAX_ENTRIES = 4096 +export const DAEMON_PTY_ENV_DELETE_MAX_BYTES = 1024 * 1024 + +const REQUESTS_WITH_SESSION_ID = new Set([ + 'cancelCreateOrAttach', + 'closeStartupQueryAuthority', + 'write', + 'resize', + 'pausePty', + 'resumePty', + 'setSessionBackground', + 'kill', + 'signal', + 'detach', + 'getCwd', + 'getForegroundProcess', + 'confirmForegroundProcess', + 'clearScrollback', + 'getSnapshot', + 'getSize', + 'takePendingOutput' +]) + +type JsonRecord = Record + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function boundedStringError( + value: unknown, + field: string, + maxBytes: number, + options: { nonEmpty?: boolean; forbidNull?: boolean } = {} +): string | null { + if (typeof value !== 'string') { + return `${field} must be a string` + } + if (options.nonEmpty && value.length === 0) { + return `${field} must not be empty` + } + if (options.forbidNull && value.includes('\0')) { + return `${field} must not contain NUL` + } + if ( + value.length > maxBytes || + measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }).exceededLimit + ) { + return `${field} exceeds ${maxBytes} bytes` + } + return null +} + +function optionalBoundedStringError( + value: unknown, + field: string, + maxBytes: number +): string | null { + return value === undefined ? null : boundedStringError(value, field, maxBytes) +} + +function optionalNullableBoundedStringError( + value: unknown, + field: string, + maxBytes: number +): string | null { + return value === undefined || value === null ? null : boundedStringError(value, field, maxBytes) +} + +function createEnvironmentError(env: unknown): string | null { + if (env === undefined) { + return null + } + if (!isRecord(env)) { + return 'createOrAttach payload.env must be a string record' + } + + let entries = 0 + let retainedBytes = 0 + for (const name in env) { + if (!Object.prototype.hasOwnProperty.call(env, name)) { + continue + } + entries += 1 + if (entries > DAEMON_PTY_ENV_MAX_ENTRIES) { + return `createOrAttach payload.env exceeds ${DAEMON_PTY_ENV_MAX_ENTRIES} entries` + } + const nameError = boundedStringError( + name, + 'createOrAttach payload.env name', + DAEMON_PTY_ENV_NAME_MAX_BYTES + ) + if (nameError) { + return nameError + } + const value = env[name] + const valueError = boundedStringError( + value, + 'createOrAttach payload.env value', + DAEMON_PTY_ENV_VALUE_MAX_BYTES + ) + if (valueError) { + return valueError + } + retainedBytes += + measureUtf8ByteLength(name).byteLength + measureUtf8ByteLength(value as string).byteLength + if (retainedBytes > DAEMON_PTY_ENV_MAX_BYTES) { + return `createOrAttach payload.env exceeds ${DAEMON_PTY_ENV_MAX_BYTES} bytes` + } + } + return null +} + +function createEnvironmentDeleteError(envToDelete: unknown): string | null { + if (envToDelete === undefined) { + return null + } + if (!Array.isArray(envToDelete)) { + return 'createOrAttach payload.envToDelete must be a string array' + } + if (envToDelete.length > DAEMON_PTY_ENV_DELETE_MAX_ENTRIES) { + return `createOrAttach payload.envToDelete exceeds ${DAEMON_PTY_ENV_DELETE_MAX_ENTRIES} entries` + } + + let retainedBytes = 0 + for (const name of envToDelete) { + const nameError = boundedStringError( + name, + 'createOrAttach payload.envToDelete entry', + DAEMON_PTY_ENV_NAME_MAX_BYTES + ) + if (nameError) { + return nameError + } + retainedBytes += measureUtf8ByteLength(name as string).byteLength + if (retainedBytes > DAEMON_PTY_ENV_DELETE_MAX_BYTES) { + return `createOrAttach payload.envToDelete exceeds ${DAEMON_PTY_ENV_DELETE_MAX_BYTES} bytes` + } + } + return null +} + +function createOrAttachPayloadError(payload: unknown): string | null { + if (!isRecord(payload)) { + return 'createOrAttach payload must be an object' + } + + const checks = [ + () => + boundedStringError( + payload.sessionId, + 'createOrAttach payload.sessionId', + DAEMON_SESSION_ID_MAX_BYTES, + { nonEmpty: true, forbidNull: true } + ), + () => + optionalBoundedStringError( + payload.cwd, + 'createOrAttach payload.cwd', + DAEMON_PTY_CWD_MAX_BYTES + ), + () => + optionalBoundedStringError( + payload.command, + 'createOrAttach payload.command', + DAEMON_PTY_COMMAND_MAX_BYTES + ), + () => + optionalBoundedStringError( + payload.historySeed, + 'createOrAttach payload.historySeed', + DAEMON_PTY_HISTORY_SEED_MAX_BYTES + ), + () => + optionalBoundedStringError( + payload.shellOverride, + 'createOrAttach payload.shellOverride', + DAEMON_PTY_CWD_MAX_BYTES + ), + () => + optionalNullableBoundedStringError( + payload.terminalWindowsWslDistro, + 'createOrAttach payload.terminalWindowsWslDistro', + DAEMON_PTY_CWD_MAX_BYTES + ), + () => + terminalSizeAdmissionError(payload.cols, payload.rows, 'createOrAttach payload', { + allowMissing: true + }), + () => createEnvironmentError(payload.env), + () => createEnvironmentDeleteError(payload.envToDelete) + ] + for (const check of checks) { + const error = check() + if (error) { + return error + } + } + return null +} + +export function daemonHelloAdmissionError(value: unknown): string | null { + if (!isRecord(value) || value.type !== 'hello') { + return 'Expected hello' + } + if (value.role !== 'control' && value.role !== 'stream') { + return 'Invalid hello role' + } + return boundedStringError(value.clientId, 'hello clientId', DAEMON_CLIENT_ID_MAX_BYTES, { + nonEmpty: true, + forbidNull: true + }) +} + +export function daemonRequestAdmissionError(value: unknown): string | null { + if (!isRecord(value)) { + return 'Daemon request must be an object' + } + const idError = boundedStringError(value.id, 'Daemon request id', DAEMON_REQUEST_ID_MAX_BYTES, { + nonEmpty: true, + forbidNull: true + }) + if (idError) { + return idError + } + const typeError = boundedStringError( + value.type, + 'Daemon request type', + DAEMON_REQUEST_TYPE_MAX_BYTES, + { nonEmpty: true } + ) + if (typeError) { + return typeError + } + if (value.type === 'createOrAttach') { + return createOrAttachPayloadError(value.payload) + } + if (!REQUESTS_WITH_SESSION_ID.has(value.type as string)) { + return null + } + if (!isRecord(value.payload)) { + return `${value.type as string} payload must be an object` + } + const sessionIdError = boundedStringError( + value.payload.sessionId, + `${value.type as string} payload.sessionId`, + DAEMON_SESSION_ID_MAX_BYTES, + { nonEmpty: true, forbidNull: true } + ) + if (sessionIdError) { + return sessionIdError + } + return value.type === 'resize' + ? terminalSizeAdmissionError(value.payload.cols, value.payload.rows, 'resize payload') + : null +} + +export function getBoundedDaemonRequestId(value: unknown): string | null { + if (!isRecord(value)) { + return null + } + return boundedStringError(value.id, 'Daemon request id', DAEMON_REQUEST_ID_MAX_BYTES, { + nonEmpty: true, + forbidNull: true + }) === null + ? (value.id as string) + : null +} diff --git a/src/main/daemon/daemon-client-outbound-admission.test.ts b/src/main/daemon/daemon-client-outbound-admission.test.ts new file mode 100644 index 00000000000..33b3a983ea5 --- /dev/null +++ b/src/main/daemon/daemon-client-outbound-admission.test.ts @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + DAEMON_CLIENT_MAX_CONTROL_BUFFERED_BYTES, + DAEMON_CLIENT_MAX_PENDING_REQUESTS, + DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES, + DaemonClient +} from './client' + +type PendingRequest = { + resolve: (value: unknown) => void + reject: (reason: Error) => void + timer: ReturnType +} + +type FakeControlSocket = { + writableLength: number + write: ReturnType + destroy: ReturnType +} + +function createConnectedClient(): { + client: DaemonClient + socket: FakeControlSocket + pendingRequests: Map +} { + const client = new DaemonClient({ socketPath: 'unused', tokenPath: 'unused' }) + const socket: FakeControlSocket = { + writableLength: 0, + write: vi.fn(() => true), + destroy: vi.fn() + } + const pendingRequests = new Map() + const state = client as unknown as { + connected: boolean + controlSocket: FakeControlSocket + pendingRequests: Map + } + state.connected = true + state.controlSocket = socket + state.pendingRequests = pendingRequests + return { client, socket, pendingRequests } +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('DaemonClient outbound admission', () => { + it('rejects before serialization when the pending request cap is full', async () => { + vi.useFakeTimers() + const { client, socket, pendingRequests } = createConnectedClient() + for (let index = 0; index < DAEMON_CLIENT_MAX_PENDING_REQUESTS; index += 1) { + pendingRequests.set(`request-${index}`, { + resolve: () => {}, + reject: () => {}, + timer: setTimeout(() => {}, 60_000) + }) + } + + await expect( + client.request('write', { data: 'x'.repeat(DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES) }) + ).rejects.toThrow('pending request limit') + + expect(socket.write).not.toHaveBeenCalled() + expect(pendingRequests.size).toBe(DAEMON_CLIENT_MAX_PENDING_REQUESTS) + for (const pending of pendingRequests.values()) { + clearTimeout(pending.timer) + } + }) + + it('rejects a request whose serialized line exceeds the daemon contract', async () => { + const { client, socket, pendingRequests } = createConnectedClient() + + await expect( + client.request('write', { + data: 'x'.repeat(DAEMON_CLIENT_MAX_REQUEST_LINE_BYTES) + }) + ).rejects.toThrow() + + expect(socket.write).not.toHaveBeenCalled() + expect(pendingRequests.size).toBe(0) + }) + + it('rejects and closes before growing a saturated control buffer', async () => { + const { client, socket, pendingRequests } = createConnectedClient() + socket.writableLength = DAEMON_CLIENT_MAX_CONTROL_BUFFERED_BYTES + + await expect(client.request('listSessions', undefined)).rejects.toThrow('control buffer limit') + + expect(socket.write).not.toHaveBeenCalled() + expect(socket.destroy).toHaveBeenCalledOnce() + expect(pendingRequests.size).toBe(0) + }) + + it('releases request admission when socket.write throws', async () => { + vi.useFakeTimers() + const { client, socket, pendingRequests } = createConnectedClient() + socket.write.mockImplementation(() => { + throw new Error('write failed') + }) + + await expect(client.request('listSessions', undefined)).rejects.toThrow('write failed') + + expect(pendingRequests.size).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('drops a notification and closes before growing a saturated control buffer', () => { + const { client, socket } = createConnectedClient() + socket.writableLength = DAEMON_CLIENT_MAX_CONTROL_BUFFERED_BYTES + + client.notify('write', { sessionId: 'session-a', data: 'hello' }) + + expect(socket.write).not.toHaveBeenCalled() + expect(socket.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/daemon/daemon-control-file-reader.test.ts b/src/main/daemon/daemon-control-file-reader.test.ts new file mode 100644 index 00000000000..63fde6f746c --- /dev/null +++ b/src/main/daemon/daemon-control-file-reader.test.ts @@ -0,0 +1,40 @@ +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + MAX_DAEMON_CONTROL_FILE_BYTES, + readDaemonControlFileText +} from './daemon-control-file-reader' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function createPath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-daemon-control-')) + roots.push(root) + return join(root, 'control') +} + +describe('readDaemonControlFileText', () => { + it('accepts a control file exactly at the byte cap', () => { + const filePath = createPath() + writeFileSync(filePath, Buffer.alloc(MAX_DAEMON_CONTROL_FILE_BYTES, 0x61)) + + expect(readDaemonControlFileText(filePath)).toHaveLength(MAX_DAEMON_CONTROL_FILE_BYTES) + }) + + it('rejects a sparse control file beyond the byte cap', () => { + const filePath = createPath() + const file = openSync(filePath, 'w') + ftruncateSync(file, MAX_DAEMON_CONTROL_FILE_BYTES + 1) + closeSync(file) + + expect(() => readDaemonControlFileText(filePath)).toThrow('exceeds') + }) +}) diff --git a/src/main/daemon/daemon-control-file-reader.ts b/src/main/daemon/daemon-control-file-reader.ts new file mode 100644 index 00000000000..f8e1e114a37 --- /dev/null +++ b/src/main/daemon/daemon-control-file-reader.ts @@ -0,0 +1,9 @@ +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const MAX_DAEMON_CONTROL_FILE_BYTES = 64 * 1024 + +export function readDaemonControlFileText(filePath: string): string { + return readNodeFileSyncWithinLimit(filePath, MAX_DAEMON_CONTROL_FILE_BYTES).buffer.toString( + 'utf8' + ) +} diff --git a/src/main/daemon/daemon-entry.ts b/src/main/daemon/daemon-entry.ts index ba98af21c5b..b422e30dc85 100644 --- a/src/main/daemon/daemon-entry.ts +++ b/src/main/daemon/daemon-entry.ts @@ -6,7 +6,6 @@ * Signals readiness to parent via IPC: { type: 'ready' } * Shuts down cleanly on SIGTERM. */ -import { readFileSync } from 'node:fs' import { startDaemon, type DaemonHandle } from './daemon-main' import { createPtySubprocess } from './pty-subprocess' import { warmWindowsConptyOnce } from './windows-conpty-warmup' @@ -17,6 +16,7 @@ import { prepareMacosTccLoginShell, probeMacosLoginSessionAlive } from '../providers/macos-tcc-login-shell' +import { readDaemonLoginSessionProbeVerdict } from './daemon-login-session-probe-verdict' import { MacosLoginSessionDeathWatch } from './macos-login-session-death-watch' import { readCurrentProcessMacSystemResolverHealth } from '../network/macos-system-resolver-health' @@ -176,13 +176,7 @@ async function main(): Promise { // 'alive' → accepted/healthy, 'dead' → rejected/unhealthy, 'hang' → // timeout-inconclusive/unhealthy (the fail-safe path), else inconclusive. const e2eProbeFile = process.env.ORCA_E2E_LOGIN_SESSION_PROBE_FILE - const readE2eVerdict = (): string => { - try { - return readFileSync(e2eProbeFile as string, 'utf8').trim() - } catch { - return '' - } - } + const readE2eVerdict = (): string => readDaemonLoginSessionProbeVerdict(e2eProbeFile as string) deathWatch = loginSessionWatch && process.platform === 'darwin' ? new MacosLoginSessionDeathWatch({ diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 5e5e63e1104..f50a2d4e0ac 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,11 +3,12 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBe(26) + expect(PROTOCOL_VERSION).toBe(27) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(23) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(24) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(25) + expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(26) }) }) diff --git a/src/main/daemon/daemon-foreground-process-protocol.ts b/src/main/daemon/daemon-foreground-process-protocol.ts index 01b050c58d3..c0d672ec159 100644 --- a/src/main/daemon/daemon-foreground-process-protocol.ts +++ b/src/main/daemon/daemon-foreground-process-protocol.ts @@ -9,3 +9,7 @@ export type GetForegroundProcessRequest = { export type ConfirmForegroundProcessRequest = Omit & { type: 'confirmForegroundProcess' } + +export type InspectProcessRequest = Omit & { + type: 'inspectProcess' +} diff --git a/src/main/daemon/daemon-health-socket-cleanup.test.ts b/src/main/daemon/daemon-health-socket-cleanup.test.ts index 6daa6cdf00b..0bb7e5f7207 100644 --- a/src/main/daemon/daemon-health-socket-cleanup.test.ts +++ b/src/main/daemon/daemon-health-socket-cleanup.test.ts @@ -10,7 +10,8 @@ const { netConnectMock } = vi.hoisted(() => ({ vi.mock('net', () => ({ connect: netConnectMock })) -import { healthCheckDaemon, killStaleDaemon } from './daemon-health' +import { checkDaemonHealth, healthCheckDaemon, killStaleDaemon } from './daemon-health' +import { DAEMON_HANDSHAKE_MAX_LINE_BYTES } from './ndjson' class FakeSocket extends EventEmitter { destroy = vi.fn() @@ -71,6 +72,21 @@ describe('daemon health socket listener cleanup', () => { expect(socket.destroy).toHaveBeenCalledTimes(1) }) + it('rejects and releases a newline-free oversized health response', async () => { + const socket = new FakeSocket() + netConnectMock.mockReturnValueOnce(socket) + + const result = checkDaemonHealth(socketPath, tokenPath) + socket.emit('connect') + socket.emit('data', Buffer.alloc(DAEMON_HANDSHAKE_MAX_LINE_BYTES + 1, 0x78)) + + await expect(result).resolves.toBe('rejected') + expect(socket.listenerCount('connect')).toBe(0) + expect(socket.listenerCount('error')).toBe(0) + expect(socket.listenerCount('data')).toBe(0) + expect(socket.destroy).toHaveBeenCalledOnce() + }) + it('removes stale-socket probe listeners after a timeout', async () => { vi.useFakeTimers() const socket = new FakeSocket() diff --git a/src/main/daemon/daemon-health.ts b/src/main/daemon/daemon-health.ts index 50e7226f73c..d27456b3ec2 100644 --- a/src/main/daemon/daemon-health.ts +++ b/src/main/daemon/daemon-health.ts @@ -4,12 +4,13 @@ import { execFile, execFileSync } from 'node:child_process' import { existsSync, readFileSync, unlinkSync } from 'node:fs' import { connect, type Socket } from 'node:net' import { promisify } from 'node:util' +import { StringDecoder } from 'node:string_decoder' import { getProcessOutputFields, iterateProcessOutputLines } from '../../shared/process-output-field-scanner' import { isStartupDiagnosticsEnabled, logStartupDiagnostic } from '../startup/startup-diagnostics' -import { encodeNdjson } from './ndjson' +import { DAEMON_HANDSHAKE_MAX_LINE_BYTES, createNdjsonParser, encodeNdjson } from './ndjson' import { getDaemonPidPath } from './daemon-spawner' import { PROTOCOL_VERSION, @@ -18,6 +19,7 @@ import { type SystemResolverHealth, type SystemResolverHealthResult } from './types' +import { readDaemonControlFileText } from './daemon-control-file-reader' const HEALTH_CHECK_TIMEOUT_MS = 3_000 const RESOLVER_HEALTH_CHECK_TIMEOUT_MS = 3_000 @@ -89,7 +91,7 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis let token: string try { - token = readFileSync(tokenPath, 'utf8').trim() + token = readDaemonControlFileText(tokenPath).trim() } catch { resolve('unreachable') return @@ -123,55 +125,42 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis } sock?.write(encodeNdjson(hello)) } - const onData = (chunk: Buffer): void => { + const onMessage = (rawMessage: unknown): void => { if (settled) { return } - buffer += chunk.toString() - for (;;) { - const newlineIdx = buffer.indexOf('\n') - if (newlineIdx === -1) { - break - } - const line = buffer.slice(0, newlineIdx) - buffer = buffer.slice(newlineIdx + 1) - if (!line) { - continue - } - - let message: Record - try { - message = JSON.parse(line) as Record - } catch { + if (!rawMessage || typeof rawMessage !== 'object') { + settle('rejected') + return + } + const message = rawMessage as Record + if (message.type === 'hello') { + if (!(message as HelloResponse).ok) { settle('rejected') return } + // Why: a protocol-live daemon with a stale cwd or node-pty helper + // will answer ping but cannot create terminals, so reuse must check + // the PTY spawn prerequisites too. + sock?.write(encodeNdjson({ id: 'health-1', type: 'ptySpawnHealth' })) + return + } - if (message.type === 'hello') { - if (!(message as HelloResponse).ok) { - settle('rejected') - return - } - // Why: a protocol-live daemon with a stale cwd or node-pty helper - // will answer ping but cannot create terminals, so reuse must check - // the PTY spawn prerequisites too. - sock?.write(encodeNdjson({ id: 'health-1', type: 'ptySpawnHealth' })) - continue - } - - if (message.id === 'health-1') { - settle(message.ok === true ? 'healthy' : 'pty-spawn-unhealthy') - return - } + if (message.id === 'health-1') { + settle(message.ok === true ? 'healthy' : 'pty-spawn-unhealthy') } } + const decoder = new StringDecoder('utf8') + const parser = createNdjsonParser(onMessage, () => settle('rejected'), { + maxLineBytes: DAEMON_HANDSHAKE_MAX_LINE_BYTES + }) + const onData = (chunk: Buffer): void => parser.feed(decoder.write(chunk)) const timer = setTimeout(() => settle('unreachable'), HEALTH_CHECK_TIMEOUT_MS) sock = connect({ path: socketPath }) sock.on('error', onError) sock.on('connect', onConnect) - let buffer = '' sock.on('data', onData) }) } @@ -201,7 +190,7 @@ export function getMacDaemonSystemResolverHealth( let token: string try { - token = readFileSync(tokenPath, 'utf8').trim() + token = readDaemonControlFileText(tokenPath).trim() } catch { resolve('unknown') return @@ -235,59 +224,46 @@ export function getMacDaemonSystemResolverHealth( } sock?.write(encodeNdjson(hello)) } - const onData = (chunk: Buffer): void => { + const onMessage = (rawMessage: unknown): void => { if (settled) { return } - buffer += chunk.toString() - for (;;) { - const newlineIdx = buffer.indexOf('\n') - if (newlineIdx === -1) { - break - } - const line = buffer.slice(0, newlineIdx) - buffer = buffer.slice(newlineIdx + 1) - if (!line) { - continue - } - - let message: Record - try { - message = JSON.parse(line) as Record - } catch { + if (!rawMessage || typeof rawMessage !== 'object') { + settle('unknown') + return + } + const message = rawMessage as Record + if (message.type === 'hello') { + if (!(message as HelloResponse).ok) { settle('unknown') return } + // Why: the daemon must report health from inside its own process; + // external launchctl bsexec probes can misclassify healthy PTYs. + sock?.write(encodeNdjson({ id: 'resolver-health-1', type: 'systemResolverHealth' })) + return + } - if (message.type === 'hello') { - if (!(message as HelloResponse).ok) { - settle('unknown') - return - } - // Why: the daemon must report health from inside its own process; - // external launchctl bsexec probes can misclassify healthy PTYs. - sock?.write(encodeNdjson({ id: 'resolver-health-1', type: 'systemResolverHealth' })) - continue - } - - if (message.id === 'resolver-health-1') { - if (!message.ok || typeof message.payload !== 'object' || message.payload === null) { - settle('unknown') - return - } - const payload = message.payload as Partial - settle(isSystemResolverHealth(payload.health) ? payload.health : 'unknown') + if (message.id === 'resolver-health-1') { + if (!message.ok || typeof message.payload !== 'object' || message.payload === null) { + settle('unknown') return } + const payload = message.payload as Partial + settle(isSystemResolverHealth(payload.health) ? payload.health : 'unknown') } } + const decoder = new StringDecoder('utf8') + const parser = createNdjsonParser(onMessage, () => settle('unknown'), { + maxLineBytes: DAEMON_HANDSHAKE_MAX_LINE_BYTES + }) + const onData = (chunk: Buffer): void => parser.feed(decoder.write(chunk)) const timer = setTimeout(() => settle('unknown'), RESOLVER_HEALTH_CHECK_TIMEOUT_MS) sock = connect({ path: socketPath }) sock.on('error', onError) sock.on('connect', onConnect) - let buffer = '' sock.on('data', onData) }) } @@ -591,7 +567,7 @@ async function readVerifiedDaemonPid( let parsedPid: ParsedDaemonPid | null try { parsedPid = parseDaemonPidFile( - readFileSync(getDaemonPidPath(runtimeDir, protocolVersion), 'utf8') + readDaemonControlFileText(getDaemonPidPath(runtimeDir, protocolVersion)) ) } catch { return null @@ -638,7 +614,7 @@ export async function killStaleDaemon( const pidPath = getDaemonPidPath(runtimeDir, protocolVersion) let killedDaemon = false try { - const parsedPid = parseDaemonPidFile(readFileSync(pidPath, 'utf8')) + const parsedPid = parseDaemonPidFile(readDaemonControlFileText(pidPath)) if ( parsedPid && (await isDaemonProcess(parsedPid.pid, socketPath, tokenPath, parsedPid.startedAtMs)) diff --git a/src/main/daemon/daemon-host-relocation.ts b/src/main/daemon/daemon-host-relocation.ts index 4618fdbdfe5..0e8e3410600 100644 --- a/src/main/daemon/daemon-host-relocation.ts +++ b/src/main/daemon/daemon-host-relocation.ts @@ -3,8 +3,7 @@ import { cpSync, existsSync, mkdirSync, - readFileSync, - readdirSync, + opendirSync, renameSync, rmSync, writeFileSync @@ -12,6 +11,7 @@ import { import { dirname, join, win32 as winPath } from 'node:path' import { app } from 'electron' import { parseDaemonPidFile, startTimeMatches } from './daemon-health' +import { readDaemonControlFileText } from './daemon-control-file-reader' /** * Relocate the terminal daemon's process image out of the app install dir into LOCAL userData so it @@ -30,6 +30,7 @@ export type RelocatedDaemonHost = { const HOST_SUBDIR = 'daemon-host' const MARKER_NAME = '.materialized.json' +const MAX_PINNED_DAEMON_VERSIONS = 1024 // LOCAL appData (not roaming) so OneDrive/roaming never syncs this ~260MB runtime. Shared with NSIS uninstall (config/nsis/daemon-host-uninstall.nsh) — keep in sync. const LOCAL_HOST_ROOT_NAME = 'Orca' @@ -184,7 +185,7 @@ function executeManifest(ops: CopyOp[], stagingRoot: string): void { function readMarker(dir: string): MaterializeMarker | null { try { const parsed = JSON.parse( - readFileSync(join(dir, MARKER_NAME), 'utf8') + readDaemonControlFileText(join(dir, MARKER_NAME)) ) as Partial if (typeof parsed.version === 'string' && typeof parsed.entryRelPath === 'string') { return { @@ -289,25 +290,41 @@ function isDaemonPidAlive(pid: number, startedAtMs: number | null): boolean { */ export function collectPinnedDaemonVersions(runtimeDir: string): Set { const pinned = new Set() - let entries + let directory: ReturnType try { - entries = readdirSync(runtimeDir, { withFileTypes: true }) + directory = opendirSync(runtimeDir) } catch { return pinned } - for (const entry of entries) { - if (!entry.isFile() || !/^daemon-v\d+\.pid$/.test(entry.name)) { - continue + try { + while (pinned.size < MAX_PINNED_DAEMON_VERSIONS) { + const entry = directory.readSync() + if (!entry) { + break + } + if (!entry.isFile() || !/^daemon-v\d+\.pid$/.test(entry.name)) { + continue + } + let parsed + try { + parsed = parseDaemonPidFile(readDaemonControlFileText(join(runtimeDir, entry.name))) + } catch { + continue + } + // appVersion null => pre-relocation daemon forked from the install dir; pins no host dir here. + if ( + parsed && + parsed.appVersion !== null && + isDaemonPidAlive(parsed.pid, parsed.startedAtMs) + ) { + pinned.add(parsed.appVersion) + } } - let parsed + } finally { try { - parsed = parseDaemonPidFile(readFileSync(join(runtimeDir, entry.name), 'utf8')) + directory.closeSync() } catch { - continue - } - // appVersion null => pre-relocation daemon forked from the install dir; pins no host dir here. - if (parsed && parsed.appVersion !== null && isDaemonPidAlive(parsed.pid, parsed.startedAtMs)) { - pinned.add(parsed.appVersion) + // Best-effort discovery cleanup. } } return pinned @@ -321,22 +338,38 @@ export function pruneOldDaemonHosts(pinnedVersions: ReadonlySet): void { if (process.platform !== 'win32' || !app.isPackaged) { return } + // Why: an incomplete saturated pin scan must fail closed rather than remove a live daemon image. + if (pinnedVersions.size >= MAX_PINNED_DAEMON_VERSIONS) { + return + } const version = app.getVersion() const root = hostRootDir() - let entries + let directory: ReturnType try { - entries = readdirSync(root, { withFileTypes: true }) + directory = opendirSync(root) } catch { return } - for (const entry of entries) { - if (!entry.isDirectory() || entry.name === version || pinnedVersions.has(entry.name)) { - continue + try { + while (true) { + const entry = directory.readSync() + if (!entry) { + break + } + if (!entry.isDirectory() || entry.name === version || pinnedVersions.has(entry.name)) { + continue + } + try { + rmSync(join(root, entry.name), { recursive: true, force: true }) + } catch { + // Still locked or already gone — retry on a future launch. + } } + } finally { try { - rmSync(join(root, entry.name), { recursive: true, force: true }) + directory.closeSync() } catch { - // Still locked or already gone — retry on a future launch. + // Best-effort pruning cleanup. } } } diff --git a/src/main/daemon/daemon-init.test.ts b/src/main/daemon/daemon-init.test.ts index 5a3003f767d..0f5d216fd81 100644 --- a/src/main/daemon/daemon-init.test.ts +++ b/src/main/daemon/daemon-init.test.ts @@ -235,6 +235,10 @@ vi.mock('fs', () => ({ writeFileSync: writeFileSyncMock })) +vi.mock('./daemon-control-file-reader', () => ({ + readDaemonControlFileText: readFileSyncMock +})) + vi.mock('child_process', () => ({ fork: forkMock })) vi.mock('net', () => ({ connect: netConnectMock })) diff --git a/src/main/daemon/daemon-init.ts b/src/main/daemon/daemon-init.ts index a748ea10a8b..b242ecca362 100644 --- a/src/main/daemon/daemon-init.ts +++ b/src/main/daemon/daemon-init.ts @@ -3,7 +3,7 @@ restart, teardown); the "swap the provider atomically" invariant keeps restart + import { join } from 'node:path' import { randomUUID } from 'node:crypto' import { app } from 'electron' -import { mkdirSync, existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs' +import { mkdirSync, existsSync, unlinkSync, writeFileSync } from 'node:fs' import { fork, type ChildProcess } from 'node:child_process' import { connect } from 'node:net' import { @@ -51,6 +51,7 @@ import { confirmSeededClaudeLivePtys, hasSeededUnconfirmedClaudePtys } from '../claude-accounts/live-pty-gate' +import { readDaemonControlFileText } from './daemon-control-file-reader' // Why: daemon init runs concurrent with window load, so an in-process t timestamp (not harness stderr timing) measures cold-start. function logDaemonMilestone(event: string, details: Record = {}): void { @@ -1015,7 +1016,7 @@ async function waitForDaemonEndpointExit(socketPath: string): Promise { function legacyDaemonProcessMayBeAlive(runtimeDir: string, protocolVersion: number): boolean { try { const parsed = parseDaemonPidFile( - readFileSync(getDaemonPidPath(runtimeDir, protocolVersion), 'utf8') + readDaemonControlFileText(getDaemonPidPath(runtimeDir, protocolVersion)) ) if (!parsed) { return false diff --git a/src/main/daemon/daemon-login-session-probe-verdict.test.ts b/src/main/daemon/daemon-login-session-probe-verdict.test.ts new file mode 100644 index 00000000000..e870d113aff --- /dev/null +++ b/src/main/daemon/daemon-login-session-probe-verdict.test.ts @@ -0,0 +1,33 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + MAX_DAEMON_LOGIN_SESSION_PROBE_BYTES, + readDaemonLoginSessionProbeVerdict +} from './daemon-login-session-probe-verdict' + +describe('daemon login-session probe verdict', () => { + const cleanupPaths: string[] = [] + + afterEach(async () => { + await Promise.all(cleanupPaths.splice(0).map((path) => rm(path, { recursive: true }))) + }) + + async function createProbe(content: string): Promise { + const directory = await mkdtemp(join(tmpdir(), 'orca-daemon-login-probe-')) + cleanupPaths.push(directory) + const path = join(directory, 'verdict') + await writeFile(path, content) + return path + } + + it('reads a normal verdict', async () => { + expect(readDaemonLoginSessionProbeVerdict(await createProbe(' alive\n'))).toBe('alive') + }) + + it('fails inconclusive when the verdict file exceeds its cap', async () => { + const path = await createProbe('x'.repeat(MAX_DAEMON_LOGIN_SESSION_PROBE_BYTES + 1)) + expect(readDaemonLoginSessionProbeVerdict(path)).toBe('') + }) +}) diff --git a/src/main/daemon/daemon-login-session-probe-verdict.ts b/src/main/daemon/daemon-login-session-probe-verdict.ts new file mode 100644 index 00000000000..16df15bb83a --- /dev/null +++ b/src/main/daemon/daemon-login-session-probe-verdict.ts @@ -0,0 +1,13 @@ +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const MAX_DAEMON_LOGIN_SESSION_PROBE_BYTES = 64 + +export function readDaemonLoginSessionProbeVerdict(filePath: string): string { + try { + return readNodeFileSyncWithinLimit(filePath, MAX_DAEMON_LOGIN_SESSION_PROBE_BYTES) + .buffer.toString('utf8') + .trim() + } catch { + return '' + } +} diff --git a/src/main/daemon/daemon-protocol-version.test.ts b/src/main/daemon/daemon-protocol-version.test.ts index 9fb4784b687..e557263697d 100644 --- a/src/main/daemon/daemon-protocol-version.test.ts +++ b/src/main/daemon/daemon-protocol-version.test.ts @@ -2,17 +2,19 @@ import { describe, expect, it } from 'vitest' import { AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, + COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION, PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './daemon-protocol-version' describe('daemon protocol version', () => { - it('ships claim and incarnation authority after startup-ingress generations', () => { - expect(PROTOCOL_VERSION).toBe(26) + it('ships strict completion inspection after claim authority', () => { + expect(PROTOCOL_VERSION).toBe(27) + expect(COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION).toBe(27) expect(AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION).toBe(26) expect(AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION).toBe(26) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toEqual( - Array.from({ length: 25 }, (_, index) => index + 1) + Array.from({ length: 26 }, (_, index) => index + 1) ) }) }) diff --git a/src/main/daemon/daemon-protocol-version.ts b/src/main/daemon/daemon-protocol-version.ts index e8c88774f53..8afcfdf3e22 100644 --- a/src/main/daemon/daemon-protocol-version.ts +++ b/src/main/daemon/daemon-protocol-version.ts @@ -1,12 +1,13 @@ // Why: daemons survive app updates, so wire behavior must be version-gated. -export const PROTOCOL_VERSION = 26 +export const PROTOCOL_VERSION = 27 +export const COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION = 27 export const PTY_STARTUP_INGRESS_PROTOCOL_VERSION = 25 export const AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION = 26 export const AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION = 26 export const GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION = 22 export const CLEAN_DISCONNECT_PROTOCOL_VERSION = 24 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ] as const export function supportsPtyStartupIngress(protocolVersion: number): boolean { diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 73c21f42f5d..b66a8d6e273 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -868,6 +868,60 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { }) describe('listProcesses', () => { + it('strips unknown daemon owner payloads before publication', () => { + const normalize = ( + adapter as unknown as { + validatedAgentSessionOwners(owners: unknown): { + agentSessionOwners?: unknown[] + } + } + ).validatedAgentSessionOwners.bind(adapter) + const owner = { + claim: { + digestVersion: 1, + keyId: 'key', + identityDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + worktreeScopeDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + agent: 'codex', + unknownPayload: 'claim payload' + }, + generation: 'generation-1', + phase: 'live', + ptyId: 'pty-1', + surface: { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed', + unknownPayload: 'surface payload' + }, + unknownPayload: 'owner payload' + } + + expect(normalize([owner])).toEqual({ + agentSessionOwners: [ + { + claim: { + digestVersion: 1, + keyId: 'key', + identityDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + worktreeScopeDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + agent: 'codex' + }, + generation: 'generation-1', + phase: 'live', + ptyId: 'pty-1', + surface: { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed' + } + } + ] + }) + }) + it('returns active sessions', async () => { await adapter.spawn({ cols: 80, @@ -885,6 +939,26 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(procs[0].cwd).toBe('/repo/owned-before-osc7') expect(procs[0].worktreeId).toBe('repo::/repo/owned-before-osc7') }) + + it('reports the daemon session WSL owner', async () => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform') + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + try { + const spawned = await adapter.spawn({ + cols: 80, + rows: 24, + cwd: '\\\\wsl.localhost\\Ubuntu\\home\\jin\\repo' + }) + + const procs = await adapter.listProcesses() + + expect(procs.find((process) => process.id === spawned.id)?.wslDistro).toBe('Ubuntu') + } finally { + if (platform) { + Object.defineProperty(process, 'platform', platform) + } + } + }) }) describe('hasChildProcesses / getForegroundProcess', () => { @@ -1292,7 +1366,6 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { it('does not schedule a checkpoint timer until a session is dirty', async () => { const adapterClass = DaemonPtyAdapter as unknown as { CHECKPOINT_INTERVAL_MS: number } const previousInterval = adapterClass.CHECKPOINT_INTERVAL_MS - const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout') adapterClass.CHECKPOINT_INTERVAL_MS = 10_000 try { @@ -1303,14 +1376,16 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { cwd: '/home/user', sessionId: 'idle-checkpoint' }) + const internals = historyAdapter as unknown as { + checkpointTimer: ReturnType | null + } - expect(setTimeoutSpy.mock.calls.some(([, delay]) => delay === 10_000)).toBe(false) + expect(internals.checkpointTimer).toBeNull() lastSubprocess._simulateData('dirty after idle\r\n') - await waitFor(() => setTimeoutSpy.mock.calls.some(([, delay]) => delay === 10_000)) + await waitFor(() => internals.checkpointTimer !== null) } finally { adapterClass.CHECKPOINT_INTERVAL_MS = previousInterval - setTimeoutSpy.mockRestore() } }) diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index fc20ae5e110..9f4bb2f809f 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -10,6 +10,7 @@ import { supportsPtyStartupBarrier } from './shell-ready' import { CODEX_SHELL_READY_TIMEOUT_MS } from './session' import { CLEAN_DISCONNECT_PROTOCOL_VERSION, + COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION, AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION, @@ -27,6 +28,8 @@ import { isAgentSessionOwnerBinding, type AgentSessionOwnerBinding } from '../../shared/agent-session-host-authority' +import { MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES } from '../../shared/claimed-agent-pty-owner' +import { cloneAgentSessionOwnerBinding } from '../../shared/claimed-agent-pty-owner-snapshot' import type { IPtyProvider, PtyBackgroundStreamEvent, @@ -43,6 +46,7 @@ import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-d import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' import type { PtyIncarnationId } from '../../shared/pty-incarnation' import { resolveSafePtyDefaultCwd } from '../providers/pty-default-cwd' +import { PtyProcessListAdmission } from '../providers/pty-process-list-admission' type ColdRestorePayload = { scrollback: string @@ -822,6 +826,18 @@ export class DaemonPtyAdapter implements IPtyProvider { return foregroundProcess !== null && !isShellProcess(foregroundProcess) } + async inspectProcess( + id: string + ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> { + if (this.protocolVersion < COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION) { + throw new Error('terminal_liveness_unavailable') + } + return this.client.request<{ + foregroundProcess: string | null + hasChildProcesses: boolean + }>('inspectProcess', { sessionId: id }) + } + async getForegroundProcess(id: string): Promise { try { const result = await this.client.request<{ foregroundProcess: string | null }>( @@ -909,21 +925,28 @@ export class DaemonPtyAdapter implements IPtyProvider { undefined, remainingRequestTimeoutMs(opts?.deadlineMs) ) - return result.sessions - .filter((s) => s.isAlive) - .map((s) => { - const { worktreeId } = parsePtySessionId(s.sessionId) - return { - id: s.sessionId, - ...(s.incarnationId ? { incarnationId: s.incarnationId } : {}), + const admission = new PtyProcessListAdmission() + const processes: PtyProcessInfo[] = [] + for (const session of result.sessions) { + if (!session.isAlive) { + continue + } + const { worktreeId } = parsePtySessionId(session.sessionId) + processes.push( + admission.admit({ + id: session.sessionId, + ...(session.incarnationId ? { incarnationId: session.incarnationId } : {}), // Why: OSC 7 may not arrive before cleanup; spawn cwd is authoritative until the daemon reports a live cwd. - cwd: s.cwd ?? this.initialCwds.get(s.sessionId) ?? '', + cwd: session.cwd ?? this.initialCwds.get(session.sessionId) ?? '', title: 'shell', ...(worktreeId ? { worktreeId } : {}), - ...(s.terminalHandle ? { terminalHandle: s.terminalHandle } : {}), - ...this.validatedAgentSessionOwners(s.agentSessionOwners) - } - }) + ...(session.terminalHandle ? { terminalHandle: session.terminalHandle } : {}), + ...(session.wslDistro !== undefined ? { wslDistro: session.wslDistro } : {}), + ...this.validatedAgentSessionOwners(session.agentSessionOwners) + }) + ) + } + return processes } private validatedAgentSessionOwners( @@ -932,10 +955,16 @@ export class DaemonPtyAdapter implements IPtyProvider { if (owners === undefined) { return {} } - if (!Array.isArray(owners) || !owners.every(isAgentSessionOwnerBinding)) { + if ( + !Array.isArray(owners) || + owners.length > MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES || + !owners.every((owner) => isAgentSessionOwnerBinding(owner) && owner.phase === 'live') + ) { throw new Error('agent_session_ownership_unknown') } - return owners.length > 0 ? { agentSessionOwners: owners } : {} + return owners.length > 0 + ? { agentSessionOwners: owners.map(cloneAgentSessionOwnerBinding) } + : {} } // Why: the Manage Sessions panel needs the full SessionInfo (pid, state, diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index e003cbede2c..155a252e538 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -7,6 +7,7 @@ import { AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION } from './types' +import { MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES } from '../providers/pty-process-list-admission' type AdapterMock = DaemonPtyAdapter & { emitData: (id: string, data: string, sequenceChars?: number) => void @@ -80,6 +81,7 @@ function createAdapter( acknowledgeDataEvent: vi.fn(), hasChildProcesses: vi.fn(async () => false), getForegroundProcess: vi.fn(async () => null), + inspectProcess: vi.fn(async () => ({ foregroundProcess: null, hasChildProcesses: false })), confirmForegroundProcess: vi.fn(async () => `${label}-confirmed`), serialize: vi.fn(async () => '{}'), revive: vi.fn(async () => {}), @@ -140,6 +142,15 @@ function createAdapter( } as unknown as AdapterMock } +it('rejects completion inspection when no daemon owns the session', async () => { + const router = new DaemonPtyRouter({ + current: createAdapter('current'), + legacy: [createAdapter('legacy')] + }) + + await expect(router.inspectProcess('unmapped-session')).rejects.toThrow('terminal_gone') +}) + describe('DaemonPtyRouter', () => { it('reports separate conservative resume and fresh-create boundaries', () => { const current = createAdapter( @@ -411,6 +422,17 @@ describe('DaemonPtyRouter', () => { await expect(router.listProcesses()).rejects.toThrow('legacy unavailable') }) + it('fails listProcesses closed when adapters amplify the aggregate listing', async () => { + const current = createAdapter( + 'current', + buildSessionIds('current', MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES) + ) + const legacy = createAdapter('legacy', ['legacy-over-cap']) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.listProcesses()).rejects.toThrow('pty_process_list_capacity') + }) + it('merges startup reconciliation and updates route mappings', async () => { const current = createAdapter('current', [], { alive: ['current-alive'], diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 4d1e3d723de..e5b71eac1d2 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -8,6 +8,10 @@ import type { PtySpawnResult } from '../providers/types' import type { PtyIncarnationId } from '../../shared/pty-incarnation' +import { + collectPtyProcessListings, + PtyProcessListAdmission +} from '../providers/pty-process-list-admission' export class DaemonPtyRouter implements IPtyProvider { private current: DaemonPtyAdapter @@ -49,10 +53,12 @@ export class DaemonPtyRouter implements IPtyProvider { } async discoverLegacySessions(): Promise { + const admission = new PtyProcessListAdmission() for (const adapter of this.legacy) { try { const sessions = await adapter.listProcesses() - for (const session of sessions) { + for (const rawSession of sessions) { + const session = admission.admit(rawSession) this.sessionAdapters.set(session.id, adapter) } } catch (error) { @@ -189,6 +195,12 @@ export class DaemonPtyRouter implements IPtyProvider { return this.adapterFor(id).getForegroundProcess(id) } + async inspectProcess( + id: string + ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> { + return this.adapterForInspection(id).inspectProcess(id) + } + async confirmForegroundProcess(id: string): Promise { return this.adapterFor(id).confirmForegroundProcess(id) } @@ -204,10 +216,9 @@ export class DaemonPtyRouter implements IPtyProvider { async listProcesses(opts?: { deadlineMs?: number }): Promise { // Why: runtime exact-stop/liveness flows must fail closed if any adapter // cannot provide a trustworthy process list. - const results = await Promise.all( - this.allAdapters().map((adapter) => adapter.listProcesses(opts)) + return await collectPtyProcessListings(this.allAdapters(), (adapter) => + adapter.listProcesses(opts) ) - return results.flat() } async getDefaultShell(): Promise { @@ -348,6 +359,17 @@ export class DaemonPtyRouter implements IPtyProvider { return this.sessionAdapters.get(sessionId) ?? this.current } + private adapterForInspection(sessionId: string): DaemonPtyAdapter { + const adapter = + this.sessionAdapters.get(sessionId) ?? + this.allAdapters().find((candidate) => candidate.hasPty(sessionId)) + if (!adapter) { + throw new Error('terminal_gone') + } + this.sessionAdapters.set(sessionId, adapter) + return adapter + } + private allAdapters(): DaemonPtyAdapter[] { return [this.current, ...this.legacy] } diff --git a/src/main/daemon/daemon-pty-size.ts b/src/main/daemon/daemon-pty-size.ts index 13f8dfe2bc3..2a4b02e96a7 100644 --- a/src/main/daemon/daemon-pty-size.ts +++ b/src/main/daemon/daemon-pty-size.ts @@ -1,13 +1,9 @@ -const DEFAULT_COLS = 80 -const DEFAULT_ROWS = 24 +import { isValidTerminalSize, normalizeTerminalSize } from '../../shared/terminal-size-limits' export function isValidPtySize(cols: number, rows: number): boolean { - return Number.isFinite(cols) && Number.isFinite(rows) && cols >= 1 && rows >= 1 + return isValidTerminalSize(cols, rows) } export function normalizePtySize(cols: number, rows: number): { cols: number; rows: number } { - if (isValidPtySize(cols, rows)) { - return { cols, rows } - } - return { cols: DEFAULT_COLS, rows: DEFAULT_ROWS } + return normalizeTerminalSize(cols, rows) } diff --git a/src/main/daemon/daemon-response-admission.ts b/src/main/daemon/daemon-response-admission.ts new file mode 100644 index 00000000000..815b5f6fb6c --- /dev/null +++ b/src/main/daemon/daemon-response-admission.ts @@ -0,0 +1,20 @@ +import type { DaemonRequest } from './types' + +export const DAEMON_MAX_RESPONSE_BYTES = 16 * 1024 * 1024 +export const DAEMON_RESPONSE_RESERVATION_BYTES = DAEMON_MAX_RESPONSE_BYTES +export const DAEMON_MAX_ACTIVE_RESPONSE_BYTES = 128 * 1024 * 1024 +export const DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT = 64 * 1024 * 1024 +export const DAEMON_CONTROL_PROCESS_MAX_BUFFERED_BYTES = 64 * 1024 * 1024 + +const REQUESTS_WITH_POTENTIALLY_LARGE_RESULTS = new Set([ + 'createOrAttach', + 'getSnapshot', + 'listSessions', + 'takePendingOutput' +]) + +export function daemonResponseReservationBytes(request: DaemonRequest): number { + return REQUESTS_WITH_POTENTIALLY_LARGE_RESULTS.has(request.type) + ? DAEMON_RESPONSE_RESERVATION_BYTES + : 0 +} diff --git a/src/main/daemon/daemon-server-admission.test.ts b/src/main/daemon/daemon-server-admission.test.ts new file mode 100644 index 00000000000..c7ec4b8d9c0 --- /dev/null +++ b/src/main/daemon/daemon-server-admission.test.ts @@ -0,0 +1,354 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { connect, type Socket } from 'node:net' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT, + DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT, + DAEMON_MAX_CONTROL_CLIENTS, + DAEMON_MAX_STREAM_ATTACHMENTS, + DAEMON_MAX_TRANSPORT_SOCKETS +} from './daemon-admission-limits' +import { + DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT, + DAEMON_RESPONSE_RESERVATION_BYTES +} from './daemon-response-admission' +import { DaemonClient } from './client' +import { DaemonServer } from './daemon-server' +import { encodeNdjson } from './ndjson' +import { getDaemonSocketPath } from './daemon-spawner' +import { PROTOCOL_VERSION, type DaemonRequest } from './types' + +type ConnectedClientState = { + clientId: string + controlSocket: Socket + streamSocket: Socket | null + activeRequestCount: number + activeRequestBytes: number + activeResponseBytes: number +} + +type DaemonServerAdmissionState = { + transportSockets: Set + clients: Map + activeRequestCount: number + activeRequestBytes: number + activeResponseBytes: number + pendingPtySpawnPreparations: Map> + dispatchRequest(socket: Socket, clientId: string, value: unknown, lineBytes: number): void +} + +function readJsonLine(socket: Socket): Promise> { + return new Promise((resolve, reject) => { + let buffered = '' + const cleanup = (): void => { + socket.off('data', onData) + socket.off('error', onError) + socket.off('close', onClose) + } + const onData = (chunk: Buffer): void => { + buffered += chunk.toString('utf8') + const newline = buffered.indexOf('\n') + if (newline === -1) { + return + } + cleanup() + resolve(JSON.parse(buffered.slice(0, newline)) as Record) + } + const onError = (error: Error): void => { + cleanup() + reject(error) + } + const onClose = (): void => { + cleanup() + reject(new Error('Socket closed before a response')) + } + socket.on('data', onData) + socket.on('error', onError) + socket.on('close', onClose) + }) +} + +describe('DaemonServer admission', () => { + let directory: string + let socketPath: string + let tokenPath: string + let server: DaemonServer + let client: DaemonClient | null + let sockets: Socket[] + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'daemon-admission-test-')) + socketPath = getDaemonSocketPath(directory) + tokenPath = join(directory, 'daemon.token') + client = null + sockets = [] + }) + + afterEach(async () => { + client?.disconnect() + for (const socket of sockets) { + socket.destroy() + } + await server?.shutdown() + rmSync(directory, { recursive: true, force: true }) + }) + + async function startServer( + preparePtySpawn?: () => Promise, + ptySpawnHealthCheck?: () => Promise + ): Promise { + server = new DaemonServer({ + socketPath, + tokenPath, + ...(preparePtySpawn ? { preparePtySpawn } : {}), + ...(ptySpawnHealthCheck ? { ptySpawnHealthCheck } : {}), + spawnSubprocess: () => { + throw new Error('Test unexpectedly spawned a subprocess') + } + }) + await server.start() + } + + async function openSocket(): Promise { + const socket = connect(socketPath) + sockets.push(socket) + await new Promise((resolve, reject) => { + socket.once('connect', resolve) + socket.once('error', reject) + }) + return socket + } + + async function hello( + role: 'control' | 'stream', + clientId: string + ): Promise<{ socket: Socket; response: Record }> { + const socket = await openSocket() + const responsePromise = readJsonLine(socket) + socket.write( + encodeNdjson({ + type: 'hello', + version: PROTOCOL_VERSION, + token: readFileSync(tokenPath, 'utf8').trim(), + clientId, + role + }) + ) + return { socket, response: await responsePromise } + } + + async function waitFor(predicate: () => boolean): Promise { + await vi.waitFor(() => expect(predicate()).toBe(true)) + } + + it('bounds pre-auth transports and releases capacity after disconnect', async () => { + await startServer() + const state = server as unknown as DaemonServerAdmissionState + + await Promise.all(Array.from({ length: DAEMON_MAX_TRANSPORT_SOCKETS }, () => openSocket())) + await waitFor(() => state.transportSockets.size === DAEMON_MAX_TRANSPORT_SOCKETS) + + const overflow = await openSocket() + await waitFor(() => overflow.destroyed) + expect(state.transportSockets.size).toBe(DAEMON_MAX_TRANSPORT_SOCKETS) + + sockets[0].destroy() + await waitFor(() => state.transportSockets.size === DAEMON_MAX_TRANSPORT_SOCKETS - 1) + await openSocket() + await waitFor(() => state.transportSockets.size === DAEMON_MAX_TRANSPORT_SOCKETS) + }) + + it('bounds control clients and stream attachments, then reuses released slots', async () => { + await startServer() + const state = server as unknown as DaemonServerAdmissionState + const controls: Socket[] = [] + + for (let index = 0; index < DAEMON_MAX_CONTROL_CLIENTS; index += 1) { + const result = await hello('control', `client-${index}`) + expect(result.response.ok).toBe(true) + controls.push(result.socket) + } + await expect(hello('control', 'control-overflow')).resolves.toMatchObject({ + response: { ok: false, retryable: true, error: expect.stringContaining('control-client') } + }) + expect(state.clients.size).toBe(DAEMON_MAX_CONTROL_CLIENTS) + + for (let index = 0; index < DAEMON_MAX_STREAM_ATTACHMENTS; index += 1) { + await expect(hello('stream', `client-${index}`)).resolves.toMatchObject({ + response: { ok: true } + }) + } + await expect(hello('stream', `client-${DAEMON_MAX_STREAM_ATTACHMENTS}`)).resolves.toMatchObject( + { + response: { + ok: false, + retryable: true, + error: expect.stringContaining('stream-attachment') + } + } + ) + + controls[0].destroy() + await waitFor(() => state.clients.size === DAEMON_MAX_CONTROL_CLIENTS - 1) + await expect(hello('control', 'replacement-control')).resolves.toMatchObject({ + response: { ok: true } + }) + await expect(hello('stream', `client-${DAEMON_MAX_STREAM_ATTACHMENTS}`)).resolves.toMatchObject( + { + response: { ok: true } + } + ) + }) + + it('bounds concurrent response construction and cleans ownership after disconnect', async () => { + let finishPreparation!: () => void + const preparation = new Promise((resolve) => { + finishPreparation = resolve + }) + const preparePtySpawn = vi.fn(() => preparation) + await startServer(preparePtySpawn) + client = new DaemonClient({ socketPath, tokenPath }) + await client.ensureConnected() + const state = server as unknown as DaemonServerAdmissionState + + const maxLargeResponses = + DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT / DAEMON_RESPONSE_RESERVATION_BYTES + const pending = Array.from({ length: maxLargeResponses }, (_, index) => + client! + .request('createOrAttach', { + sessionId: `pending-${index}`, + cols: 80, + rows: 24 + }) + .catch((error: unknown) => error) + ) + await waitFor(() => preparePtySpawn.mock.calls.length === maxLargeResponses) + await expect( + client.request('createOrAttach', { + sessionId: 'request-overflow', + cols: 80, + rows: 24 + }) + ).rejects.toThrow('request capacity exceeded') + expect(state.activeRequestCount).toBe(maxLargeResponses) + expect(state.activeResponseBytes).toBe(DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT) + expect(state.pendingPtySpawnPreparations.size).toBe(maxLargeResponses) + + client.disconnect() + finishPreparation() + await Promise.all(pending) + await waitFor( + () => + state.activeRequestCount === 0 && + state.activeRequestBytes === 0 && + state.activeResponseBytes === 0 && + state.pendingPtySpawnPreparations.size === 0 + ) + }) + + it('keeps the independent active-request cap for small response methods', async () => { + let finishHealthCheck!: () => void + const healthCheck = new Promise((resolve) => { + finishHealthCheck = resolve + }) + const ptySpawnHealthCheck = vi.fn(() => healthCheck) + await startServer(undefined, ptySpawnHealthCheck) + const { socket, response } = await hello('control', 'active-request-cap-client') + expect(response.ok).toBe(true) + const state = server as unknown as DaemonServerAdmissionState + + for (let index = 0; index < DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT; index += 1) { + socket.write( + encodeNdjson({ id: `pending-${index}`, type: 'ptySpawnHealth', payload: undefined }) + ) + } + await waitFor( + () => ptySpawnHealthCheck.mock.calls.length === DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT + ) + const overflowResponse = readJsonLine(socket) + socket.write(encodeNdjson({ id: 'overflow', type: 'ptySpawnHealth', payload: undefined })) + await expect(overflowResponse).resolves.toMatchObject({ + id: 'overflow', + ok: false, + error: expect.stringContaining('request capacity exceeded') + }) + + socket.destroy() + finishHealthCheck() + await waitFor(() => state.activeRequestCount === 0 && state.activeRequestBytes === 0) + }) + + it('enforces the per-client retained-byte budget independently of request count', async () => { + let finishPreparation!: () => void + const preparation = new Promise((resolve) => { + finishPreparation = resolve + }) + const preparePtySpawn = vi.fn(() => preparation) + await startServer(preparePtySpawn) + client = new DaemonClient({ socketPath, tokenPath }) + await client.ensureConnected() + const state = server as unknown as DaemonServerAdmissionState + const connected = [...state.clients.values()][0] + const request = (id: string): DaemonRequest => ({ + id, + type: 'createOrAttach', + payload: { sessionId: id, cols: 80, rows: 24 } + }) + + state.dispatchRequest( + connected.controlSocket, + connected.clientId, + request('within-byte-budget'), + DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT - 1024 + ) + await waitFor(() => preparePtySpawn.mock.calls.length === 1) + state.dispatchRequest( + connected.controlSocket, + connected.clientId, + request('over-byte-budget'), + 2048 + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(preparePtySpawn).toHaveBeenCalledOnce() + expect(connected.activeRequestCount).toBe(1) + + client.disconnect() + finishPreparation() + await waitFor(() => state.activeRequestCount === 0 && state.activeRequestBytes === 0) + }) + + it('holds idle retirement until admitted work releases after disconnect', async () => { + let finishHealthCheck!: () => void + const healthCheck = new Promise((resolve) => { + finishHealthCheck = resolve + }) + const ptySpawnHealthCheck = vi.fn(() => healthCheck) + const onIdleShutdown = vi.fn() + server = new DaemonServer({ + socketPath, + tokenPath, + ptySpawnHealthCheck, + onIdleShutdown, + spawnSubprocess: () => { + throw new Error('Test unexpectedly spawned a subprocess') + } + }) + await server.start() + client = new DaemonClient({ socketPath, tokenPath }) + await client.ensureConnected() + const state = server as unknown as DaemonServerAdmissionState + + const pending = client.request('ptySpawnHealth', undefined).catch((error: unknown) => error) + await waitFor(() => ptySpawnHealthCheck.mock.calls.length === 1) + client.disconnect() + await pending + await waitFor(() => state.transportSockets.size === 0) + expect(state.activeRequestCount).toBe(1) + expect(onIdleShutdown).not.toHaveBeenCalled() + + finishHealthCheck() + await waitFor(() => state.activeRequestCount === 0 && onIdleShutdown.mock.calls.length === 1) + }) +}) diff --git a/src/main/daemon/daemon-server.ts b/src/main/daemon/daemon-server.ts index 814a27b269e..5c38c4d2837 100644 --- a/src/main/daemon/daemon-server.ts +++ b/src/main/daemon/daemon-server.ts @@ -4,7 +4,12 @@ import { randomUUID } from 'node:crypto' import { performance } from 'node:perf_hooks' import { writeFileSync, chmodSync } from 'node:fs' import { StringDecoder } from 'node:string_decoder' -import { encodeNdjson, createNdjsonParser } from './ndjson' +import { + DAEMON_HANDSHAKE_MAX_LINE_BYTES, + encodeBoundedNdjson, + encodeNdjson, + createNdjsonParser +} from './ndjson' import { TerminalHost } from './terminal-host' import { DaemonStreamDataBatcher } from './daemon-stream-data-batcher' import { @@ -36,6 +41,27 @@ import { isAgentSessionExecutionClaim, isAgentSessionSurfaceBinding } from '../../shared/agent-session-host-authority' +import { + DAEMON_CONTROL_SOCKET_MAX_BUFFERED_BYTES, + DAEMON_HANDSHAKE_TIMEOUT_MS, + DAEMON_MAX_ACTIVE_REQUEST_BYTES, + DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT, + DAEMON_MAX_ACTIVE_REQUESTS, + DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT, + DAEMON_MAX_CONTROL_CLIENTS, + DAEMON_MAX_STREAM_ATTACHMENTS, + DAEMON_MAX_TRANSPORT_SOCKETS, + daemonHelloAdmissionError, + daemonRequestAdmissionError, + getBoundedDaemonRequestId +} from './daemon-admission-limits' +import { + DAEMON_CONTROL_PROCESS_MAX_BUFFERED_BYTES, + DAEMON_MAX_ACTIVE_RESPONSE_BYTES, + DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT, + DAEMON_MAX_RESPONSE_BYTES, + daemonResponseReservationBytes +} from './daemon-response-admission' export type DaemonServerOptions = { socketPath: string @@ -77,6 +103,9 @@ type ConnectedClient = { controlSocket: Socket streamSocket: Socket | null authenticatedPairEstablished: boolean + activeRequestCount: number + activeRequestBytes: number + activeResponseBytes: number } type PendingPtySpawnPreparation = { @@ -109,6 +138,9 @@ export class DaemonServer { private preparePtySpawn: () => Promise private log: DaemonFileLog private transportSockets = new Set() + private activeRequestCount = 0 + private activeRequestBytes = 0 + private activeResponseBytes = 0 private createOrAttachInFlight = 0 private idleShutdownState: 'running' | 'idle-shutdown-pending' | 'shutting-down' = 'running' private initialAdoptionTimer: unknown | null = null @@ -304,6 +336,7 @@ export class DaemonServer { return ( this.transportSockets.size === 0 && this.clients.size === 0 && + this.activeRequestCount === 0 && this.createOrAttachInFlight === 0 && this.host.listSessions().length === 0 ) @@ -382,6 +415,12 @@ export class DaemonServer { } private handleConnection(socket: Socket): void { + socket.on('error', () => socket.destroy()) + if (this.transportSockets.size >= DAEMON_MAX_TRANSPORT_SOCKETS) { + socket.destroy() + return + } + this.cancelInitialAdoptionTimer() this.transportSockets.add(socket) const removeTransport = (): void => { @@ -389,7 +428,6 @@ export class DaemonServer { this.reevaluateIdleShutdown() } socket.once('close', removeTransport) - socket.on('error', () => socket.destroy()) if (this.idleShutdownState !== 'running') { // Why: a connection accepted just before close() gets an explicit retry signal instead of dying mid-auth. @@ -403,45 +441,67 @@ export class DaemonServer { ) return } + + let handshakeComplete = false + const handshakeTimer = setTimeout(() => socket.destroy(), DAEMON_HANDSHAKE_TIMEOUT_MS) + handshakeTimer.unref() + socket.once('close', () => clearTimeout(handshakeTimer)) // Why: keep UTF-8 sequences intact across socket chunks before NDJSON parsing. const decoder = new StringDecoder('utf8') const parser = createNdjsonParser( - (msg) => this.handleFirstMessage(socket, msg, parser), + (msg) => { + if (handshakeComplete) { + return + } + handshakeComplete = true + clearTimeout(handshakeTimer) + this.handleFirstMessage(socket, msg) + }, () => { socket.destroy() - } + }, + { maxLineBytes: DAEMON_HANDSHAKE_MAX_LINE_BYTES } ) socket.on('data', (chunk) => parser.feed(decoder.write(chunk))) } - private handleFirstMessage( - socket: Socket, - msg: unknown, - _parser: ReturnType - ): void { - const hello = msg as HelloMessage - if (hello.type !== 'hello') { - this.log.log('client-hello-rejected', { reason: 'expected-hello' }) - socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Expected hello' })) - socket.destroy() + private handleFirstMessage(socket: Socket, msg: unknown): void { + const helloError = daemonHelloAdmissionError(msg) + if (helloError) { + this.log.log('client-hello-rejected', { reason: 'invalid-hello' }) + this.rejectHello(socket, helloError) return } + const hello = msg as HelloMessage if (hello.version !== this.protocolVersion) { this.log.log('client-hello-rejected', { reason: 'protocol-mismatch', clientVersion: hello.version }) - socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Protocol version mismatch' })) - socket.destroy() + this.rejectHello(socket, 'Protocol version mismatch') return } if (hello.token !== this.token) { this.log.log('client-hello-rejected', { reason: 'invalid-token', role: hello.role }) - socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Invalid token' })) - socket.destroy() + this.rejectHello(socket, 'Invalid token') + return + } + + const previous = hello.role === 'control' ? this.clients.get(hello.clientId) : undefined + const streamClient = hello.role === 'stream' ? this.clients.get(hello.clientId) : undefined + if (hello.role === 'control' && !previous && this.clients.size >= DAEMON_MAX_CONTROL_CLIENTS) { + this.rejectHello(socket, 'Daemon control-client capacity exceeded; reconnect', true) + return + } + if ( + streamClient && + streamClient.streamSocket === null && + this.streamAttachmentCount() >= DAEMON_MAX_STREAM_ATTACHMENTS + ) { + this.rejectHello(socket, 'Daemon stream-attachment capacity exceeded; reconnect', true) return } @@ -463,12 +523,14 @@ export class DaemonServer { ) if (hello.role === 'control') { - const previous = this.clients.get(hello.clientId) const client: ConnectedClient = { clientId: hello.clientId, controlSocket: socket, streamSocket: null, - authenticatedPairEstablished: false + authenticatedPairEstablished: false, + activeRequestCount: 0, + activeRequestBytes: 0, + activeResponseBytes: 0 } this.clients.set(hello.clientId, client) this.setupControlSocket(socket, hello.clientId) @@ -480,30 +542,50 @@ export class DaemonServer { previous.streamSocket?.destroy() previous.controlSocket.destroy() } - } else if (hello.role === 'stream') { - const client = this.clients.get(hello.clientId) - if (!client) { - // Why: a stream socket is meaningless without its control socket; drop the orphan. - socket.destroy() - return - } - this.setupStreamSocket(socket, client) - client.authenticatedPairEstablished = true + } else if (streamClient) { + this.setupStreamSocket(socket, streamClient) + streamClient.authenticatedPairEstablished = true // Why: one-shot health probes authenticate only a control socket; they are not fresh app activity. this.onAuthenticatedClientPair() // A complete app connection (unlike a probe) re-owns the endpoint and cancels pending retirement. this.initialAdoptionDeadlineMs = null this.retirementRequested = false this.cancelInitialAdoptionTimer() + } else { + // Why: preserve the legacy authenticated handshake before dropping a stream with no owner. + socket.destroy() } } + private rejectHello(socket: Socket, error: string, retryable = false): void { + socket.end( + encodeNdjson({ + type: 'hello', + ok: false, + error, + ...(retryable ? { retryable: true } : {}) + }), + () => socket.destroy() + ) + } + + private streamAttachmentCount(): number { + let count = 0 + for (const client of this.clients.values()) { + if (client.streamSocket) { + count += 1 + } + } + return count + } + private setupControlSocket(socket: Socket, clientId: string): void { // Why: decode as a UTF-8 stream so emoji/Unicode split across chunks isn't corrupted. const decoder = new StringDecoder('utf8') const parser = createNdjsonParser( - (msg) => this.handleRequest(socket, clientId, msg as DaemonRequest), - () => {} // Ignore parse errors + (msg, lineBytes) => this.dispatchRequest(socket, clientId, msg, lineBytes ?? 0), + () => socket.destroy(), + { includeLineBytes: true } ) // Remove the initial data listener and replace with the RPC parser @@ -569,6 +651,86 @@ export class DaemonServer { } } + private dispatchRequest( + socket: Socket, + clientId: string, + value: unknown, + lineBytes: number + ): void { + const client = this.clients.get(clientId) + if (!client || client.controlSocket !== socket) { + return + } + + const requestId = getBoundedDaemonRequestId(value) + const admissionError = daemonRequestAdmissionError(value) + if (admissionError) { + this.writeRequestError(socket, requestId, admissionError) + return + } + const request = value as DaemonRequest + const responseReservationBytes = daemonResponseReservationBytes(request) + if ( + client.activeRequestCount >= DAEMON_MAX_ACTIVE_REQUESTS_PER_CLIENT || + this.activeRequestCount >= DAEMON_MAX_ACTIVE_REQUESTS || + client.activeRequestBytes + lineBytes > DAEMON_MAX_ACTIVE_REQUEST_BYTES_PER_CLIENT || + this.activeRequestBytes + lineBytes > DAEMON_MAX_ACTIVE_REQUEST_BYTES || + client.activeResponseBytes + responseReservationBytes > + DAEMON_MAX_ACTIVE_RESPONSE_BYTES_PER_CLIENT || + this.activeResponseBytes + responseReservationBytes > DAEMON_MAX_ACTIVE_RESPONSE_BYTES + ) { + this.writeRequestError(socket, request.id, 'Daemon request capacity exceeded; retry') + return + } + + client.activeRequestCount += 1 + client.activeRequestBytes += lineBytes + this.activeRequestCount += 1 + this.activeRequestBytes += lineBytes + client.activeResponseBytes += responseReservationBytes + this.activeResponseBytes += responseReservationBytes + void this.handleRequest(socket, clientId, request).finally(() => { + client.activeRequestCount -= 1 + client.activeRequestBytes -= lineBytes + this.activeRequestCount -= 1 + this.activeRequestBytes -= lineBytes + client.activeResponseBytes -= responseReservationBytes + this.activeResponseBytes -= responseReservationBytes + this.reevaluateIdleShutdown() + }) + } + + private writeRequestError(socket: Socket, requestId: string | null, error: string): void { + if (!requestId || requestId.startsWith(NOTIFY_PREFIX)) { + return + } + this.writeControlMessage(socket, { id: requestId, ok: false, error }) + } + + private writeControlMessage(socket: Socket, message: unknown, onFlushed?: () => void): void { + const encoded = encodeBoundedNdjson(message, DAEMON_MAX_RESPONSE_BYTES) + const encodedBytes = Buffer.byteLength(encoded, 'utf8') + if ( + socket.writableLength + encodedBytes > DAEMON_CONTROL_SOCKET_MAX_BUFFERED_BYTES || + this.controlSocketBufferedBytes() + encodedBytes > DAEMON_CONTROL_PROCESS_MAX_BUFFERED_BYTES + ) { + socket.destroy() + return + } + socket.write(encoded, onFlushed) + } + + private controlSocketBufferedBytes(): number { + let total = 0 + for (const client of this.clients.values()) { + total += Math.max(0, client.controlSocket.writableLength) + if (total > DAEMON_CONTROL_PROCESS_MAX_BUFFERED_BYTES) { + break + } + } + return total + } + private async handleRequest( socket: Socket, clientId: string, @@ -582,19 +744,17 @@ export class DaemonServer { const pendingShutdown = this.pendingShutdownReplies.get( this.shutdownReplyKey(clientId, request.id) ) - socket.write(encodeNdjson({ id: request.id, ok: true, payload: result }), () => { + this.writeControlMessage(socket, { id: request.id, ok: true, payload: result }, () => { pendingShutdown?.start() }) } } catch (err) { if (!isNotify) { - socket.write( - encodeNdjson({ - id: request.id, - ok: false, - error: err instanceof Error ? err.message : String(err) - }) - ) + this.writeControlMessage(socket, { + id: request.id, + ok: false, + error: err instanceof Error ? err.message : String(err) + }) } } } @@ -909,6 +1069,9 @@ export class DaemonServer { case 'getForegroundProcess': return { foregroundProcess: this.host.getForegroundProcess(request.payload.sessionId) } + case 'inspectProcess': + return this.host.inspectProcess(request.payload.sessionId) + case 'confirmForegroundProcess': return { foregroundProcess: await this.host.confirmForegroundProcess(request.payload.sessionId) @@ -927,6 +1090,7 @@ export class DaemonServer { authenticatedClient !== undefined && authenticatedClient.streamSocket !== null && this.clients.size === 1 && + this.activeRequestCount <= 1 && this.createOrAttachInFlight === 0 && this.host.listSessions().length === 0 && [...this.transportSockets].every( diff --git a/src/main/daemon/daemon-spawner.ts b/src/main/daemon/daemon-spawner.ts index ba0ea239c67..2a900d03cde 100644 --- a/src/main/daemon/daemon-spawner.ts +++ b/src/main/daemon/daemon-spawner.ts @@ -1,7 +1,8 @@ import { createHash, randomUUID } from 'node:crypto' -import { constants, copyFileSync, existsSync, readFileSync, renameSync, unlinkSync } from 'node:fs' +import { constants, copyFileSync, existsSync, renameSync, unlinkSync } from 'node:fs' import { join } from 'node:path' import { PROTOCOL_VERSION } from './types' +import { readDaemonControlFileText } from './daemon-control-file-reader' export type DaemonConnectionInfo = { socketPath: string @@ -141,7 +142,7 @@ function claimAndUnlinkOwnedFile( return false } try { - if (ownsContent(readFileSync(claimedPath, 'utf8'))) { + if (ownsContent(readDaemonControlFileText(claimedPath))) { unlinkSync(claimedPath) return true } diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 87dc721db0f..a8211b3615c 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -113,6 +113,16 @@ function createDaemonAdapter( } as unknown as DaemonPtyAdapter & ProviderMock } +it('rejects completion inspection instead of borrowing the fallback provider', async () => { + const provider = new DegradedDaemonPtyProvider({ + current: createDaemonAdapter('daemon'), + legacy: [], + fallback: createProvider('fallback') + }) + + await expect(provider.inspectProcess('unmapped-session')).rejects.toThrow('terminal_gone') +}) + describe('DegradedDaemonPtyProvider', () => { it('only delegates owner-listing authority to the provider that owns the id', async () => { const current = createDaemonAdapter('daemon', ['daemon-session']) diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 0dd1577a614..47c5375bad4 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -1,14 +1,13 @@ import type { DaemonPtyAdapter } from './daemon-pty-adapter' import { shutdownDegradedFallbackSessions } from './degraded-daemon-fallback-shutdown' -import type { - IPtyProvider, - PtyBackgroundStreamEvent, - PtyDataEvent, - PtyProviderBufferSnapshot, - PtyProcessInfo, - PtySpawnOptions, - PtySpawnResult -} from '../providers/types' +import { inspectPtyProviderProcess } from '../providers/pty-process-inspection' +import type { IPtyProvider, PtyBackgroundStreamEvent } from '../providers/types' +import type { PtyDataEvent, PtyProviderBufferSnapshot } from '../providers/types' +import type { PtyProcessInfo, PtySpawnOptions, PtySpawnResult } from '../providers/types' +import { + collectPtyProcessListings, + PtyProcessListAdmission +} from '../providers/pty-process-list-admission' export class DegradedDaemonPtyProvider implements IPtyProvider { readonly routesFreshSpawnsToLocalProvider = true @@ -50,11 +49,12 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { } async discoverDaemonSessions(): Promise { + const admission = new PtyProcessListAdmission() for (const adapter of this.allDaemonAdapters()) { try { const sessions = await adapter.listProcesses() for (const session of sessions) { - this.sessionProviders.set(session.id, adapter) + this.sessionProviders.set(admission.admit(session).id, adapter) } } catch (error) { console.warn('[daemon] Failed to discover degraded daemon sessions', error) @@ -158,7 +158,11 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { async getForegroundProcess(id: string): Promise { return this.providerFor(id).getForegroundProcess(id) } - + inspectProcess(id: string) { + return this.hasPty(id) + ? inspectPtyProviderProcess(this.providerFor(id), id) + : Promise.reject(new Error('terminal_gone')) + } async confirmForegroundProcess(id: string): Promise { return this.providerFor(id).confirmForegroundProcess?.(id) ?? null } @@ -171,12 +175,8 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { await this.fallback.revive(state) } - async listProcesses(opts?: { deadlineMs?: number }): Promise { - const results = await Promise.all( - this.allProviders().map((provider) => provider.listProcesses(opts)) - ) - return results.flat() - } + listProcesses = (opts?: { deadlineMs?: number }): Promise => + collectPtyProcessListings(this.allProviders(), (provider) => provider.listProcesses(opts)) async getDefaultShell(): Promise { return this.fallback.getDefaultShell() diff --git a/src/main/daemon/headless-emulator-size-limits.test.ts b/src/main/daemon/headless-emulator-size-limits.test.ts new file mode 100644 index 00000000000..38e4f096cd6 --- /dev/null +++ b/src/main/daemon/headless-emulator-size-limits.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'vitest' +import { MAX_TERMINAL_COLS } from '../../shared/terminal-size-limits' +import { HeadlessEmulator } from './headless-emulator' + +describe('HeadlessEmulator terminal size limits', () => { + it('uses safe defaults for oversized persisted construction and resize dimensions', () => { + const emulator = new HeadlessEmulator({ cols: MAX_TERMINAL_COLS + 1, rows: 24 }) + expect(emulator.getAppliedSize()).toEqual({ cols: 80, rows: 24 }) + + emulator.resize(80, Number.MAX_SAFE_INTEGER) + expect(emulator.getAppliedSize()).toEqual({ cols: 80, rows: 24 }) + emulator.dispose() + }) +}) diff --git a/src/main/daemon/headless-emulator.ts b/src/main/daemon/headless-emulator.ts index 7cd36fe80f3..4f3797e6f51 100644 --- a/src/main/daemon/headless-emulator.ts +++ b/src/main/daemon/headless-emulator.ts @@ -20,6 +20,7 @@ import { } from './terminal-view-attribute-responder' import type { TerminalSnapshot, TerminalModes } from './types' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' +import { normalizeTerminalSize } from '../../shared/terminal-size-limits' export type HeadlessEmulatorOptions = { cols: number @@ -69,6 +70,7 @@ export class HeadlessEmulator { private partialEscapeTail = '' constructor(opts: HeadlessEmulatorOptions) { + const size = normalizeTerminalSize(opts.cols, opts.rows) this.pathFlavor = opts.pathFlavor this.remotePosixFileUriAuthority = opts.remotePosixFileUriAuthority === true this.oscText = new TerminalOscCwdTitleScanner({ @@ -77,8 +79,8 @@ export class HeadlessEmulator { wslDistro: opts.wslDistro }) this.terminal = new Terminal({ - cols: opts.cols, - rows: opts.rows, + cols: size.cols, + rows: size.rows, scrollback: opts.scrollback ?? DEFAULT_SCROLLBACK, allowProposedApi: true, logLevel: 'off', @@ -224,7 +226,8 @@ export class HeadlessEmulator { return } this.restoredOscLinks = [] - this.terminal.resize(cols, rows) + const size = normalizeTerminalSize(cols, rows) + this.terminal.resize(size.cols, size.rows) } // Why: these dims proxy the child's real size, so they stay stale on a dropped resize the renderer must detect. diff --git a/src/main/daemon/history-manager-memory.test.ts b/src/main/daemon/history-manager-memory.test.ts new file mode 100644 index 00000000000..c472cb3a55b --- /dev/null +++ b/src/main/daemon/history-manager-memory.test.ts @@ -0,0 +1,68 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { getHistorySessionDirName } from './history-paths' +import { HistoryManager } from './history-manager' +import type { TerminalSnapshot } from './types' + +const directories: string[] = [] + +function createManager(): { directory: string; manager: HistoryManager } { + const directory = mkdtempSync(join(tmpdir(), 'orca-history-manager-memory-')) + directories.push(directory) + return { directory, manager: new HistoryManager(directory) } +} + +function snapshot(snapshotAnsi: string): TerminalSnapshot { + return { + snapshotAnsi, + scrollbackAnsi: '', + rehydrateSequences: '', + cwd: '/workspace', + cols: 80, + rows: 24, + modes: { + bracketedPaste: false, + mouseTracking: false, + applicationCursor: false, + alternateScreen: false + }, + scrollbackLines: 0 + } +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('terminal history write memory limits', () => { + it('rejects an oversized escaped checkpoint before writing it', async () => { + const { directory, manager } = createManager() + await manager.openSession('checkpoint', { cwd: '/workspace', cols: 80, rows: 24 }) + + await manager.checkpoint('checkpoint', snapshot('\n'.repeat(9 * 1024 * 1024))) + + expect(manager.isSessionDisabled('checkpoint')).toBe(true) + expect( + existsSync(join(directory, getHistorySessionDirName('checkpoint'), 'checkpoint.json')) + ).toBe(false) + }) + + it('rejects oversized escaped metadata before writing it', async () => { + const { directory, manager } = createManager() + + await manager.openSession('metadata', { + cwd: '\n'.repeat(40 * 1024), + cols: 80, + rows: 24 + }) + + expect(manager.isSessionDisabled('metadata')).toBe(true) + expect(existsSync(join(directory, getHistorySessionDirName('metadata'), 'meta.json'))).toBe( + false + ) + }) +}) diff --git a/src/main/daemon/history-manager.ts b/src/main/daemon/history-manager.ts index 1af27025182..9fc5a140dfc 100644 --- a/src/main/daemon/history-manager.ts +++ b/src/main/daemon/history-manager.ts @@ -2,55 +2,39 @@ import { join } from 'node:path' import { mkdirSync, writeFileSync, - readFileSync, existsSync, rmSync, unlinkSync, - openSync, - closeSync, - readSync, - fstatSync, promises as fsPromises } from 'node:fs' import { getHistorySessionDirName } from './history-paths' import { - decodeLogHeader, - encodeLogBatch, + encodeLogBatchWithinLimit, encodeLogHeader, LOG_HEADER_BYTES } from './terminal-history-log' import type { PendingOutputRecord, TerminalCheckpointFile, TerminalSnapshot } from './types' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES, + TERMINAL_HISTORY_LOG_MAX_BYTES +} from './terminal-history-file-limits' +import { + readTerminalHistorySessionMeta, + stringifyTerminalHistorySessionMeta, + type HistoryManagerOptions, + type OpenSessionOptions, + type SessionMeta +} from './terminal-history-session-metadata' +import { + resolveTerminalHistoryLogState, + type TerminalHistoryLogState +} from './terminal-history-log-state' -// Why 5MB: bounds cold-restore replay time and per-session disk; hitting the cap triggers one checkpoint that resets the log. -const LOG_MAX_BYTES = 5 * 1024 * 1024 +export type { HistoryManagerOptions, OpenSessionOptions, SessionMeta } -export type SessionMeta = { - cwd: string - cols: number - rows: number - startedAt: string - endedAt: string | null - exitCode: number | null -} - -export type OpenSessionOptions = { - cwd: string - cols: number - rows: number -} - -type SessionWriter = { +type SessionWriter = TerminalHistoryLogState & { dir: string - checkpointPath: string - logPath: string - /** Generation of the on-disk log header. Null until lazily resolved on first append after a warm registerWriter. */ - logGeneration: number | null - /** Current log file size. Null until lazily resolved alongside generation. */ - logBytes: number | null -} - -export type HistoryManagerOptions = { - onWriteError?: (sessionId: string, error: Error) => void } export class HistoryManager { @@ -78,7 +62,7 @@ export class HistoryManager { endedAt: null, exitCode: null } - writeFileSync(join(dir, 'meta.json'), JSON.stringify(meta, null, 2)) + writeFileSync(join(dir, 'meta.json'), stringifyTerminalHistorySessionMeta(meta)) // Why: clear stale recovery files (incl. legacy scrollback.bin) so a crash before the first checkpoint can't replay a prior session's content. const checkpointPath = join(dir, 'checkpoint.json') @@ -153,11 +137,15 @@ export class HistoryManager { return 'ok' } try { - this.resolveLogState(writer) - const batch = encodeLogBatch(seq, records) + resolveTerminalHistoryLogState(writer) // Why max(..., header): a fresh log's header (written below) must count toward the projected size or the cap overshoots. - const projectedBytes = Math.max(writer.logBytes ?? 0, LOG_HEADER_BYTES) + batch.length - if (projectedBytes > LOG_MAX_BYTES) { + const existingBytes = Math.max(writer.logBytes ?? 0, LOG_HEADER_BYTES) + const batch = encodeLogBatchWithinLimit( + seq, + records, + TERMINAL_HISTORY_LOG_MAX_BYTES - existingBytes + ) + if (!batch) { return 'needs-checkpoint' } if (writer.logBytes === 0) { @@ -192,7 +180,7 @@ export class HistoryManager { effectiveCwd = meta?.cwd ?? null } - this.resolveLogState(writer) + resolveTerminalHistoryLogState(writer) const generation = (writer.logGeneration ?? 0) + 1 const checkpointFile: TerminalCheckpointFile = { snapshotAnsi: snapshot.snapshotAnsi, @@ -207,7 +195,10 @@ export class HistoryManager { generation, checkpointedAt: new Date().toISOString() } - const data = JSON.stringify(checkpointFile) + const data = stringifyJsonWithinByteLimit( + checkpointFile, + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES + ).serialized // Why: tmp+rename is atomic (corrupt checkpoint > stale); async so a sync ~MB write can't stall IPC (worse under Windows AV). // The adapter's checkpointInFlight guard serializes checkpoints, so concurrent async writes can't collide on the fixed .tmp path. const tmpPath = `${writer.checkpointPath}.tmp` @@ -222,46 +213,6 @@ export class HistoryManager { } } - // Why: a warm registerWriter may attach to an existing log; read generation/size once so appends continue it, not clobber it. - private resolveLogState(writer: SessionWriter): void { - if (writer.logBytes !== null && writer.logGeneration !== null) { - return - } - let headerGeneration: number | null = null - let size = 0 - try { - const fd = openSync(writer.logPath, 'r') - try { - size = fstatSync(fd).size - const header = Buffer.alloc(LOG_HEADER_BYTES) - if (readSync(fd, header, 0, LOG_HEADER_BYTES, 0) === LOG_HEADER_BYTES) { - headerGeneration = decodeLogHeader(header) - } - } finally { - closeSync(fd) - } - } catch { - // Missing log file — fresh state below. - } - if (headerGeneration !== null) { - writer.logGeneration = headerGeneration - writer.logBytes = size - return - } - // Missing/unreadable header: logBytes = 0 makes the next append truncate-rewrite, so a garbage file can't be extended. - writer.logBytes = 0 - writer.logGeneration = this.readCheckpointGeneration(writer) ?? 0 - } - - private readCheckpointGeneration(writer: SessionWriter): number | null { - try { - const checkpoint = JSON.parse(readFileSync(writer.checkpointPath, 'utf-8')) - return typeof checkpoint.generation === 'number' ? checkpoint.generation : null - } catch { - return null - } - } - async closeSession(sessionId: string, exitCode: number): Promise { const writer = this.writers.get(sessionId) if (!writer) { @@ -306,7 +257,7 @@ export class HistoryManager { return null } try { - return JSON.parse(readFileSync(metaPath, 'utf-8')) + return readTerminalHistorySessionMeta(metaPath) } catch { return null } @@ -333,7 +284,7 @@ export class HistoryManager { private readMetaFromDir(dir: string): SessionMeta | null { const metaPath = join(dir, 'meta.json') try { - return JSON.parse(readFileSync(metaPath, 'utf-8')) + return readTerminalHistorySessionMeta(metaPath) } catch { return null } @@ -343,11 +294,11 @@ export class HistoryManager { const metaPath = join(dir, 'meta.json') let meta: SessionMeta try { - meta = JSON.parse(readFileSync(metaPath, 'utf-8')) + meta = readTerminalHistorySessionMeta(metaPath) } catch { return } Object.assign(meta, updates) - writeFileSync(metaPath, JSON.stringify(meta, null, 2)) + writeFileSync(metaPath, stringifyTerminalHistorySessionMeta(meta)) } } diff --git a/src/main/daemon/history-reader-memory.test.ts b/src/main/daemon/history-reader-memory.test.ts new file mode 100644 index 00000000000..23663c8fe59 --- /dev/null +++ b/src/main/daemon/history-reader-memory.test.ts @@ -0,0 +1,123 @@ +import { + closeSync, + ftruncateSync, + mkdirSync, + mkdtempSync, + openSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { HistoryReader } from './history-reader' +import { getHistorySessionDirName } from './history-paths' +import { + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES, + TERMINAL_HISTORY_LOG_MAX_BYTES, + TERMINAL_HISTORY_META_MAX_BYTES +} from './terminal-history-file-limits' +import { + readTerminalHistoryJson, + TERMINAL_HISTORY_JSON_MAX_STRUCTURAL_TOKENS +} from './terminal-history-file-reader' + +const directories: string[] = [] + +function createSession(sessionId: string): { basePath: string; sessionPath: string } { + const basePath = mkdtempSync(join(tmpdir(), 'orca-history-memory-')) + directories.push(basePath) + const sessionPath = join(basePath, getHistorySessionDirName(sessionId)) + mkdirSync(sessionPath, { recursive: true }) + writeFileSync( + join(sessionPath, 'meta.json'), + JSON.stringify({ + cwd: '/workspace', + cols: 80, + rows: 24, + startedAt: '2026-01-01T00:00:00.000Z', + endedAt: null, + exitCode: null + }) + ) + return { basePath, sessionPath } +} + +function createSparseFile(path: string, bytes: number): void { + const descriptor = openSync(path, 'w') + ftruncateSync(descriptor, bytes) + closeSync(descriptor) +} + +function checkpoint(): string { + return JSON.stringify({ + snapshotAnsi: 'safe checkpoint', + scrollbackAnsi: '', + rehydrateSequences: '', + cwd: '/workspace', + cols: 80, + rows: 24, + modes: { + bracketedPaste: false, + mouseTracking: false, + applicationCursor: false, + alternateScreen: false + } + }) +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true }) + } +}) + +describe('terminal history restore memory limits', () => { + it('falls back to a valid checkpoint when the incremental log is oversized', () => { + const { basePath, sessionPath } = createSession('oversized-log') + writeFileSync(join(sessionPath, 'checkpoint.json'), checkpoint()) + createSparseFile(join(sessionPath, 'output.log'), TERMINAL_HISTORY_LOG_MAX_BYTES + 1) + + expect(new HistoryReader(basePath).detectColdRestore('oversized-log')?.snapshotAnsi).toBe( + 'safe checkpoint' + ) + }) + + it('ignores oversized checkpoint and metadata files before parsing', () => { + const checkpointSession = createSession('oversized-checkpoint') + createSparseFile( + join(checkpointSession.sessionPath, 'checkpoint.json'), + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES + 1 + ) + expect( + new HistoryReader(checkpointSession.basePath).detectColdRestore('oversized-checkpoint') + ).toBeNull() + + const metadataSession = createSession('oversized-metadata') + createSparseFile( + join(metadataSession.sessionPath, 'meta.json'), + TERMINAL_HISTORY_META_MAX_BYTES + 1 + ) + expect( + new HistoryReader(metadataSession.basePath).hasRestorableHistory('oversized-metadata') + ).toBe(false) + }) + + it('rejects structurally amplified checkpoints before parsing', () => { + const { sessionPath } = createSession('amplified-checkpoint') + const checkpointPath = join(sessionPath, 'checkpoint.json') + writeFileSync( + checkpointPath, + `{"snapshotAnsi":"","values":[${'0,'.repeat(TERMINAL_HISTORY_JSON_MAX_STRUCTURAL_TOKENS)}0]}` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + try { + expect(() => + readTerminalHistoryJson(checkpointPath, TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES) + ).toThrow(/JSON structure exceeds/) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) +}) diff --git a/src/main/daemon/history-reader.ts b/src/main/daemon/history-reader.ts index e219d58cfc4..3269b8d2b09 100644 --- a/src/main/daemon/history-reader.ts +++ b/src/main/daemon/history-reader.ts @@ -1,11 +1,26 @@ import { join } from 'node:path' -import { readFileSync, existsSync, readdirSync } from 'node:fs' +import { existsSync, opendirSync } from 'node:fs' import type { SessionMeta } from './history-manager' import type { TerminalCheckpointFile, TerminalModes } from './types' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' import { getHistorySessionDirName } from './history-paths' import { decodeTerminalHistoryLog } from './terminal-history-log' import { HeadlessEmulator } from './headless-emulator' +import { + readTerminalHistoryBuffer, + readTerminalHistoryJson, + readTerminalHistoryText +} from './terminal-history-file-reader' +import { + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES, + TERMINAL_HISTORY_LEGACY_SCROLLBACK_MAX_BYTES, + TERMINAL_HISTORY_LOG_MAX_BYTES, + TERMINAL_HISTORY_META_MAX_BYTES +} from './terminal-history-file-limits' +import { + retainNewestRestorableTerminalHistorySessions, + type RestorableTerminalHistorySession +} from './terminal-history-restorable-retention' export type ColdRestoreInfo = { snapshotAnsi: string @@ -59,7 +74,10 @@ export class HistoryReader { let checkpoint: TerminalCheckpointFile | null = null if (checkpointExists) { try { - checkpoint = JSON.parse(readFileSync(checkpointPath, 'utf-8')) + checkpoint = readTerminalHistoryJson( + checkpointPath, + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES + ) } catch { checkpoint = null } @@ -88,31 +106,55 @@ export class HistoryReader { return [] } - let entries: { isDirectory(): boolean; name: string }[] + let directory: ReturnType try { - entries = readdirSync(this.basePath, { withFileTypes: true }) + directory = opendirSync(this.basePath) } catch { return [] } - const restorable: string[] = [] - for (const entry of entries) { - if (!entry.isDirectory()) { - continue - } - let sessionId: string - try { - sessionId = decodeURIComponent(entry.name) - } catch { - continue - } - const meta = this.readMeta(sessionId) - if (meta && meta.endedAt === null) { - restorable.push(sessionId) + const sessions = function* ( + reader: HistoryReader + ): Generator { + let order = 0 + while (true) { + const entry = directory.readSync() + if (!entry) { + return + } + if (!entry.isDirectory()) { + continue + } + let sessionId: string + try { + sessionId = decodeURIComponent(entry.name) + } catch { + continue + } + const meta = reader.readMeta(sessionId) + if (meta && meta.endedAt === null) { + const parsedStartedAt = Date.parse(meta.startedAt) + yield { + sessionId, + startedAtMs: Number.isFinite(parsedStartedAt) ? parsedStartedAt : 0, + order + } + order += 1 + } } } - return restorable + try { + return retainNewestRestorableTerminalHistorySessions(sessions(this)) + } catch { + return [] + } finally { + try { + directory.closeSync() + } catch { + // Best effort after a directory read failure. + } + } } // Why a scratch emulator: replaying base + raw records through the same @@ -127,7 +169,10 @@ export class HistoryReader { ): ColdRestoreInfo | null { let logBuffer: Buffer try { - logBuffer = readFileSync(join(sessionDir, 'output.log')) + logBuffer = readTerminalHistoryBuffer( + join(sessionDir, 'output.log'), + TERMINAL_HISTORY_LOG_MAX_BYTES + ) } catch { return null } @@ -228,7 +273,7 @@ export class HistoryReader { return null } try { - return JSON.parse(readFileSync(metaPath, 'utf-8')) + return readTerminalHistoryJson(metaPath, TERMINAL_HISTORY_META_MAX_BYTES) } catch { return null } @@ -249,7 +294,10 @@ export class HistoryReader { return null } try { - const scrollback = readFileSync(scrollbackPath, 'utf-8') + const scrollback = readTerminalHistoryText( + scrollbackPath, + TERMINAL_HISTORY_LEGACY_SCROLLBACK_MAX_BYTES + ) const truncated = this.truncateAltScreen(scrollback) return { snapshotAnsi: truncated, diff --git a/src/main/daemon/ndjson.test.ts b/src/main/daemon/ndjson.test.ts index 3d1d2c1512b..a7075f5194a 100644 --- a/src/main/daemon/ndjson.test.ts +++ b/src/main/daemon/ndjson.test.ts @@ -1,7 +1,20 @@ import { describe, expect, it, vi } from 'vitest' -import { encodeNdjson, createNdjsonParser, NDJSON_MAX_LINE_BYTES } from './ndjson' +import { + encodeBoundedNdjson, + encodeNdjson, + createNdjsonParser, + NDJSON_MAX_LINE_BYTES, + NDJSON_MAX_STRUCTURAL_TOKENS +} from './ndjson' describe('encodeNdjson', () => { + it('bounds response serialization while preserving admitted wire bytes', () => { + expect(encodeBoundedNdjson({ ok: true }, 12)).toBe('{"ok":true}\n') + expect(() => encodeBoundedNdjson({ value: 'x'.repeat(100) }, 32)).toThrow( + 'JSON output exceeds 31 bytes' + ) + }) + it('encodes an object as a JSON line ending with newline', () => { const result = encodeNdjson({ type: 'hello', version: 1 }) expect(result).toBe('{"type":"hello","version":1}\n') @@ -32,6 +45,32 @@ describe('createNdjsonParser', () => { expect(onError).not.toHaveBeenCalled() }) + it('optionally reports each parsed line byte length without re-serializing it', () => { + const onMessage = vi.fn() + const parser = createNdjsonParser(onMessage, undefined, { includeLineBytes: true }) + + parser.feed('{"text":"é"}\n') + + expect(onMessage).toHaveBeenCalledWith({ text: 'é' }, Buffer.byteLength('{"text":"é"}')) + }) + + it('rejects structurally amplified lines before parsing', () => { + const onMessage = vi.fn() + const onError = vi.fn() + const parser = createNdjsonParser(onMessage, onError) + const parseSpy = vi.spyOn(JSON, 'parse') + try { + parser.feed(`{"values":[${'0,'.repeat(NDJSON_MAX_STRUCTURAL_TOKENS)}0]}\n`) + expect(onMessage).not.toHaveBeenCalled() + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('JSON structure exceeds') }) + ) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) + it('parses multiple messages in a single chunk', () => { const onMessage = vi.fn() const parser = createNdjsonParser(onMessage) @@ -56,6 +95,20 @@ describe('createNdjsonParser', () => { expect(onMessage).toHaveBeenCalledWith({ type: 'hello', version: 1 }) }) + it('parses a line delivered in more than 100,000 one-character fragments', () => { + const onMessage = vi.fn() + const parser = createNdjsonParser(onMessage, undefined, { includeLineBytes: true }) + const message = { value: 'x'.repeat(100_000) } + const line = JSON.stringify(message) + + for (const character of line) { + parser.feed(character) + } + parser.feed('\n') + + expect(onMessage).toHaveBeenCalledWith(message, Buffer.byteLength(line)) + }) + it('handles a chunk that ends mid-line followed by more data', () => { const onMessage = vi.fn() const parser = createNdjsonParser(onMessage) diff --git a/src/main/daemon/ndjson.ts b/src/main/daemon/ndjson.ts index 17b37cb8d11..60e1c5359df 100644 --- a/src/main/daemon/ndjson.ts +++ b/src/main/daemon/ndjson.ts @@ -1,8 +1,22 @@ +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' + export function encodeNdjson(msg: unknown): string { return `${JSON.stringify(msg)}\n` } +export function encodeBoundedNdjson(msg: unknown, maxBytes: number): string { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new RangeError('NDJSON byte limit must be a positive safe integer') + } + return `${stringifyJsonWithinByteLimit(msg, maxBytes - 1).serialized}\n` +} + export const NDJSON_MAX_LINE_BYTES = 16 * 1024 * 1024 +export const DAEMON_HANDSHAKE_MAX_LINE_BYTES = 64 * 1024 +export const NDJSON_MAX_STRUCTURAL_TOKENS = 1_000_000 +export const NDJSON_MAX_NESTING_DEPTH = 128 export type NdjsonParser = { feed(chunk: string): void @@ -11,21 +25,20 @@ export type NdjsonParser = { export type NdjsonParserOptions = { maxLineBytes?: number + includeLineBytes?: boolean } export function createNdjsonParser( - onMessage: (msg: unknown) => void, + onMessage: (msg: unknown, lineBytes?: number) => void, onError?: (err: Error) => void, options: NdjsonParserOptions = {} ): NdjsonParser { - let buffer = '' - let bufferBytes = 0 + const buffer = new GrowingByteBuffer() let discardingOversizedLine = false const maxLineBytes = Math.max(1, options.maxLineBytes ?? NDJSON_MAX_LINE_BYTES) const clearBuffer = (): void => { - buffer = '' - bufferBytes = 0 + buffer.clear() } const reportOversizedLine = (observedBytes: number): void => { @@ -53,8 +66,8 @@ export function createNdjsonParser( return } - const segmentBytes = Buffer.byteLength(segment, 'utf8') - const nextLineBytes = bufferBytes + segmentBytes + const segmentBytes = Buffer.from(segment, 'utf8') + const nextLineBytes = buffer.byteLength + segmentBytes.byteLength // Why: daemon sockets are local but persistent; a peer that never sends // a newline must not grow the parser buffer without bound. if (nextLineBytes > maxLineBytes) { @@ -67,21 +80,29 @@ export function createNdjsonParser( continue } - buffer += segment - bufferBytes = nextLineBytes + buffer.append(segmentBytes) if (!hasNewline) { return } - const line = buffer - clearBuffer() + const lineBytes = buffer.byteLength + const line = buffer.takeString('utf8') if (line.length === 0) { continue } try { - onMessage(JSON.parse(line)) + assertJsonTextStructureWithinLimits(line, { + structuralTokens: NDJSON_MAX_STRUCTURAL_TOKENS, + nestingDepth: NDJSON_MAX_NESTING_DEPTH + }) + const parsed = JSON.parse(line) + if (options.includeLineBytes) { + onMessage(parsed, lineBytes) + } else { + onMessage(parsed) + } } catch (err) { onError?.(err instanceof Error ? err : new Error(String(err))) } diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index 8066770c526..db721e4d6a4 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -66,6 +66,7 @@ import { assertSafeAgentStartupCwd, resolveSafePtyDefaultCwd } from '../provider import { ORCA_HERMES_STARTUP_QUERY_ENV } from '../../shared/hermes-startup-query' import type { TuiAgent } from '../../shared/types' import { forceKillPosixPtyProcessGroups } from '../pty/posix-pty-process-groups' +import { appendCompactedStringChunk } from '../../shared/string-chunk-compaction' const PANE_IDENTITY_ENV_KEYS = [ 'ORCA_PANE_KEY', @@ -810,7 +811,7 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl const bufferPreListenerData = (data: string): void => { // Why: Windows shell-arg startup commands can print before Session wires this subprocess in; preserve that spawn-time race window. - pendingPreListenerData.push(data) + appendCompactedStringChunk(pendingPreListenerData, data) pendingPreListenerDataChars += data.length while (pendingPreListenerDataChars > PENDING_PRE_LISTENER_DATA_MAX_CHARS) { const removed = pendingPreListenerData.shift() diff --git a/src/main/daemon/session-pending-output.test.ts b/src/main/daemon/session-pending-output.test.ts index 446c080a504..3d1d8b7de6d 100644 --- a/src/main/daemon/session-pending-output.test.ts +++ b/src/main/daemon/session-pending-output.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest' -import { Session } from './session' +import { PENDING_OUTPUT_MAX_RECORDS, Session } from './session' // Coverage for the incremental-checkpoint record stream (issue #5096): every // PTY byte, resize, and clear is recorded so the 5s checkpoint can persist @@ -115,6 +115,25 @@ describe('Session pending output', () => { expect(recovered!.records).toEqual([{ kind: 'output', data: 'post-overflow' }]) }) + it('flags overflow when tiny records reach the metadata cap', () => { + const subprocess = createMockSubprocess() + const live = createSession(subprocess) + + for (let index = 0; index <= PENDING_OUTPUT_MAX_RECORDS / 2; index += 1) { + subprocess.simulateData('x') + live.clearScrollback() + } + + const overflowed = live.takePendingOutput(false) + expect(overflowed!.overflowed).toBe(true) + expect(overflowed!.records).toEqual([]) + + subprocess.simulateData('post-overflow') + expect(live.takePendingOutput(false)!.records).toEqual([ + { kind: 'output', data: 'post-overflow' } + ]) + }) + it('returns the snapshot and drops records in the same take when requested', () => { const subprocess = createMockSubprocess() const live = createSession(subprocess) diff --git a/src/main/daemon/session-pre-ready-input-memory.test.ts b/src/main/daemon/session-pre-ready-input-memory.test.ts new file mode 100644 index 00000000000..4aadf16895a --- /dev/null +++ b/src/main/daemon/session-pre-ready-input-memory.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { PRE_READY_STDIN_MAX_SEGMENTS, Session, type SubprocessHandle } from './session' + +function createSubprocess(): SubprocessHandle { + return { + pid: 12345, + getForegroundProcess: () => null, + write() {}, + resize() {}, + kill() {}, + forceKill() {}, + signal() {}, + onData() {}, + onExit() {}, + dispose() {} + } +} + +describe('Session pre-ready input memory', () => { + it('bounds one-character segments independently of the byte cap', () => { + const session = new Session({ + sessionId: 'pre-ready-segment-test', + cols: 80, + rows: 24, + subprocess: createSubprocess(), + shellReadySupported: true + }) + + try { + for (let index = 0; index < PRE_READY_STDIN_MAX_SEGMENTS; index += 1) { + session.write('x') + } + expect(() => session.write('x')).toThrow('safe memory limit') + } finally { + session.dispose() + } + }) +}) diff --git a/src/main/daemon/session.test.ts b/src/main/daemon/session.test.ts index bc4b45484f3..3a68c1c584f 100644 --- a/src/main/daemon/session.test.ts +++ b/src/main/daemon/session.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PRE_READY_STDIN_MAX_CODE_UNITS } from './session' import { PRODUCER_PAUSE_FAILSAFE_MS, SESSION_FORCE_KILL_RETRY_MS, Session } from './session' import type { SessionState, ShellReadyState } from './types' import type { TuiAgent } from '../../shared/types' @@ -352,6 +353,19 @@ describe('Session', () => { expect(subprocess.written).toEqual(['first\n', 'second\n']) }) + it('bounds aggregate input retained before shell readiness without changing admitted writes', () => { + createSession({ shellReadySupported: true }) + const chunk = 'x'.repeat(16 * 1024) + const admittedChunks = PRE_READY_STDIN_MAX_CODE_UNITS / chunk.length + + for (let index = 0; index < admittedChunks; index += 1) { + session.write(chunk) + } + expect(() => session.write('overflow')).toThrow('safe memory limit') + vi.advanceTimersByTime(15_000) + expect(subprocess.written).toEqual(Array(admittedChunks).fill(chunk)) + }) + it('uses the short settle path when marker and prompt bytes arrive together', () => { createSession({ shellReadySupported: true }) session.write('codex\n') diff --git a/src/main/daemon/session.ts b/src/main/daemon/session.ts index 7edaa500663..cbec43de1e4 100644 --- a/src/main/daemon/session.ts +++ b/src/main/daemon/session.ts @@ -37,6 +37,9 @@ const SESSION_FORCE_KILL_MAX_ATTEMPTS = 2 // Why: bounds in-memory pending output when no client drains it; past the cap we drop records and flag // overflow so the next take falls back to one full snapshot. UTF-16 units; worst-case wire is ~6x, under NDJSON_MAX_LINE_BYTES (16MB). const PENDING_OUTPUT_MAX_BYTES = 2 * 1024 * 1024 +export const PENDING_OUTPUT_MAX_RECORDS = 4_096 +export const PRE_READY_STDIN_MAX_CODE_UNITS = 16 * 1024 * 1024 +export const PRE_READY_STDIN_MAX_SEGMENTS = 4_096 // Why: pause is a fire-and-forget notify, so a resume can be lost (main crash, dropped socket); a lost // resume must never wedge a shell, so auto-resume after this window — a still-flooded main re-pauses. export const PRODUCER_PAUSE_FAILSAFE_MS = 5_000 @@ -112,6 +115,7 @@ export class Session { private readonly onSessionExit?: (code: number) => void private attachedClients: AttachedClient[] = [] private preReadyStdinQueue: string[] = [] + private preReadyStdinCodeUnits = 0 private shellReadyScanState: ShellReadyScanState | null = null private shellReadyTimer: ReturnType | null = null private killTimer: ReturnType | null = null @@ -218,7 +222,17 @@ export class Session { // Why: keep queuing during the post-ready flush-gate window ('ready' but not yet flushed); a // direct write would race fresh input ahead of the buffered startup command. if (this._shellState === 'pending' || this.postReadyFlushGate.isPending) { + const nextCodeUnits = this.preReadyStdinCodeUnits + data.length + if ( + nextCodeUnits > PRE_READY_STDIN_MAX_CODE_UNITS || + this.preReadyStdinQueue.length >= PRE_READY_STDIN_MAX_SEGMENTS + ) { + throw new Error( + 'Terminal input queued before shell readiness exceeds the safe memory limit.' + ) + } this.preReadyStdinQueue.push(data) + this.preReadyStdinCodeUnits = nextCodeUnits return } @@ -523,6 +537,7 @@ export class Session { this.attachedClients = [] this.preReadyStdinQueue = [] + this.preReadyStdinCodeUnits = 0 this.postReadyFlushGate.clear() this.emulator.dispose() @@ -567,6 +582,7 @@ export class Session { } this.shellReadyScanState = null this.preReadyStdinQueue = [] + this.preReadyStdinCodeUnits = 0 this.postReadyFlushGate.clear() this.disposeSubprocessHandle() } @@ -589,15 +605,20 @@ export class Session { return } const bytes = record.kind === 'output' ? record.data.length : 8 - if (this.pendingOutputBytes + bytes > PENDING_OUTPUT_MAX_BYTES) { + const last = this.pendingOutputRecords.at(-1) + const canCoalesce = + record.kind === 'output' && last?.kind === 'output' && last.data.length < 64 * 1024 + if ( + this.pendingOutputBytes + bytes > PENDING_OUTPUT_MAX_BYTES || + (!canCoalesce && this.pendingOutputRecords.length >= PENDING_OUTPUT_MAX_RECORDS) + ) { this.pendingOutputRecords = [] this.pendingOutputBytes = 0 this.pendingOutputOverflowed = true return } // Why: coalesce the thousands of tiny TUI chunks per tick to keep take RPC/log frames compact; 64KB cap bounds append cost. - const last = this.pendingOutputRecords.at(-1) - if (record.kind === 'output' && last?.kind === 'output' && last.data.length < 64 * 1024) { + if (canCoalesce) { last.data += record.data } else { this.pendingOutputRecords.push(record) @@ -720,6 +741,7 @@ export class Session { private flushPreReadyQueue(): void { const queued = this.preReadyStdinQueue this.preReadyStdinQueue = [] + this.preReadyStdinCodeUnits = 0 for (const data of queued) { this.subprocess.write(data) } diff --git a/src/main/daemon/terminal-history-file-limits.ts b/src/main/daemon/terminal-history-file-limits.ts new file mode 100644 index 00000000000..3a8e4a1891e --- /dev/null +++ b/src/main/daemon/terminal-history-file-limits.ts @@ -0,0 +1,4 @@ +export const TERMINAL_HISTORY_META_MAX_BYTES = 64 * 1024 +export const TERMINAL_HISTORY_LOG_MAX_BYTES = 5 * 1024 * 1024 +export const TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES = 16 * 1024 * 1024 +export const TERMINAL_HISTORY_LEGACY_SCROLLBACK_MAX_BYTES = 16 * 1024 * 1024 diff --git a/src/main/daemon/terminal-history-file-reader.ts b/src/main/daemon/terminal-history-file-reader.ts new file mode 100644 index 00000000000..dd6595e10e9 --- /dev/null +++ b/src/main/daemon/terminal-history-file-reader.ts @@ -0,0 +1,22 @@ +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' + +export const TERMINAL_HISTORY_JSON_MAX_STRUCTURAL_TOKENS = 1_000_000 +export const TERMINAL_HISTORY_JSON_MAX_NESTING_DEPTH = 128 + +export function readTerminalHistoryBuffer(filePath: string, maxBytes: number): Buffer { + return readNodeFileSyncWithinLimit(filePath, maxBytes).buffer +} + +export function readTerminalHistoryText(filePath: string, maxBytes: number): string { + return readTerminalHistoryBuffer(filePath, maxBytes).toString('utf8') +} + +export function readTerminalHistoryJson(filePath: string, maxBytes: number): T { + const text = readTerminalHistoryText(filePath, maxBytes) + assertJsonTextStructureWithinLimits(text, { + structuralTokens: TERMINAL_HISTORY_JSON_MAX_STRUCTURAL_TOKENS, + nestingDepth: TERMINAL_HISTORY_JSON_MAX_NESTING_DEPTH + }) + return JSON.parse(text) as T +} diff --git a/src/main/daemon/terminal-history-log-state.ts b/src/main/daemon/terminal-history-log-state.ts new file mode 100644 index 00000000000..6ad44e0977a --- /dev/null +++ b/src/main/daemon/terminal-history-log-state.ts @@ -0,0 +1,55 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { decodeLogHeader, LOG_HEADER_BYTES } from './terminal-history-log' +import { TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES } from './terminal-history-file-limits' +import { readTerminalHistoryJson } from './terminal-history-file-reader' + +export type TerminalHistoryLogState = { + checkpointPath: string + logPath: string + /** Null until a warm writer resolves the existing log on first append. */ + logGeneration: number | null + /** Null until resolved alongside the generation. */ + logBytes: number | null +} + +export function resolveTerminalHistoryLogState(writer: TerminalHistoryLogState): void { + if (writer.logBytes !== null && writer.logGeneration !== null) { + return + } + let headerGeneration: number | null = null + let size = 0 + try { + const fd = openSync(writer.logPath, 'r') + try { + size = fstatSync(fd).size + const header = Buffer.alloc(LOG_HEADER_BYTES) + if (readSync(fd, header, 0, LOG_HEADER_BYTES, 0) === LOG_HEADER_BYTES) { + headerGeneration = decodeLogHeader(header) + } + } finally { + closeSync(fd) + } + } catch { + // Missing log file starts a fresh generation below. + } + if (headerGeneration !== null) { + writer.logGeneration = headerGeneration + writer.logBytes = size + return + } + // Why: a zero byte count makes the next append replace a corrupt or headerless log. + writer.logBytes = 0 + writer.logGeneration = readCheckpointGeneration(writer.checkpointPath) ?? 0 +} + +function readCheckpointGeneration(checkpointPath: string): number | null { + try { + const checkpoint = readTerminalHistoryJson<{ generation?: unknown }>( + checkpointPath, + TERMINAL_HISTORY_CHECKPOINT_MAX_BYTES + ) + return typeof checkpoint.generation === 'number' ? checkpoint.generation : null + } catch { + return null + } +} diff --git a/src/main/daemon/terminal-history-log.test.ts b/src/main/daemon/terminal-history-log.test.ts index 426056d4725..0fd6fea07d3 100644 --- a/src/main/daemon/terminal-history-log.test.ts +++ b/src/main/daemon/terminal-history-log.test.ts @@ -1,8 +1,9 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { decodeLogHeader, decodeTerminalHistoryLog, encodeLogBatch, + encodeLogBatchWithinLimit, encodeLogHeader, LOG_HEADER_BYTES } from './terminal-history-log' @@ -44,6 +45,16 @@ describe('terminal history log codec', () => { expect(log!.batches).toEqual([{ seq: 3, records }]) }) + it('rejects an oversized batch before allocating its encoded buffer', () => { + const allocation = vi.spyOn(Buffer, 'allocUnsafe') + + expect( + encodeLogBatchWithinLimit(1, [{ kind: 'output', data: 'escaped 🐋 output' }], 12) + ).toBeNull() + expect(allocation).not.toHaveBeenCalled() + allocation.mockRestore() + }) + it('decodes multiple contiguous batches', () => { const log = decodeTerminalHistoryLog( buildLog(1, [ diff --git a/src/main/daemon/terminal-history-log.ts b/src/main/daemon/terminal-history-log.ts index 4f7866c37d6..ba260d32dca 100644 --- a/src/main/daemon/terminal-history-log.ts +++ b/src/main/daemon/terminal-history-log.ts @@ -23,6 +23,8 @@ const FRAME_BATCH = 0x01 const FRAME_OUTPUT = 0x02 const FRAME_RESIZE = 0x03 const FRAME_CLEAR = 0x04 +const FRAME_HEADER_BYTES = 5 +const BATCH_FRAME_BYTES = FRAME_HEADER_BYTES + 4 export type TerminalHistoryLogBatch = { seq: number @@ -61,20 +63,75 @@ export function decodeLogHeader(buffer: Buffer): number | null { } export function encodeLogBatch(seq: number, records: PendingOutputRecord[]): Buffer { - const frames: Buffer[] = [encodeFrame(FRAME_BATCH, encodeSeqPayload(seq))] + const byteLength = measureLogBatchBytes(records) + return encodeLogBatchWithByteLength(seq, records, byteLength) +} + +export function encodeLogBatchWithinLimit( + seq: number, + records: PendingOutputRecord[], + maxBytes: number +): Buffer | null { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + return null + } + const byteLength = measureLogBatchBytes(records, maxBytes) + return byteLength === null ? null : encodeLogBatchWithByteLength(seq, records, byteLength) +} + +function measureLogBatchBytes(records: PendingOutputRecord[]): number +function measureLogBatchBytes(records: PendingOutputRecord[], maxBytes: number): number | null +function measureLogBatchBytes(records: PendingOutputRecord[], maxBytes?: number): number | null { + let byteLength = BATCH_FRAME_BYTES + if (maxBytes !== undefined && byteLength > maxBytes) { + return null + } for (const record of records) { if (record.kind === 'output') { - frames.push(encodeFrame(FRAME_OUTPUT, Buffer.from(record.data, 'utf8'))) + const payloadBytes = Buffer.byteLength(record.data, 'utf8') + if (maxBytes !== undefined && payloadBytes > maxBytes - byteLength - FRAME_HEADER_BYTES) { + return null + } + byteLength += FRAME_HEADER_BYTES + payloadBytes } else if (record.kind === 'resize') { - const payload = Buffer.alloc(4) - payload.writeUInt16LE(clampU16(record.cols), 0) - payload.writeUInt16LE(clampU16(record.rows), 2) - frames.push(encodeFrame(FRAME_RESIZE, payload)) + byteLength += FRAME_HEADER_BYTES + 4 } else { - frames.push(encodeFrame(FRAME_CLEAR, Buffer.alloc(0))) + byteLength += FRAME_HEADER_BYTES + } + if (maxBytes !== undefined && byteLength > maxBytes) { + return null } } - return Buffer.concat(frames) + return byteLength +} + +function encodeLogBatchWithByteLength( + seq: number, + records: PendingOutputRecord[], + byteLength: number +): Buffer { + const batch = Buffer.allocUnsafe(byteLength) + let offset = 0 + offset = writeFrameHeader(batch, offset, FRAME_BATCH, 4) + batch.writeUInt32LE(seq >>> 0, offset) + offset += 4 + + for (const record of records) { + if (record.kind === 'output') { + const payloadBytes = Buffer.byteLength(record.data, 'utf8') + offset = writeFrameHeader(batch, offset, FRAME_OUTPUT, payloadBytes) + batch.write(record.data, offset, payloadBytes, 'utf8') + offset += payloadBytes + } else if (record.kind === 'resize') { + offset = writeFrameHeader(batch, offset, FRAME_RESIZE, 4) + batch.writeUInt16LE(clampU16(record.cols), offset) + batch.writeUInt16LE(clampU16(record.rows), offset + 2) + offset += 4 + } else { + offset = writeFrameHeader(batch, offset, FRAME_CLEAR, 0) + } + } + return batch } /** Returns null for missing magic / unknown format version — callers fall @@ -147,17 +204,15 @@ export function decodeTerminalHistoryLog(buffer: Buffer): TerminalHistoryLogCont return { generation, batches, truncatedTail } } -function encodeFrame(kind: number, payload: Buffer): Buffer { - const header = Buffer.alloc(5) - header.writeUInt8(kind, 0) - header.writeUInt32LE(payload.length, 1) - return Buffer.concat([header, payload]) -} - -function encodeSeqPayload(seq: number): Buffer { - const payload = Buffer.alloc(4) - payload.writeUInt32LE(seq >>> 0, 0) - return payload +function writeFrameHeader( + buffer: Buffer, + offset: number, + kind: number, + payloadBytes: number +): number { + buffer.writeUInt8(kind, offset) + buffer.writeUInt32LE(payloadBytes, offset + 1) + return offset + FRAME_HEADER_BYTES } function clampU16(value: number): number { diff --git a/src/main/daemon/terminal-history-restorable-retention.test.ts b/src/main/daemon/terminal-history-restorable-retention.test.ts new file mode 100644 index 00000000000..f713389b6ec --- /dev/null +++ b/src/main/daemon/terminal-history-restorable-retention.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { retainNewestRestorableTerminalHistorySessions } from './terminal-history-restorable-retention' + +describe('retainNewestRestorableTerminalHistorySessions', () => { + it('preserves enumeration order exactly at the cap', () => { + const sessions = [ + { sessionId: 'middle', startedAtMs: 2, order: 0 }, + { sessionId: 'oldest', startedAtMs: 1, order: 1 }, + { sessionId: 'newest', startedAtMs: 3, order: 2 } + ] + + expect(retainNewestRestorableTerminalHistorySessions(sessions, sessions.length)).toEqual([ + 'middle', + 'oldest', + 'newest' + ]) + }) + + it('retains the newest sessions while preserving their relative enumeration order', () => { + const sessions = [ + { sessionId: 'middle', startedAtMs: 2, order: 0 }, + { sessionId: 'oldest', startedAtMs: 1, order: 1 }, + { sessionId: 'newest', startedAtMs: 4, order: 2 }, + { sessionId: 'newer', startedAtMs: 3, order: 3 } + ] + + expect(retainNewestRestorableTerminalHistorySessions(sessions, 2)).toEqual(['newest', 'newer']) + }) +}) diff --git a/src/main/daemon/terminal-history-restorable-retention.ts b/src/main/daemon/terminal-history-restorable-retention.ts new file mode 100644 index 00000000000..c9287dcb840 --- /dev/null +++ b/src/main/daemon/terminal-history-restorable-retention.ts @@ -0,0 +1,69 @@ +export const MAX_RESTORABLE_TERMINAL_HISTORY_SESSIONS = 10_000 + +export type RestorableTerminalHistorySession = { + sessionId: string + startedAtMs: number + order: number +} + +function compareRecency( + left: RestorableTerminalHistorySession, + right: RestorableTerminalHistorySession +): number { + return left.startedAtMs - right.startedAtMs || left.order - right.order +} + +function siftDownOldest(heap: RestorableTerminalHistorySession[], startIndex: number): void { + let index = startIndex + while (true) { + const left = index * 2 + 1 + if (left >= heap.length) { + return + } + const right = left + 1 + const oldestChild = + right < heap.length && compareRecency(heap[right], heap[left]) < 0 ? right : left + if (compareRecency(heap[index], heap[oldestChild]) <= 0) { + return + } + const current = heap[index] + heap[index] = heap[oldestChild] + heap[oldestChild] = current + index = oldestChild + } +} + +function heapifyOldestFirst(heap: RestorableTerminalHistorySession[]): void { + for (let index = Math.floor(heap.length / 2) - 1; index >= 0; index--) { + siftDownOldest(heap, index) + } +} + +export function retainNewestRestorableTerminalHistorySessions( + sessions: Iterable, + limit = MAX_RESTORABLE_TERMINAL_HISTORY_SESSIONS +): string[] { + const retained: RestorableTerminalHistorySession[] = [] + let overflowed = false + + for (const session of sessions) { + if (retained.length < limit) { + retained.push(session) + continue + } + if (!overflowed) { + heapifyOldestFirst(retained) + overflowed = true + } + if (compareRecency(session, retained[0]) <= 0) { + continue + } + retained[0] = session + siftDownOldest(retained, 0) + } + + if (overflowed) { + retained.sort((left, right) => left.order - right.order) + } + return retained.map((session) => session.sessionId) +} diff --git a/src/main/daemon/terminal-history-session-metadata.ts b/src/main/daemon/terminal-history-session-metadata.ts new file mode 100644 index 00000000000..fb83c033314 --- /dev/null +++ b/src/main/daemon/terminal-history-session-metadata.ts @@ -0,0 +1,36 @@ +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { TERMINAL_HISTORY_META_MAX_BYTES } from './terminal-history-file-limits' +import { readTerminalHistoryJson } from './terminal-history-file-reader' + +export type SessionMeta = { + cwd: string + cols: number + rows: number + startedAt: string + endedAt: string | null + exitCode: number | null +} + +export type OpenSessionOptions = { + cwd: string + cols: number + rows: number +} + +export type HistoryManagerOptions = { + onWriteError?: (sessionId: string, error: Error) => void +} + +export function stringifyTerminalHistorySessionMeta(meta: SessionMeta): string { + // Preflight compact JSON before materializing the larger pretty representation. + stringifyJsonWithinByteLimit(meta, TERMINAL_HISTORY_META_MAX_BYTES) + const serialized = JSON.stringify(meta, null, 2) + if (Buffer.byteLength(serialized, 'utf8') > TERMINAL_HISTORY_META_MAX_BYTES) { + throw new Error('Terminal history metadata exceeds its byte limit') + } + return serialized +} + +export function readTerminalHistorySessionMeta(filePath: string): SessionMeta { + return readTerminalHistoryJson(filePath, TERMINAL_HISTORY_META_MAX_BYTES) +} diff --git a/src/main/daemon/terminal-host-session-listing.ts b/src/main/daemon/terminal-host-session-listing.ts index 2f0e99c332d..3ae2d5e845e 100644 --- a/src/main/daemon/terminal-host-session-listing.ts +++ b/src/main/daemon/terminal-host-session-listing.ts @@ -19,6 +19,7 @@ export function listLiveTerminalHostSessions( shellState: session.shellState, isAlive: true, ...(session.terminalHandle ? { terminalHandle: session.terminalHandle } : {}), + wslDistro: session.wslDistro, pid: session.pid, cwd: session.getCwd(), cols: size?.cols ?? 0, diff --git a/src/main/daemon/terminal-host.test.ts b/src/main/daemon/terminal-host.test.ts index e7e83e31a31..c1e9f09ebc1 100644 --- a/src/main/daemon/terminal-host.test.ts +++ b/src/main/daemon/terminal-host.test.ts @@ -84,6 +84,9 @@ describe('TerminalHost', () => { await host.dispose() }) + it('rejects missing strict inspection', () => + expect(() => host.inspectProcess('missing-session')).toThrow('not found')) + describe('createOrAttach', () => { it('creates a new session when none exists', async () => { const result = await host.createOrAttach({ diff --git a/src/main/daemon/terminal-host.ts b/src/main/daemon/terminal-host.ts index 6be96d27d8e..ae348a3854e 100644 --- a/src/main/daemon/terminal-host.ts +++ b/src/main/daemon/terminal-host.ts @@ -16,6 +16,7 @@ import { resolveTerminalHostSessionCwd } from './terminal-host-session-cwd' import { TerminalHostTombstones } from './terminal-host-tombstones' import { listLiveTerminalHostSessions } from './terminal-host-session-listing' import { createOrAttachTerminalSession } from './terminal-host-session-create' +import { isShellProcess } from '../../shared/agent-detection' export type { CreateOrAttachOptions, CreateOrAttachResult } from './terminal-host-create-contract' export type { TerminalHostOptions } from './terminal-host-options' @@ -146,6 +147,17 @@ export class TerminalHost { return session.getForegroundProcess() } + inspectProcess(sessionId: string): { + foregroundProcess: string | null + hasChildProcesses: boolean + } { + const foregroundProcess = this.getAliveSession(sessionId).getForegroundProcess() + return { + foregroundProcess, + hasChildProcesses: foregroundProcess !== null && !isShellProcess(foregroundProcess) + } + } + async confirmForegroundProcess(sessionId: string): Promise { const session = this.sessions.get(sessionId) if (!session || !session.isAlive) { diff --git a/src/main/daemon/types.ts b/src/main/daemon/types.ts index bec511d8e1e..2ee1ebd9535 100644 --- a/src/main/daemon/types.ts +++ b/src/main/daemon/types.ts @@ -1,11 +1,13 @@ import type { ConfirmForegroundProcessRequest, - GetForegroundProcessRequest + GetForegroundProcessRequest, + InspectProcessRequest } from './daemon-foreground-process-protocol' export type { ConfirmForegroundProcessRequest, - GetForegroundProcessRequest + GetForegroundProcessRequest, + InspectProcessRequest } from './daemon-foreground-process-protocol' // ─── Protocol Version ──────────────────────────────────────────────── @@ -24,6 +26,7 @@ export { AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, CLEAN_DISCONNECT_PROTOCOL_VERSION, + COMPLETION_PROCESS_INSPECTION_PROTOCOL_VERSION, GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION, PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION, @@ -302,6 +305,7 @@ export type DaemonRequest = | DetachRequest | GetCwdRequest | GetForegroundProcessRequest + | InspectProcessRequest | ConfirmForegroundProcessRequest | ClearScrollbackRequest | ShutdownRequest @@ -355,6 +359,7 @@ export type SessionInfo = { shellState: ShellReadyState isAlive: boolean terminalHandle?: string + wslDistro?: string | null pid: number | null cwd: string | null cols: number diff --git a/src/main/devin/hook-config-json.ts b/src/main/devin/hook-config-json.ts index 6a316ebc6aa..2c9592c0d39 100644 --- a/src/main/devin/hook-config-json.ts +++ b/src/main/devin/hook-config-json.ts @@ -1,5 +1,7 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { parse as parseJsonc, type ParseError } from 'jsonc-parser' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { AGENT_HOOK_CONFIG_MAX_BYTES } from '../agent-hooks/agent-hook-file-limits' import { isPlainObject, type HooksConfig } from '../agent-hooks/installer-utils' /** Devin documents config.json as JSONC; stock JSON.parse rejects comments. */ @@ -9,7 +11,10 @@ export function readDevinHooksConfig(configPath: string): HooksConfig | null { } try { - const text = readFileSync(configPath, 'utf-8') + const text = readNodeFileSyncWithinLimit( + configPath, + AGENT_HOOK_CONFIG_MAX_BYTES + ).buffer.toString('utf8') return parseDevinHooksConfigText(text, 'Devin config.json') } catch { return null diff --git a/src/main/diagnostics/main-thread-churn-probe.test.ts b/src/main/diagnostics/main-thread-churn-probe.test.ts index cfc159fc1fb..deefc95273b 100644 --- a/src/main/diagnostics/main-thread-churn-probe.test.ts +++ b/src/main/diagnostics/main-thread-churn-probe.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { MAIN_THREAD_DIAGNOSTICS_ENV, + SUBPROCESS_SPAWN_STATS_MAX_ENTRIES, classifySubprocessCommand, drainSubprocessSpawnStats, isMainThreadDiagnosticsEnabled, @@ -71,4 +72,22 @@ describe('recordSubprocessSpawn', () => { }) expect(drainSubprocessSpawnStats()).toEqual({}) }) + + it('bounds unique diagnostic buckets and aggregates overflow', () => { + vi.stubEnv(MAIN_THREAD_DIAGNOSTICS_ENV, '1') + for (let index = 0; index < 200; index += 1) { + recordSubprocessSpawn(`tool-${index}`, [], 1) + } + + const drained = drainSubprocessSpawnStats() + expect(Object.keys(drained)).toHaveLength(SUBPROCESS_SPAWN_STATS_MAX_ENTRIES) + expect(drained.other).toEqual({ count: 73, blockMsTotal: 73, blockMsMax: 1 }) + }) + + it('materializes a bounded binary bucket from an oversized command path', () => { + const command = `/tmp/${'x'.repeat(1024 * 1024)}` + const classified = classifySubprocessCommand(command, []) + expect(classified).toHaveLength(64) + expect(classified).toBe('x'.repeat(64)) + }) }) diff --git a/src/main/diagnostics/main-thread-churn-probe.ts b/src/main/diagnostics/main-thread-churn-probe.ts index f4fe900cf8c..97dc30a93eb 100644 --- a/src/main/diagnostics/main-thread-churn-probe.ts +++ b/src/main/diagnostics/main-thread-churn-probe.ts @@ -8,6 +8,9 @@ export const MAIN_THREAD_DIAGNOSTICS_ENV = 'ORCA_MAIN_THREAD_DIAGNOSTICS' // reported in issue #7576. const TICK_MS = 25 const REPORT_EVERY_MS = 5_000 +const SUBPROCESS_BINARY_NAME_MAX_CHARS = 64 +export const SUBPROCESS_SPAWN_STATS_MAX_ENTRIES = 128 +const SUBPROCESS_SPAWN_STATS_OVERFLOW_KEY = 'other' export function isMainThreadDiagnosticsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { return env[MAIN_THREAD_DIAGNOSTICS_ENV] === '1' @@ -30,32 +33,39 @@ const SUBCOMMAND_BINARIES = new Set(['git', 'gh', 'glab']) // Split on both separators so Windows-style paths classify correctly even // when the classifier itself runs in a posix test environment. function binaryName(command: string): string { - const leaf = command.split(/[\\/]/).pop() ?? command - return leaf.replace(/\.exe$/i, '').toLowerCase() + const slash = command.lastIndexOf('/') + const backslash = command.lastIndexOf('\\') + const leaf = command.slice( + Math.max(slash, backslash) + 1, + Math.max(slash, backslash) + 1 + SUBPROCESS_BINARY_NAME_MAX_CHARS + ) + return leaf + .replace(/\.exe$/i, '') + .toLowerCase() + .replace(/$/u, '') } export function classifySubprocessCommand(command: string, args: readonly string[]): string { let binary = binaryName(command) - const rest = [...args] + let firstArgIndex = 0 if (binary === 'wsl') { - while (rest.length > 0) { - const arg = rest.shift() - if (arg === '--') { - break - } + const separatorIndex = args.indexOf('--') + if (separatorIndex < 0) { + return 'wsl' } - const unwrapped = rest.shift() + const unwrapped = args[separatorIndex + 1] if (!unwrapped) { return 'wsl' } binary = binaryName(unwrapped) + firstArgIndex = separatorIndex + 2 } if (!SUBCOMMAND_BINARIES.has(binary)) { return binary } let subcommand: string | null = null - for (let i = 0; i < rest.length; i++) { - const arg = rest[i] + for (let i = firstArgIndex; i < args.length; i++) { + const arg = args[i] if (!arg.startsWith('-')) { subcommand = arg break @@ -69,7 +79,7 @@ export function classifySubprocessCommand(command: string, args: readonly string if (!subcommand) { return binary } - return `${binary} ${subcommand.slice(0, 40)}` + return `${binary} ${subcommand.slice(0, 40).replace(/$/u, '')}` } export type SubprocessSpawnStats = { @@ -95,7 +105,12 @@ export function recordSubprocessSpawn( if (!isMainThreadDiagnosticsEnabled()) { return } - const key = classifySubprocessCommand(command, args) + const classifiedKey = classifySubprocessCommand(command, args) + const key = + spawnStatsByCommand.has(classifiedKey) || + spawnStatsByCommand.size < SUBPROCESS_SPAWN_STATS_MAX_ENTRIES - 1 + ? classifiedKey + : SUBPROCESS_SPAWN_STATS_OVERFLOW_KEY const stats = spawnStatsByCommand.get(key) if (stats) { stats.count++ diff --git a/src/main/emulator/android/android-device-inventory.test.ts b/src/main/emulator/android/android-device-inventory.test.ts index d70d35a4510..a608a53b518 100644 --- a/src/main/emulator/android/android-device-inventory.test.ts +++ b/src/main/emulator/android/android-device-inventory.test.ts @@ -2,9 +2,11 @@ import { describe, expect, it, vi } from 'vitest' import type { AndroidCommandResult, AndroidCommandRunner } from './android-command-runner' import type { AndroidSdkPaths } from './android-sdk-discovery' import { + ANDROID_AVD_NAME_LOOKUP_CONCURRENCY, findRunningAvdSerial, listAndroidDevices, - mergeAndroidDevices + mergeAndroidDevices, + resolveRunningAvdNames } from './android-device-inventory' import { parseAdbDevices } from './adb-devices' @@ -88,4 +90,41 @@ describe('findRunningAvdSerial', () => { expect(await findRunningAvdSerial(fake, SDK, 'Pixel_7', running)).toBe('emulator-5554') expect(await findRunningAvdSerial(fake, SDK, 'Other', running)).toBeNull() }) + + it.each([ + ['at the limit', ANDROID_AVD_NAME_LOOKUP_CONCURRENCY], + ['above the limit', ANDROID_AVD_NAME_LOOKUP_CONCURRENCY + 1] + ])('bounds AVD name probes %s', async (_, count) => { + let active = 0 + let peak = 0 + let started = 0 + const releases: (() => void)[] = [] + const fake: AndroidCommandRunner = async (_binary, args) => { + started++ + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + return ok(`${args[1]}\nOK`) + } + const running = parseAdbDevices( + `List of devices attached\n${Array.from( + { length: count }, + (_, index) => `emulator-${index}\tdevice` + ).join('\n')}` + ) + + const result = resolveRunningAvdNames(fake, SDK, running) + await vi.waitFor(() => + expect(started).toBe(Math.min(count, ANDROID_AVD_NAME_LOOKUP_CONCURRENCY)) + ) + if (count > ANDROID_AVD_NAME_LOOKUP_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(started).toBe(count)) + } + releases.splice(0).forEach((release) => release()) + + expect((await result).size).toBe(count) + expect(peak).toBe(Math.min(count, ANDROID_AVD_NAME_LOOKUP_CONCURRENCY)) + }) }) diff --git a/src/main/emulator/android/android-device-inventory.ts b/src/main/emulator/android/android-device-inventory.ts index 63c3b17a2e8..1df03fce44c 100644 --- a/src/main/emulator/android/android-device-inventory.ts +++ b/src/main/emulator/android/android-device-inventory.ts @@ -3,6 +3,9 @@ import type { AndroidSdkPaths } from './android-sdk-discovery' import { adbDevicesArgs, parseAdbDevices, type AndroidAdbDevice } from './adb-devices' import { listAvdsArgs, parseAvdList } from './avd-manager' import type { EmulatorDevice } from '../backends/emulator-backend' +import { mapWithConcurrency } from '../../../shared/map-with-concurrency' + +export const ANDROID_AVD_NAME_LOOKUP_CONCURRENCY = 4 // Android device discovery: turns raw `adb`/`emulator` output into the // cross-backend EmulatorDevice list and resolves AVD names to running serials. @@ -23,17 +26,19 @@ export async function resolveRunningAvdNames( running: AndroidAdbDevice[] ): Promise> { const names = new Map() - await Promise.all( - running - .filter((device) => device.isEmulator) - .map(async (device) => { - const out = await runner(sdk.adb, ['-s', device.serial, 'emu', 'avd', 'name']) - const name = firstNonStatusLine(out.stdout) - if (name) { - names.set(device.serial, name) - } - }) + const resolved = await mapWithConcurrency( + running.filter((device) => device.isEmulator), + ANDROID_AVD_NAME_LOOKUP_CONCURRENCY, + async (device) => { + const out = await runner(sdk.adb, ['-s', device.serial, 'emu', 'avd', 'name']) + return { serial: device.serial, name: firstNonStatusLine(out.stdout) } + } ) + for (const { serial, name } of resolved) { + if (name) { + names.set(serial, name) + } + } return names } diff --git a/src/main/emulator/android/scrcpy-stream-session.test.ts b/src/main/emulator/android/scrcpy-stream-session.test.ts new file mode 100644 index 00000000000..59fefd4e762 --- /dev/null +++ b/src/main/emulator/android/scrcpy-stream-session.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { + appendScrcpyServerLogPreview, + SCRCPY_SERVER_LOG_PREVIEW_CHARS +} from './scrcpy-stream-session' + +describe('appendScrcpyServerLogPreview', () => { + it('preserves the diagnostic prefix while bounding a chatty long-lived server', () => { + const prefix = 'startup\n' + const first = appendScrcpyServerLogPreview(prefix, Buffer.from('x'.repeat(100_000))) + const complete = appendScrcpyServerLogPreview(first, Buffer.from('ignored forever')) + + expect(complete).toBe(first) + expect(complete).toHaveLength(SCRCPY_SERVER_LOG_PREVIEW_CHARS) + expect(complete.startsWith(prefix)).toBe(true) + }) +}) diff --git a/src/main/emulator/android/scrcpy-stream-session.ts b/src/main/emulator/android/scrcpy-stream-session.ts index b7cec131e2c..552c1bfc3b5 100644 --- a/src/main/emulator/android/scrcpy-stream-session.ts +++ b/src/main/emulator/android/scrcpy-stream-session.ts @@ -29,6 +29,7 @@ import { emulatorProbe, emulatorProbeError } from '../emulator-probe' const DEVICE_NAME_BYTES = 64 const DUMMY_BYTE = 1 const DYNAMIC_FORWARD_PORT = 0 +export const SCRCPY_SERVER_LOG_PREVIEW_CHARS = 1_000 export type ScrcpyStreamCallbacks = { onMeta: (meta: ScrcpyVideoMeta) => void @@ -46,6 +47,14 @@ export type ScrcpyStreamOptions = { maxSize?: number } +export function appendScrcpyServerLogPreview(current: string, chunk: Buffer): string { + const remaining = SCRCPY_SERVER_LOG_PREVIEW_CHARS - current.length + if (remaining <= 0) { + return current + } + return current + chunk.toString().slice(0, remaining) +} + function newScid(): string { // scrcpy parses scid as a SIGNED 32-bit hex int, so mask to 31 bits and pad to // 8 hex digits to match the server's own %08x format. @@ -127,13 +136,13 @@ export class ScrcpyStreamSession { }) let serverLog = '' const capture = (chunk: Buffer): void => { - serverLog += chunk.toString() + serverLog = appendScrcpyServerLogPreview(serverLog, chunk) } this.server.stdout?.on('data', capture) this.server.stderr?.on('data', capture) this.server.on('error', (error) => this.fail(error.message)) this.server.on('exit', (code) => { - emulatorProbe('scrcpy.server.exit', { code, log: serverLog.slice(0, 1000).trim() }) + emulatorProbe('scrcpy.server.exit', { code, log: serverLog.trim() }) if (!this.metaSeen) { this.fail('scrcpy server exited before the video stream started') return diff --git a/src/main/emulator/android/uiautomator-tree.test.ts b/src/main/emulator/android/uiautomator-tree.test.ts index 041c1a4abbd..9d9ec7665f1 100644 --- a/src/main/emulator/android/uiautomator-tree.test.ts +++ b/src/main/emulator/android/uiautomator-tree.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest' import { EmulatorError } from '../emulator-errors' -import { parseAndroidBounds, parseUiAutomatorXml } from './uiautomator-tree' +import { + ANDROID_UIAUTOMATOR_XML_MAX_ATTRIBUTES_PER_ELEMENT, + ANDROID_UIAUTOMATOR_XML_MAX_DEPTH, + ANDROID_UIAUTOMATOR_XML_MAX_ELEMENTS, + parseAndroidBounds, + parseUiAutomatorXml +} from './uiautomator-tree' // Realistic `uiautomator dump` output: an XML prolog, a root, and // nested self-describing elements (mix of container + self-closing leaf). @@ -97,6 +103,23 @@ describe('parseUiAutomatorXml', () => { expect(() => parseUiAutomatorXml('')).toThrowError(EmulatorError) expect(() => parseUiAutomatorXml('')).toThrowError(EmulatorError) }) + + it('rejects element-count, nesting, and per-element attribute amplification', () => { + const tooManyElements = `${''.repeat( + ANDROID_UIAUTOMATOR_XML_MAX_ELEMENTS + )}` + const tooDeep = `${''.repeat(ANDROID_UIAUTOMATOR_XML_MAX_DEPTH + 1)}${''.repeat( + ANDROID_UIAUTOMATOR_XML_MAX_DEPTH + 1 + )}` + const tooManyAttributes = ` `a${index}="x"` + ).join(' ')} />` + + expect(() => parseUiAutomatorXml(tooManyElements)).toThrow('exceeds 50000 elements') + expect(() => parseUiAutomatorXml(tooDeep)).toThrow('exceeds 256 levels') + expect(() => parseUiAutomatorXml(tooManyAttributes)).toThrow('exceeds 64 attributes') + }) }) describe('parseAndroidBounds', () => { diff --git a/src/main/emulator/android/uiautomator-tree.ts b/src/main/emulator/android/uiautomator-tree.ts index 3a3bb7d6e6a..f63299db528 100644 --- a/src/main/emulator/android/uiautomator-tree.ts +++ b/src/main/emulator/android/uiautomator-tree.ts @@ -18,6 +18,11 @@ export type AndroidAxNode = { // Raw element produced by the parser before mapping to the typed Android node. type RawElement = { tag: string; attributes: Record; children: RawElement[] } +export const ANDROID_UIAUTOMATOR_XML_MAX_BYTES = 16 * 1024 * 1024 +export const ANDROID_UIAUTOMATOR_XML_MAX_ELEMENTS = 50_000 +export const ANDROID_UIAUTOMATOR_XML_MAX_DEPTH = 256 +export const ANDROID_UIAUTOMATOR_XML_MAX_ATTRIBUTES_PER_ELEMENT = 64 + // Parses an Android bounds string "[left,top][right,bottom]" -> AndroidAxBounds, // or null when the format doesn't match. Coordinates may be negative (off-screen). export function parseAndroidBounds(value: string): AndroidAxBounds | null { @@ -40,6 +45,9 @@ export function parseUiAutomatorXml(xml: string): AndroidAxNode { if (xml.trim() === '') { throw new EmulatorError('emulator_error', 'Cannot parse empty uiautomator XML') } + if (Buffer.byteLength(xml) > ANDROID_UIAUTOMATOR_XML_MAX_BYTES) { + throw new EmulatorError('emulator_error', 'uiautomator XML exceeds the 16 MiB limit') + } let root: RawElement try { root = parseDocument(xml) @@ -107,6 +115,7 @@ function setBool( function parseDocument(xml: string): RawElement { let i = 0 const n = xml.length + let elementCount = 0 const fail = (message: string): never => { throw new EmulatorError('emulator_error', `${message} at offset ${i}`) @@ -156,7 +165,14 @@ function parseDocument(xml: string): RawElement { } } - const parseElement = (): RawElement => { + const parseElement = (depth: number): RawElement => { + if (depth > ANDROID_UIAUTOMATOR_XML_MAX_DEPTH) { + fail(`uiautomator XML exceeds ${ANDROID_UIAUTOMATOR_XML_MAX_DEPTH} levels`) + } + elementCount += 1 + if (elementCount > ANDROID_UIAUTOMATOR_XML_MAX_ELEMENTS) { + fail(`uiautomator XML exceeds ${ANDROID_UIAUTOMATOR_XML_MAX_ELEMENTS} elements`) + } if (xml[i] !== '<') { fail('Expected element start') } @@ -166,6 +182,7 @@ function parseDocument(xml: string): RawElement { fail('Expected tag name') } const attributes: Record = {} + let attributeCount = 0 for (;;) { skipWs() if (i >= n) { @@ -186,6 +203,12 @@ function parseDocument(xml: string): RawElement { if (name === '') { fail('Expected attribute name') } + attributeCount += 1 + if (attributeCount > ANDROID_UIAUTOMATOR_XML_MAX_ATTRIBUTES_PER_ELEMENT) { + fail( + `uiautomator XML element exceeds ${ANDROID_UIAUTOMATOR_XML_MAX_ATTRIBUTES_PER_ELEMENT} attributes` + ) + } skipWs() if (xml[i] !== '=') { fail("Expected '=' after attribute name") @@ -207,10 +230,14 @@ function parseDocument(xml: string): RawElement { attributes[name] = decodeEntities(xml.slice(start, i)) i++ } - return parseChildren(tag, attributes) + return parseChildren(tag, attributes, depth) } - const parseChildren = (tag: string, attributes: Record): RawElement => { + const parseChildren = ( + tag: string, + attributes: Record, + depth: number + ): RawElement => { const children: RawElement[] = [] for (;;) { if (i >= n) { @@ -243,7 +270,7 @@ function parseDocument(xml: string): RawElement { } else if (startsWith('', 'processing instruction') } else { - children.push(parseElement()) + children.push(parseElement(depth + 1)) } } } @@ -252,7 +279,7 @@ function parseDocument(xml: string): RawElement { if (i >= n || xml[i] !== '<') { fail('No root element found') } - return parseElement() + return parseElement(1) } // Decodes the predefined XML entities plus numeric character references. diff --git a/src/main/emulator/mjpeg-frame-stream-reconnect.test.ts b/src/main/emulator/mjpeg-frame-stream-reconnect.test.ts new file mode 100644 index 00000000000..ad8816d3893 --- /dev/null +++ b/src/main/emulator/mjpeg-frame-stream-reconnect.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +type FakeResponseLike = { + destroy: ReturnType + destroyed: boolean + emit: (eventName: string, ...args: unknown[]) => boolean +} + +type FakeRequestLike = { + destroy: ReturnType + destroyed: boolean + emit: (eventName: string, ...args: unknown[]) => boolean + respond: (statusCode: number) => FakeResponseLike +} + +const requestState = vi.hoisted(() => ({ + requests: [] as FakeRequestLike[] +})) + +vi.mock('node:http', async () => { + const { EventEmitter } = await import('node:events') + + class FakeResponse extends EventEmitter implements FakeResponseLike { + destroyed = false + readonly resume = vi.fn() + readonly destroy = vi.fn(() => { + this.destroyed = true + return this + }) + + constructor(readonly statusCode: number) { + super() + } + } + + class FakeRequest extends EventEmitter implements FakeRequestLike { + destroyed = false + private response: FakeResponse | null = null + readonly end = vi.fn() + readonly destroy = vi.fn((error?: Error) => { + if (this.destroyed) { + return this + } + this.destroyed = true + this.response?.destroy() + if (error) { + this.emit('error', error) + } + return this + }) + + constructor(private readonly onResponse: (response: FakeResponse) => void) { + super() + } + + respond(statusCode: number): FakeResponse { + const response = new FakeResponse(statusCode) + this.response = response + this.onResponse(response) + return response + } + } + + return { + request: vi.fn( + ( + _url: unknown, + _options: unknown, + onResponse: (response: FakeResponse) => void + ): FakeRequest => { + const request = new FakeRequest(onResponse) + requestState.requests.push(request) + return request + } + ) + } +}) + +import { MjpegFrameStream } from './mjpeg-frame-stream' + +const JPEG = Buffer.from([0xff, 0xd8, 0x01, 0x02, 0xff, 0xd9]) +const streams: MjpegFrameStream[] = [] + +function makeStream() { + const callbacks = { + onError: vi.fn(), + onFrame: vi.fn() + } + const stream = new MjpegFrameStream('http://127.0.0.1:3100/stream.mjpeg', callbacks) + streams.push(stream) + return { callbacks, stream } +} + +beforeEach(() => { + vi.useFakeTimers() + requestState.requests.length = 0 +}) + +afterEach(() => { + for (const stream of streams) { + stream.stop() + } + streams.length = 0 + vi.useRealTimers() +}) + +describe('MjpegFrameStream reconnect ownership', () => { + it('closes an endless error response before reconnecting and ignores stale errors', () => { + const { callbacks, stream } = makeStream() + stream.start() + stream.start() + expect(requestState.requests).toHaveLength(1) + + const firstRequest = requestState.requests[0] + const errorResponse = firstRequest.respond(503) + expect(firstRequest.destroyed).toBe(true) + expect(errorResponse.destroyed).toBe(true) + expect(callbacks.onError).toHaveBeenCalledWith('Simulator stream returned HTTP 503.') + + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + + firstRequest.emit('error', new Error('late failure')) + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + expect(callbacks.onError).toHaveBeenCalledTimes(1) + + requestState.requests[1].emit('timeout') + expect(callbacks.onError).toHaveBeenLastCalledWith('Simulator stream timed out.') + stream.stop() + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + }) + + it('delivers a valid response and reconnects once after it ends', () => { + const { callbacks, stream } = makeStream() + stream.start() + const firstRequest = requestState.requests[0] + const response = firstRequest.respond(200) + + response.emit('data', JPEG) + expect(callbacks.onFrame).toHaveBeenCalledWith(JPEG) + + response.emit('end') + expect(firstRequest.destroyed).toBe(true) + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + + response.emit('error', new Error('late response error')) + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + expect(callbacks.onError).not.toHaveBeenCalled() + }) + + it('destroys a response that arrives after its request is stale', () => { + const { callbacks, stream } = makeStream() + stream.start() + const firstRequest = requestState.requests[0] + + firstRequest.emit('error', new Error('connection reset')) + vi.runOnlyPendingTimers() + expect(requestState.requests).toHaveLength(2) + + const staleResponse = firstRequest.respond(200) + staleResponse.emit('data', JPEG) + expect(staleResponse.destroyed).toBe(true) + expect(callbacks.onFrame).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/emulator/mjpeg-frame-stream.ts b/src/main/emulator/mjpeg-frame-stream.ts index 315b13f303d..1df5817839c 100644 --- a/src/main/emulator/mjpeg-frame-stream.ts +++ b/src/main/emulator/mjpeg-frame-stream.ts @@ -56,6 +56,9 @@ export class MjpegFrameStream { } start(): void { + if (this.stopped || this.request || this.reconnectTimer) { + return + } this.openRequest() } @@ -70,10 +73,12 @@ export class MjpegFrameStream { this.pending = Buffer.alloc(0) } - private scheduleReconnect(): void { - if (this.stopped || this.reconnectTimer) { + private scheduleReconnect(request: ClientRequest): void { + if (this.stopped || this.request !== request || this.reconnectTimer) { return } + this.request = null + request.destroy() this.reconnectTimer = setTimeout(() => { this.reconnectTimer = null this.openRequest() @@ -81,34 +86,45 @@ export class MjpegFrameStream { } private openRequest(): void { - if (this.stopped) { + if (this.stopped || this.request) { return } const req = requestForUrl(this.streamUrl, (res) => { + if (this.stopped || this.request !== req) { + res.destroy() + return + } if (res.statusCode && res.statusCode >= 400) { + this.scheduleReconnect(req) + res.destroy() this.callbacks.onError(`Simulator stream returned HTTP ${res.statusCode}.`) - res.resume() - this.scheduleReconnect() return } - res.on('data', (chunk: Buffer) => this.handleChunk(chunk)) - res.on('end', () => this.scheduleReconnect()) + res.on('data', (chunk: Buffer) => { + if (!this.stopped && this.request === req) { + this.handleChunk(chunk) + } + }) + res.on('end', () => this.scheduleReconnect(req)) res.on('error', (error) => { + if (this.stopped || this.request !== req) { + return + } + this.scheduleReconnect(req) this.callbacks.onError(error.message) - this.scheduleReconnect() }) }) this.request = req req.on('timeout', () => req.destroy(new Error('Simulator stream timed out.'))) req.on('error', (error) => { - if (this.stopped) { + if (this.stopped || this.request !== req) { return } + this.scheduleReconnect(req) this.callbacks.onError(error.message) - this.scheduleReconnect() }) req.end() } diff --git a/src/main/emulator/scrcpy-video-registry-memory.test.ts b/src/main/emulator/scrcpy-video-registry-memory.test.ts new file mode 100644 index 00000000000..805c7f9bb83 --- /dev/null +++ b/src/main/emulator/scrcpy-video-registry-memory.test.ts @@ -0,0 +1,194 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + SCRCPY_VIDEO_MAX_GOP_FRAMES, + SCRCPY_VIDEO_MAX_REGISTRY_ENTRIES, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL, + SCRCPY_VIDEO_MAX_SUBSCRIBERS, + scrcpyVideoRegistry, + type ScrcpyVideoFrameMessage +} from './scrcpy-video-registry' + +const MEBIBYTE = 1024 * 1024 +const registeredDevices = new Set() + +function registerDevice(deviceId: string, close = vi.fn()): void { + scrcpyVideoRegistry.register(deviceId, close) + registeredDevices.add(deviceId) +} + +function frame( + pts: string, + byteLength: number, + options: { config?: boolean; keyFrame?: boolean } = {} +): ScrcpyVideoFrameMessage { + return { + config: options.config === true, + keyFrame: options.keyFrame === true, + pts, + bytes: { byteLength } as ArrayBuffer + } +} + +function replayFrames(deviceId: string): ScrcpyVideoFrameMessage[] { + const frames: ScrcpyVideoFrameMessage[] = [] + const unsubscribe = scrcpyVideoRegistry.subscribe(deviceId, (event) => { + if (event.type === 'frame') { + frames.push(event.frame) + } + }) + unsubscribe() + return frames +} + +function replayBytes(deviceId: string): number { + return replayFrames(deviceId).reduce((total, replayed) => total + replayed.bytes.byteLength, 0) +} + +afterEach(() => { + for (const deviceId of registeredDevices) { + scrcpyVideoRegistry.stop(deviceId) + } + registeredDevices.clear() +}) + +describe('scrcpy video replay retention', () => { + it('sheds an overloaded GOP until a fresh keyframe arrives', () => { + registerDevice('frame-cap') + scrcpyVideoRegistry.pushFrame('frame-cap', frame('key', 1, { keyFrame: true })) + for (let index = 1; index <= SCRCPY_VIDEO_MAX_GOP_FRAMES + 10; index += 1) { + scrcpyVideoRegistry.pushFrame('frame-cap', frame(String(index), 1)) + } + + expect(replayFrames('frame-cap')).toEqual([]) + + scrcpyVideoRegistry.pushFrame('frame-cap', frame('recovery-key', 1, { keyFrame: true })) + expect(replayFrames('frame-cap').map((replayed) => replayed.pts)).toEqual(['recovery-key']) + }) + + it('sheds only cached deltas under per-device byte pressure', () => { + registerDevice('device-bytes') + const liveFrames: string[] = [] + const unsubscribe = scrcpyVideoRegistry.subscribe('device-bytes', (event) => { + if (event.type === 'frame') { + liveFrames.push(event.frame.pts) + } + }) + + scrcpyVideoRegistry.pushFrame('device-bytes', frame('config', MEBIBYTE, { config: true })) + scrcpyVideoRegistry.pushFrame('device-bytes', frame('key', 8 * MEBIBYTE, { keyFrame: true })) + for (let index = 1; index <= 5; index += 1) { + scrcpyVideoRegistry.pushFrame('device-bytes', frame(`delta-${index}`, 8 * MEBIBYTE)) + } + unsubscribe() + + expect(liveFrames).toEqual([ + 'config', + 'key', + 'delta-1', + 'delta-2', + 'delta-3', + 'delta-4', + 'delta-5' + ]) + expect(replayFrames('device-bytes').map((replayed) => replayed.pts)).toEqual(['config']) + expect(replayBytes('device-bytes')).toBeLessThanOrEqual( + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE + ) + scrcpyVideoRegistry.pushFrame( + 'device-bytes', + frame('recovery-key', 8 * MEBIBYTE, { keyFrame: true }) + ) + expect(replayFrames('device-bytes').map((replayed) => replayed.pts)).toEqual([ + 'config', + 'recovery-key' + ]) + }) + + it('delivers an oversized frame live without retaining invalid replay', () => { + registerDevice('oversized') + const liveFrames: string[] = [] + const unsubscribe = scrcpyVideoRegistry.subscribe('oversized', (event) => { + if (event.type === 'frame') { + liveFrames.push(event.frame.pts) + } + }) + scrcpyVideoRegistry.pushFrame('oversized', frame('config', 1, { config: true })) + scrcpyVideoRegistry.pushFrame( + 'oversized', + frame('key', SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE, { keyFrame: true }) + ) + unsubscribe() + + expect(liveFrames).toEqual(['config', 'key']) + expect(replayFrames('oversized').map((replayed) => replayed.pts)).toEqual(['config']) + }) + + it('bounds replay bytes across devices while protecting the newest device', () => { + const perEntryBytes = 16 * MEBIBYTE + const deviceCount = Math.floor(SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL / perEntryBytes) + 1 + const deviceIds = Array.from({ length: deviceCount }, (_, index) => `aggregate-${index}`) + + for (const deviceId of deviceIds) { + registerDevice(deviceId) + scrcpyVideoRegistry.pushFrame(deviceId, frame('config', MEBIBYTE, { config: true })) + scrcpyVideoRegistry.pushFrame( + deviceId, + frame('key', perEntryBytes - MEBIBYTE, { keyFrame: true }) + ) + } + + const totalReplayBytes = deviceIds.reduce((total, deviceId) => total + replayBytes(deviceId), 0) + expect(totalReplayBytes).toBeLessThanOrEqual(SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL) + expect(replayFrames(deviceIds.at(-1)!).map((replayed) => replayed.pts)).toEqual([ + 'config', + 'key' + ]) + for (const deviceId of deviceIds) { + expect(replayFrames(deviceId).map((replayed) => replayed.pts)).toEqual( + expect.arrayContaining(['config']) + ) + } + }) + + it('cleans accounting before a reentrant close callback', () => { + const close = vi.fn(() => scrcpyVideoRegistry.stop('reentrant')) + registerDevice('reentrant', close) + scrcpyVideoRegistry.pushFrame( + 'reentrant', + frame('key', SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE, { keyFrame: true }) + ) + + scrcpyVideoRegistry.stop('reentrant') + + expect(close).toHaveBeenCalledOnce() + expect(scrcpyVideoRegistry.has('reentrant')).toBe(false) + }) + + it('bounds active registry entries without closing admitted streams', () => { + const closes: ReturnType[] = [] + for (let index = 0; index < SCRCPY_VIDEO_MAX_REGISTRY_ENTRIES; index += 1) { + const close = vi.fn() + closes.push(close) + registerDevice(`entry-${index}`, close) + } + + expect(() => registerDevice('entry-overflow')).toThrow(/active scrcpy video streams/) + for (const close of closes) { + expect(close).not.toHaveBeenCalled() + } + }) + + it('bounds subscriber callbacks and releases admission on unsubscribe', () => { + registerDevice('subscriber-cap') + const unsubscribes = Array.from({ length: SCRCPY_VIDEO_MAX_SUBSCRIBERS }, () => + scrcpyVideoRegistry.subscribe('subscriber-cap', () => {}) + ) + + expect(() => scrcpyVideoRegistry.subscribe('subscriber-cap', () => {})).toThrow( + /video subscribers/ + ) + unsubscribes[0]() + expect(() => scrcpyVideoRegistry.subscribe('subscriber-cap', () => {})).not.toThrow() + }) +}) diff --git a/src/main/emulator/scrcpy-video-registry.ts b/src/main/emulator/scrcpy-video-registry.ts index 417317ac550..47a4215f0e9 100644 --- a/src/main/emulator/scrcpy-video-registry.ts +++ b/src/main/emulator/scrcpy-video-registry.ts @@ -8,8 +8,13 @@ import type { ScrcpyVideoMeta } from './android/scrcpy-video-frame-parser' // scrcpy keyframes ~every 10s; high-motion content can otherwise buffer // hundreds of deltas. Cap the replayed GOP so memory stays bounded and a -// late subscriber isn't flooded — always keep the keyframe at index 0. -const MAX_GOP_FRAMES = 120 +// late subscriber isn't flooded. An overloaded GOP is shed as a unit because +// dropping an interior delta would make the retained suffix undecodable. +export const SCRCPY_VIDEO_MAX_GOP_FRAMES = 120 +export const SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE = 32 * 1024 * 1024 +export const SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL = 128 * 1024 * 1024 +export const SCRCPY_VIDEO_MAX_REGISTRY_ENTRIES = 16 +export const SCRCPY_VIDEO_MAX_SUBSCRIBERS = 8 export type ScrcpyVideoFrameMessage = { config: boolean @@ -30,13 +35,27 @@ type RegistryEntry = { gop: ScrcpyVideoFrameMessage[] subscribers: Set close: () => void + retainedBytes: number } class ScrcpyVideoRegistry { private readonly entries = new Map() + private retainedBytes = 0 + private subscriberCount = 0 register(deviceId: string, close: () => void): void { - this.entries.set(deviceId, { subscribers: new Set(), gop: [], close }) + if (!this.entries.has(deviceId) && this.entries.size >= SCRCPY_VIDEO_MAX_REGISTRY_ENTRIES) { + throw new Error( + `Orca can have at most ${SCRCPY_VIDEO_MAX_REGISTRY_ENTRIES} active scrcpy video streams.` + ) + } + this.stop(deviceId) + this.entries.set(deviceId, { + subscribers: new Set(), + gop: [], + close, + retainedBytes: 0 + }) } pushMeta(deviceId: string, meta: ScrcpyVideoMeta): void { @@ -56,20 +75,21 @@ class ScrcpyVideoRegistry { return } if (frame.config) { - entry.config = frame + this.replaceConfig(entry, frame) } else if (frame.keyFrame) { // A keyframe starts a fresh decodeable GOP; buffer it + the following // deltas so a late subscriber can decode immediately on replay. - entry.gop = [frame] + this.clearGop(entry) + entry.gop.push(frame) + this.adjustRetainedBytes(entry, frame.bytes.byteLength) } else if (entry.gop.length > 0) { // Only buffer deltas once a keyframe anchors the GOP (a delta alone is // undecodable); deltas before the first keyframe are still sent live below. entry.gop.push(frame) - // Drop the oldest delta (never index 0, the keyframe) so replay stays decodable. - if (entry.gop.length > MAX_GOP_FRAMES) { - entry.gop.splice(1, 1) - } + this.adjustRetainedBytes(entry, frame.bytes.byteLength) } + this.trimEntryReplay(entry) + this.trimAggregateReplay(deviceId) for (const subscriber of entry.subscribers) { subscriber({ type: 'frame', frame }) } @@ -82,6 +102,11 @@ class ScrcpyVideoRegistry { if (!entry) { return () => {} } + if (this.subscriberCount >= SCRCPY_VIDEO_MAX_SUBSCRIBERS) { + throw new Error( + `Orca can have at most ${SCRCPY_VIDEO_MAX_SUBSCRIBERS} scrcpy video subscribers.` + ) + } if (entry.meta) { subscriber({ type: 'meta', meta: entry.meta }) } @@ -92,8 +117,16 @@ class ScrcpyVideoRegistry { for (const frame of entry.gop) { subscriber({ type: 'frame', frame }) } + if (this.entries.get(deviceId) !== entry) { + return () => {} + } entry.subscribers.add(subscriber) - return () => entry.subscribers.delete(subscriber) + this.subscriberCount += 1 + return () => { + if (entry.subscribers.delete(subscriber)) { + this.subscriberCount -= 1 + } + } } stop(deviceId: string): void { @@ -101,14 +134,90 @@ class ScrcpyVideoRegistry { if (!entry) { return } - entry.close() - entry.subscribers.clear() this.entries.delete(deviceId) + this.subscriberCount -= entry.subscribers.size + entry.subscribers.clear() + this.clearGop(entry) + this.clearConfig(entry) + entry.close() } has(deviceId: string): boolean { return this.entries.has(deviceId) } + + private adjustRetainedBytes(entry: RegistryEntry, delta: number): void { + entry.retainedBytes += delta + this.retainedBytes += delta + } + + private replaceConfig(entry: RegistryEntry, frame: ScrcpyVideoFrameMessage): void { + this.clearConfig(entry) + entry.config = frame + this.adjustRetainedBytes(entry, frame.bytes.byteLength) + } + + private clearConfig(entry: RegistryEntry): void { + if (!entry.config) { + return + } + this.adjustRetainedBytes(entry, -entry.config.bytes.byteLength) + entry.config = undefined + } + + private clearGop(entry: RegistryEntry): void { + for (const frame of entry.gop) { + this.adjustRetainedBytes(entry, -frame.bytes.byteLength) + } + entry.gop = [] + } + + private trimEntryReplay(entry: RegistryEntry): void { + if ( + entry.gop.length > SCRCPY_VIDEO_MAX_GOP_FRAMES || + entry.retainedBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE + ) { + this.clearGop(entry) + } + if (entry.retainedBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE) { + this.clearConfig(entry) + } + } + + private trimAggregateReplay(preferredDeviceId: string): void { + if (this.retainedBytes <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL) { + return + } + const preferred = this.entries.get(preferredDeviceId) + + for (const [deviceId, entry] of this.entries) { + if (deviceId === preferredDeviceId) { + continue + } + if (this.retainedBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL && entry.gop.length > 0) { + this.clearGop(entry) + } + } + if ( + preferred && + this.retainedBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL && + preferred.gop.length > 0 + ) { + this.clearGop(preferred) + } + for (const [deviceId, entry] of this.entries) { + if (this.retainedBytes <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL) { + break + } + if (deviceId === preferredDeviceId) { + continue + } + this.clearConfig(entry) + } + if (preferred && this.retainedBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_TOTAL) { + this.clearConfig(preferred) + } + } } export const scrcpyVideoRegistry = new ScrcpyVideoRegistry() diff --git a/src/main/emulator/serve-sim-accessibility-tree.test.ts b/src/main/emulator/serve-sim-accessibility-tree.test.ts index 48314017a87..2976554f0ee 100644 --- a/src/main/emulator/serve-sim-accessibility-tree.test.ts +++ b/src/main/emulator/serve-sim-accessibility-tree.test.ts @@ -4,7 +4,10 @@ const { netFetchMock } = vi.hoisted(() => ({ netFetchMock: vi.fn() })) vi.mock('electron', () => ({ net: { fetch: netFetchMock } })) -import { requestServeSimAccessibilityTree } from './serve-sim-accessibility-tree' +import { + MAX_SERVE_SIM_AX_RESPONSE_BYTES, + requestServeSimAccessibilityTree +} from './serve-sim-accessibility-tree' const AX_URL = 'http://127.0.0.1:3100/ax' @@ -85,6 +88,25 @@ describe('requestServeSimAccessibilityTree', () => { }) }) + it('rejects oversized or excessively nested payloads before parsing', async () => { + netFetchMock.mockResolvedValueOnce( + new Response('[]', { + status: 200, + headers: { 'content-length': String(MAX_SERVE_SIM_AX_RESPONSE_BYTES + 1) } + }) + ) + await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({ + code: 'emulator_helper_failed' + }) + + netFetchMock.mockResolvedValueOnce( + new Response(`${'['.repeat(129)}0${']'.repeat(129)}`, { status: 200 }) + ) + await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({ + code: 'emulator_error' + }) + }) + it('maps a network failure to a helper error', async () => { netFetchMock.mockRejectedValue(new Error('connect ECONNREFUSED')) diff --git a/src/main/emulator/serve-sim-accessibility-tree.ts b/src/main/emulator/serve-sim-accessibility-tree.ts index dab53d0e382..cde73fdc646 100644 --- a/src/main/emulator/serve-sim-accessibility-tree.ts +++ b/src/main/emulator/serve-sim-accessibility-tree.ts @@ -1,16 +1,22 @@ import { net } from 'electron' import { EmulatorError } from './emulator-errors' import { normalizeServeSimAxTree, type NormalizedAxNode } from './serve-sim-ax-normalization' +import { + API_RESPONSE_JSON_LIMITS, + readFetchResponseTextWithinLimit +} from '../../shared/fetch-response-body' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' const AX_REQUEST_TIMEOUT_MS = 5_000 const MAX_ERROR_BODY_LENGTH = 512 +export const MAX_SERVE_SIM_AX_RESPONSE_BYTES = 16 * 1024 * 1024 export async function requestServeSimAccessibilityTree(axUrl: string): Promise { try { const response = await net.fetch(axUrl, { signal: AbortSignal.timeout(AX_REQUEST_TIMEOUT_MS) }) - const body = await response.text() + const body = await readFetchResponseTextWithinLimit(response, MAX_SERVE_SIM_AX_RESPONSE_BYTES) if (!response.ok) { const detail = body.slice(0, MAX_ERROR_BODY_LENGTH) || response.statusText const retry = response.status === 503 ? ' Accessibility may still be warming up; retry.' : '' @@ -22,6 +28,7 @@ export async function requestServeSimAccessibilityTree(axUrl: string): Promise { + return { + device: udid, + streamUrl: 'http://127.0.0.1:3100/stream.mjpeg', + wsUrl: 'ws://127.0.0.1:3100/ws' + } +} + +afterEach(() => { + for (const filePath of stateFiles.splice(0)) { + rmSync(filePath, { force: true }) + } +}) + describe('parseServeSimDetachedSession', () => { it('uses serve-sim streamUrl when present', () => { const info = parseServeSimDetachedSession( @@ -56,6 +83,27 @@ describe('parseServeSimDetachedSession', () => { expect(info.streamUrl).toBe('http://127.0.0.1:3100/stream.mjpeg') }) + + it('reads a helper PID state file at the exact byte boundary', () => { + const udid = `orca-boundary-${process.pid}-${Date.now()}` + const statePath = stateFileFor(udid) + const state = '{"pid":4321}' + writeFileSync( + statePath, + state + ' '.repeat(SERVE_SIM_STATE_FILE_MAX_BYTES - Buffer.byteLength(state)) + ) + + expect(parseServeSimDetachedSession(detachedPayload(udid), udid).helperPid).toBe(4321) + }) + + it('ignores a sparse helper PID state file one byte over the boundary', () => { + const udid = `orca-oversized-${process.pid}-${Date.now()}` + const statePath = stateFileFor(udid) + writeFileSync(statePath, '{"pid":4321}') + truncateSync(statePath, SERVE_SIM_STATE_FILE_MAX_BYTES + 1) + + expect(parseServeSimDetachedSession(detachedPayload(udid), udid).helperPid).toBeUndefined() + }) }) describe('deriveAxUrlFromStreamUrl', () => { diff --git a/src/main/emulator/serve-sim-detached-session.ts b/src/main/emulator/serve-sim-detached-session.ts index d45bde41ff0..2d579d06322 100644 --- a/src/main/emulator/serve-sim-detached-session.ts +++ b/src/main/emulator/serve-sim-detached-session.ts @@ -1,10 +1,11 @@ -import { existsSync, readFileSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import { EmulatorError } from './emulator-errors' import type { EmulatorSessionInfo } from './emulator-types' const MJPEG_STREAM_SUFFIX = '/stream.mjpeg' +export const SERVE_SIM_STATE_FILE_MAX_BYTES = 8 * 1024 function streamUrlFromServeSimUrl(url: string): string { return url.endsWith(MJPEG_STREAM_SUFFIX) ? url : `${url.replace(/\/$/, '')}${MJPEG_STREAM_SUFFIX}` @@ -42,11 +43,11 @@ export function parseServeSimDetachedSession(raw: unknown, udid: string): Emulat } try { const statePath = join(tmpdir(), 'serve-sim', `server-${info.deviceUdid}.json`) - if (existsSync(statePath)) { - const state = JSON.parse(readFileSync(statePath, 'utf8')) as { pid?: unknown } - if (typeof state.pid === 'number') { - info.helperPid = state.pid - } + const state = JSON.parse( + readNodeFileSyncWithinLimit(statePath, SERVE_SIM_STATE_FILE_MAX_BYTES).buffer.toString('utf8') + ) as { pid?: unknown } + if (typeof state.pid === 'number') { + info.helperPid = state.pid } } catch {} return info diff --git a/src/main/emulator/serve-sim-execution.ts b/src/main/emulator/serve-sim-execution.ts index 5bdf0e34cc2..9207fdfd913 100644 --- a/src/main/emulator/serve-sim-execution.ts +++ b/src/main/emulator/serve-sim-execution.ts @@ -1,16 +1,9 @@ import { execFile } from 'node:child_process' -import { - accessSync, - chmodSync, - constants, - existsSync, - mkdirSync, - readFileSync, - writeFileSync -} from 'node:fs' +import { accessSync, chmodSync, constants, existsSync, mkdirSync, writeFileSync } from 'node:fs' import { app } from 'electron' import { platform, tmpdir } from 'node:os' import { delimiter, dirname, join } from 'node:path' +import { nodeFileContentsEqualSync } from '../../shared/node-file-content-equality' import { EmulatorError } from './emulator-errors' import { materializeServeSimRuntime } from './serve-sim-runtime-materializer' @@ -45,8 +38,10 @@ function ensureMacOpenShim(): string | null { } try { mkdirSync(MAC_OPEN_SHIM_DIR, { recursive: true }) - const current = existsSync(MAC_OPEN_SHIM_PATH) ? readFileSync(MAC_OPEN_SHIM_PATH, 'utf8') : '' - if (current !== MAC_OPEN_SHIM) { + if ( + !existsSync(MAC_OPEN_SHIM_PATH) || + !nodeFileContentsEqualSync(MAC_OPEN_SHIM_PATH, MAC_OPEN_SHIM) + ) { writeFileSync(MAC_OPEN_SHIM_PATH, MAC_OPEN_SHIM, { mode: 0o755 }) } chmodSync(MAC_OPEN_SHIM_PATH, 0o755) diff --git a/src/main/emulator/serve-sim-runtime-materializer.ts b/src/main/emulator/serve-sim-runtime-materializer.ts index 26abff4a6be..c7519c79cb1 100644 --- a/src/main/emulator/serve-sim-runtime-materializer.ts +++ b/src/main/emulator/serve-sim-runtime-materializer.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { chmodSync, cpSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync } from 'node:fs' +import { chmodSync, cpSync, existsSync, mkdirSync, opendirSync, renameSync, rmSync } from 'node:fs' import { join } from 'node:path' export type ServeSimRuntimeMaterializerOptions = { @@ -13,6 +13,7 @@ const EXECUTABLE_RELATIVE_PATHS = [ join('bin', 'serve-sim-bin'), join('dist', 'simcam', 'serve-sim-camera-helper') ] +export const SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES = 4_096 function defaultClearQuarantine(dir: string): void { if (process.platform !== 'darwin') { @@ -28,21 +29,44 @@ function defaultClearQuarantine(dir: string): void { execFileSync('/usr/bin/xattr', ['-rd', 'com.apple.quarantine', dir], { timeout: 30_000 }) } -function pruneStaleServeSimRuntimes(targetRootDir: string, keepVersion: string): void { - let entries: string[] +export function pruneStaleServeSimRuntimes( + targetRootDir: string, + keepVersion: string, + requestedMaxEntries = SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES +): void { + const maxEntries = + Number.isSafeInteger(requestedMaxEntries) && requestedMaxEntries >= 0 + ? Math.min(requestedMaxEntries, SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES) + : SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES + let directory: ReturnType try { - entries = readdirSync(targetRootDir) + directory = opendirSync(targetRootDir, { bufferSize: 32 }) } catch { return } - for (const entryName of entries) { - if (entryName === keepVersion) { - continue + + let scannedEntries = 0 + try { + while (scannedEntries < maxEntries) { + const entry = directory.readSync() + if (entry === null) { + return + } + scannedEntries += 1 + if (entry.name === keepVersion) { + continue + } + try { + rmSync(join(targetRootDir, entry.name), { recursive: true, force: true }) + } catch { + // Old-version cleanup is best-effort; a locked file must not block materialization. + } } + } finally { try { - rmSync(join(targetRootDir, entryName), { recursive: true, force: true }) + directory.closeSync() } catch { - // Old-version cleanup is best-effort; a locked file must not block materialization. + // A failed close cannot make already bounded cleanup unsafe. } } } diff --git a/src/main/emulator/serve-sim-runtime-pruning-bounds.test.ts b/src/main/emulator/serve-sim-runtime-pruning-bounds.test.ts new file mode 100644 index 00000000000..d3990cb350a --- /dev/null +++ b/src/main/emulator/serve-sim-runtime-pruning-bounds.test.ts @@ -0,0 +1,79 @@ +import type * as NodeFs from 'node:fs' +import { join } from 'node:path' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { opendirSyncMock, rmSyncMock } = vi.hoisted(() => ({ + opendirSyncMock: vi.fn(), + rmSyncMock: vi.fn() +})) + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), + opendirSync: opendirSyncMock, + rmSync: rmSyncMock +})) + +import { + pruneStaleServeSimRuntimes, + SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES +} from './serve-sim-runtime-materializer' + +function useEntryNames(names: string[]): { + closeSync: ReturnType + readSync: ReturnType +} { + let index = 0 + const directory = { + closeSync: vi.fn(), + readSync: vi.fn(() => { + const name = names[index] + index += 1 + return name === undefined ? null : { name } + }) + } + opendirSyncMock.mockReturnValue(directory) + return directory +} + +describe('serve-sim stale runtime pruning bounds', () => { + beforeEach(() => { + opendirSyncMock.mockReset() + rmSyncMock.mockReset() + }) + + it('processes the exact requested entry limit in stream order', () => { + const directory = useEntryNames(['old-a', 'keep', 'old-b', 'old-c']) + + pruneStaleServeSimRuntimes('/runtime', 'keep', 3) + + expect(rmSyncMock).toHaveBeenCalledTimes(2) + expect(rmSyncMock).toHaveBeenNthCalledWith(1, join('/runtime', 'old-a'), { + recursive: true, + force: true + }) + expect(rmSyncMock).toHaveBeenNthCalledWith(2, join('/runtime', 'old-b'), { + recursive: true, + force: true + }) + expect(directory.readSync).toHaveBeenCalledTimes(3) + expect(directory.closeSync).toHaveBeenCalledOnce() + }) + + it('stops at the production cap without reading or retaining the next entry', () => { + let nextEntry = 0 + const directory = { + closeSync: vi.fn(), + readSync: vi.fn(() => { + nextEntry += 1 + return { name: `old-${nextEntry}` } + }) + } + opendirSyncMock.mockReturnValue(directory) + + pruneStaleServeSimRuntimes('/runtime', 'keep') + + expect(directory.readSync).toHaveBeenCalledTimes(SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES) + expect(rmSyncMock).toHaveBeenCalledTimes(SERVE_SIM_RUNTIME_MAX_PRUNE_ENTRIES) + expect(directory.closeSync).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/filesystem-provider-bounded-text.test.ts b/src/main/filesystem-provider-bounded-text.test.ts new file mode 100644 index 00000000000..71acf33d3ed --- /dev/null +++ b/src/main/filesystem-provider-bounded-text.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it, vi } from 'vitest' +import type { IFilesystemProvider } from './providers/types' +import { readFilesystemProviderBoundedText } from './filesystem-provider-bounded-text' + +const limits = { maxBytes: 8, maxCodeUnits: 8 } + +function provider(args: { + size: number + content?: string + isBinary?: boolean +}): IFilesystemProvider { + return { + stat: vi.fn().mockResolvedValue({ + size: args.size, + type: 'file', + mtime: 0 + }), + readFile: vi.fn().mockResolvedValue({ + content: args.content ?? '', + isBinary: args.isBinary ?? false + }) + } as unknown as IFilesystemProvider +} + +describe('readFilesystemProviderBoundedText', () => { + it('reads a file at the exact byte boundary', async () => { + const fsProvider = provider({ size: limits.maxBytes, content: '12345678' }) + + await expect( + readFilesystemProviderBoundedText(fsProvider, '/repo/orca.yaml', limits) + ).resolves.toEqual({ kind: 'text', content: '12345678' }) + expect(fsProvider.readFile).toHaveBeenCalledOnce() + }) + + it('rejects a +1 stat before remote materialization', async () => { + const fsProvider = provider({ size: limits.maxBytes + 1, content: 'not read' }) + + await expect( + readFilesystemProviderBoundedText(fsProvider, '/repo/orca.yaml', limits) + ).resolves.toEqual({ kind: 'oversized' }) + expect(fsProvider.readFile).not.toHaveBeenCalled() + }) + + it('rechecks UTF-8 bytes after a stat/read race', async () => { + const fsProvider = provider({ size: 2, content: '🐋🐋🐋' }) + + await expect( + readFilesystemProviderBoundedText(fsProvider, '/repo/orca.yaml', limits) + ).resolves.toEqual({ kind: 'oversized' }) + }) + + it('rechecks code units after a stat/read race', async () => { + const fsProvider = provider({ size: 2, content: '123456789' }) + + await expect( + readFilesystemProviderBoundedText(fsProvider, '/repo/orca.yaml', limits) + ).resolves.toEqual({ kind: 'oversized' }) + }) + + it('preserves binary classification', async () => { + const fsProvider = provider({ size: 2, content: 'xx', isBinary: true }) + + await expect( + readFilesystemProviderBoundedText(fsProvider, '/repo/orca.yaml', limits) + ).resolves.toEqual({ kind: 'binary' }) + }) +}) diff --git a/src/main/filesystem-provider-bounded-text.ts b/src/main/filesystem-provider-bounded-text.ts new file mode 100644 index 00000000000..e255e60605b --- /dev/null +++ b/src/main/filesystem-provider-bounded-text.ts @@ -0,0 +1,34 @@ +import { measureUtf8ByteLength } from '../shared/utf8-byte-limits' +import type { IFilesystemProvider } from './providers/types' + +export type FilesystemProviderBoundedText = + | { kind: 'text'; content: string } + | { kind: 'binary' } + | { kind: 'oversized' } + +export async function readFilesystemProviderBoundedText( + provider: IFilesystemProvider, + filePath: string, + limits: { maxBytes: number; maxCodeUnits: number } +): Promise { + const fileStat = await provider.stat(filePath) + if ( + !Number.isSafeInteger(fileStat.size) || + fileStat.size < 0 || + fileStat.size > limits.maxBytes + ) { + return { kind: 'oversized' } + } + + const result = await provider.readFile(filePath) + if (result.isBinary) { + return { kind: 'binary' } + } + if ( + result.content.length > limits.maxCodeUnits || + measureUtf8ByteLength(result.content, { stopAfterBytes: limits.maxBytes }).exceededLimit + ) { + return { kind: 'oversized' } + } + return { kind: 'text', content: result.content } +} diff --git a/src/main/generated-node-bounded-file-reader.test.ts b/src/main/generated-node-bounded-file-reader.test.ts new file mode 100644 index 00000000000..462be297bb7 --- /dev/null +++ b/src/main/generated-node-bounded-file-reader.test.ts @@ -0,0 +1,86 @@ +import { + closeSync, + mkdtempSync, + openSync, + readSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { runInNewContext } from 'node:vm' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + GENERATED_NODE_MANAGED_FILE_MAX_BYTES, + getGeneratedNodeBoundedFileReaderSourceLines +} from './generated-node-bounded-file-reader' + +type GeneratedReader = ( + fs: { + closeSync: typeof closeSync + openSync: typeof openSync + readSync: typeof readSync + }, + path: string +) => string + +const temporaryDirectories: string[] = [] + +function buildGeneratedReader(): GeneratedReader { + const module = { exports: undefined as GeneratedReader | undefined } + runInNewContext( + [ + ...getGeneratedNodeBoundedFileReaderSourceLines(), + 'module.exports = readOrcaManagedFileWithinLimit;' + ].join('\n'), + { Buffer, module } + ) + if (!module.exports) { + throw new Error('generated bounded reader did not export') + } + return module.exports +} + +function createFile(content: string): string { + const directory = mkdtempSync(join(tmpdir(), 'orca-generated-node-read-')) + temporaryDirectories.push(directory) + const path = join(directory, 'managed-file') + writeFileSync(path, content) + return path +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('generated Node bounded file reader', () => { + it('preserves ordinary managed-file contents', () => { + const path = createFile('stable 🐋 endpoint') + + expect(buildGeneratedReader()({ openSync, readSync, closeSync }, path)).toBe( + 'stable 🐋 endpoint' + ) + }) + + it('accepts a file at the exact byte cap', () => { + const path = createFile('x'.repeat(GENERATED_NODE_MANAGED_FILE_MAX_BYTES)) + + expect( + Buffer.byteLength(buildGeneratedReader()({ openSync, readSync, closeSync }, path), 'utf8') + ).toBe(GENERATED_NODE_MANAGED_FILE_MAX_BYTES) + }) + + it('rejects a sparse oversized file and always closes its descriptor', () => { + const path = createFile('') + truncateSync(path, GENERATED_NODE_MANAGED_FILE_MAX_BYTES + 1) + const close = vi.fn(closeSync) + + expect(() => buildGeneratedReader()({ openSync, readSync, closeSync: close }, path)).toThrow( + `Managed Orca file exceeds ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES} bytes` + ) + expect(close).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/generated-node-bounded-file-reader.ts b/src/main/generated-node-bounded-file-reader.ts new file mode 100644 index 00000000000..7f483557e5e --- /dev/null +++ b/src/main/generated-node-bounded-file-reader.ts @@ -0,0 +1,30 @@ +export const GENERATED_NODE_MANAGED_FILE_MAX_BYTES = 64 * 1024 + +// Why: emitted runtimes cannot import Orca's reader; fixed capacity also catches growth after stat. +export function getGeneratedNodeBoundedFileReaderSourceLines(options?: { + typed?: boolean +}): string[] { + const signature = options?.typed + ? `function readOrcaManagedFileWithinLimit(fs: any, path: string, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES}): string {` + : `function readOrcaManagedFileWithinLimit(fs, path, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES}) {` + return [ + signature, + " const descriptor = fs.openSync(path, 'r');", + ' try {', + ' const buffer = Buffer.allocUnsafe(maxBytes + 1);', + ' let offset = 0;', + ' while (offset < buffer.length) {', + ' const bytesRead = fs.readSync(descriptor, buffer, offset, buffer.length - offset, null);', + ' if (bytesRead === 0) break;', + ' offset += bytesRead;', + ' }', + ' if (offset > maxBytes) {', + " throw Object.assign(new Error('Managed Orca file exceeds ' + maxBytes + ' bytes'), { code: 'EFBIG' });", + ' }', + " return buffer.toString('utf8', 0, offset);", + ' } finally {', + ' fs.closeSync(descriptor);', + ' }', + '}' + ] +} diff --git a/src/main/ghostty/index.test.ts b/src/main/ghostty/index.test.ts index 37d059c91fd..8c2485425ed 100644 --- a/src/main/ghostty/index.test.ts +++ b/src/main/ghostty/index.test.ts @@ -1,22 +1,36 @@ import type { Store } from '../persistence' import type { GlobalSettings } from '../../shared/types' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' +import type * as GhosttyParser from './parser' import { afterEach, describe, expect, it, vi } from 'vitest' -const { statMock, readFileMock } = vi.hoisted(() => ({ +const { statMock, readNodeFileWithinLimitMock, parseGhosttyConfigMock } = vi.hoisted(() => ({ statMock: vi.fn(), - readFileMock: vi.fn() + readNodeFileWithinLimitMock: vi.fn(), + parseGhosttyConfigMock: vi.fn() })) vi.mock('fs/promises', () => ({ - stat: statMock, - readFile: readFileMock + stat: statMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => ({ + ...(await importOriginal()), + readNodeFileWithinLimit: readNodeFileWithinLimitMock +})) + +vi.mock('./parser', async (importOriginal) => { + const actual = await importOriginal() + parseGhosttyConfigMock.mockImplementation(actual.parseGhosttyConfig) + return { ...actual, parseGhosttyConfig: parseGhosttyConfigMock } +}) + vi.mock('os', () => ({ platform: vi.fn(() => 'darwin'), homedir: vi.fn(() => '/Users/alice') })) +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import { previewGhosttyImport } from './index' const originalXdgConfigHome = process.env.XDG_CONFIG_HOME @@ -36,6 +50,13 @@ function createStore(settings: Record = {}): Store { } as Store } +function fileRead(content: string, size = Buffer.byteLength(content)) { + return { + buffer: Buffer.from(content), + stats: { isFile: () => true, size } + } +} + describe('previewGhosttyImport', () => { it('returns found false when no config exists', async () => { statMock.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })) @@ -52,11 +73,13 @@ describe('previewGhosttyImport', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue(` + readNodeFileWithinLimitMock.mockResolvedValue( + fileRead(` font-family = JetBrains Mono font-size = 14 background = #1a1a1a `) + ) const result = await previewGhosttyImport( createStore({ @@ -88,12 +111,12 @@ background = #1a1a1a } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockImplementation(async (p: string) => { + readNodeFileWithinLimitMock.mockImplementation(async (p: string) => { if (p === '/Users/alice/.config/ghostty/config.ghostty') { - return 'font-size = 22\nbackground = #1a1a1a\n' + return fileRead('font-size = 22\nbackground = #1a1a1a\n') } if (p === '/Users/alice/.config/ghostty/config') { - return 'font-family = JetBrains Mono\nfont-size = 18\n' + return fileRead('font-family = JetBrains Mono\nfont-size = 18\n') } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) @@ -121,7 +144,7 @@ background = #1a1a1a } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('font-family = Menlo\nfont-size = 12\n') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('font-family = Menlo\nfont-size = 12\n')) const result = await previewGhosttyImport( createStore({ @@ -142,7 +165,9 @@ background = #1a1a1a } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('background = #1a1a1a\nforeground = #e0e0e0\n') + readNodeFileWithinLimitMock.mockResolvedValue( + fileRead('background = #1a1a1a\nforeground = #e0e0e0\n') + ) const result = await previewGhosttyImport( createStore({ @@ -162,7 +187,9 @@ background = #1a1a1a } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('background = #1a1a1a\nforeground = #e0e0e0\n') + readNodeFileWithinLimitMock.mockResolvedValue( + fileRead('background = #1a1a1a\nforeground = #e0e0e0\n') + ) const result = await previewGhosttyImport( createStore({ @@ -192,7 +219,7 @@ background = #1a1a1a } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('font-family = JetBrains Mono\n') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('font-family = JetBrains Mono\n')) // Why: Replace timer globals temporarily to detect any polling setup. const originalSetInterval = globalThis.setInterval @@ -212,4 +239,44 @@ background = #1a1a1a expect(setIntervalMock).not.toHaveBeenCalled() expect(setTimeoutMock).not.toHaveBeenCalled() }) + + it('accepts a config at the exact byte limit', async () => { + const configPath = '/Users/alice/Library/Application Support/com.mitchellh.ghostty/config' + statMock.mockImplementation(async (p: string) => { + if (p === configPath) { + return { isFile: () => true, size: 1_000_000 } + } + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('font-size = 14\n', 1_000_000)) + + const result = await previewGhosttyImport(createStore()) + + expect(result.found).toBe(true) + expect(result.diff).toEqual({ terminalFontSize: 14 }) + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith(configPath, 1_000_000) + }) + + it('rejects config growth beyond the limit before parsing', async () => { + const configPath = '/Users/alice/Library/Application Support/com.mitchellh.ghostty/config' + statMock.mockImplementation(async (p: string) => { + if (p === configPath) { + return { isFile: () => true, size: 128 } + } + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + readNodeFileWithinLimitMock.mockRejectedValue( + new NodeFileReadTooLargeError(1_000_001, 1_000_000) + ) + + const result = await previewGhosttyImport(createStore()) + + expect(result).toMatchObject({ + found: false, + diff: {}, + unsupportedKeys: [], + error: 'Config file is too large to import (1000001 bytes, limit 1000000).' + }) + expect(parseGhosttyConfigMock).not.toHaveBeenCalled() + }) }) diff --git a/src/main/ghostty/index.ts b/src/main/ghostty/index.ts index 8557193fe9f..da406241aca 100644 --- a/src/main/ghostty/index.ts +++ b/src/main/ghostty/index.ts @@ -1,6 +1,9 @@ -import { readFile, stat } from 'node:fs/promises' import { platform } from 'node:os' import type { GlobalSettings, GhosttyImportPreview } from '../../shared/types' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' import type { Store } from '../persistence' import { findGhosttyConfigPaths } from './discovery' import { parseGhosttyConfig } from './parser' @@ -110,17 +113,25 @@ export async function previewGhosttyImport(store: Store): Promise MAX_CONFIG_BYTES) { + const result = await readNodeFileWithinLimit(configPath, MAX_CONFIG_BYTES) + if (!result.stats.isFile()) { return { found: false, diff: {}, unsupportedKeys: [], - error: `Config file is too large to import (${info.size} bytes, limit ${MAX_CONFIG_BYTES}).` + error: 'Could not read config: Config path is not a file' } } - content = await readFile(configPath, 'utf-8') + content = result.buffer.toString('utf8') } catch (err) { + if (err instanceof NodeFileReadTooLargeError) { + return { + found: false, + diff: {}, + unsupportedKeys: [], + error: `Config file is too large to import (${err.observedBytes} bytes, limit ${MAX_CONFIG_BYTES}).` + } + } const message = err instanceof Error ? err.message : 'Could not read config file' return { found: false, diff --git a/src/main/ghostty/theme-import.test.ts b/src/main/ghostty/theme-import.test.ts index c2d2ec61ab9..3698c88f369 100644 --- a/src/main/ghostty/theme-import.test.ts +++ b/src/main/ghostty/theme-import.test.ts @@ -1,15 +1,20 @@ import type { Store } from '../persistence' import type { GlobalSettings } from '../../shared/types' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { statMock, readFileMock } = vi.hoisted(() => ({ +const { statMock, readNodeFileWithinLimitMock } = vi.hoisted(() => ({ statMock: vi.fn(), - readFileMock: vi.fn() + readNodeFileWithinLimitMock: vi.fn() })) vi.mock('fs/promises', () => ({ - stat: statMock, - readFile: readFileMock + stat: statMock +})) + +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => ({ + ...(await importOriginal()), + readNodeFileWithinLimit: readNodeFileWithinLimitMock })) vi.mock('os', () => ({ @@ -47,6 +52,13 @@ function createStore(settings: Record = {}): Store { } as Store } +function fileRead(content: string) { + return { + buffer: Buffer.from(content), + stats: { isFile: () => true, size: Buffer.byteLength(content) } + } +} + describe('previewGhosttyImport theme references', () => { it('resolves a theme reference into color overrides', async () => { const configPath = '/Users/alice/Library/Application Support/com.mitchellh.ghostty/config' @@ -57,11 +69,11 @@ describe('previewGhosttyImport theme references', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockImplementation(async (p: string) => { + readNodeFileWithinLimitMock.mockImplementation(async (p: string) => { if (p === themePath) { - return 'palette = 1=#d54e53\nbackground = #000000\nforeground = #eaeaea\n' + return fileRead('palette = 1=#d54e53\nbackground = #000000\nforeground = #eaeaea\n') } - return 'theme = Tomorrow Night Bright\nfont-size = 14\n' + return fileRead('theme = Tomorrow Night Bright\nfont-size = 14\n') }) const result = await previewGhosttyImport(createStore()) @@ -86,11 +98,11 @@ describe('previewGhosttyImport theme references', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockImplementation(async (p: string) => { + readNodeFileWithinLimitMock.mockImplementation(async (p: string) => { if (p === themePath) { - return 'palette = 1=#d54e53\npalette = 2=#b9ca4a\nbackground = #000000\n' + return fileRead('palette = 1=#d54e53\npalette = 2=#b9ca4a\nbackground = #000000\n') } - return 'theme = night\nbackground = #101010\npalette = 1=#ff0000\n' + return fileRead('theme = night\nbackground = #101010\npalette = 1=#ff0000\n') }) const result = await previewGhosttyImport(createStore()) @@ -115,11 +127,11 @@ describe('previewGhosttyImport theme references', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockImplementation(async (p: string) => { + readNodeFileWithinLimitMock.mockImplementation(async (p: string) => { if (p === themePath) { - return 'background = #202020\nforeground = #f0f0f0\n' + return fileRead('background = #202020\nforeground = #f0f0f0\n') } - return `theme = ${themePath}\n` + return fileRead(`theme = ${themePath}\n`) }) const result = await previewGhosttyImport(createStore()) @@ -145,7 +157,7 @@ describe('previewGhosttyImport theme references', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('theme = Missing Theme\n') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('theme = Missing Theme\n')) const result = await previewGhosttyImport(createStore()) @@ -161,7 +173,9 @@ describe('previewGhosttyImport theme references', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('theme = light:Tomorrow,dark:Tomorrow Night\n') + readNodeFileWithinLimitMock.mockResolvedValue( + fileRead('theme = light:Tomorrow,dark:Tomorrow Night\n') + ) const result = await previewGhosttyImport(createStore()) diff --git a/src/main/ghostty/theme-resolution.test.ts b/src/main/ghostty/theme-resolution.test.ts index 607e69a39e8..f31e8bbcd05 100644 --- a/src/main/ghostty/theme-resolution.test.ts +++ b/src/main/ghostty/theme-resolution.test.ts @@ -1,20 +1,34 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' +import type * as GhosttyParser from './parser' -const { statMock, readFileMock } = vi.hoisted(() => ({ +const { statMock, readNodeFileWithinLimitMock, parseGhosttyConfigMock } = vi.hoisted(() => ({ statMock: vi.fn(), - readFileMock: vi.fn() + readNodeFileWithinLimitMock: vi.fn(), + parseGhosttyConfigMock: vi.fn() })) vi.mock('fs/promises', () => ({ - stat: statMock, - readFile: readFileMock + stat: statMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => ({ + ...(await importOriginal()), + readNodeFileWithinLimit: readNodeFileWithinLimitMock +})) + +vi.mock('./parser', async (importOriginal) => { + const actual = await importOriginal() + parseGhosttyConfigMock.mockImplementation(actual.parseGhosttyConfig) + return { ...actual, parseGhosttyConfig: parseGhosttyConfigMock } +}) + vi.mock('os', () => ({ platform: vi.fn(() => 'darwin'), homedir: vi.fn(() => '/Users/alice') })) +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import { getGhosttyThemeSearchDirs, resolveGhosttyThemeColors } from './theme-resolution' const originalXdgConfigHome = process.env.XDG_CONFIG_HOME @@ -49,6 +63,13 @@ selection-background = #424242 selection-foreground = #eaeaea ` +function fileRead(content: string, size = Buffer.byteLength(content)) { + return { + buffer: Buffer.from(content), + stats: { isFile: () => true, size } + } +} + describe('getGhosttyThemeSearchDirs', () => { it('probes XDG, then the Ghostty.app resources dir', () => { delete process.env.XDG_CONFIG_HOME @@ -80,7 +101,7 @@ describe('resolveGhosttyThemeColors', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue(THEME_FILE) + readNodeFileWithinLimitMock.mockResolvedValue(fileRead(THEME_FILE)) const result = await resolveGhosttyThemeColors('Tomorrow Night Bright') expect(result).toEqual({ @@ -100,7 +121,7 @@ describe('resolveGhosttyThemeColors', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('background = #1d1f21') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('background = #1d1f21')) const result = await resolveGhosttyThemeColors('Tomorrow') expect(result).toEqual({ background: '#1d1f21' }) @@ -113,7 +134,7 @@ describe('resolveGhosttyThemeColors', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('background = #1d1f21') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('background = #1d1f21')) const result = await resolveGhosttyThemeColors('/Users/alice/themes/work') expect(result).toEqual({ background: '#1d1f21' }) @@ -127,7 +148,7 @@ describe('resolveGhosttyThemeColors', () => { } throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }) - readFileMock.mockResolvedValue('background = #101010\nfont-size = 20') + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('background = #101010\nfont-size = 20')) const result = await resolveGhosttyThemeColors('custom') expect(result).toEqual({ background: '#101010' }) @@ -149,7 +170,7 @@ describe('resolveGhosttyThemeColors', () => { it('rejects oversized theme files', async () => { statMock.mockImplementation(async () => ({ isFile: () => true, size: 10_000_000 })) expect(await resolveGhosttyThemeColors('huge')).toBeNull() - expect(readFileMock).not.toHaveBeenCalled() + expect(readNodeFileWithinLimitMock).not.toHaveBeenCalled() expect(statMock).toHaveBeenCalledTimes(1) }) @@ -165,7 +186,35 @@ describe('resolveGhosttyThemeColors', () => { }) expect(await resolveGhosttyThemeColors('night')).toBeNull() - expect(readFileMock).not.toHaveBeenCalled() + expect(readNodeFileWithinLimitMock).not.toHaveBeenCalled() expect(statMock).toHaveBeenCalledTimes(1) }) + + it('accepts a theme at the exact byte limit', async () => { + const themePath = '/Users/alice/.config/ghostty/themes/boundary' + statMock.mockImplementation(async (p: string) => { + if (p === themePath) { + return { isFile: () => true, size: 262_144 } + } + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + readNodeFileWithinLimitMock.mockResolvedValue(fileRead('background = #1d1f21', 262_144)) + + expect(await resolveGhosttyThemeColors('boundary')).toEqual({ background: '#1d1f21' }) + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith(themePath, 262_144) + }) + + it('rejects theme growth beyond the limit before parsing', async () => { + const themePath = '/Users/alice/.config/ghostty/themes/growing' + statMock.mockImplementation(async (p: string) => { + if (p === themePath) { + return { isFile: () => true, size: 128 } + } + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + readNodeFileWithinLimitMock.mockRejectedValue(new NodeFileReadTooLargeError(262_145, 262_144)) + + expect(await resolveGhosttyThemeColors('growing')).toBeNull() + expect(parseGhosttyConfigMock).not.toHaveBeenCalled() + }) }) diff --git a/src/main/ghostty/theme-resolution.ts b/src/main/ghostty/theme-resolution.ts index eedb2c1592d..8354a397a94 100644 --- a/src/main/ghostty/theme-resolution.ts +++ b/src/main/ghostty/theme-resolution.ts @@ -1,6 +1,7 @@ import { homedir, platform } from 'node:os' import path from 'node:path' -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' import { parseGhosttyConfig } from './parser' // Why: theme files ship a few dozen short lines; anything larger is not a @@ -86,7 +87,11 @@ async function readThemeColors(themePath: string): Promise { if (!info.isFile() || info.size > MAX_THEME_BYTES) { return { status: 'invalid' } } - content = await readFile(themePath, 'utf-8') + const result = await readNodeFileWithinLimit(themePath, MAX_THEME_BYTES) + if (!result.stats.isFile()) { + return { status: 'invalid' } + } + content = result.buffer.toString('utf8') } catch (err) { return { status: isMissingThemeError(err) ? 'missing' : 'invalid' } } diff --git a/src/main/git/gh-rate-limit-breaker.ts b/src/main/git/gh-rate-limit-breaker.ts index 8bc4276490a..36c15077490 100644 --- a/src/main/git/gh-rate-limit-breaker.ts +++ b/src/main/git/gh-rate-limit-breaker.ts @@ -12,6 +12,8 @@ * the github rate-limit prober can use it without an import cycle. */ +import { cacheIdentityDigest } from '../cache-identity-digest' + export type GhRateLimitBucket = 'core' | 'search' | 'graphql' // Why: a primary 403 does not carry the reset time. The search window resets @@ -62,7 +64,7 @@ const blockedUntilMsByScopeAndBucket = new Map() let resetProbe: ((bucket: GhRateLimitBucket, scope: string) => void) | null = null function breakerKey(bucket: GhRateLimitBucket, scope = DEFAULT_SCOPE): string { - return `${scope}\0${bucket}` + return cacheIdentityDigest([scope, bucket]) } // gh api flags that take a separate value, so the endpoint arg can be found. diff --git a/src/main/git/git-capability-state.test.ts b/src/main/git/git-capability-state.test.ts index 0d499aea163..c26d68350f2 100644 --- a/src/main/git/git-capability-state.test.ts +++ b/src/main/git/git-capability-state.test.ts @@ -2,7 +2,9 @@ import { beforeEach, describe, expect, it } from 'vitest' import { clearGitCapabilityStateForTests, getLocalGitCapabilityCache, - getSshGitCapabilityCache + getSshGitCapabilityCache, + LOCAL_GIT_CAPABILITY_HOST_KEY_MAX_BYTES, + LOCAL_GIT_CAPABILITY_HOST_MAX_ENTRIES } from './git-capability-state' describe('Git capability execution-host state', () => { @@ -34,4 +36,27 @@ describe('Git capability execution-host state', () => { getSshGitCapabilityCache(replacementProvider) ) }) + + it('evicts the oldest local execution host after the host ceiling', () => { + const oldest = getLocalGitCapabilityCache({ wslDistro: 'Distro-0' }) + let newest = oldest + for (let index = 1; index <= LOCAL_GIT_CAPABILITY_HOST_MAX_ENTRIES; index++) { + newest = getLocalGitCapabilityCache({ wslDistro: `Distro-${index}` }) + } + + expect(getLocalGitCapabilityCache({ wslDistro: 'Distro-0' })).not.toBe(oldest) + expect( + getLocalGitCapabilityCache({ + wslDistro: `Distro-${LOCAL_GIT_CAPABILITY_HOST_MAX_ENTRIES}` + }) + ).toBe(newest) + }) + + it('does not retain oversized execution-host keys', () => { + const oversizedDistro = 'x'.repeat(LOCAL_GIT_CAPABILITY_HOST_KEY_MAX_BYTES + 1) + + expect(getLocalGitCapabilityCache({ wslDistro: oversizedDistro })).not.toBe( + getLocalGitCapabilityCache({ wslDistro: oversizedDistro }) + ) + }) }) diff --git a/src/main/git/git-capability-state.ts b/src/main/git/git-capability-state.ts index aec7a369a26..9898cfffc9b 100644 --- a/src/main/git/git-capability-state.ts +++ b/src/main/git/git-capability-state.ts @@ -1,4 +1,5 @@ import { GitCapabilityCache } from '../../shared/git-capability-cache' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' import { parseWslUncPath } from '../../shared/wsl-paths' type LocalGitCapabilityTarget = { @@ -6,6 +7,8 @@ type LocalGitCapabilityTarget = { wslDistro?: string } +export const LOCAL_GIT_CAPABILITY_HOST_MAX_ENTRIES = 64 +export const LOCAL_GIT_CAPABILITY_HOST_KEY_MAX_BYTES = 4 * 1024 const localCapabilitiesByExecutionHost = new Map() // Why: reconnecting creates a new provider, while concurrent IPC/runtime users // of one SSH connection must share the same remote Git capability results. @@ -21,10 +24,24 @@ export function getLocalGitCapabilityCache( target: LocalGitCapabilityTarget = {} ): GitCapabilityCache { const executionHost = getLocalGitExecutionHostKey(target) + if ( + measureUtf8ByteLength(executionHost, { + stopAfterBytes: LOCAL_GIT_CAPABILITY_HOST_KEY_MAX_BYTES + }).exceededLimit + ) { + return new GitCapabilityCache() + } let cache = localCapabilitiesByExecutionHost.get(executionHost) if (!cache) { cache = new GitCapabilityCache() localCapabilitiesByExecutionHost.set(executionHost, cache) + while (localCapabilitiesByExecutionHost.size > LOCAL_GIT_CAPABILITY_HOST_MAX_ENTRIES) { + const oldestHost = localCapabilitiesByExecutionHost.keys().next().value + if (oldestHost === undefined) { + break + } + localCapabilitiesByExecutionHost.delete(oldestHost) + } } return cache } diff --git a/src/main/git/huge-folder-ignore.test.ts b/src/main/git/huge-folder-ignore.test.ts index 60b107ce34c..74f93e71474 100644 --- a/src/main/git/huge-folder-ignore.test.ts +++ b/src/main/git/huge-folder-ignore.test.ts @@ -84,6 +84,35 @@ describe('appendFolderToGitignore', () => { expect(wrote).toBe(false) }) + it('streams a large single-line file while preserving append semantics', async () => { + const gitignorePath = path.join(dir, '.gitignore') + await fs.writeFile(gitignorePath, 'x') + await fs.truncate(gitignorePath, 8 * 1024 * 1024) + + expect(await appendFolderToGitignore(dir, 'dist')).toBe(true) + + const handle = await fs.open(gitignorePath, 'r') + try { + const info = await handle.stat() + const tail = Buffer.alloc(7) + await handle.read(tail, 0, tail.length, info.size - tail.length) + expect(tail.toString('utf8')).toBe('\ndist/\n') + } finally { + await handle.close() + } + }) + + it('preserves trimmed-line matching across streamed chunks', async () => { + const gitignorePath = path.join(dir, '.gitignore') + await fs.writeFile(gitignorePath, 'x') + await fs.truncate(gitignorePath, 8 * 1024 * 1024) + await fs.appendFile(gitignorePath, '\n\u00a0node_modules/\u00a0\n') + const sizeBefore = (await fs.stat(gitignorePath)).size + + expect(await appendFolderToGitignore(dir, 'node_modules')).toBe(false) + expect((await fs.stat(gitignorePath)).size).toBe(sizeBefore) + }) + it('rejects folder names outside the known allowlist (injection guard)', async () => { await expect(appendFolderToGitignore(dir, 'node_modules\n/etc/passwd')).rejects.toThrow( /Refusing to add/ diff --git a/src/main/git/huge-folder-ignore.ts b/src/main/git/huge-folder-ignore.ts index a115a38f6d8..507def3fc1f 100644 --- a/src/main/git/huge-folder-ignore.ts +++ b/src/main/git/huge-folder-ignore.ts @@ -1,6 +1,7 @@ -import { existsSync } from 'node:fs' -import { appendFile, readFile, stat } from 'node:fs/promises' +import { createReadStream, existsSync } from 'node:fs' +import { appendFile, stat } from 'node:fs/promises' import * as path from 'node:path' +import { StringDecoder } from 'node:string_decoder' import { checkIgnoredPaths } from './check-ignored-paths' import type { GitRuntimeOptions } from './git-runtime-options' @@ -9,6 +10,78 @@ import type { GitRuntimeOptions } from './git-runtime-options' // to ignore these by name (matching the well-known offenders) the way a mature // SCM does, rather than asking the user to hand-edit .gitignore. const KNOWN_HUGE_FOLDER_NAMES = ['node_modules', '.next', 'dist', 'build', 'target', 'vendor'] +const GITIGNORE_SCAN_CHUNK_BYTES = 64 * 1024 + +type GitignoreScanResult = { + alreadyListed: boolean + hasContent: boolean + endsWithNewline: boolean +} + +async function scanGitignoreForFolder( + gitignorePath: string, + folderName: string +): Promise { + const targets = [folderName, `${folderName}/`] + const decoder = new StringDecoder('utf8') + const input = createReadStream(gitignorePath, { highWaterMark: GITIGNORE_SCAN_CHUNK_BYTES }) + let phase: 'leading' | 'candidate' | 'trailing' | 'rejected' = 'leading' + let candidate = '' + let alreadyListed = false + let hasContent = false + let endsWithNewline = false + + const finishLine = (): void => { + if (phase === 'trailing' || (phase === 'candidate' && targets.includes(candidate))) { + alreadyListed = true + } + phase = 'leading' + candidate = '' + } + const acceptText = (text: string): void => { + for (const character of text) { + if (character === '\n') { + finishLine() + continue + } + const whitespace = character.trim().length === 0 + if (phase === 'leading') { + if (whitespace) { + continue + } + candidate = character + phase = targets.some((target) => target.startsWith(candidate)) ? 'candidate' : 'rejected' + continue + } + if (phase === 'candidate') { + if (whitespace) { + phase = targets.includes(candidate) ? 'trailing' : 'rejected' + continue + } + candidate += character + if (!targets.some((target) => target.startsWith(candidate))) { + phase = 'rejected' + } + continue + } + if (phase === 'trailing' && !whitespace) { + phase = 'rejected' + } + } + } + + for await (const chunk of input) { + const buffer = chunk as Buffer + hasContent ||= buffer.length > 0 + if (buffer.length > 0) { + endsWithNewline = buffer.at(-1) === 0x0a + } + acceptText(decoder.write(buffer)) + } + acceptText(decoder.end()) + finishLine() + return { alreadyListed, hasContent, endsWithNewline } +} /** * Return the relative names of known-huge folders that exist in the worktree and @@ -58,21 +131,21 @@ export async function appendFolderToGitignore( } const gitignorePath = path.join(worktreePath, '.gitignore') const line = `${safeFolderName}/` - let existingContent = '' + let scan: GitignoreScanResult = { + alreadyListed: false, + hasContent: false, + endsWithNewline: false + } try { - existingContent = await readFile(gitignorePath, 'utf-8') + scan = await scanGitignoreForFolder(gitignorePath, safeFolderName) } catch { // .gitignore doesn't exist yet — we'll create it below } - const alreadyListed = existingContent - .split(/\r?\n/) - .map((l) => l.trim()) - .some((l) => l === safeFolderName || l === line) - if (alreadyListed) { + if (scan.alreadyListed) { return false } // Why: keep a clean trailing newline whether or not the file ended with one. - const needsLeadingNewline = existingContent.length > 0 && !existingContent.endsWith('\n') + const needsLeadingNewline = scan.hasContent && !scan.endsWithNewline await appendFile(gitignorePath, `${needsLeadingNewline ? '\n' : ''}${line}\n`, 'utf-8') return true } diff --git a/src/main/git/repo-metadata-bounds.test.ts b/src/main/git/repo-metadata-bounds.test.ts new file mode 100644 index 00000000000..f9374a2d618 --- /dev/null +++ b/src/main/git/repo-metadata-bounds.test.ts @@ -0,0 +1,61 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { mkdirSync, mkdtempSync, truncateSync, writeFileSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { isGitRepo, MAX_GIT_MARKER_FILE_BYTES } from './repo' + +describe.sequential('Git marker fallback metadata bounds', () => { + let root: string + let originalPath: string | undefined + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-git-marker-bounds-')) + originalPath = process.env.PATH + process.env.PATH = '' + }) + + afterEach(async () => { + if (originalPath === undefined) { + delete process.env.PATH + } else { + process.env.PATH = originalPath + } + await rm(root, { recursive: true, force: true }) + }) + + function writeOversizedSparseFile(filePath: string): void { + writeFileSync(filePath, 'x') + truncateSync(filePath, MAX_GIT_MARKER_FILE_BYTES + 1) + } + + it('rejects an oversized .git pointer without loading it', () => { + const checkout = join(root, 'checkout') + mkdirSync(checkout) + writeOversizedSparseFile(join(checkout, '.git')) + + expect(isGitRepo(checkout)).toBe(false) + }) + + it('rejects an oversized linked-worktree commondir pointer', () => { + const checkout = join(root, 'checkout') + const adminDir = join(root, 'admin') + mkdirSync(checkout) + mkdirSync(adminDir) + writeFileSync(join(checkout, '.git'), `gitdir: ${adminDir}\n`) + writeFileSync(join(adminDir, 'HEAD'), 'ref: refs/heads/main\n') + writeOversizedSparseFile(join(adminDir, 'commondir')) + + expect(isGitRepo(checkout)).toBe(false) + }) + + it('checks a bare marker without retaining an oversized config', () => { + const bareRepo = join(root, 'bare.git') + mkdirSync(join(bareRepo, 'objects'), { recursive: true }) + mkdirSync(join(bareRepo, 'refs')) + writeFileSync(join(bareRepo, 'HEAD'), 'ref: refs/heads/main\n') + writeOversizedSparseFile(join(bareRepo, 'config')) + + expect(isGitRepo(bareRepo)).toBe(true) + }) +}) diff --git a/src/main/git/repo.ts b/src/main/git/repo.ts index df5b2865132..52c8a65aedd 100644 --- a/src/main/git/repo.ts +++ b/src/main/git/repo.ts @@ -1,5 +1,5 @@ /* oxlint-disable max-lines */ -import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs' +import { existsSync, realpathSync, statSync } from 'node:fs' import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path' import { gitExecFileSync, gitExecFileAsync } from './runner' import type { BaseRefSearchResult } from '../../shared/types' @@ -10,6 +10,7 @@ import { parseWslUncPath } from '../../shared/wsl-paths' import { toWindowsWslPath } from '../wsl' import { buildHostedRemoteCommitUrl, buildHostedRemoteFileUrl } from './hosted-remote-url' import { getLocalGitCapabilityCache } from './git-capability-state' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' type LocalGitExecOptions = { wslDistro?: string @@ -21,6 +22,7 @@ type LocalDefaultBaseRefGitOptions = { } const DEFAULT_BASE_REF_PROBE_TIMEOUT_MS = 15_000 +export const MAX_GIT_MARKER_FILE_BYTES = 64 * 1024 type GitRepoProbeResult = 'repo' | 'not-repo' | 'indeterminate' type GitMarkerScanResult = { status: 'valid'; rootPath: string } | { status: 'absent' | 'invalid' } @@ -276,7 +278,10 @@ function scanWorktreeMarkerSync(worktreePath: string): GitMarkerScanResult { if (marker.isFile()) { let gitDir: string | null try { - gitDir = parseGitdirFile(worktreePath, readFileSync(dotGit, 'utf8')) + gitDir = parseGitdirFile( + worktreePath, + readNodeFileSyncWithinLimit(dotGit, MAX_GIT_MARKER_FILE_BYTES).buffer.toString('utf8') + ) } catch { return { status: 'invalid' } } @@ -331,7 +336,10 @@ function hasValidLinkedWorktreeGitDirectorySync(gitDir: string): boolean { } const commonDir = resolveGitMetadataPath( gitDir, - readFileSync(join(gitDir, 'commondir'), 'utf8') + readNodeFileSyncWithinLimit( + join(gitDir, 'commondir'), + MAX_GIT_MARKER_FILE_BYTES + ).buffer.toString('utf8') ) return commonDir !== null && hasValidCommonGitDirectorySync(commonDir) } catch { @@ -345,7 +353,10 @@ function hasValidBareRepoMarkerSync(path: string): boolean { function gitConfigDeclaresNonBare(gitDir: string): boolean { try { - const config = readFileSync(join(gitDir, 'config'), 'utf8') + const config = readNodeFileSyncWithinLimit( + join(gitDir, 'config'), + MAX_GIT_MARKER_FILE_BYTES + ).buffer.toString('utf8') let inCoreSection = false for (const line of config.split(/\r?\n/)) { const section = line.match(/^\s*\[([^\]]+)\]/) diff --git a/src/main/git/runner-command-exec.test.ts b/src/main/git/runner-command-exec.test.ts index 97f8eb8254e..18a58770740 100644 --- a/src/main/git/runner-command-exec.test.ts +++ b/src/main/git/runner-command-exec.test.ts @@ -15,6 +15,7 @@ vi.mock('node:child_process', () => ({ import { commandExecFileAsync, + DEFAULT_GIT_MAX_BUFFER, ghExecFileAsync, gitExecFileAsync, gitStreamStdout, @@ -155,6 +156,35 @@ describe('commandExecFileAsync Windows command shims', () => { }) }) + it('applies the default output cap to Windows .cmd shim executions', async () => { + await withPlatform('win32', async () => { + const command = createMockChildProcess(1234) + const taskkill = createMockTaskkillProcess() + spawnMock.mockImplementation((cmd: string) => (cmd === 'taskkill' ? taskkill : command)) + const toString = vi.fn(() => 'should not decode') + const oversizedChunk = { + byteLength: DEFAULT_GIT_MAX_BUFFER + 1, + toString + } as unknown as Buffer + + const promise = commandExecFileAsync('C:\\tools\\pnpm.cmd', ['store', 'prune'], { + cwd: 'C:\\repo' + }) + const rejection = expect(promise).rejects.toThrow( + 'C:\\tools\\pnpm.cmd stdout exceeded maxBuffer.' + ) + command.stdout.emit('data', oversizedChunk) + + await rejection + expect(toString).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledWith( + 'taskkill', + ['/pid', '1234', '/t', '/f'], + expect.objectContaining({ stdio: 'ignore', windowsHide: true }) + ) + }) + }) + it('removes listeners after successful Windows .cmd shim executions', async () => { await withPlatform('win32', async () => { const command = createMockChildProcess(1234) @@ -174,6 +204,26 @@ describe('commandExecFileAsync Windows command shims', () => { expect(command.listenerCount('close')).toBe(0) }) }) + + it('captures 100,000 one-byte fragments from a Windows command shim', async () => { + await withPlatform('win32', async () => { + const command = createMockChildProcess(1234) + spawnMock.mockReturnValue(command) + const promise = commandExecFileAsync('C:\\tools\\pnpm.cmd', ['--version'], { + cwd: 'C:\\repo' + }) + const fragment = Buffer.from('x') + + for (let index = 0; index < 100_000; index += 1) { + command.stdout.emit('data', fragment) + } + command.emit('close', 0) + + const result = await promise + expect(result.stdout).toHaveLength(100_000) + expect(result.stdout.slice(-4)).toBe('xxxx') + }) + }) }) describe('runner execFile timeout handling', () => { @@ -645,6 +695,21 @@ describe('gitStreamStdout', () => { await rejection }) + it('captures streamed stderr delivered as 100,000 one-byte fragments', async () => { + const child = createMockChildProcess(1234) + spawnMock.mockReturnValue(child) + const promise = gitStreamStdout(['status'], { cwd: '/repo', onStdout: () => {} }) + const fragment = Buffer.from(' ') + + for (let index = 0; index < 100_000; index += 1) { + child.stderr.emit('data', fragment) + } + child.stderr.emit('data', Buffer.from('fatal: fragmented')) + child.emit('close', 128) + + await expect(promise).rejects.toThrow(/fatal: fragmented$/) + }) + it('rejects (not crashes) when the onStdout callback throws', async () => { const child = createMockChildProcess(1234) spawnMock.mockReturnValue(child) diff --git a/src/main/git/runner.ts b/src/main/git/runner.ts index ae4e93174e1..2f5e3952110 100644 --- a/src/main/git/runner.ts +++ b/src/main/git/runner.ts @@ -33,6 +33,7 @@ import { appendGitConfigEnv, gitCredentialPromptGuardEnv } from '../../shared/git-credential-prompt-env' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import { getSpawnArgsForWindows, isWindowsBatchScript, resolveWindowsCommand } from '../win32-utils' import { buildWslLoginShellCommand, @@ -497,6 +498,7 @@ async function spawnCommandCapture( args: string[], options: CommandExecOptions ): Promise<{ stdout: string; stderr: string }> { + const maxBuffer = options.maxBuffer ?? DEFAULT_GIT_MAX_BUFFER const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(command, args) return new Promise((resolve, reject) => { if (options.signal?.aborted) { @@ -504,10 +506,8 @@ async function spawnCommandCapture( return } let settled = false - let stdout = '' - let stderr = '' - let stdoutBytes = 0 - let stderrBytes = 0 + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() const spawnStartedAt = performance.now() const child = spawn(spawnCmd, spawnArgs, { cwd: options.cwd, @@ -538,11 +538,15 @@ async function spawnCommandCapture( } settled = true cleanupListeners() + const stdoutText = stdout.toString(options.encoding ?? 'utf-8') + const stderrText = stderr.toString(options.encoding ?? 'utf-8') + stdout.clear() + stderr.clear() if (error) { - reject(Object.assign(error, { stdout, stderr })) + reject(Object.assign(error, { stdout: stdoutText, stderr: stderrText })) return } - resolve({ stdout, stderr }) + resolve({ stdout: stdoutText, stderr: stderrText }) } timer = options.timeout ? setTimeout(() => { @@ -552,22 +556,20 @@ async function spawnCommandCapture( : null options.signal?.addEventListener('abort', onAbort, { once: true }) function onStdoutData(chunk: Buffer): void { - stdoutBytes += chunk.byteLength - if (options.maxBuffer && stdoutBytes > options.maxBuffer) { + if (stdout.byteLength + chunk.byteLength > maxBuffer) { void killSpawnedCommandTree(child) finish(new Error(`${command} stdout exceeded maxBuffer.`)) return } - stdout += chunk.toString(options.encoding ?? 'utf-8') + stdout.append(chunk) } function onStderrData(chunk: Buffer): void { - stderrBytes += chunk.byteLength - if (options.maxBuffer && stderrBytes > options.maxBuffer) { + if (stderr.byteLength + chunk.byteLength > maxBuffer) { void killSpawnedCommandTree(child) finish(new Error(`${command} stderr exceeded maxBuffer.`)) return } - stderr += chunk.toString(options.encoding ?? 'utf-8') + stderr.append(chunk) } function onError(error: Error): void { finish(error) @@ -972,11 +974,9 @@ export async function gitStreamStdout( let settled = false let stoppedEarly = false let stdoutBytes = 0 - let stderr = '' - let stderrBytes = 0 + const stderr = new GrowingByteBuffer() // Why: decode statefully so a multibyte UTF-8 char split across chunks isn't corrupted into replacement chars. const stdoutDecoder = new StringDecoder('utf8') - const stderrDecoder = new StringDecoder('utf8') const cleanup = (): void => { child.stdout?.off('data', onStdoutData) @@ -986,7 +986,6 @@ export async function gitStreamStdout( options.signal?.removeEventListener('abort', onAbort) // Flush any bytes the decoders were holding for an incomplete sequence. stdoutDecoder.end() - stderrDecoder.end() } const finish = (error: Error | null): void => { if (settled) { @@ -994,8 +993,10 @@ export async function gitStreamStdout( } settled = true cleanup() + const stderrText = stderr.toString() + stderr.clear() if (error) { - reject(Object.assign(error, { stderr })) + reject(Object.assign(error, { stderr: stderrText })) return } resolve({ stoppedEarly }) @@ -1029,13 +1030,12 @@ export async function gitStreamStdout( } } function onStderrData(chunk: Buffer): void { - stderrBytes += chunk.byteLength - if (stderrBytes > maxBuffer) { + if (stderr.byteLength + chunk.byteLength > maxBuffer) { void killSpawnedCommandTree(child) finish(new Error('git stderr exceeded maxBuffer.')) return } - stderr += stderrDecoder.write(chunk) + stderr.append(chunk) } function onError(error: Error): void { finish(error) @@ -1045,7 +1045,7 @@ export async function gitStreamStdout( finish(null) return } - finish(new Error(`git exited with ${code}: ${stderr}`)) + finish(new Error(`git exited with ${code}: ${stderr.toString()}`)) } function onAbort(): void { if (!child.pid) { diff --git a/src/main/git/status-git-pointer-bounds.test.ts b/src/main/git/status-git-pointer-bounds.test.ts new file mode 100644 index 00000000000..6f62d91527b --- /dev/null +++ b/src/main/git/status-git-pointer-bounds.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { resolveGitDir } from './status' + +describe('resolveGitDir metadata bounds', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + async function makeWorktree(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-resolve-git-dir-')) + roots.push(root) + const worktreePath = join(root, 'checkout') + await mkdir(worktreePath) + return worktreePath + } + + it('preserves a normal linked-worktree pointer', async () => { + const worktreePath = await makeWorktree() + await writeFile(join(worktreePath, '.git'), 'gitdir: ../common/.git/worktrees/checkout\n') + + await expect(resolveGitDir(worktreePath)).resolves.toBe( + join(worktreePath, '..', 'common', '.git', 'worktrees', 'checkout') + ) + }) + + it('falls back to the .git path for an oversized sparse pointer', async () => { + const worktreePath = await makeWorktree() + const dotGitPath = join(worktreePath, '.git') + await writeFile(dotGitPath, 'x') + await truncate(dotGitPath, 64 * 1024 + 1) + + await expect(resolveGitDir(worktreePath)).resolves.toBe(dotGitPath) + }) +}) diff --git a/src/main/git/status-submodule-path-payload-bounds.test.ts b/src/main/git/status-submodule-path-payload-bounds.test.ts new file mode 100644 index 00000000000..c0e1cf9b042 --- /dev/null +++ b/src/main/git/status-submodule-path-payload-bounds.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { gitExecFileAsyncMock } = vi.hoisted(() => ({ gitExecFileAsyncMock: vi.fn() })) + +vi.mock('./runner', () => ({ + gitExecFileAsync: gitExecFileAsyncMock, + gitOptionalLocksDisabledEnv: (env: NodeJS.ProcessEnv = process.env) => ({ + ...env, + GIT_OPTIONAL_LOCKS: '0' + }) +})) + +import { + clearSubmodulePathsCacheForTests, + getSubmodulePathsCacheCodeUnitsForTests, + getSubmodulePathsCacheCountForTests, + listSubmodulePaths, + MAX_SUBMODULE_PATH_CODE_UNITS, + MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS, + MAX_SUBMODULE_PATHS_PER_REPO +} from './status' + +describe('submodule path payload bounds', () => { + beforeEach(() => { + clearSubmodulePathsCacheForTests() + gitExecFileAsyncMock.mockReset() + }) + + it('preserves under-limit path order and cache reuse', async () => { + gitExecFileAsyncMock.mockResolvedValue({ + stdout: 'submodule.one.path packages/one\nsubmodule.two.path packages/two/\n' + }) + + await expect(listSubmodulePaths('/repo')).resolves.toEqual(['packages/one', 'packages/two']) + await expect(listSubmodulePaths('/repo')).resolves.toEqual(['packages/one', 'packages/two']) + expect(gitExecFileAsyncMock).toHaveBeenCalledOnce() + }) + + it('fails closed above the per-repository path count', async () => { + gitExecFileAsyncMock.mockResolvedValue({ + stdout: Array.from( + { length: MAX_SUBMODULE_PATHS_PER_REPO + 1 }, + (_, index) => `submodule.s${index}.path modules/${index}\n` + ).join('') + }) + + await expect(listSubmodulePaths('/repo')).resolves.toEqual([]) + }) + + it('fails closed on a single oversized path', async () => { + gitExecFileAsyncMock.mockResolvedValue({ + stdout: `submodule.large.path ${'x'.repeat(MAX_SUBMODULE_PATH_CODE_UNITS + 1)}\n` + }) + + await expect(listSubmodulePaths('/repo')).resolves.toEqual([]) + }) + + it('evicts old payloads at the aggregate retained-code-unit budget', async () => { + const segment = 'x'.repeat(60 * 1024) + const stdout = Array.from( + { length: 16 }, + (_, index) => `submodule.s${index}.path ${segment}${index}\n` + ).join('') + gitExecFileAsyncMock.mockResolvedValue({ stdout }) + + for (let index = 0; index < 18; index += 1) { + await listSubmodulePaths(`/repo-${index}`) + } + + expect(getSubmodulePathsCacheCodeUnitsForTests()).toBeLessThanOrEqual( + MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS + ) + expect(getSubmodulePathsCacheCountForTests()).toBeLessThan(18) + }) + + it('does not retain an oversized cache key', async () => { + gitExecFileAsyncMock.mockResolvedValue({ stdout: 'submodule.one.path packages/one\n' }) + const worktreePath = `/${'r'.repeat(64 * 1024)}` + + await listSubmodulePaths(worktreePath) + await listSubmodulePaths(worktreePath) + + expect(gitExecFileAsyncMock).toHaveBeenCalledTimes(2) + expect(getSubmodulePathsCacheCountForTests()).toBe(0) + expect(getSubmodulePathsCacheCodeUnitsForTests()).toBe(0) + }) +}) diff --git a/src/main/git/status-upstream-cache-key-bounds.test.ts b/src/main/git/status-upstream-cache-key-bounds.test.ts new file mode 100644 index 00000000000..a163ccd12a0 --- /dev/null +++ b/src/main/git/status-upstream-cache-key-bounds.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' + +const { existsSyncMock, gitExecFileAsyncMock, readFileMock } = vi.hoisted(() => ({ + existsSyncMock: vi.fn(), + gitExecFileAsyncMock: vi.fn(), + readFileMock: vi.fn() +})) + +vi.mock('./runner', () => ({ + gitExecFileAsync: gitExecFileAsyncMock, + gitStreamStdout: async ( + args: string[], + options: { onStdout: (chunk: string) => boolean | void } + ) => { + const { stdout } = await gitExecFileAsyncMock(args) + return { stoppedEarly: options.onStdout(stdout ?? '') === true } + }, + gitOptionalLocksDisabledEnv: (env: NodeJS.ProcessEnv = process.env) => ({ + ...env, + GIT_OPTIONAL_LOCKS: '0' + }) +})) + +vi.mock('fs/promises', () => ({ readFile: readFileMock })) +vi.mock('fs', () => ({ existsSync: existsSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readNodeFileWithinLimit: async (filePath: string, maxBytes: number) => { + const value = await readFileMock(filePath) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.length, maxBytes) + } + return { buffer, stats: { isFile: () => true, size: buffer.length } } + } + } +}) + +import { + clearEffectiveUpstreamNegativeStatusCache, + clearEffectiveUpstreamStatusCacheForTests, + getEffectiveUpstreamStatusCacheCountForTests, + getEffectiveUpstreamStatusGenerationCountForTests, + getStatus, + MAX_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES +} from './status' + +describe('effective-upstream cache key bounds', () => { + beforeEach(() => { + clearEffectiveUpstreamStatusCacheForTests() + existsSyncMock.mockReset() + gitExecFileAsyncMock.mockReset() + readFileMock.mockReset() + existsSyncMock.mockReturnValue(false) + readFileMock.mockResolvedValue('gitdir: /repo/.git/worktrees/feature\n') + gitExecFileAsyncMock.mockImplementation(async (args: string[]) => { + if (args.includes('status')) { + return { stdout: '# branch.oid abcdef1234567890\n# branch.head feature\n' } + } + if (args[0] === 'symbolic-ref') { + return { stdout: 'feature\n' } + } + if (args[0] === 'config' && args[1] === '--list') { + return { stdout: 'core.repositoryformatversion\n0\0' } + } + if (args[0] === 'rev-parse') { + throw new Error('missing upstream') + } + throw new Error(`unexpected git command: ${args.join(' ')}`) + }) + }) + + it('re-probes without retaining keys above 64 KiB', async () => { + const worktreePath = `/${'p'.repeat(MAX_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES)}` + + await getStatus(worktreePath) + await getStatus(worktreePath) + + const upstreamProbes = gitExecFileAsyncMock.mock.calls.filter( + ([args]) => (args as string[])[0] === 'rev-parse' && (args as string[]).includes('HEAD@{u}') + ) + expect(upstreamProbes).toHaveLength(2) + expect(getEffectiveUpstreamStatusCacheCountForTests()).toBe(0) + expect(getEffectiveUpstreamStatusGenerationCountForTests()).toBe(0) + + clearEffectiveUpstreamNegativeStatusCache({ worktreePath, branchName: 'feature' }) + expect(getEffectiveUpstreamStatusGenerationCountForTests()).toBe(0) + }) +}) diff --git a/src/main/git/status-upstream-negative-cache.test.ts b/src/main/git/status-upstream-negative-cache.test.ts index 263842810a2..a92e849180d 100644 --- a/src/main/git/status-upstream-negative-cache.test.ts +++ b/src/main/git/status-upstream-negative-cache.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' const { existsSyncMock, gitExecFileAsyncMock, readFileMock } = vi.hoisted(() => ({ existsSyncMock: vi.fn(), @@ -30,6 +31,21 @@ vi.mock('fs', () => ({ existsSync: existsSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readNodeFileWithinLimit: async (filePath: string, maxBytes: number) => { + const value = await readFileMock(filePath) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.length, maxBytes) + } + return { buffer, stats: { isFile: () => true, size: buffer.length } } + } + } +}) + function isConfigListSnapshotCommand(args: string[]): boolean { return args[0] === 'config' && args[1] === '--list' && args[2] === '-z' } diff --git a/src/main/git/status-upstream-probe-churn.test.ts b/src/main/git/status-upstream-probe-churn.test.ts index 49f6913e0af..705f75f3521 100644 --- a/src/main/git/status-upstream-probe-churn.test.ts +++ b/src/main/git/status-upstream-probe-churn.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' // Repro command: // pnpm exec vitest run --config config/vitest.config.ts src/main/git/status-upstream-probe-churn.test.ts -t "missing-upstream polling churn" @@ -35,6 +36,21 @@ vi.mock('fs', () => ({ existsSync: existsSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readNodeFileWithinLimit: async (filePath: string, maxBytes: number) => { + const value = await readFileMock(filePath) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.length, maxBytes) + } + return { buffer, stats: { isFile: () => true, size: buffer.length } } + } + } +}) + import { clearEffectiveUpstreamStatusCacheForTests, getStatus } from './status' function getGitArgs(call: unknown[]): string[] { diff --git a/src/main/git/status.test.ts b/src/main/git/status.test.ts index 6084d67f59c..2cdc41dc367 100644 --- a/src/main/git/status.test.ts +++ b/src/main/git/status.test.ts @@ -1,6 +1,7 @@ /* eslint-disable max-lines -- Why: git status/discard/chunking behavior is verified together here to keep the command contract readable in one place. */ import { beforeEach, describe, expect, it, vi } from 'vitest' import type * as NodeFs from 'node:fs' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' import path from 'node:path' import { MAX_RENDERED_DIFF_COMBINED_CHARACTERS, @@ -62,6 +63,33 @@ vi.mock('fs', () => ({ existsSync: existsSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readNodeFileWithinLimit: async (filePath: string, maxBytes: number) => { + if (maxBytes === 64 * 1024) { + const value = await readFileMock(filePath) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.length, maxBytes) + } + return { buffer, stats: { isFile: () => true, size: buffer.length } } + } + const stats = await statMock(filePath) + if (stats.size > maxBytes) { + throw new actual.NodeFileReadTooLargeError(stats.size, maxBytes) + } + const value = await readFileMock(filePath) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new actual.NodeFileReadTooLargeError(buffer.length, maxBytes) + } + return { buffer, stats } + } + } +}) + import { abortMerge, abortRebase, diff --git a/src/main/git/status.ts b/src/main/git/status.ts index 34c5d750476..c90555a9247 100644 --- a/src/main/git/status.ts +++ b/src/main/git/status.ts @@ -1,6 +1,5 @@ /* eslint-disable max-lines */ import { existsSync } from 'node:fs' -import { readFile, stat } from 'node:fs/promises' import * as path from 'node:path' import type { GitBranchChangeEntry, @@ -31,6 +30,8 @@ import { type GitLineStats } from '../../shared/git-uncommitted-line-stats' import { decodeGitCQuotedPath } from '../../shared/git-cquoted-path' +import { iterateNulDelimitedFields } from '../../shared/nul-delimited-fields' +import { iterateProcessOutputLines } from '../../shared/process-output-field-scanner' import { gitExecFileAsync, gitExecFileAsyncBuffer, @@ -57,12 +58,18 @@ import { clearGitStatusLineStatsCacheKey, reuseOrRecomputeGitStatusLineStats } from '../../shared/git-status-line-stats-cache' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' const MAX_GIT_SHOW_BYTES = 10 * 1024 * 1024 +const MAX_GIT_POINTER_FILE_BYTES = 64 * 1024 const MAX_STAGED_COMMIT_CONTEXT_BYTES = MAX_GIT_SHOW_BYTES const BULK_CHUNK_SIZE = 100 const EFFECTIVE_UPSTREAM_NEGATIVE_CACHE_TTL_MS = 5 * 60_000 const MAX_EFFECTIVE_UPSTREAM_NEGATIVE_CACHE_ENTRIES = 512 +export const MAX_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES = 64 * 1024 type EffectiveUpstreamStatusCacheEntry = { expiresAt: number @@ -71,9 +78,19 @@ type EffectiveUpstreamStatusCacheEntry = { const SUBMODULE_PATHS_CACHE_TTL_MS = 5_000 export const MAX_SUBMODULE_PATHS_CACHE_ENTRIES = 512 -type SubmodulePathsCacheEntry = { paths: string[]; expiresAt: number } +export const MAX_SUBMODULE_PATHS_PER_REPO = 10_000 +export const MAX_SUBMODULE_PATH_CODE_UNITS = 64 * 1024 +export const MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS = 4 * 1024 * 1024 +export const MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS = 16 * 1024 * 1024 +const MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES = 64 * 1024 +type SubmodulePathsCacheEntry = { + paths: string[] + expiresAt: number + retainedCodeUnits: number +} const submodulePathsCache = new Map() let submodulePathsCacheGeneration = 0 +let submodulePathsCacheCodeUnits = 0 // Why: cache the upstream name to skip its 4-5-spawn resolution chain each poll; revalidate via one rev-list (issue #7576). const RESOLVED_UPSTREAM_NAME_CACHE_TTL_MS = 60_000 @@ -117,6 +134,7 @@ export function clearSubmodulePathsCacheForTests(): void { function clearSubmodulePathsCache(): void { submodulePathsCache.clear() + submodulePathsCacheCodeUnits = 0 // Why: bump the generation so a pre-mutation read can't repopulate the invalidated cache. submodulePathsCacheGeneration += 1 } @@ -125,6 +143,10 @@ export function getSubmodulePathsCacheCountForTests(): number { return submodulePathsCache.size } +export function getSubmodulePathsCacheCodeUnitsForTests(): number { + return submodulePathsCacheCodeUnits +} + function gitRuntimeOptionsKey(options: GitRuntimeOptions): readonly unknown[] { return [options.wslDistro ?? null] } @@ -137,18 +159,30 @@ function getSubmodulePathsCacheKey(worktreePath: string, options: GitRuntimeOpti function pruneExpiredSubmodulePathsCache(now: number): void { for (const [cacheKey, entry] of submodulePathsCache) { if (entry.expiresAt <= now) { - submodulePathsCache.delete(cacheKey) + deleteSubmodulePathsCacheEntry(cacheKey) } } } +function deleteSubmodulePathsCacheEntry(cacheKey: string): void { + const entry = submodulePathsCache.get(cacheKey) + if (!entry) { + return + } + submodulePathsCache.delete(cacheKey) + submodulePathsCacheCodeUnits -= entry.retainedCodeUnits +} + function trimSubmodulePathsCache(): void { - while (submodulePathsCache.size > MAX_SUBMODULE_PATHS_CACHE_ENTRIES) { + while ( + submodulePathsCache.size > MAX_SUBMODULE_PATHS_CACHE_ENTRIES || + submodulePathsCacheCodeUnits > MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS + ) { const oldestKey = submodulePathsCache.keys().next().value if (oldestKey === undefined) { break } - submodulePathsCache.delete(oldestKey) + deleteSubmodulePathsCacheEntry(oldestKey) } } @@ -158,7 +192,7 @@ function getCachedSubmodulePaths(cacheKey: string, now: number): string[] | null return null } if (cached.expiresAt <= now) { - submodulePathsCache.delete(cacheKey) + deleteSubmodulePathsCacheEntry(cacheKey) return null } submodulePathsCache.delete(cacheKey) @@ -167,8 +201,23 @@ function getCachedSubmodulePaths(cacheKey: string, now: number): string[] | null } function rememberSubmodulePaths(cacheKey: string, paths: string[], now: number): void { - submodulePathsCache.delete(cacheKey) - submodulePathsCache.set(cacheKey, { paths, expiresAt: now + SUBMODULE_PATHS_CACHE_TTL_MS }) + if (Buffer.byteLength(cacheKey, 'utf8') > MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES) { + deleteSubmodulePathsCacheEntry(cacheKey) + return + } + const retainedCodeUnits = + cacheKey.length + paths.reduce((total, submodulePath) => total + submodulePath.length, 0) + if (retainedCodeUnits > MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS) { + deleteSubmodulePathsCacheEntry(cacheKey) + return + } + deleteSubmodulePathsCacheEntry(cacheKey) + submodulePathsCache.set(cacheKey, { + paths, + expiresAt: now + SUBMODULE_PATHS_CACHE_TTL_MS, + retainedCodeUnits + }) + submodulePathsCacheCodeUnits += retainedCodeUnits trimSubmodulePathsCache() } @@ -585,6 +634,10 @@ function getEffectiveUpstreamStatusCacheKey( return [worktreePath, options.wslDistro ?? 'host', branchName, upstreamName ?? ''].join('\0') } +function canRetainEffectiveUpstreamCacheKey(cacheKey: string): boolean { + return Buffer.byteLength(cacheKey, 'utf8') <= MAX_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES +} + export function clearEffectiveUpstreamNegativeStatusCache(identity: { worktreePath: string branchName: string @@ -601,6 +654,9 @@ export function clearEffectiveUpstreamNegativeStatusCache(identity: { effectiveUpstreamStatusCache.delete(cacheKey) effectiveUpstreamStatusInFlight.delete(cacheKey) resolvedUpstreamNameCache.delete(cacheKey) + if (!canRetainEffectiveUpstreamCacheKey(cacheKey)) { + return + } effectiveUpstreamStatusWriteGeneration.set( cacheKey, (effectiveUpstreamStatusWriteGeneration.get(cacheKey) ?? 0) + 1 @@ -666,6 +722,11 @@ function rememberEffectiveUpstreamStatus( probedSameNameOriginRef: boolean, writeGeneration: number ): void { + if (!canRetainEffectiveUpstreamCacheKey(cacheKey)) { + effectiveUpstreamStatusCache.delete(cacheKey) + effectiveUpstreamStatusWriteGeneration.delete(cacheKey) + return + } // Why: hasConfiguredPushTarget gates a write action; re-probe each poll rather than cache a stale positive. if (status.hasUpstream || status.hasConfiguredPushTarget) { effectiveUpstreamStatusCache.delete(cacheKey) @@ -702,7 +763,8 @@ async function readOrProbeEffectiveUpstreamStatus( options: GitRuntimeOptions = {}, bypassCache = false ): Promise { - if (!bypassCache) { + const cacheable = !bypassCache && canRetainEffectiveUpstreamCacheKey(cacheKey) + if (cacheable) { const cached = readCachedEffectiveUpstreamStatus(cacheKey, Date.now()) if (cached) { return cached @@ -721,7 +783,7 @@ async function readOrProbeEffectiveUpstreamStatus( worktreePath, branchName, options, - bypassCache + !cacheable ).then((result) => { rememberEffectiveUpstreamStatus( cacheKey, @@ -732,7 +794,7 @@ async function readOrProbeEffectiveUpstreamStatus( ) return result.status }) - if (!bypassCache) { + if (cacheable) { effectiveUpstreamStatusInFlight.set(cacheKey, probe) } try { @@ -773,7 +835,11 @@ async function probeOrRevalidateEffectiveUpstreamStatus( } } const result = await probeEffectiveUpstreamStatus(worktreePath, branchName, options) - if (result.status.hasUpstream && result.status.upstreamName) { + if ( + canRetainEffectiveUpstreamCacheKey(cacheKey) && + result.status.hasUpstream && + result.status.upstreamName + ) { resolvedUpstreamNameCache.set(cacheKey, { upstreamName: result.status.upstreamName, expiresAt: Date.now() + RESOLVED_UPSTREAM_NAME_CACHE_TTL_MS @@ -973,7 +1039,9 @@ export async function resolveGitDir(worktreePath: string): Promise { const dotGitPath = path.join(worktreePath, '.git') try { - const dotGitContents = await readFile(dotGitPath, 'utf-8') + const dotGitContents = ( + await readNodeFileWithinLimit(dotGitPath, MAX_GIT_POINTER_FILE_BYTES) + ).buffer.toString('utf-8') const match = dotGitContents.match(/^gitdir:\s*(.+)\s*$/m) if (match) { return path.resolve(worktreePath, match[1]) @@ -989,6 +1057,37 @@ export async function resolveGitDir(worktreePath: string): Promise { * List configured submodule paths (relative, forward-slash) for a worktree, cached * briefly. Read from `.gitmodules` to avoid an index-wide `ls-files` scan. */ +function parseSubmodulePaths(stdout: string): string[] | null { + const paths: string[] = [] + let retainedCodeUnits = 0 + for (const line of iterateProcessOutputLines(stdout)) { + if (line.length > MAX_SUBMODULE_PATH_CODE_UNITS + 4_096) { + return null + } + const spaceIndex = line.indexOf(' ') + const submodulePath = + spaceIndex === -1 + ? '' + : line + .slice(spaceIndex + 1) + .trim() + .replace(/\/+$/, '') + if (!submodulePath) { + continue + } + if ( + paths.length >= MAX_SUBMODULE_PATHS_PER_REPO || + submodulePath.length > MAX_SUBMODULE_PATH_CODE_UNITS || + submodulePath.length > MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS - retainedCodeUnits + ) { + return null + } + paths.push(submodulePath) + retainedCodeUnits += submodulePath.length + } + return paths +} + export async function listSubmodulePaths( worktreePath: string, options: GitRuntimeOptions = {} @@ -1008,18 +1107,7 @@ export async function listSubmodulePaths( ['config', '--file', '.gitmodules', '--get-regexp', '^submodule\\..*\\.path$'], { ...gitOptionsForWorktree(worktreePath, options), env: gitOptionalLocksDisabledEnv() } ) - paths = stdout - .split(/\r?\n/) - .map((line) => { - const spaceIndex = line.indexOf(' ') - return spaceIndex === -1 - ? '' - : line - .slice(spaceIndex + 1) - .trim() - .replace(/\/+$/, '') - }) - .filter((value) => value.length > 0) + paths = parseSubmodulePaths(stdout) ?? [] } catch { // No .gitmodules (or git config failure) — treat as a repo without submodules. paths = [] @@ -1770,31 +1858,23 @@ async function readGitBlobAtOidPath( } async function readWorkingTreeFile(filePath: string): Promise { - let fileStat try { - fileStat = await stat(filePath) + const { buffer, stats: fileStat } = await readNodeFileWithinLimit(filePath, MAX_GIT_SHOW_BYTES) + if (!fileStat.isFile()) { + return { content: '', isBinary: false, exists: false } + } + return bufferToBlob(buffer, filePath) } catch (error) { - // Why: only ENOENT is a real deletion; other stat errors are read failures, not absence. + if (error instanceof NodeFileReadTooLargeError) { + return { content: '', isBinary: true, exists: true } + } + // Why: only ENOENT is a real deletion; other read errors are failures, not absence. return { content: '', isBinary: false, exists: (error as NodeJS.ErrnoException)?.code !== 'ENOENT' } } - if (!fileStat.isFile()) { - return { content: '', isBinary: false, exists: false } - } - if (fileStat.size > MAX_GIT_SHOW_BYTES) { - // Why: mirror git's maxBuffer cap for working-tree reads so readFile can't pull in huge assets. - return { content: '', isBinary: true, exists: true } - } - try { - const buffer = await readFile(filePath) - return bufferToBlob(buffer, filePath) - } catch { - // Why: the file exists but could not be read — a read failure, not a deletion. - return { content: '', isBinary: false, exists: true } - } } function bufferToBlob(buffer: Buffer, filePath?: string): GitBlobReadResult { @@ -2069,7 +2149,7 @@ async function listTrackedPathSpecs( } ) // Why: a tracked directory can hold enough paths to exceed the JS argument limit. - for (const trackedPath of stdout.split('\0')) { + for (const trackedPath of iterateNulDelimitedFields(stdout)) { if (trackedPath) { trackedPaths.push(trackedPath) } diff --git a/src/main/git/worktree.ts b/src/main/git/worktree.ts index 08decaefcaf..a802aea6986 100644 --- a/src/main/git/worktree.ts +++ b/src/main/git/worktree.ts @@ -18,6 +18,7 @@ import { isSubmoduleWorktreeRemovalRefusal } from '../../shared/worktree-submodu import { decodeGitCQuotedPath } from '../../shared/git-cquoted-path' import { parseGitRevListAheadBehindCounts } from '../../shared/git-rev-list-output' import { parseWslUncPath } from '../../shared/wsl-paths' +import { iterateNulDelimitedFields } from '../../shared/nul-delimited-fields' import { hasUnsupportedRevParsePathFormatEcho, isUnsupportedRevParsePathFormatError, @@ -548,7 +549,7 @@ function splitNulWorktreeList(output: string): string[][] { const blocks: string[][] = [] let currentBlock: string[] = [] - for (const field of output.split('\0')) { + for (const field of iterateNulDelimitedFields(output)) { if (field) { currentBlock.push(field) continue @@ -1376,10 +1377,12 @@ function hasOnlyIgnoredUntrackedStatus( ) .filter((entry) => entry && !entry.split('/').includes('..')) ) - return status - .split('\0') - .filter(Boolean) - .every((entry) => entry.startsWith('?? ') && ignored.has(entry.slice(3).replace(/\\/g, '/'))) + for (const entry of iterateNulDelimitedFields(status)) { + if (entry && (!entry.startsWith('?? ') || !ignored.has(entry.slice(3).replace(/\\/g, '/')))) { + return false + } + } + return true } function translateWorktreePath( diff --git a/src/main/gitea/client.test.ts b/src/main/gitea/client.test.ts index eada3d6cd95..112c03ab6c9 100644 --- a/src/main/gitea/client.test.ts +++ b/src/main/gitea/client.test.ts @@ -17,7 +17,11 @@ import { } from './client' import { _resetGiteaRepoRefCache } from './repository-ref' import { + GITEA_SCAN_CACHE_ENTRY_MAX_BYTES, + GITEA_SCAN_MAX_IN_FLIGHT, + GITEA_SCAN_REPO_KEY_MAX_BYTES, _getGiteaPullRequestScanCacheSize, + _getGiteaPullRequestScanState, _resetGiteaPullRequestScanCache, scanGiteaPullRequests } from './pull-request-scan-cache' @@ -299,6 +303,62 @@ describe('Gitea client', () => { } }) + it('skips retained state for oversized scan keys and payloads', async () => { + const exactKey = 'k'.repeat(GITEA_SCAN_REPO_KEY_MAX_BYTES) + const oversizedKey = `${exactKey}x` + await scanGiteaPullRequests(exactKey, async () => [], 50, 5) + await scanGiteaPullRequests(oversizedKey, async () => [], 50, 5) + const oversizedListing = [giteaPr(1, 'x'.repeat(GITEA_SCAN_CACHE_ENTRY_MAX_BYTES))] + await expect( + scanGiteaPullRequests('oversized-payload', async () => oversizedListing, 50, 5) + ).resolves.toEqual(oversizedListing) + + expect(_getGiteaPullRequestScanCacheSize()).toBe(1) + }) + + it('caps distinct in-flight scans and recovers tracked capacity', async () => { + let release!: () => void + const gate = new Promise((resolve) => { + release = resolve + }) + const scans = Array.from({ length: GITEA_SCAN_MAX_IN_FLIGHT }, (_, index) => + scanGiteaPullRequests( + `in-flight-${index}`, + async () => { + await gate + return [] + }, + 50, + 5 + ) + ) + let overflowCalls = 0 + await scanGiteaPullRequests( + 'overflow', + async () => { + overflowCalls += 1 + return [] + }, + 50, + 5 + ) + await scanGiteaPullRequests( + 'overflow', + async () => { + overflowCalls += 1 + return [] + }, + 50, + 5 + ) + + expect(_getGiteaPullRequestScanState().inFlight).toBe(GITEA_SCAN_MAX_IN_FLIGHT) + expect(overflowCalls).toBe(2) + release() + await Promise.all(scans) + expect(_getGiteaPullRequestScanState().inFlight).toBe(0) + }) + it('does not let an in-flight scan re-cache results from before an invalidation', async () => { let releaseFirstScan!: () => void const firstScanGate = new Promise((resolve) => { diff --git a/src/main/gitea/client.ts b/src/main/gitea/client.ts index ba9e7c3ed8f..ca805222106 100644 --- a/src/main/gitea/client.ts +++ b/src/main/gitea/client.ts @@ -14,6 +14,7 @@ import { type HostedReviewExecutionOptions } from '../source-control/hosted-review-git-options' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' const REQUEST_TIMEOUT_MS = 5000 // Why: self-hosted Forgejo can take ~5s to serve one /pulls page (it loads @@ -102,7 +103,7 @@ async function requestJsonAtBase( } return null } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } catch (error) { if (throwOnFailure) { throw error diff --git a/src/main/gitea/pull-request-scan-cache.ts b/src/main/gitea/pull-request-scan-cache.ts index 8864f087d6d..b06f21846aa 100644 --- a/src/main/gitea/pull-request-scan-cache.ts +++ b/src/main/gitea/pull-request-scan-cache.ts @@ -1,4 +1,5 @@ import type { RawGiteaPullRequest } from './pull-request-mappers' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' export type GiteaPullRequestPageFetcher = (page: number) => Promise @@ -16,7 +17,11 @@ const SCAN_TTL_MS = 30_000 const FAILED_SCAN_RETRY_MS = 3_000 // Why: each entry can retain hundreds of full PR payloads, so TTL alone is not // enough protection when many repositories are opened during one app session. -const MAX_SCAN_CACHE_ENTRIES = 32 +export const GITEA_SCAN_CACHE_MAX_ENTRIES = 32 +export const GITEA_SCAN_MAX_IN_FLIGHT = 32 +export const GITEA_SCAN_REPO_KEY_MAX_BYTES = 4 * 1024 +export const GITEA_SCAN_CACHE_ENTRY_MAX_PULL_REQUESTS = 250 +export const GITEA_SCAN_CACHE_ENTRY_MAX_BYTES = 512 * 1024 const scanCache = new Map() const inFlightScans = new Map>() @@ -41,6 +46,9 @@ function rememberScanCacheEntry( ttlMs: number ): void { removeScanCacheEntry(repoKey) + if (!isRetainablePullRequestListing(pullRequests)) { + return + } let entry!: GiteaPullRequestScanEntry const expirationTimer = setTimeout(() => removeScanCacheEntry(repoKey, entry), ttlMs) expirationTimer.unref() @@ -50,7 +58,7 @@ function rememberScanCacheEntry( pullRequests } scanCache.set(repoKey, entry) - while (scanCache.size > MAX_SCAN_CACHE_ENTRIES) { + while (scanCache.size > GITEA_SCAN_CACHE_MAX_ENTRIES) { const oldestKey = scanCache.keys().next().value if (oldestKey === undefined) { break @@ -59,6 +67,65 @@ function rememberScanCacheEntry( } } +function isRetainableRepoKey(repoKey: string): boolean { + return !measureUtf8ByteLength(repoKey, { + stopAfterBytes: GITEA_SCAN_REPO_KEY_MAX_BYTES + }).exceededLimit +} + +function isRetainablePullRequestListing(pullRequests: RawGiteaPullRequest[]): boolean { + if (pullRequests.length > GITEA_SCAN_CACHE_ENTRY_MAX_PULL_REQUESTS) { + return false + } + let remainingBytes = GITEA_SCAN_CACHE_ENTRY_MAX_BYTES - pullRequests.length * 128 + if (remainingBytes < 0) { + return false + } + for (const pullRequest of pullRequests) { + const values = [ + pullRequest.title, + pullRequest.state, + pullRequest.html_url, + pullRequest.updated_at, + pullRequest.head?.ref, + pullRequest.head?.label, + pullRequest.head?.sha + ] + for (const value of values) { + if (typeof value !== 'string') { + continue + } + const measured = measureUtf8ByteLength(value, { stopAfterBytes: remainingBytes }) + if (measured.exceededLimit) { + return false + } + remainingBytes -= measured.byteLength + } + } + return true +} + +async function collectPullRequests( + fetchPage: GiteaPullRequestPageFetcher, + pageLimit: number, + maxPages: number +): Promise<{ completed: boolean; pullRequests: RawGiteaPullRequest[] }> { + const pullRequests: RawGiteaPullRequest[] = [] + let completed = true + for (let page = 1; page <= maxPages; page++) { + const list = await fetchPage(page) + if (!list) { + completed = false + break + } + pullRequests.push(...list) + if (list.length < pageLimit) { + break + } + } + return { completed, pullRequests } +} + function reusableScanCacheEntry(repoKey: string): GiteaPullRequestScanEntry | null { const entry = scanCache.get(repoKey) if (!entry) { @@ -89,6 +156,9 @@ export async function scanGiteaPullRequests( pageLimit: number, maxPages: number ): Promise { + if (!isRetainableRepoKey(repoKey)) { + return (await collectPullRequests(fetchPage, pageLimit, maxPages)).pullRequests + } const cached = reusableScanCacheEntry(repoKey) if (cached) { return cached.pullRequests @@ -97,22 +167,13 @@ export async function scanGiteaPullRequests( if (running) { return running } + if (inFlightScans.size >= GITEA_SCAN_MAX_IN_FLIGHT) { + return (await collectPullRequests(fetchPage, pageLimit, maxPages)).pullRequests + } const generation = scanGenerations.get(repoKey) ?? 0 activeScanCounts.set(repoKey, (activeScanCounts.get(repoKey) ?? 0) + 1) const scan = (async () => { - const pullRequests: RawGiteaPullRequest[] = [] - let completed = true - for (let page = 1; page <= maxPages; page++) { - const list = await fetchPage(page) - if (!list) { - completed = false - break - } - pullRequests.push(...list) - if (list.length < pageLimit) { - break - } - } + const { completed, pullRequests } = await collectPullRequests(fetchPage, pageLimit, maxPages) if ((scanGenerations.get(repoKey) ?? 0) === generation) { rememberScanCacheEntry(repoKey, pullRequests, completed ? SCAN_TTL_MS : FAILED_SCAN_RETRY_MS) } @@ -159,3 +220,17 @@ export function _resetGiteaPullRequestScanCache(): void { export function _getGiteaPullRequestScanCacheSize(): number { return scanCache.size } + +export function _getGiteaPullRequestScanState(): { + cached: number + inFlight: number + active: number + generations: number +} { + return { + cached: scanCache.size, + inFlight: inFlightScans.size, + active: activeScanCounts.size, + generations: scanGenerations.size + } +} diff --git a/src/main/gitea/repository-ref.ts b/src/main/gitea/repository-ref.ts index f5a9acc8e5f..29ba407f909 100644 --- a/src/main/gitea/repository-ref.ts +++ b/src/main/gitea/repository-ref.ts @@ -1,5 +1,9 @@ import { gitExecFileAsync } from '../git/runner' import { getSshGitProvider } from '../providers/ssh-git-dispatch' +import { + buildRepositoryRefCacheKey, + RepositoryRefCache +} from '../source-control/repository-ref-cache' export type GiteaRepoRef = { host: string @@ -20,8 +24,7 @@ const KNOWN_NON_GITEA_HOSTS = new Set([ 'dev.azure.com', 'ssh.dev.azure.com' ]) -const REPO_REF_CACHE_MAX_ENTRIES = 512 -const repoRefCache = new Map() +const repoRefCache = new RepositoryRefCache() /** @internal - exposed for tests only */ export function _resetGiteaRepoRefCache(): void { @@ -33,17 +36,6 @@ export function _getGiteaRepoRefCacheSize(): number { return repoRefCache.size } -function rememberRepoRefCacheEntry(cacheKey: string, value: GiteaRepoRef | null): void { - repoRefCache.set(cacheKey, value) - while (repoRefCache.size > REPO_REF_CACHE_MAX_ENTRIES) { - const oldestKey = repoRefCache.keys().next().value - if (oldestKey === undefined) { - return - } - repoRefCache.delete(oldestKey) - } -} - function decodeSegment(value: string): string { try { return decodeURIComponent(value) @@ -148,9 +140,10 @@ export async function getGiteaRepoRefForRemote( localGitOptions: LocalGitExecOptions = {} ): Promise { const runtimeKey = connectionId ?? `local:${localGitOptions.wslDistro ?? 'host'}` - const cacheKey = `${runtimeKey}\0${repoPath}\0${remoteName}` - if (repoRefCache.has(cacheKey)) { - return repoRefCache.get(cacheKey)! + const cacheKey = buildRepositoryRefCacheKey([runtimeKey, repoPath, remoteName]) + const cached = repoRefCache.get(cacheKey) + if (cached.found) { + return cached.value } try { const sshGitProvider = connectionId ? getSshGitProvider(connectionId) : null @@ -164,7 +157,11 @@ export async function getGiteaRepoRefForRemote( ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) }) const result = parseGiteaRepoRef(stdout) - rememberRepoRefCacheEntry(cacheKey, result) + repoRefCache.remember( + cacheKey, + result, + result ? [result.host, result.owner, result.repo, result.apiBaseUrl, result.webBaseUrl] : [] + ) return result } catch { if (connectionId) { @@ -172,7 +169,7 @@ export async function getGiteaRepoRefForRemote( // caching them as "not Gitea" would poison the repo for the session. return null } - rememberRepoRefCacheEntry(cacheKey, null) + repoRefCache.remember(cacheKey, null, []) return null } } diff --git a/src/main/github/client.ts b/src/main/github/client.ts index 5229c187f11..311484d0c4a 100644 --- a/src/main/github/client.ts +++ b/src/main/github/client.ts @@ -34,7 +34,7 @@ import { GITHUB_WORK_ITEMS_SSH_REMOTE_REQUIRED_MESSAGE, sortWorkItemsByNumber } from '../../shared/work-items' -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { sliceCheckLogTail } from './check-job-log-tail-slice' @@ -43,8 +43,6 @@ import { safePRRefreshErrorMessage } from './pr-refresh-error-classification' import { getPRConflictSummary } from './conflict-summary' -import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' -import { joinWorktreeRelativePath } from '../runtime/runtime-relative-paths' import { splitRemoteBranchName } from '../../shared/git-effective-upstream' import { execFileAsync, @@ -74,6 +72,9 @@ import { } from '../source-control/hosted-review-git-options' import { shouldHideNonOpenReviewOnDefaultBranch } from '../source-control/repo-default-branch' import { readLocalGitConfigSignature } from './local-git-config-signature' +import { readHostedReviewTemplate } from '../source-control/pull-request-template' +import { cacheIdentityDigest } from '../cache-identity-digest' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' import { getGitHubApiRepositoryForRemote, getIssueGitHubApiRepository, @@ -1773,41 +1774,6 @@ async function findOpenPRByHeadBase(args: { return { number: list[0].number, url: list[0].url } } -async function readPullRequestTemplate( - repoPath: string, - connectionId?: string | null -): Promise { - const relativeCandidates = [ - '.github/pull_request_template.md', - '.github/PULL_REQUEST_TEMPLATE.md', - 'pull_request_template.md', - 'PULL_REQUEST_TEMPLATE.md', - 'docs/pull_request_template.md', - 'docs/PULL_REQUEST_TEMPLATE.md' - ] - const remoteProvider = connectionId ? getSshFilesystemProvider(connectionId) : undefined - if (connectionId && !remoteProvider) { - return '' - } - for (const relativeCandidate of relativeCandidates) { - try { - if (remoteProvider) { - const result = await remoteProvider.readFile( - joinWorktreeRelativePath(repoPath, relativeCandidate) - ) - if (result.isBinary) { - continue - } - return result.content - } - return await readFile(join(repoPath, relativeCandidate), 'utf8') - } catch { - // Try the next conventional PR template path. - } - } - return '' -} - export async function createGitHubPullRequest( repoPath: string, input: CreateHostedReviewInput, @@ -1862,7 +1828,7 @@ export async function createGitHubPullRequest( try { const body = input.useTemplate && !input.body?.trim() - ? await readPullRequestTemplate(repoPath, connectionId) + ? await readHostedReviewTemplate(repoPath, connectionId, 'github') : (input.body ?? '') await writeFile(bodyPath, body, 'utf8') const createArgs = [ @@ -2216,7 +2182,7 @@ async function detectRepositoryMergeMetadata( branchName: string | undefined, ghOptions: GhExecOptions ): Promise { - const cacheKey = `${githubRepoIdentityKey(ownerRepo)}:${branchName ?? '__repo__'}` + const cacheKey = cacheIdentityDigest([githubRepoIdentityKey(ownerRepo), branchName ?? '__repo__']) pruneRepositoryMergeMetadataCache() const cached = repositoryMergeMetadataCache.get(cacheKey) if (cached) { @@ -2390,11 +2356,16 @@ type TrackedUpstreamBranch = { const TRACKED_UPSTREAM_SNAPSHOT_CACHE_TTL_MS = 30_000 const TRACKED_UPSTREAM_SNAPSHOT_CACHE_MAX_ENTRIES = 512 +export const TRACKED_UPSTREAM_SNAPSHOT_MAX_IN_FLIGHT = 32 +export const TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES = 4096 +export const TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES = 2 * 1024 * 1024 +export const TRACKED_UPSTREAM_SNAPSHOT_CACHE_MAX_BYTES = 32 * 1024 * 1024 type TrackedUpstreamSnapshotCacheEntry = { expiresAt: number gitConfigSignature?: string upstreamsByBranchName: Map + retainedBytes: number } type TrackedUpstreamSnapshotProbeResult = { @@ -2402,6 +2373,7 @@ type TrackedUpstreamSnapshotProbeResult = { gitConfigSignature?: string probeFailed: boolean upstreamsByBranchName: Map + retainedBytes: number } const trackedUpstreamSnapshotCache = new Map() @@ -2410,6 +2382,16 @@ const trackedUpstreamSnapshotInFlight = new Map< Promise >() const trackedUpstreamSnapshotGenerations = new Map() +let trackedUpstreamSnapshotCacheBytes = 0 + +function deleteTrackedUpstreamSnapshot(cacheKey: string): void { + const cached = trackedUpstreamSnapshotCache.get(cacheKey) + if (!cached) { + return + } + trackedUpstreamSnapshotCacheBytes -= cached.retainedBytes + trackedUpstreamSnapshotCache.delete(cacheKey) +} function beginTrackedUpstreamSnapshotProbe(cacheKey: string): symbol { const generation = Symbol() @@ -2427,16 +2409,19 @@ function finishTrackedUpstreamSnapshotProbe(cacheKey: string, generation: symbol function pruneTrackedUpstreamSnapshotCache(now: number): void { for (const [cacheKey, cached] of trackedUpstreamSnapshotCache) { if (cached.expiresAt <= now) { - trackedUpstreamSnapshotCache.delete(cacheKey) + deleteTrackedUpstreamSnapshot(cacheKey) } } // Why: workspace/runtime churn can create unbounded unique keys within one TTL window, so expiry sweeping alone isn't a memory bound. - while (trackedUpstreamSnapshotCache.size > TRACKED_UPSTREAM_SNAPSHOT_CACHE_MAX_ENTRIES) { + while ( + trackedUpstreamSnapshotCache.size > TRACKED_UPSTREAM_SNAPSHOT_CACHE_MAX_ENTRIES || + trackedUpstreamSnapshotCacheBytes > TRACKED_UPSTREAM_SNAPSHOT_CACHE_MAX_BYTES + ) { const oldestKey = trackedUpstreamSnapshotCache.keys().next().value if (oldestKey === undefined) { break } - trackedUpstreamSnapshotCache.delete(oldestKey) + deleteTrackedUpstreamSnapshot(oldestKey) } } @@ -2456,6 +2441,7 @@ export function __resetTrackedUpstreamBranchCacheForTests(): void { trackedUpstreamSnapshotCache.clear() trackedUpstreamSnapshotInFlight.clear() trackedUpstreamSnapshotGenerations.clear() + trackedUpstreamSnapshotCacheBytes = 0 } function parseTrackedUpstreamBranch(upstreamRef: string): TrackedUpstreamBranch | null { @@ -2463,7 +2449,10 @@ function parseTrackedUpstreamBranch(upstreamRef: string): TrackedUpstreamBranch if (!parsed) { return null } - return parsed + return { + remoteName: parsed.remoteName.replace(/$/u, ''), + branchName: parsed.branchName.replace(/$/u, '') + } } function shouldRetryTrackedUpstreamBranch( @@ -2504,10 +2493,10 @@ async function getTrackedUpstreamBranch( ) { return cached.upstreamsByBranchName.get(branchName) ?? null } - trackedUpstreamSnapshotCache.delete(cacheKey) + deleteTrackedUpstreamSnapshot(cacheKey) } if (cached) { - trackedUpstreamSnapshotCache.delete(cacheKey) + deleteTrackedUpstreamSnapshot(cacheKey) } const inFlight = trackedUpstreamSnapshotInFlight.get(cacheKey) @@ -2524,18 +2513,31 @@ async function getTrackedUpstreamBranch( } } + if (trackedUpstreamSnapshotInFlight.size >= TRACKED_UPSTREAM_SNAPSHOT_MAX_IN_FLIGHT) { + const result = await probeTrackedUpstreamSnapshot( + repoPath, + connectionId, + localGitOptions, + branchName + ) + return result.upstreamsByBranchName.get(branchName) ?? null + } + // Why: PR polling asks about hundreds of branches at once; read all upstreams in one git process per repo/runtime, not one probe per branch. const probeGeneration = beginTrackedUpstreamSnapshotProbe(cacheKey) - const probe = probeTrackedUpstreamSnapshot(repoPath, connectionId, localGitOptions) + const probe = probeTrackedUpstreamSnapshot(repoPath, connectionId, localGitOptions, branchName) trackedUpstreamSnapshotInFlight.set(cacheKey, probe) try { const result = await probe if (result.cacheable && trackedUpstreamSnapshotGenerations.get(cacheKey) === probeGeneration) { + deleteTrackedUpstreamSnapshot(cacheKey) trackedUpstreamSnapshotCache.set(cacheKey, { ...(result.gitConfigSignature ? { gitConfigSignature: result.gitConfigSignature } : {}), upstreamsByBranchName: getCacheableTrackedUpstreamSnapshot(result.upstreamsByBranchName), + retainedBytes: result.retainedBytes, expiresAt: Date.now() + TRACKED_UPSTREAM_SNAPSHOT_CACHE_TTL_MS }) + trackedUpstreamSnapshotCacheBytes += result.retainedBytes pruneTrackedUpstreamSnapshotCache(Date.now()) } if (trackedUpstreamSnapshotGenerations.get(cacheKey) !== probeGeneration) { @@ -2556,7 +2558,8 @@ async function getTrackedUpstreamBranch( async function probeTrackedUpstreamSnapshot( repoPath: string, connectionId?: string | null, - localGitOptions: { wslDistro?: string } = {} + localGitOptions: { wslDistro?: string } = {}, + requestedBranchName?: string ): Promise { const startingGitConfigSignature = await readLocalGitConfigSignature({ repoPath, @@ -2566,8 +2569,10 @@ async function probeTrackedUpstreamSnapshot( const { probeFailed, upstreamsByBranchName } = await probeTrackedUpstreamBranches( repoPath, connectionId, - localGitOptions + localGitOptions, + requestedBranchName ) + const retainedBytes = measureTrackedUpstreamSnapshotBytes(upstreamsByBranchName) const endingGitConfigSignature = await readLocalGitConfigSignature({ repoPath, connectionId: connectionId ?? null, @@ -2583,7 +2588,8 @@ async function probeTrackedUpstreamSnapshot( cacheable: !configSignatureChanged && !probeFailed, probeFailed, ...(gitConfigSignature ? { gitConfigSignature } : {}), - upstreamsByBranchName + upstreamsByBranchName, + retainedBytes } } @@ -2626,13 +2632,14 @@ function getTrackedUpstreamBranchCacheKey( const runtimeKey = connectionId ? `ssh:${connectionId}` : `local:${localGitOptions.wslDistro ?? 'host'}` - return [runtimeKey, repoPath].join('\0') + return cacheIdentityDigest([runtimeKey, repoPath]) } async function probeTrackedUpstreamBranches( repoPath: string, connectionId?: string | null, - localGitOptions: { wslDistro?: string } = {} + localGitOptions: { wslDistro?: string } = {}, + requestedBranchName?: string ): Promise<{ probeFailed: boolean upstreamsByBranchName: Map @@ -2648,29 +2655,106 @@ async function probeTrackedUpstreamBranches( }) return { probeFailed: false, - upstreamsByBranchName: parseTrackedUpstreamBranches(result.stdout) + upstreamsByBranchName: parseTrackedUpstreamBranches(result.stdout, requestedBranchName) } } catch { return { probeFailed: true, upstreamsByBranchName: new Map() } } } -function parseTrackedUpstreamBranches(stdout: string): Map { +function parseTrackedUpstreamBranches( + stdout: string, + requestedBranchName?: string +): Map { const upstreamsByBranchName = new Map() - for (const line of stdout.split(/\r?\n/)) { + let retainedBytes = 0 + let lineStart = 0 + while (lineStart <= stdout.length) { + const newline = stdout.indexOf('\n', lineStart) + const lineEnd = newline === -1 ? stdout.length : newline + const line = stdout.slice( + lineStart, + lineEnd > lineStart && stdout[lineEnd - 1] === '\r' ? lineEnd - 1 : lineEnd + ) + lineStart = newline === -1 ? stdout.length + 1 : newline + 1 if (!line) { continue } - const [branchName, upstreamRef] = line.split('\0') - const localBranchName = branchName?.replace(/^refs\/heads\//, '') + const separator = line.indexOf('\0') + const branchName = separator === -1 ? line : line.slice(0, separator) + const upstreamRef = separator === -1 ? '' : line.slice(separator + 1) + const localBranchName = branchName.startsWith('refs/heads/') + ? branchName.slice('refs/heads/'.length) + : branchName if (!localBranchName) { continue } - upstreamsByBranchName.set(localBranchName, parseTrackedUpstreamRef(upstreamRef ?? '')) + const parsedUpstream = parseTrackedUpstreamRef(upstreamRef) + const entryBytes = measureTrackedUpstreamEntryBytes(localBranchName, parsedUpstream) + if (entryBytes === null) { + continue + } + const isRequested = localBranchName === requestedBranchName + while ( + isRequested && + upstreamsByBranchName.size > 0 && + (upstreamsByBranchName.size >= TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES || + retainedBytes + entryBytes > TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES) + ) { + const oldest = upstreamsByBranchName.keys().next().value + if (oldest === undefined) { + break + } + retainedBytes -= + measureTrackedUpstreamEntryBytes(oldest, upstreamsByBranchName.get(oldest) ?? null) ?? 0 + upstreamsByBranchName.delete(oldest) + } + if ( + upstreamsByBranchName.size >= TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES || + retainedBytes + entryBytes > TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES + ) { + continue + } + upstreamsByBranchName.set(localBranchName.replace(/$/u, ''), parsedUpstream) + retainedBytes += entryBytes } return upstreamsByBranchName } +export function _parseTrackedUpstreamBranchesForTests( + stdout: string, + requestedBranchName?: string +): Map { + return parseTrackedUpstreamBranches(stdout, requestedBranchName) +} + +function measureTrackedUpstreamEntryBytes( + branchName: string, + upstream: TrackedUpstreamBranch | null +): number | null { + let remainingBytes = TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES - 64 + let bytes = 64 + for (const value of [branchName, upstream?.remoteName ?? '', upstream?.branchName ?? '']) { + const measured = measureUtf8ByteLength(value, { stopAfterBytes: remainingBytes }) + if (measured.exceededLimit) { + return null + } + remainingBytes -= measured.byteLength + bytes += measured.byteLength + } + return bytes <= TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES ? bytes : null +} + +function measureTrackedUpstreamSnapshotBytes( + upstreamsByBranchName: Map +): number { + let bytes = 0 + for (const [branchName, upstream] of upstreamsByBranchName) { + bytes += measureTrackedUpstreamEntryBytes(branchName, upstream) ?? 0 + } + return bytes +} + function parseTrackedUpstreamRef(upstreamRef: string): TrackedUpstreamBranch | null { const remoteRefPrefix = 'refs/remotes/' const normalizedRef = upstreamRef.trim() @@ -3787,7 +3871,9 @@ async function attachFailedJobLogTails( // Why: cap log fetches so failed-job details stay a bounded follow-up, not a burst of hosted log downloads. for (const job of failedJobs) { const jobCacheKey = getCheckJobLogTailCacheKey(job) - const cacheKey = jobCacheKey ? `${githubRepoIdentityKey(ownerRepo)}:${jobCacheKey}` : null + const cacheKey = jobCacheKey + ? cacheIdentityDigest([githubRepoIdentityKey(ownerRepo), jobCacheKey]) + : null if (!cacheKey) { continue } diff --git a/src/main/github/conflict-summary-bounds.test.ts b/src/main/github/conflict-summary-bounds.test.ts new file mode 100644 index 00000000000..5196f66726b --- /dev/null +++ b/src/main/github/conflict-summary-bounds.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' +import { + parseMergeTreeNameOnlyOutput, + PR_CONFLICT_FILES_MAX_BYTES, + PR_CONFLICT_FILES_MAX_ENTRIES +} from './conflict-summary' + +describe('PR conflict summary retention bounds', () => { + it('caps retained conflict file count', () => { + const stdout = [ + 'tree-oid', + ...Array.from({ length: PR_CONFLICT_FILES_MAX_ENTRIES + 1 }, (_, index) => `file-${index}`) + ].join('\0') + const files = parseMergeTreeNameOnlyOutput(stdout) + expect(files).toHaveLength(PR_CONFLICT_FILES_MAX_ENTRIES) + expect(files.at(-1)).toBe(`file-${PR_CONFLICT_FILES_MAX_ENTRIES - 1}`) + }) + + it('stops before retaining an oversized path', () => { + const files = parseMergeTreeNameOnlyOutput( + `tree-oid\0${'x'.repeat(PR_CONFLICT_FILES_MAX_BYTES + 1)}\0later` + ) + expect(files).toEqual([]) + }) +}) diff --git a/src/main/github/conflict-summary-cache.ts b/src/main/github/conflict-summary-cache.ts index 99a2715ce6d..654df1005ee 100644 --- a/src/main/github/conflict-summary-cache.ts +++ b/src/main/github/conflict-summary-cache.ts @@ -1,4 +1,5 @@ import type { PRConflictSummary } from '../../shared/types' +import { cacheIdentityDigest } from '../cache-identity-digest' // Why 60s: the hottest coordinator cadences that re-derive a CONFLICTING PR // (10s mergeability-pending, 2.5s manual-pending) previously each ran a @@ -13,6 +14,7 @@ export const CONFLICT_SUMMARY_BASE_FETCH_WINDOW_MS = 60_000 // that stop refreshing can't accumulate entries forever. const BASE_OID_CACHE_MAX = 64 const SUMMARY_CACHE_MAX = 128 +const CONFLICT_SUMMARY_MAX_IN_FLIGHT = 32 export type FreshBaseTipResolution = | { kind: 'resolved'; oid: string } @@ -42,11 +44,10 @@ export function getConflictSummaryGitRuntimeKey(wslDistro: string | undefined): return wslDistro ? `wsl:${wslDistro}` : 'local:host' } -// Why JSON: repo paths and git ref names may contain any printable joiner -// character (git allows `|` in branch names), so a delimiter-joined key could -// alias distinct identities onto one cache entry. +// Why digest: keys include arbitrary repo paths and refs; a length-framed +// digest prevents delimiter aliases without retaining those strings. export function buildConflictSummaryCacheKey(...parts: string[]): string { - return JSON.stringify(parts) + return cacheIdentityDigest(parts) } export function readFreshBaseTipResolution(baseKey: string): FreshBaseTipResolution | null { @@ -121,8 +122,14 @@ function dedupeInFlight( if (existing) { return existing } - const promise = factory().finally(() => { - map.delete(key) + if (map.size >= CONFLICT_SUMMARY_MAX_IN_FLIGHT) { + return factory() + } + let promise!: Promise + promise = factory().finally(() => { + if (map.get(key) === promise) { + map.delete(key) + } }) map.set(key, promise) return promise diff --git a/src/main/github/conflict-summary.ts b/src/main/github/conflict-summary.ts index 73ec657977c..392c41a6c30 100644 --- a/src/main/github/conflict-summary.ts +++ b/src/main/github/conflict-summary.ts @@ -4,6 +4,7 @@ import { isUnsupportedMergeTreeWriteTreeError } from '../../shared/git-merge-tree-capability' import { gitExecFileAsync } from '../git/runner' +import { iterateNulDelimitedFields } from '../../shared/nul-delimited-fields' import { clearGitCapabilityStateForTests, getLocalGitCapabilityCache @@ -20,11 +21,15 @@ import { storeResolvedBaseTip, storeCachedSummary } from './conflict-summary-cache' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' type LocalGitExecOptions = { wslDistro?: string } +export const PR_CONFLICT_FILES_MAX_ENTRIES = 512 +export const PR_CONFLICT_FILES_MAX_BYTES = 256 * 1024 + export function __resetPRConflictSummaryCachesForTests(): void { clearGitCapabilityStateForTests() __resetPRConflictSummaryDerivationCachesForTests() @@ -301,13 +306,27 @@ async function loadConflictingFilesWithLegacyMergeTree( } } -function parseMergeTreeNameOnlyOutput(stdout: string): string[] { - const entries = stdout.split('\0').filter(Boolean) - if (entries.length === 0) { - return [] +export function parseMergeTreeNameOnlyOutput(stdout: string): string[] { + const files: string[] = [] + let skippedTreeId = false + let retainedBytes = 0 + for (const entry of iterateNulDelimitedFields(stdout)) { + if (!entry) { + continue + } + if (!skippedTreeId) { + skippedTreeId = true + continue + } + const measured = measureUtf8ByteLength(entry, { + stopAfterBytes: PR_CONFLICT_FILES_MAX_BYTES - retainedBytes + }) + if (measured.exceededLimit || files.length >= PR_CONFLICT_FILES_MAX_ENTRIES) { + break + } + files.push(entry.replace(/$/u, '')) + retainedBytes += measured.byteLength } - - const [, ...files] = entries return files } diff --git a/src/main/github/gh-utils.test.ts b/src/main/github/gh-utils.test.ts index 04c76eb5e53..23f3879c896 100644 --- a/src/main/github/gh-utils.test.ts +++ b/src/main/github/gh-utils.test.ts @@ -678,6 +678,35 @@ describe('github owner/repo resolution', () => { await rm(repoPath, { recursive: true, force: true }) } }) + + it('parses exact-limit config files and safely skips oversized include graphs', async () => { + const repoPath = await mkdtemp(join(tmpdir(), 'orca-gh-utils-')) + const gitDir = join(repoPath, '.git') + const includedConfigPath = join(repoPath, 'included.gitconfig') + const configPath = join(gitDir, 'config') + const maxConfigBytes = 4 * 1024 * 1024 + await mkdir(gitDir) + await writeFile(includedConfigPath, '[user]\n\tname = first\n') + const includePrefix = `[include]\n\tpath = "${includedConfigPath}"\n#` + await writeFile(configPath, includePrefix + 'x'.repeat(maxConfigBytes - includePrefix.length)) + try { + const exactFirst = await readLocalGitConfigSignature({ repoPath, connectionId: null }) + await writeFile(includedConfigPath, '[user]\n\tname = exact-limit-change\n') + const exactSecond = await readLocalGitConfigSignature({ repoPath, connectionId: null }) + expect(exactSecond).not.toEqual(exactFirst) + + await writeFile( + configPath, + includePrefix + 'x'.repeat(maxConfigBytes + 1 - includePrefix.length) + ) + const oversizedFirst = await readLocalGitConfigSignature({ repoPath, connectionId: null }) + await writeFile(includedConfigPath, '[user]\n\tname = ignored-oversized-change\n') + const oversizedSecond = await readLocalGitConfigSignature({ repoPath, connectionId: null }) + expect(oversizedSecond).toEqual(oversizedFirst) + } finally { + await rm(repoPath, { recursive: true, force: true }) + } + }) }) describe('resolveIssueSource', () => { diff --git a/src/main/github/gh-utils.ts b/src/main/github/gh-utils.ts index b294e90ac1a..ab81e4e3630 100644 --- a/src/main/github/gh-utils.ts +++ b/src/main/github/gh-utils.ts @@ -4,6 +4,7 @@ import { gitExecFileAsync, ghExecFileAsync } from '../git/runner' // Pure error-parsing helpers come from the lightweight module (not `runner`) so // tests that mock `../git/runner` still resolve the real implementations. import { extractExecError, parseRetryAfterMs } from '../git/exec-error' +import { IntegrationApiConcurrencyGate } from '../integration-api-concurrency' // Why: legacy generic execFile wrapper - only used by callers that don't need // WSL-aware routing. Repo-scoped callers should use the runner exports below. @@ -34,26 +35,12 @@ export type { } from './github-repository-identity' const MAX_CONCURRENT = 4 -let running = 0 -const queue: (() => void)[] = [] +const concurrencyGate = new IntegrationApiConcurrencyGate(MAX_CONCURRENT) export function acquire(): Promise { - if (running < MAX_CONCURRENT) { - running += 1 - return Promise.resolve() - } - return new Promise((resolve) => - queue.push(() => { - running += 1 - resolve() - }) - ) + return concurrencyGate.acquire() } export function release(): void { - running -= 1 - const next = queue.shift() - if (next) { - next() - } + concurrencyGate.release() } diff --git a/src/main/github/github-api-repository-validation.ts b/src/main/github/github-api-repository-validation.ts new file mode 100644 index 00000000000..a3ab47471f8 --- /dev/null +++ b/src/main/github/github-api-repository-validation.ts @@ -0,0 +1,14 @@ +import type { GitHubOwnerRepo } from '../../shared/types' + +const GITHUB_OWNER_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/ +const GITHUB_REPO_SLUG_RE = /^[A-Za-z0-9._-]+$/ + +// Why: renderer/RPC overrides are interpolated into authenticated REST paths. +export function isValidGitHubApiRepository(repository: GitHubOwnerRepo): boolean { + return ( + GITHUB_OWNER_SLUG_RE.test(repository.owner) && + GITHUB_REPO_SLUG_RE.test(repository.repo) && + repository.repo !== '.' && + repository.repo !== '..' + ) +} diff --git a/src/main/github/github-api-repository.ts b/src/main/github/github-api-repository.ts index 08461f849e9..16965705148 100644 --- a/src/main/github/github-api-repository.ts +++ b/src/main/github/github-api-repository.ts @@ -14,6 +14,8 @@ import { getEnterpriseGitHubRepoSlugForRemote, isGitHubHostAuthenticated } from './github-enterprise-repository' +import { isValidGitHubApiRepository } from './github-api-repository-validation' +import { cacheIdentityDigest } from '../cache-identity-digest' export type GitHubApiRepository = GitHubOwnerRepo export type GitHubRepoExecOptions = ReturnType & { host?: string } @@ -28,25 +30,12 @@ type GitHubApiRepositoryResolution = | undefined | (() => Promise) -// Why: renderer/RPC repository overrides are interpolated into REST paths. -// Reject path syntax before an authenticated gh process can target it. -const GITHUB_OWNER_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/ -const GITHUB_REPO_SLUG_RE = /^[A-Za-z0-9._-]+$/ - -function isValidGitHubApiRepository(repository: GitHubApiRepository): boolean { - return ( - GITHUB_OWNER_SLUG_RE.test(repository.owner) && - GITHUB_REPO_SLUG_RE.test(repository.repo) && - repository.repo !== '.' && - repository.repo !== '..' - ) -} - // Why: the enterprise branch spawns an uncached `git remote get-url` (an SSH // round trip on connection-backed repos) — hot paths like per-file contents // and viewed-state toggles resolve per call, so cache like ownerRepoCache does. const ORIGIN_REPO_CACHE_TTL_MS = 30_000 const ORIGIN_REPO_CACHE_MAX_ENTRIES = 512 +const ORIGIN_REPO_MAX_IN_FLIGHT = 32 const originRepoCache = new Map() const originRepoInFlight = new Map>() @@ -56,7 +45,12 @@ function originRepoCacheKey( connectionId?: string | null, localGitOptions: LocalGitExecOptions = {} ): string { - return `${connectionId ?? 'local'}\0${localGitOptions.wslDistro ?? ''}\0${repoPath}\0${remoteName}` + return cacheIdentityDigest([ + connectionId ?? 'local', + localGitOptions.wslDistro ?? '', + repoPath, + remoteName + ]) } /** @internal - exposed for tests only */ @@ -136,6 +130,9 @@ export async function getGitHubApiRepositoryForRemote( } return slug ?? null })() + if (originRepoInFlight.size >= ORIGIN_REPO_MAX_IN_FLIGHT) { + return probe + } originRepoInFlight.set(cacheKey, probe) try { return await probe diff --git a/src/main/github/github-enterprise-repository.ts b/src/main/github/github-enterprise-repository.ts index f0277068471..87d5acde4ea 100644 --- a/src/main/github/github-enterprise-repository.ts +++ b/src/main/github/github-enterprise-repository.ts @@ -13,6 +13,7 @@ import { type LocalGitExecOptions } from './github-repository-identity' import { parseWslPath } from '../wsl' +import { cacheIdentityDigest } from '../cache-identity-digest' export type GitHubEnterpriseRepoSlug = GitHubOwnerRepo & { host: string } @@ -22,6 +23,8 @@ export type GitHubEnterpriseRepoSlug = GitHubOwnerRepo & { host: string } // GHES remote is not left to fall through to Gitea (#8312). const HOST_AUTH_TTL_MS = 60_000 const HOST_AUTH_CACHE_MAX_ENTRIES = 512 +const HOST_AUTH_MAX_IN_FLIGHT = 16 +const GITHUB_HOST_MAX_CHARS = 1024 type HostAuthCacheEntry = { authenticatedHost: string | null @@ -82,6 +85,9 @@ type NormalizedGitHubHost = { } function normalizeGitHubHost(host: string): NormalizedGitHubHost | null { + if (host.length > GITHUB_HOST_MAX_CHARS) { + return null + } const match = host .trim() .toLowerCase() @@ -128,8 +134,14 @@ async function resolveAuthenticatedGitHubHost( connectionId?: string | null, localGitOptions: LocalGitExecOptions = {} ): Promise { - const normalizedHost = normalizeGitHubHost(host)?.authority ?? host.trim().toLowerCase() - const cacheKey = `${runtimeCacheKey(repoPath, localGitOptions.wslDistro)}\0${normalizedHost}` + const normalizedHost = normalizeGitHubHost(host)?.authority + if (!normalizedHost) { + return null + } + const cacheKey = cacheIdentityDigest([ + runtimeCacheKey(repoPath, localGitOptions.wslDistro), + normalizedHost + ]) const now = Date.now() pruneHostAuthCache(now) const cached = hostAuthCache.get(cacheKey) @@ -168,6 +180,9 @@ async function resolveAuthenticatedGitHubHost( pruneHostAuthCache(Date.now()) return authenticatedHost })() + if (hostAuthInFlight.size >= HOST_AUTH_MAX_IN_FLIGHT) { + return probe + } hostAuthInFlight.set(cacheKey, probe) try { return await probe diff --git a/src/main/github/github-remote-identity-parsing.test.ts b/src/main/github/github-remote-identity-parsing.test.ts index c0b8b599520..0f7c70249c1 100644 --- a/src/main/github/github-remote-identity-parsing.test.ts +++ b/src/main/github/github-remote-identity-parsing.test.ts @@ -1,8 +1,22 @@ import { describe, expect, it } from 'vitest' -import { parseGitHubOwnerRepo, parseGitHubRemoteIdentity } from './github-remote-identity-parsing' +import { + GITHUB_REMOTE_REPO_MAX_BYTES, + GITHUB_REMOTE_URL_MAX_BYTES, + parseGitHubOwnerRepo, + parseGitHubRemoteIdentity +} from './github-remote-identity-parsing' describe('parseGitHubRemoteIdentity', () => { + it('rejects oversized remote URLs and identity fields before retention', () => { + expect(parseGitHubRemoteIdentity('x'.repeat(GITHUB_REMOTE_URL_MAX_BYTES + 1))).toBeNull() + expect( + parseGitHubRemoteIdentity( + `git@github.com:owner/${'r'.repeat(GITHUB_REMOTE_REPO_MAX_BYTES + 1)}.git` + ) + ).toBeNull() + }) + it('parses a plain github.com https remote', () => { expect(parseGitHubRemoteIdentity('https://github.com/team/orca.git')).toEqual({ host: 'github.com', diff --git a/src/main/github/github-remote-identity-parsing.ts b/src/main/github/github-remote-identity-parsing.ts index 6f13e62014d..7ec2f10dd33 100644 --- a/src/main/github/github-remote-identity-parsing.ts +++ b/src/main/github/github-remote-identity-parsing.ts @@ -1,6 +1,15 @@ import type { GitHubOwnerRepo } from '../../shared/types' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' export type GitHubRemoteIdentity = GitHubOwnerRepo & { host: string } +export const GITHUB_REMOTE_URL_MAX_BYTES = 64 * 1024 +export const GITHUB_REMOTE_HOST_MAX_BYTES = 1024 +export const GITHUB_REMOTE_OWNER_MAX_BYTES = 256 +export const GITHUB_REMOTE_REPO_MAX_BYTES = 1024 + +function fitsRemoteField(value: string, maxBytes: number): boolean { + return !measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }).exceededLimit +} function normalizeGitHubRemoteHost(host: string): string { const normalizedHost = host.toLowerCase() @@ -22,16 +31,31 @@ function parseGitHubRemotePath(path: string): Pick { const context = githubRepoContext(repoPath, connectionId, localGitOptions) const runtimeKey = context.connectionId ?? `local:${context.wslDistro ?? 'host'}` - const cacheKey = `${runtimeKey}\0${context.repoPath}\0${remoteName}` + const cacheKey = cacheIdentityDigest([runtimeKey, context.repoPath, remoteName]) const now = Date.now() pruneOwnerRepoCache(now) const cached = ownerRepoCache.get(cacheKey) @@ -158,6 +160,9 @@ export async function getOwnerRepoForRemote( // Why: startup can resolve issue sources, PR candidates, and repo metadata // for the same repo concurrently. Coalesce missing-remote probes. const probe = resolveOwnerRepoForRemote(context, remoteName, cacheKey, nextConfigSignature) + if (ownerRepoInFlight.size >= OWNER_REPO_MAX_IN_FLIGHT) { + return probe + } ownerRepoInFlight.set(cacheKey, probe) try { return await probe diff --git a/src/main/github/local-git-config-signature.ts b/src/main/github/local-git-config-signature.ts index 1a3c297d5f7..350650d78fc 100644 --- a/src/main/github/local-git-config-signature.ts +++ b/src/main/github/local-git-config-signature.ts @@ -1,7 +1,11 @@ -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' +import { createHash } from 'node:crypto' import { homedir } from 'node:os' import { dirname, isAbsolute, join, resolve } from 'node:path' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' import type { GitHubRepoContext } from './github-repository-identity' +import { cacheIdentityDigest } from '../cache-identity-digest' type LocalGitConfigPaths = { commonConfigPath: string @@ -9,6 +13,18 @@ type LocalGitConfigPaths = { } const localGitConfigSignatureInFlight = new Map>() +const LOCAL_GIT_CONFIG_SIGNATURE_MAX_IN_FLIGHT = 32 +const MAX_GIT_CONFIG_BYTES = 4 * 1024 * 1024 +const MAX_GIT_POINTER_FILE_BYTES = 64 * 1024 +const MAX_INCLUDED_CONFIG_FILES = 256 +const MAX_INCLUDED_CONFIG_DEPTH = 8 +const MAX_INCLUDED_CONFIG_PATH_BYTES = 16 * 1024 +const MAX_INCLUDED_CONFIG_AGGREGATE_PATH_BYTES = 2 * 1024 * 1024 + +type ConfigSignatureBudget = { + admittedFiles: number + pathBytes: number +} export async function readLocalGitConfigSignature( context: GitHubRepoContext @@ -18,13 +34,16 @@ export async function readLocalGitConfigSignature( // runtimes are already separated by cache key and probed through git. return undefined } - const cacheKey = context.repoPath + const cacheKey = cacheIdentityDigest([context.repoPath]) const inFlight = localGitConfigSignatureInFlight.get(cacheKey) if (inFlight) { return inFlight } const read = readUncachedLocalGitConfigSignature(context.repoPath) + if (localGitConfigSignatureInFlight.size >= LOCAL_GIT_CONFIG_SIGNATURE_MAX_IN_FLIGHT) { + return read + } localGitConfigSignatureInFlight.set(cacheKey, read) try { return await read @@ -48,31 +67,54 @@ async function readUncachedLocalGitConfigSignature(repoPath: string): Promise() + visited = new Set(), + budget: ConfigSignatureBudget = { admittedFiles: 0, pathBytes: 0 }, + depth = 0 ): Promise { if (visited.has(configPath)) { return [] } + const measuredPath = measureUtf8ByteLength(configPath, { + stopAfterBytes: MAX_INCLUDED_CONFIG_PATH_BYTES + }) + if ( + measuredPath.exceededLimit || + depth > MAX_INCLUDED_CONFIG_DEPTH || + budget.admittedFiles >= MAX_INCLUDED_CONFIG_FILES || + budget.pathBytes + measuredPath.byteLength > MAX_INCLUDED_CONFIG_AGGREGATE_PATH_BYTES + ) { + return [] + } visited.add(configPath) + budget.admittedFiles += 1 + budget.pathBytes += measuredPath.byteLength const ownSignature = await readConfigPathSignature(configPath) let configText: string try { - configText = await readFile(configPath, 'utf8') + configText = (await readNodeFileWithinLimit(configPath, MAX_GIT_CONFIG_BYTES)).buffer.toString( + 'utf8' + ) } catch { return [ownSignature] } const includedPaths = parseIncludedConfigPaths(configText, dirname(configPath)) - const includedSignatures = await Promise.all( - includedPaths.map((includedPath) => readConfigPathSignatures(includedPath, visited)) - ) - return [ownSignature, ...includedSignatures.flat()] + const signatures = [ownSignature] + for (const includedPath of includedPaths) { + signatures.push(...(await readConfigPathSignatures(includedPath, visited, budget, depth + 1))) + } + return signatures } async function readConfigPathSignature(configPath: string): Promise { @@ -103,6 +145,9 @@ function parseIncludedConfigPaths(configText: string, baseDir: string): string[] const includePath = parseIncludedConfigPath(line) if (includePath) { includedPaths.push(resolveIncludedConfigPath(includePath, baseDir)) + if (includedPaths.length >= MAX_INCLUDED_CONFIG_FILES) { + break + } } } return includedPaths @@ -214,7 +259,9 @@ async function resolveLocalGitConfigPaths(repoPath: string): Promise[1] @@ -61,7 +62,11 @@ export async function isCommitPartOfMergedPR(args: { } const owner = args.ownerRepo.owner const repo = args.ownerRepo.repo - const cacheKey = `${githubRepoIdentityKey(args.ownerRepo)}#${args.prNumber}@${oid}` + const cacheKey = cacheIdentityDigest([ + githubRepoIdentityKey(args.ownerRepo), + String(args.prNumber), + oid + ]) const ghOptions = { ...args.ghOptions, ...githubHostExecOptions(args.ownerRepo) } const now = Date.now() pruneMergedPRCommitMembershipCache(now) diff --git a/src/main/github/pr-refresh-coordinator-memory.test.ts b/src/main/github/pr-refresh-coordinator-memory.test.ts new file mode 100644 index 00000000000..57472c2e50a --- /dev/null +++ b/src/main/github/pr-refresh-coordinator-memory.test.ts @@ -0,0 +1,84 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { GitHubPRRefreshCandidate } from '../../shared/types' +import { + PR_REFRESH_ALIAS_LIMIT, + PR_REFRESH_QUEUE_ENTRY_LIMIT +} from '../../shared/pr-refresh-memory-limits' + +const { sendToTrustedUIRendererMock } = vi.hoisted(() => ({ + sendToTrustedUIRendererMock: vi.fn() +})) + +vi.mock('electron', () => ({ + webContents: { getAllWebContents: () => [] } +})) +vi.mock('./client', () => ({ getPRForBranchOutcome: vi.fn() })) +vi.mock('./github-api-repository', () => ({ getOriginGitHubApiRepository: vi.fn() })) +vi.mock('./rate-limit', () => ({ + getRateLimit: vi.fn(), + noteRepositoryRateLimitSpend: vi.fn(), + repositoryRateLimitGuard: vi.fn(() => ({ blocked: false })), + spendsSharedGitHubComQuota: vi.fn(() => false) +})) +vi.mock('../crash-reporting/crash-breadcrumb-store', () => ({ + recordCoalescedCrashBreadcrumb: vi.fn() +})) +vi.mock('../ipc/ui', () => ({ sendToTrustedUIRenderer: sendToTrustedUIRendererMock })) + +function candidate(index: number, linkedPRNumber?: number): GitHubPRRefreshCandidate { + return { + cacheKey: `/repo-${index}::feature-${index}`, + repoPath: linkedPRNumber === undefined ? `/repo-${index}` : '/repo', + branch: `feature-${index}`, + repoKind: 'git', + repoId: `repo-${index}`, + worktreeId: `worktree-${index}`, + cachedFetchedAt: Date.now(), + linkedPRNumber + } +} + +describe('PR refresh coordinator memory admission', () => { + beforeEach(() => { + vi.resetModules() + vi.useFakeTimers() + vi.setSystemTime(1_000) + sendToTrustedUIRendererMock.mockReset() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('caps distinct queued refreshes while GitHub work is delayed', async () => { + const { enqueuePRRefresh, _getPRRefreshQueueSizeForTests } = + await import('./pr-refresh-coordinator') + + for (let index = 0; index <= PR_REFRESH_QUEUE_ENTRY_LIMIT; index += 1) { + enqueuePRRefresh(candidate(index), 'visible', 40, 1) + } + + expect(_getPRRefreshQueueSizeForTests()).toBe(PR_REFRESH_QUEUE_ENTRY_LIMIT) + expect(sendToTrustedUIRendererMock).toHaveBeenLastCalledWith( + 'gh:prRefreshEvent', + expect.objectContaining({ status: 'skipped', skippedReason: 'capacity' }) + ) + }) + + it('caps aliases coalesced behind one linked review', async () => { + const { enqueuePRRefresh, _getPRRefreshAliasCountForTests } = + await import('./pr-refresh-coordinator') + + for (let index = 0; index <= PR_REFRESH_ALIAS_LIMIT; index += 1) { + enqueuePRRefresh(candidate(index, 42), 'visible', 40, 1) + } + + expect(_getPRRefreshAliasCountForTests('local::runtime:host::/repo::pr::42')).toBe( + PR_REFRESH_ALIAS_LIMIT + ) + expect(sendToTrustedUIRendererMock).toHaveBeenCalledWith( + 'gh:prRefreshEvent', + expect.objectContaining({ status: 'skipped', skippedReason: 'capacity' }) + ) + }) +}) diff --git a/src/main/github/pr-refresh-coordinator.ts b/src/main/github/pr-refresh-coordinator.ts index 70b13ec2e69..a4197c50539 100644 --- a/src/main/github/pr-refresh-coordinator.ts +++ b/src/main/github/pr-refresh-coordinator.ts @@ -11,6 +11,14 @@ import type { import { getPRForBranchOutcome, type GitHubPRBranchLookupOptions } from './client' import { getOriginGitHubApiRepository } from './github-api-repository' import { ghRepoExecOptions, githubRepoContext } from './gh-utils' +import { + boundedVisiblePRRefreshCandidates, + PR_REFRESH_ACTIVE_SCOPE_LIMIT, + PR_REFRESH_QUEUE_ENTRY_LIMIT, + PR_REFRESH_RETRY_STATE_LIMIT, + retainPRRefreshAlias, + retainPRRefreshState +} from './pr-refresh-memory-bounds' import { getRateLimit, noteRepositoryRateLimitSpend, @@ -97,8 +105,19 @@ let lastBackgroundStartAt = 0 * Only a rate-limit outcome carrying `retryDisabledUntil` sets a gate; any other settled outcome clears it. */ function noteManualRetryGate(key: string, outcome: PRRefreshOutcome): void { + const now = Date.now() + for (const [gateKey, retryAt] of manualRetryGates) { + if (retryAt <= now) { + manualRetryGates.delete(gateKey) + } + } if (outcome.kind === 'upstream-error' && outcome.retryDisabledUntil !== undefined) { - manualRetryGates.set(key, outcome.retryDisabledUntil) + retainPRRefreshState( + manualRetryGates, + key, + outcome.retryDisabledUntil, + PR_REFRESH_RETRY_STATE_LIMIT + ) } else { manualRetryGates.delete(key) } @@ -118,6 +137,14 @@ const diagnosticsCounters = { backgroundPauses: 0 } +function setBoundedQueueEntry(entry: QueueEntry): boolean { + if (!queue.has(entry.key) && queue.size >= PR_REFRESH_QUEUE_ENTRY_LIMIT) { + return false + } + queue.set(entry.key, entry) + return true +} + export function setPRRefreshOutcomeObserver(observer: PRRefreshOutcomeObserver | null): void { outcomeObserver = observer } @@ -158,6 +185,26 @@ function recordPRRefreshQueueDiagnostic( }) } +function broadcastCapacitySkip( + aliases: GitHubPRRefreshAlias[], + reason: GitHubPRRefreshReason +): void { + diagnosticsCounters.skipped += 1 + recordPRRefreshQueueDiagnostic('skipped', reason, 'capacity') + broadcast({ aliases, reason, status: 'skipped', skippedReason: 'capacity' }) +} + +function addBoundedQueueAlias( + entry: QueueEntry, + alias: GitHubPRRefreshAlias, + reason: GitHubPRRefreshReason +): void { + const evicted = retainPRRefreshAlias(entry.aliases, alias, entry.candidate.cacheKey) + if (evicted) { + broadcastCapacitySkip([evicted], reason) + } +} + function clearActiveBurstWindow(windowId: number): void { const windowPrefix = `${windowId}::` for (const scope of Array.from(activeStartsByScope.keys())) { @@ -364,15 +411,18 @@ function visibleCandidateAfterOutcome( } } -function setVisibleFollowUp(entry: QueueEntry): void { +function setVisibleFollowUp(entry: QueueEntry): boolean { const existing = queue.get(entry.key) if (!existing) { - queue.set(entry.key, entry) - return + if (!setBoundedQueueEntry(entry)) { + broadcastCapacitySkip(Array.from(entry.aliases.values()), entry.reason) + return false + } + return true } for (const alias of entry.aliases.values()) { - existing.aliases.set(alias.cacheKey, alias) + addBoundedQueueAlias(existing, alias, entry.reason) } // Why: a user activation can arrive while a background refresh awaits gh; the follow-up must not overwrite that pending active/manual work. @@ -381,13 +431,14 @@ function setVisibleFollowUp(entry: QueueEntry): void { existing.priority > entry.priority || existing.dueAt <= entry.dueAt ) { - return + return true } - queue.set(entry.key, { + setBoundedQueueEntry({ ...entry, aliases: existing.aliases }) + return true } function removeQueuedAliasForInvalidCandidate(key: string, alias: GitHubPRRefreshAlias): void { @@ -426,7 +477,7 @@ function nextVisibleErrorRetryAt(key: string): number { const failures = (errorBackoff.get(key)?.failures ?? 0) + 1 const retryAt = Date.now() + Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 4)) - errorBackoff.set(key, { failures, retryAt }) + retainPRRefreshState(errorBackoff, key, { failures, retryAt }, PR_REFRESH_RETRY_STATE_LIMIT) return retryAt } @@ -463,7 +514,7 @@ function scheduleVisibleFollowUp( if (outcome.kind === 'upstream-error') { // Why: reuse the retry time already computed for the broadcast so the same failure isn't counted twice against the backoff. const retryAt = options?.plannedRetryAt ?? nextVisibleErrorRetryAt(key) - setVisibleFollowUp({ + const retained = setVisibleFollowUp({ key, candidate, aliases: new Map(aliases.map((alias) => [alias.cacheKey, alias])), @@ -473,6 +524,10 @@ function scheduleVisibleFollowUp( queuedAt: nextQueueOrder(), windowId }) + if (!retained) { + resetKeyRetryState(key) + return + } // Why: this is a delayed retry, not active work; a spinner would make visible worktrees look stuck until backoff expires. scheduleDrain(retryAt - Date.now()) return @@ -489,7 +544,7 @@ function scheduleVisibleFollowUp( ? regularDueAt : Math.min(regularDueAt, pendingMergeabilityDueAt) // Why: a coalesced linked-PR refresh may represent several branches; preserve every alias so all cache entries keep getting updates. - setVisibleFollowUp({ + const retained = setVisibleFollowUp({ key, candidate: followUpCandidate, aliases: new Map(aliases.map((alias) => [alias.cacheKey, alias])), @@ -501,6 +556,10 @@ function scheduleVisibleFollowUp( bypassBackgroundBudget: pendingMergeabilityDueAt !== null, windowId }) + if (!retained) { + resetKeyRetryState(key) + return + } scheduleDrain(Math.max(0, dueAt - Date.now())) } @@ -595,6 +654,12 @@ function pruneActiveStarts(scope: string, now: number): number[] { return activeStarts } +function pruneExpiredActiveScopes(now: number): void { + for (const scope of Array.from(activeStartsByScope.keys())) { + pruneActiveStarts(scope, now) + } +} + function nextActiveBurstDelay(entry: QueueEntry): number { const now = Date.now() const activeStarts = pruneActiveStarts(activeBurstScope(entry), now) @@ -607,9 +672,10 @@ function nextActiveBurstDelay(entry: QueueEntry): number { function noteActiveStart(entry: QueueEntry): void { const now = Date.now() const scope = activeBurstScope(entry) + pruneExpiredActiveScopes(now) const activeStarts = pruneActiveStarts(scope, now) activeStarts.push(now) - activeStartsByScope.set(scope, activeStarts) + retainPRRefreshState(activeStartsByScope, scope, activeStarts, PR_REFRESH_ACTIVE_SCOPE_LIMIT) } function activeOrder(a: QueueEntry, b: QueueEntry): number { @@ -758,7 +824,12 @@ async function drainQueue(): Promise { .find((guard) => guard.blocked) if (blockedGuard?.blocked) { const retryAt = blockedGuard.resetAt * 1000 - queue.set(next.key, { ...next, dueAt: retryAt }) + const retained = setBoundedQueueEntry({ ...next, dueAt: retryAt }) + if (!retained) { + resetKeyRetryState(next.key) + broadcastCapacitySkip(aliases, next.reason) + continue + } broadcast({ aliases, reason: next.reason, @@ -845,7 +916,7 @@ export function enqueuePRRefresh( const freshDueAt = shouldSkipFresh(candidate, reason) ? freshRetryAt(candidate) : null const dueAt = freshDueAt ?? Date.now() + (reason === 'post-push' ? POST_PUSH_DELAY_MS : 0) if (existing) { - existing.aliases.set(alias.cacheKey, alias) + addBoundedQueueAlias(existing, alias, reason) diagnosticsCounters.coalesced += 1 recordPRRefreshQueueDiagnostic('coalesced', reason) const shouldPromoteExisting = @@ -871,9 +942,13 @@ export function enqueuePRRefresh( } } } else { + if (queue.size >= PR_REFRESH_QUEUE_ENTRY_LIMIT) { + broadcastCapacitySkip([alias], reason) + return + } diagnosticsCounters.enqueued += 1 recordPRRefreshQueueDiagnostic('enqueued', reason) - queue.set(key, { + setBoundedQueueEntry({ key, candidate, aliases: new Map([[alias.cacheKey, alias]]), @@ -900,9 +975,13 @@ export function reportVisiblePRRefreshCandidates( if (existingVisible && generation < existingVisible.generation) { return } - visibleByWindow.set(windowId, { generation, keys: new Set(candidates.map(refreshKey)) }) + const retainedCandidates = boundedVisiblePRRefreshCandidates(candidates) + visibleByWindow.set(windowId, { + generation, + keys: new Set(retainedCandidates.map(refreshKey)) + }) removeInvisibleVisibleRefreshes() - for (const candidate of candidates) { + for (const candidate of retainedCandidates) { enqueuePRRefresh(candidate, 'visible', 40, windowId) } } @@ -928,7 +1007,14 @@ export async function refreshPRNow(candidate: GitHubPRRefreshCandidate): Promise const key = refreshKey(candidate) const existing = queue.get(key) const aliasMap = new Map(existing ? existing.aliases : []) - aliasMap.set(alias.cacheKey, alias) + const evictedAlias = retainPRRefreshAlias( + aliasMap, + alias, + existing?.candidate.cacheKey ?? alias.cacheKey + ) + if (evictedAlias) { + broadcastCapacitySkip([evictedAlias], 'manual') + } const aliases = Array.from(aliasMap.values()) const skippedReason = validateCandidate(candidate) if (skippedReason) { @@ -963,7 +1049,7 @@ export async function refreshPRNow(candidate: GitHubPRRefreshCandidate): Promise if (gateUntil > Date.now()) { const retryAt = gateUntil // Why: paused maps `pausedUntil` into the renderer's auto-retry, so requeue at reset (finding 12) — don't advertise an unscheduled retry. - queue.set(key, { + const retained = setBoundedQueueEntry({ key, candidate, aliases: aliasMap, @@ -972,6 +1058,16 @@ export async function refreshPRNow(candidate: GitHubPRRefreshCandidate): Promise dueAt: retryAt, queuedAt: nextQueueOrder() }) + if (!retained) { + broadcastCapacitySkip(aliases, 'manual') + return { + kind: 'upstream-error', + errorType: 'rate_limited', + message: 'GitHub is temporarily limiting requests. Try again after the limit resets.', + fetchedAt: Date.now(), + retryDisabledUntil: retryAt + } + } broadcast({ aliases, reason: 'manual', diff --git a/src/main/github/pr-refresh-memory-bounds.test.ts b/src/main/github/pr-refresh-memory-bounds.test.ts new file mode 100644 index 00000000000..9e703a6895c --- /dev/null +++ b/src/main/github/pr-refresh-memory-bounds.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' +import type { GitHubPRRefreshAlias, GitHubPRRefreshCandidate } from '../../shared/types' +import { + boundedVisiblePRRefreshCandidates, + PR_REFRESH_ALIAS_LIMIT, + PR_REFRESH_RETRY_STATE_LIMIT, + retainPRRefreshAlias, + retainPRRefreshState +} from './pr-refresh-memory-bounds' +import { PR_REFRESH_VISIBLE_CANDIDATE_LIMIT } from '../../shared/pr-refresh-memory-limits' + +function alias(index: number): GitHubPRRefreshAlias { + return { + cacheKey: `cache-${index}`, + repoId: 'repo-1', + repoPath: '/repo', + branch: `feature-${index}`, + worktreeId: `worktree-${index}`, + connectionId: null, + currentHeadOid: null, + linkedPRNumber: 42, + fallbackPRNumber: null, + fallbackPRSource: null + } +} + +function candidate(index: number): GitHubPRRefreshCandidate { + return { + cacheKey: `cache-${index}`, + repoId: 'repo-1', + repoPath: '/repo', + branch: `feature-${index}`, + worktreeId: `worktree-${index}`, + connectionId: null, + currentHeadOid: null, + linkedPRNumber: 42, + fallbackPRNumber: null, + fallbackPRSource: null, + repoKind: 'git', + cachedFetchedAt: null + } +} + +describe('PR refresh memory bounds', () => { + it('retains ordinary aliases without changing their identity or order', () => { + const aliases = new Map() + + expect(retainPRRefreshAlias(aliases, alias(0), 'cache-0')).toBeNull() + expect(retainPRRefreshAlias(aliases, alias(1), 'cache-0')).toBeNull() + + expect(Array.from(aliases.keys())).toEqual(['cache-0', 'cache-1']) + }) + + it('caps aliases while preserving the representative candidate', () => { + const aliases = new Map() + for (let index = 0; index < PR_REFRESH_ALIAS_LIMIT; index += 1) { + retainPRRefreshAlias(aliases, alias(index), 'cache-0') + } + + const evicted = retainPRRefreshAlias(aliases, alias(PR_REFRESH_ALIAS_LIMIT), 'cache-0') + + expect(aliases).toHaveLength(PR_REFRESH_ALIAS_LIMIT) + expect(aliases.has('cache-0')).toBe(true) + expect(aliases.has(`cache-${PR_REFRESH_ALIAS_LIMIT}`)).toBe(true) + expect(evicted?.cacheKey).toBe('cache-1') + }) + + it('caps retry state and refreshes existing keys without growing', () => { + const states = new Map() + for (let index = 0; index < PR_REFRESH_RETRY_STATE_LIMIT; index += 1) { + retainPRRefreshState(states, index, index, PR_REFRESH_RETRY_STATE_LIMIT) + } + + expect(retainPRRefreshState(states, 1, 99, PR_REFRESH_RETRY_STATE_LIMIT)).toBeNull() + expect( + retainPRRefreshState(states, PR_REFRESH_RETRY_STATE_LIMIT, 1, PR_REFRESH_RETRY_STATE_LIMIT) + ).toBe(0) + expect(states).toHaveLength(PR_REFRESH_RETRY_STATE_LIMIT) + expect(states.get(1)).toBe(99) + }) + + it('passes ordinary visible candidates through and truncates adversarial batches', () => { + const ordinary = [candidate(0), candidate(1)] + expect(boundedVisiblePRRefreshCandidates(ordinary)).toBe(ordinary) + + const oversized = Array.from({ length: PR_REFRESH_VISIBLE_CANDIDATE_LIMIT + 1 }, (_, index) => + candidate(index) + ) + expect(boundedVisiblePRRefreshCandidates(oversized)).toHaveLength( + PR_REFRESH_VISIBLE_CANDIDATE_LIMIT + ) + }) +}) diff --git a/src/main/github/pr-refresh-memory-bounds.ts b/src/main/github/pr-refresh-memory-bounds.ts new file mode 100644 index 00000000000..27f57ae4fdc --- /dev/null +++ b/src/main/github/pr-refresh-memory-bounds.ts @@ -0,0 +1,75 @@ +import type { GitHubPRRefreshAlias, GitHubPRRefreshCandidate } from '../../shared/types' +import { + PR_REFRESH_ALIAS_LIMIT, + PR_REFRESH_VISIBLE_CANDIDATE_LIMIT +} from '../../shared/pr-refresh-memory-limits' + +export { + PR_REFRESH_ACTIVE_SCOPE_LIMIT, + PR_REFRESH_ALIAS_LIMIT, + PR_REFRESH_QUEUE_ENTRY_LIMIT, + PR_REFRESH_RETRY_STATE_LIMIT, + PR_REFRESH_VISIBLE_CANDIDATE_LIMIT +} from '../../shared/pr-refresh-memory-limits' + +export function retainPRRefreshAlias( + aliases: Map, + alias: GitHubPRRefreshAlias, + protectedCacheKey: string +): GitHubPRRefreshAlias | null { + if (aliases.has(alias.cacheKey)) { + aliases.delete(alias.cacheKey) + aliases.set(alias.cacheKey, alias) + return null + } + if (aliases.size < PR_REFRESH_ALIAS_LIMIT) { + aliases.set(alias.cacheKey, alias) + return null + } + + let evictionKey: string | undefined + for (const cacheKey of aliases.keys()) { + if (cacheKey !== protectedCacheKey) { + evictionKey = cacheKey + break + } + } + if (evictionKey === undefined) { + return alias + } + const evicted = aliases.get(evictionKey) ?? null + aliases.delete(evictionKey) + aliases.set(alias.cacheKey, alias) + return evicted +} + +export function retainPRRefreshState( + entries: Map, + key: K, + value: V, + limit: number +): K | null { + if (entries.has(key)) { + entries.delete(key) + entries.set(key, value) + return null + } + let evictedKey: K | null = null + if (entries.size >= limit) { + const oldest = entries.keys().next() + if (!oldest.done) { + evictedKey = oldest.value + entries.delete(oldest.value) + } + } + entries.set(key, value) + return evictedKey +} + +export function boundedVisiblePRRefreshCandidates( + candidates: GitHubPRRefreshCandidate[] +): GitHubPRRefreshCandidate[] { + return candidates.length <= PR_REFRESH_VISIBLE_CANDIDATE_LIMIT + ? candidates + : candidates.slice(0, PR_REFRESH_VISIBLE_CANDIDATE_LIMIT) +} diff --git a/src/main/github/pr-refresh-validation-backoff.ts b/src/main/github/pr-refresh-validation-backoff.ts index 8f8ea56b4b6..0f077d80029 100644 --- a/src/main/github/pr-refresh-validation-backoff.ts +++ b/src/main/github/pr-refresh-validation-backoff.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto' import { resolve } from 'node:path' import { recordCoalescedCrashBreadcrumb } from '../crash-reporting/crash-breadcrumb-store' +import { cacheIdentityDigest } from '../cache-identity-digest' const VALIDATION_BACKOFF_TTL_MS = 5 * 60_000 const MAX_VALIDATION_BACKOFF_ENTRIES = 256 @@ -35,7 +36,7 @@ const counters: ValidationBackoffCounters = { } function validationIdentityKey(identity: ValidationBackoffIdentity): string { - return [identity.repoId ?? '', resolve(identity.repoPath), identity.reason].join('\0') + return cacheIdentityDigest([identity.repoId ?? '', resolve(identity.repoPath), identity.reason]) } function validationIdentityToken(key: string): string { diff --git a/src/main/github/project-view.ts b/src/main/github/project-view.ts index 2e53eff357c..2a42a3b8e69 100644 --- a/src/main/github/project-view.ts +++ b/src/main/github/project-view.ts @@ -52,6 +52,7 @@ import { isGitHubProjectRefInputTooLarge } from '../../shared/github-project-ref-input' import { githubProjectHost } from '../../shared/github-project-identity' +import { cacheIdentityDigest } from '../cache-identity-digest' // Re-export the public API so existing `./project-view` call sites keep working; the split is internal-only. export { isValidOwnerSlug, isValidRepoSlug, isValidSlug } from './project-view/internals' @@ -129,12 +130,11 @@ const parentFieldProbeInFlight = new Map>() // host's probe result can't leak into another. Normalize github.com so // host-less callers share the same probe state as explicitly pinned calls. function ownerScopeKey(owner: string, ownerType: GitHubProjectOwnerType, host?: string): string { - const base = `${owner}\u0000${ownerType}` - return `${base}\u0000${githubProjectHost(host)}` + return cacheIdentityDigest([owner, ownerType, githubProjectHost(host)]) } function ownerTypeCacheKey(owner: string, host?: string): string { - return `${owner}\u0000${githubProjectHost(host)}` + return cacheIdentityDigest([owner, githubProjectHost(host)]) } function rememberOwnerType( diff --git a/src/main/github/project-view/internals.ts b/src/main/github/project-view/internals.ts index 5952118ee4c..c3842da5f9e 100644 --- a/src/main/github/project-view/internals.ts +++ b/src/main/github/project-view/internals.ts @@ -68,15 +68,23 @@ export async function projectHostAuthenticationError( const OWNER_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/ const REPO_SLUG_RE = /^[A-Za-z0-9._-]+$/ const REPO_SLUG_RESERVED = new Set(['.', '..']) +const OWNER_SLUG_MAX_CHARS = 256 +const REPO_SLUG_MAX_CHARS = 1024 export function isValidOwnerSlug(value: unknown): value is string { - return typeof value === 'string' && value.length > 0 && OWNER_SLUG_RE.test(value) + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= OWNER_SLUG_MAX_CHARS && + OWNER_SLUG_RE.test(value) + ) } export function isValidRepoSlug(value: unknown): value is string { return ( typeof value === 'string' && value.length > 0 && + value.length <= REPO_SLUG_MAX_CHARS && REPO_SLUG_RE.test(value) && !REPO_SLUG_RESERVED.has(value) ) diff --git a/src/main/github/rate-limit.ts b/src/main/github/rate-limit.ts index 2ca3f5cad43..9409fa6fc20 100644 --- a/src/main/github/rate-limit.ts +++ b/src/main/github/rate-limit.ts @@ -22,6 +22,7 @@ import { registerGhRateLimitResetProbe, type GhRateLimitBucket } from '../git/gh-rate-limit-breaker' +import { cacheIdentityDigest } from '../cache-identity-digest' // Why: GET /rate_limit is exempt from limits, so caching only avoids a gh subprocess per render; 30s stays live while absorbing 1/s polling. const RATE_LIMIT_CACHE_TTL_MS = 30_000 @@ -196,15 +197,21 @@ const DEFAULT_BREAKER_SCOPE = ghRateLimitScopeKey('native', 'github.com') const scopeRefinementInFlight = new Map>() const scopeProbeFailureAtMs = new Map() const SCOPE_PROBE_FAILURE_MAX_ENTRIES = 512 +const SCOPE_REFINEMENT_MAX_IN_FLIGHT = 16 + +function scopeRetentionKey(scope: string): string { + return cacheIdentityDigest([scope]) +} function rememberScopeProbeFailure(scope: string, failedAt: number): void { + const retentionKey = scopeRetentionKey(scope) for (const [key, at] of scopeProbeFailureAtMs) { if (failedAt - at >= RATE_LIMIT_CACHE_TTL_MS) { scopeProbeFailureAtMs.delete(key) } } - scopeProbeFailureAtMs.delete(scope) - scopeProbeFailureAtMs.set(scope, failedAt) + scopeProbeFailureAtMs.delete(retentionKey) + scopeProbeFailureAtMs.set(retentionKey, failedAt) while (scopeProbeFailureAtMs.size > SCOPE_PROBE_FAILURE_MAX_ENTRIES) { const oldestKey = scopeProbeFailureAtMs.keys().next().value if (oldestKey === undefined) { @@ -221,16 +228,20 @@ function refineBreakerForScope(scope: string): void { return } const parts = parseGhRateLimitScopeKey(scope) - if (!parts || scopeRefinementInFlight.has(scope)) { + const retentionKey = scopeRetentionKey(scope) + if (!parts || scopeRefinementInFlight.has(retentionKey)) { return } // Why: GHES with rate limiting disabled 404s every probe. Fail open (the // fallback block stands) and don't re-probe in a tight loop while the // breaker keeps tripping. - const failedAt = scopeProbeFailureAtMs.get(scope) + const failedAt = scopeProbeFailureAtMs.get(retentionKey) if (failedAt !== undefined && Date.now() - failedAt < RATE_LIMIT_CACHE_TTL_MS) { return } + if (scopeRefinementInFlight.size >= SCOPE_REFINEMENT_MAX_IN_FLIGHT) { + return + } const probe = (async () => { try { await acquire() @@ -243,7 +254,7 @@ function refineBreakerForScope(scope: string): void { ...(parts.runtime === 'wsl' ? { wslDistro: parts.wslDistro } : {}) }) const parsed = JSON.parse(stdout) as GhRateLimitPayload - scopeProbeFailureAtMs.delete(scope) + scopeProbeFailureAtMs.delete(retentionKey) // Why: mirrors the default-scope refinement, but records into the // per-scope breaker only — the shared snapshot must keep describing // native github.com exclusively. @@ -263,10 +274,10 @@ function refineBreakerForScope(scope: string): void { // negative cache so repeated failing hosts cannot accumulate forever. rememberScopeProbeFailure(scope, Date.now()) } finally { - scopeRefinementInFlight.delete(scope) + scopeRefinementInFlight.delete(retentionKey) } })() - scopeRefinementInFlight.set(scope, probe) + scopeRefinementInFlight.set(retentionKey, probe) } registerGhRateLimitResetProbe((_bucket, scope) => refineBreakerForScope(scope)) diff --git a/src/main/github/tracked-upstream-snapshot-bounds.test.ts b/src/main/github/tracked-upstream-snapshot-bounds.test.ts new file mode 100644 index 00000000000..32e3fe92dd5 --- /dev/null +++ b/src/main/github/tracked-upstream-snapshot-bounds.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { + _parseTrackedUpstreamBranchesForTests, + TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES, + TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES +} from './client' + +describe('tracked upstream snapshot bounds', () => { + it('caps branch count while preserving the currently requested branch', () => { + const requested = `branch-${TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES}` + const stdout = Array.from( + { length: TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES + 1 }, + (_, index) => `refs/heads/branch-${index}\0refs/remotes/origin/branch-${index}\n` + ).join('') + + const parsed = _parseTrackedUpstreamBranchesForTests(stdout, requested) + expect(parsed.size).toBe(TRACKED_UPSTREAM_SNAPSHOT_MAX_BRANCHES) + expect(parsed.get(requested)).toEqual({ + remoteName: 'origin', + branchName: requested + }) + expect(parsed.has('branch-0')).toBe(false) + }) + + it('skips an individual branch that cannot fit the byte budget', () => { + const oversized = 'x'.repeat(TRACKED_UPSTREAM_SNAPSHOT_MAX_BYTES + 1) + const parsed = _parseTrackedUpstreamBranchesForTests( + `refs/heads/${oversized}\0refs/remotes/origin/main\n`, + oversized + ) + expect(parsed.size).toBe(0) + }) +}) diff --git a/src/main/gitlab/client-mr.test.ts b/src/main/gitlab/client-mr.test.ts index 91273d7a83f..559b0b32719 100644 --- a/src/main/gitlab/client-mr.test.ts +++ b/src/main/gitlab/client-mr.test.ts @@ -55,6 +55,7 @@ import { closeMR, diagnoseAuth, getRateLimit, + GITLAB_RATE_LIMIT_CACHE_HOST_MAX_BYTES, listMergeRequests, listWorkItems, mergeMR, @@ -339,6 +340,15 @@ describe('gitlab client — MR operations', () => { expect(_getGitLabRateLimitCacheSize()).toBe(64) }) + + it('does not retain an oversized host snapshot', async () => { + glabApiWithHeadersMock.mockResolvedValue({ body: '{}', headers: {} }) + await getRateLimit({ + host: 'h'.repeat(GITLAB_RATE_LIMIT_CACHE_HOST_MAX_BYTES + 1), + force: true + }) + expect(_getGitLabRateLimitCacheSize()).toBe(0) + }) }) describe('getMergeRequestForBranch', () => { diff --git a/src/main/gitlab/client.ts b/src/main/gitlab/client.ts index d2f30b2d531..4c0b994502d 100644 --- a/src/main/gitlab/client.ts +++ b/src/main/gitlab/client.ts @@ -46,6 +46,8 @@ import { type HostedReviewExecutionOptions } from '../source-control/hosted-review-git-options' import { shouldHideNonOpenReviewOnDefaultBranch } from '../source-control/repo-default-branch' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' +import { cacheIdentityDigest } from '../cache-identity-digest' // Why: glab REST addresses projects by URL-encoded path; escapes slashes for nested groups. function encodedProject(projectPath: string): string { @@ -54,6 +56,7 @@ function encodedProject(projectPath: string): string { const GITLAB_RATE_LIMIT_CACHE_TTL_MS = 30_000 const GITLAB_RATE_LIMIT_CACHE_MAX_ENTRIES = 64 +export const GITLAB_RATE_LIMIT_CACHE_HOST_MAX_BYTES = 1024 const gitLabRateLimitCache = new Map() type HostedReviewLocalGitOptions = ReturnType @@ -210,9 +213,14 @@ export async function getRateLimit(options?: { host?: string | null }): Promise { const host = options?.host?.trim() || null - const cacheKey = host ?? 'default' + const retainableHost = + host === null || + !measureUtf8ByteLength(host, { + stopAfterBytes: GITLAB_RATE_LIMIT_CACHE_HOST_MAX_BYTES + }).exceededLimit + const cacheKey = cacheIdentityDigest([host ?? 'default']) pruneGitLabRateLimitCache() - const cached = gitLabRateLimitCache.get(cacheKey) + const cached = retainableHost ? gitLabRateLimitCache.get(cacheKey) : undefined if (!options?.force && cached && Date.now() - cached.fetchedAt < GITLAB_RATE_LIMIT_CACHE_TTL_MS) { return { ok: true, snapshot: cached } } @@ -223,7 +231,9 @@ export async function getRateLimit(options?: { const args = host ? ['--hostname', host, 'user'] : ['user'] const { headers } = await glabApiWithHeaders(args) const snapshot = parseGitLabRateLimitSnapshot(headers, host) - rememberGitLabRateLimitSnapshot(cacheKey, snapshot) + if (retainableHost) { + rememberGitLabRateLimitSnapshot(cacheKey, snapshot) + } return { ok: true, snapshot } } catch (err) { const message = err instanceof Error ? err.message : String(err) diff --git a/src/main/gitlab/gitlab-known-host-probe.ts b/src/main/gitlab/gitlab-known-host-probe.ts index 1756df4ce96..920ba77f761 100644 --- a/src/main/gitlab/gitlab-known-host-probe.ts +++ b/src/main/gitlab/gitlab-known-host-probe.ts @@ -7,24 +7,160 @@ export type LocalGitExecOptions = { } const GLAB_KNOWN_HOSTS_TIMEOUT_MS = 10_000 -const knownHostsCacheByExecutionContext = new Map() +export const GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES = 1024 * 1024 +export const GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES = 64 +export const GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES = 4 * 1024 +export const GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES = 128 * 1024 +export const GLAB_KNOWN_HOSTS_MAX_ENTRIES = 64 +export const GLAB_KNOWN_HOST_MAX_BYTES = 1024 +export const GLAB_KNOWN_HOSTS_MAX_BYTES = 32 * 1024 +export const GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT = 16 + +type KnownHostsCacheEntry = { + hosts: readonly string[] + keyBytes: number +} + +type KnownHostRetention = { + hosts: string[] + retainedBytes: number + protectedEntries: number +} + +const knownHostsCacheByExecutionContext = new Map() const knownHostsInFlightByExecutionContext = new Map>() +let knownHostsCachedContextKeyBytes = 0 + +function utf8Bytes(value: string): number { + return Buffer.byteLength(value, 'utf8') +} function knownHostsExecutionKey( connectionId?: string | null, localGitOptions: LocalGitExecOptions = {} -): string { +): string | null { + let key: string if (connectionId) { + if (utf8Bytes(connectionId) > GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES) { + return null + } // Why: reconnecting can replace the SSH/relay execution host under the same id. - return `connection:${connectionId}:${getSshGitProviderGeneration(connectionId)}` + key = `connection:${connectionId}:${getSshGitProviderGeneration(connectionId)}` + } else if (localGitOptions.wslDistro) { + if (utf8Bytes(localGitOptions.wslDistro) > GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES) { + return null + } + key = `wsl:${localGitOptions.wslDistro}` + } else { + key = 'native' } - return localGitOptions.wslDistro ? `wsl:${localGitOptions.wslDistro}` : 'native' + return utf8Bytes(key) <= GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES ? key : null +} + +function deleteCachedContext(key: string): void { + const cached = knownHostsCacheByExecutionContext.get(key) + if (!cached) { + return + } + knownHostsCachedContextKeyBytes -= cached.keyBytes + knownHostsCacheByExecutionContext.delete(key) +} + +function getCachedHosts(key: string): readonly string[] | undefined { + const cached = knownHostsCacheByExecutionContext.get(key) + if (!cached) { + return undefined + } + knownHostsCacheByExecutionContext.delete(key) + knownHostsCacheByExecutionContext.set(key, cached) + return cached.hosts +} + +function createKnownHostRetention(includeDefaults: boolean): KnownHostRetention { + const hosts = includeDefaults ? [...DEFAULT_GITLAB_HOSTS] : [] + return { + hosts, + retainedBytes: hosts.reduce((total, host) => total + utf8Bytes(host), 0), + protectedEntries: hosts.length + } +} + +function retainKnownHost(retention: KnownHostRetention, host: string): void { + if (utf8Bytes(host) > GLAB_KNOWN_HOST_MAX_BYTES) { + return + } + const normalized = normalizeGitLabHost(host) + const hostBytes = utf8Bytes(normalized) + if ( + hostBytes === 0 || + hostBytes > GLAB_KNOWN_HOST_MAX_BYTES || + retention.hosts.includes(normalized) + ) { + return + } + while ( + retention.hosts.length > retention.protectedEntries && + (retention.hosts.length >= GLAB_KNOWN_HOSTS_MAX_ENTRIES || + retention.retainedBytes + hostBytes > GLAB_KNOWN_HOSTS_MAX_BYTES) + ) { + const removed = retention.hosts.splice(retention.protectedEntries, 1)[0] + retention.retainedBytes -= utf8Bytes(removed) + } + if ( + retention.hosts.length >= GLAB_KNOWN_HOSTS_MAX_ENTRIES || + retention.retainedBytes + hostBytes > GLAB_KNOWN_HOSTS_MAX_BYTES + ) { + return + } + retention.hosts.push(normalized) + retention.retainedBytes += hostBytes +} + +function retainKnownHosts(hosts: Iterable): readonly string[] { + const retention = createKnownHostRetention(true) + for (const host of hosts) { + retainKnownHost(retention, host) + } + return retention.hosts +} + +function cacheKnownHosts(key: string, hosts: Iterable): readonly string[] { + const retained = retainKnownHosts(hosts) + const keyBytes = utf8Bytes(key) + deleteCachedContext(key) + while ( + knownHostsCacheByExecutionContext.size >= GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES || + knownHostsCachedContextKeyBytes + keyBytes > GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES + ) { + const oldestKey = knownHostsCacheByExecutionContext.keys().next().value + if (oldestKey === undefined) { + break + } + deleteCachedContext(oldestKey) + } + knownHostsCacheByExecutionContext.set(key, { hosts: retained, keyBytes }) + knownHostsCachedContextKeyBytes += keyBytes + return retained } /** @internal - exposed for tests only */ export function _resetKnownHostsCache(): void { knownHostsCacheByExecutionContext.clear() knownHostsInFlightByExecutionContext.clear() + knownHostsCachedContextKeyBytes = 0 +} + +/** @internal - exposed for tests only */ +export function _getKnownHostsCacheState(): { + cachedContexts: number + cachedContextKeyBytes: number + inFlightContexts: number +} { + return { + cachedContexts: knownHostsCacheByExecutionContext.size, + cachedContextKeyBytes: knownHostsCachedContextKeyBytes, + inFlightContexts: knownHostsInFlightByExecutionContext.size + } } export function rememberGlabKnownHost( @@ -32,13 +168,19 @@ export function rememberGlabKnownHost( connectionId?: string | null, localGitOptions: LocalGitExecOptions = {} ): void { - const normalizedHost = normalizeGitLabHost(host) const key = knownHostsExecutionKey(connectionId, localGitOptions) - const cached = knownHostsCacheByExecutionContext.get(key) - if (!cached || cached.map(normalizeGitLabHost).includes(normalizedHost)) { + if (!key || utf8Bytes(host) > GLAB_KNOWN_HOST_MAX_BYTES) { return } - knownHostsCacheByExecutionContext.set(key, [...cached, normalizedHost]) + const normalizedHost = normalizeGitLabHost(host) + if (utf8Bytes(normalizedHost) > GLAB_KNOWN_HOST_MAX_BYTES) { + return + } + const cached = getCachedHosts(key) + if (!cached || cached.includes(normalizedHost)) { + return + } + cacheKnownHosts(key, [...cached, normalizedHost]) } export async function getGlabKnownHosts( @@ -46,7 +188,10 @@ export async function getGlabKnownHosts( localGitOptions: LocalGitExecOptions = {} ): Promise { const key = knownHostsExecutionKey(connectionId, localGitOptions) - const cached = knownHostsCacheByExecutionContext.get(key) + if (!key) { + return [...DEFAULT_GITLAB_HOSTS] + } + const cached = getCachedHosts(key) if (cached) { return cached } @@ -54,6 +199,9 @@ export async function getGlabKnownHosts( if (inFlight) { return inFlight } + if (knownHostsInFlightByExecutionContext.size >= GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT) { + return [...DEFAULT_GITLAB_HOSTS] + } const probe = probeGlabKnownHosts(key, connectionId, localGitOptions) knownHostsInFlightByExecutionContext.set(key, probe) try { @@ -75,14 +223,13 @@ async function probeGlabKnownHosts( // or reconnected SSH/relay results, and bound an otherwise global probe. const { stdout, stderr } = await glabExecFileAsync(['auth', 'status'], { timeout: GLAB_KNOWN_HOSTS_TIMEOUT_MS, + maxBuffer: GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, ...(!connectionId && localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) }) const hosts = parseGlabAuthStatusHosts(`${stdout}\n${stderr}`) - const merged = Array.from(new Set([...DEFAULT_GITLAB_HOSTS, ...hosts])) - knownHostsCacheByExecutionContext.set(key, merged) - return merged + return cacheKnownHosts(key, hosts) } catch { // Keep failures uncached so auth or tunnel recovery is discovered later. return [...DEFAULT_GITLAB_HOSTS] @@ -90,11 +237,11 @@ async function probeGlabKnownHosts( } export function parseGlabAuthStatusHosts(output: string): string[] { - const hosts = new Set() + const retention = createKnownHostRetention(false) // Why: self-hosted GitLab can run on a non-default port; preserve it so // services on the same hostname remain distinct downstream. for (const match of output.matchAll(/logged in to ([a-zA-Z0-9.-]+(?::\d+)?)/gi)) { - hosts.add(match[1].toLowerCase()) + retainKnownHost(retention, match[1].toLowerCase()) } for (const line of output.split('\n')) { const bareLine = line.trim() @@ -103,8 +250,8 @@ export function parseGlabAuthStatusHosts(output: string): string[] { line === bareLine && /^[a-zA-Z0-9](?:[a-zA-Z0-9.-]*[a-zA-Z0-9])?(?::\d+)?$/.test(hostLine) ) { - hosts.add(hostLine.toLowerCase()) + retainKnownHost(retention, hostLine.toLowerCase()) } } - return Array.from(hosts) + return retention.hosts } diff --git a/src/main/gitlab/gitlab-project-ref-resolution.ts b/src/main/gitlab/gitlab-project-ref-resolution.ts index 218a41869c9..d17108ecb9b 100644 --- a/src/main/gitlab/gitlab-project-ref-resolution.ts +++ b/src/main/gitlab/gitlab-project-ref-resolution.ts @@ -2,6 +2,10 @@ import { gitExecFileAsync, glabExecFileAsync } from '../git/runner' import type { IssueSourcePreference } from '../../shared/types' import { getSshGitProvider } from '../providers/ssh-git-dispatch' import { clearProjectRefInFlight, runProjectRefProbeOnce } from './project-ref-inflight' +import { + buildRepositoryRefCacheKey, + RepositoryRefCache +} from '../source-control/repository-ref-cache' import { parseGlabAuthStatusHosts, rememberGlabKnownHost, @@ -18,14 +22,23 @@ import { export { DEFAULT_GITLAB_HOSTS, parseGitLabProjectRef } export type { ProjectRef } export { + GLAB_KNOWN_HOST_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_BYTES, + GLAB_KNOWN_HOSTS_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT, + GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, + _getKnownHostsCacheState, _resetKnownHostsCache, getGlabKnownHosts, - parseGlabAuthStatusHosts + parseGlabAuthStatusHosts, + rememberGlabKnownHost } from './gitlab-known-host-probe' export type { LocalGitExecOptions } from './gitlab-known-host-probe' -const PROJECT_REF_CACHE_MAX_ENTRIES = 512 -const projectRefCache = new Map() +const projectRefCache = new RepositoryRefCache() /** @internal - exposed for tests only */ export function _resetProjectRefCache(): void { @@ -38,17 +51,6 @@ export function _getProjectRefCacheSize(): number { return projectRefCache.size } -function rememberProjectRefCacheEntry(cacheKey: string, value: ProjectRef | null): void { - projectRefCache.set(cacheKey, value) - while (projectRefCache.size > PROJECT_REF_CACHE_MAX_ENTRIES) { - const oldestKey = projectRefCache.keys().next().value - if (oldestKey === undefined) { - return - } - projectRefCache.delete(oldestKey) - } -} - export async function getProjectRefForRemote( repoPath: string, remoteName: string, @@ -57,12 +59,13 @@ export async function getProjectRefForRemote( localGitOptions: LocalGitExecOptions = {} ): Promise { const runtimeKey = connectionId ?? `local:${localGitOptions.wslDistro ?? 'host'}` - const cacheKey = `${runtimeKey}\0${repoPath}\0${remoteName}\0${knownHosts.join(',')}` - if (projectRefCache.has(cacheKey)) { - return projectRefCache.get(cacheKey)! + const cacheKey = buildRepositoryRefCacheKey([runtimeKey, repoPath, remoteName, ...knownHosts]) + const cached = projectRefCache.get(cacheKey) + if (cached.found) { + return cached.value } - return runProjectRefProbeOnce(cacheKey, () => + const probe = () => resolveProjectRefForRemote( repoPath, remoteName, @@ -71,7 +74,7 @@ export async function getProjectRefForRemote( cacheKey, localGitOptions ) - ) + return cacheKey === null ? probe() : runProjectRefProbeOnce(cacheKey, probe) } async function resolveProjectRefForRemote( @@ -79,7 +82,7 @@ async function resolveProjectRefForRemote( remoteName: string, knownHosts: readonly string[], connectionId: string | null | undefined, - cacheKey: string, + cacheKey: string | null, localGitOptions: LocalGitExecOptions ): Promise { try { @@ -95,7 +98,7 @@ async function resolveProjectRefForRemote( }) const result = parseGitLabProjectRef(stdout, knownHosts) if (result) { - rememberProjectRefCacheEntry(cacheKey, result) + projectRefCache.remember(cacheKey, result, [result.host, result.path]) return result } const remoteCandidate = parseRemoteProjectRefCandidate(stdout) @@ -109,7 +112,10 @@ async function resolveProjectRefForRemote( )) ) { rememberGlabKnownHost(remoteCandidate.host, connectionId, localGitOptions) - rememberProjectRefCacheEntry(cacheKey, remoteCandidate) + projectRefCache.remember(cacheKey, remoteCandidate, [ + remoteCandidate.host, + remoteCandidate.path + ]) return remoteCandidate } } catch { @@ -117,7 +123,7 @@ async function resolveProjectRefForRemote( return null } } - rememberProjectRefCacheEntry(cacheKey, null) + projectRefCache.remember(cacheKey, null, []) return null } diff --git a/src/main/gitlab/gl-utils.test.ts b/src/main/gitlab/gl-utils.test.ts index 9ab99ddc441..fc34280f8e8 100644 --- a/src/main/gitlab/gl-utils.test.ts +++ b/src/main/gitlab/gl-utils.test.ts @@ -12,6 +12,15 @@ vi.mock('../git/runner', () => ({ })) import { + GLAB_KNOWN_HOST_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_BYTES, + GLAB_KNOWN_HOSTS_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT, + GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, + _getKnownHostsCacheState, _getProjectRefCacheSize, _resetKnownHostsCache, _resetProjectRefCache, @@ -23,6 +32,7 @@ import { getProjectRefForRemote, parseGlabApiResponse, parseGlabAuthStatusHosts, + rememberGlabKnownHost, resolveIssueSource } from './gl-utils' import { registerSshGitProvider, unregisterSshGitProvider } from '../providers/ssh-git-dispatch' @@ -434,7 +444,10 @@ describe('getGlabKnownHosts', () => { }) await expect(getGlabKnownHosts()).resolves.toEqual(['gitlab.com', 'gitlab.example.com']) - expect(glabExecFileAsyncMock).toHaveBeenCalledWith(['auth', 'status'], { timeout: 10_000 }) + expect(glabExecFileAsyncMock).toHaveBeenCalledWith(['auth', 'status'], { + timeout: 10_000, + maxBuffer: GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES + }) }) it('falls back to default when glab auth status fails', async () => { @@ -495,10 +508,12 @@ describe('getGlabKnownHosts', () => { expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(4) expect(glabExecFileAsyncMock).toHaveBeenNthCalledWith(1, ['auth', 'status'], { timeout: 10_000, + maxBuffer: GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, wslDistro: 'Ubuntu' }) expect(glabExecFileAsyncMock).toHaveBeenNthCalledWith(2, ['auth', 'status'], { timeout: 10_000, + maxBuffer: GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, wslDistro: 'Debian' }) }) @@ -597,4 +612,144 @@ describe('getGlabKnownHosts', () => { expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(2) unregisterSshGitProvider(connectionId) }) + + it('keeps the most recently used execution contexts at the cache limit', async () => { + glabExecFileAsyncMock.mockResolvedValue({ stdout: '', stderr: '' }) + + for (let index = 0; index < GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES; index += 1) { + await getGlabKnownHosts(undefined, { wslDistro: `Distro-${index}` }) + } + expect(_getKnownHostsCacheState().cachedContexts).toBe(GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES) + + await getGlabKnownHosts(undefined, { wslDistro: 'Distro-0' }) + await getGlabKnownHosts(undefined, { wslDistro: 'Distro-overflow' }) + expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES + 1) + + await getGlabKnownHosts(undefined, { wslDistro: 'Distro-1' }) + await getGlabKnownHosts(undefined, { wslDistro: 'Distro-0' }) + expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES + 2) + }) + + it('accepts an exact-limit execution key and skips an oversized context', async () => { + glabExecFileAsyncMock.mockResolvedValue({ stdout: '', stderr: '' }) + const prefixBytes = Buffer.byteLength('wsl:', 'utf8') + const exactDistro = 'x'.repeat(GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES - prefixBytes) + + await expect(getGlabKnownHosts(undefined, { wslDistro: exactDistro })).resolves.toEqual([ + 'gitlab.com' + ]) + expect(_getKnownHostsCacheState().cachedContextKeyBytes).toBe( + GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES + ) + + await expect(getGlabKnownHosts(undefined, { wslDistro: `${exactDistro}x` })).resolves.toEqual([ + 'gitlab.com' + ]) + expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(1) + }) + + it('evicts old large context keys at the aggregate byte limit', async () => { + glabExecFileAsyncMock.mockResolvedValue({ stdout: '', stderr: '' }) + const prefixBytes = Buffer.byteLength('wsl:', 'utf8') + const retainedContexts = + GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES / GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES + const distroAt = (index: number): string => { + const suffix = String(index).padStart(3, '0') + return `${'x'.repeat(GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES - prefixBytes - suffix.length)}${suffix}` + } + + for (let index = 0; index <= retainedContexts; index += 1) { + await getGlabKnownHosts(undefined, { wslDistro: distroAt(index) }) + } + expect(_getKnownHostsCacheState()).toMatchObject({ + cachedContexts: retainedContexts, + cachedContextKeyBytes: GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES + }) + + await getGlabKnownHosts(undefined, { wslDistro: distroAt(0) }) + expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(retainedContexts + 2) + }) + + it('bounds parsed hosts while retaining the newest values', () => { + const hosts = Array.from( + { length: GLAB_KNOWN_HOSTS_MAX_ENTRIES + 6 }, + (_, index) => `gitlab-${index}.example.com` + ) + + expect(parseGlabAuthStatusHosts(hosts.join('\n'))).toEqual( + hosts.slice(-GLAB_KNOWN_HOSTS_MAX_ENTRIES) + ) + }) + + it('preserves gitlab.com and the newest remembered hosts at the count limit', async () => { + glabExecFileAsyncMock.mockResolvedValueOnce({ stdout: '', stderr: '' }) + await getGlabKnownHosts() + const customHosts = Array.from( + { length: GLAB_KNOWN_HOSTS_MAX_ENTRIES + 6 }, + (_, index) => `gitlab-${index}.example.com` + ) + for (const host of customHosts) { + rememberGlabKnownHost(host) + } + + await expect(getGlabKnownHosts()).resolves.toEqual([ + 'gitlab.com', + ...customHosts.slice(-(GLAB_KNOWN_HOSTS_MAX_ENTRIES - 1)) + ]) + }) + + it('accepts an exact-limit host and ignores an oversized host', async () => { + glabExecFileAsyncMock.mockResolvedValueOnce({ stdout: '', stderr: '' }) + await getGlabKnownHosts() + const exactHost = `${'x'.repeat(GLAB_KNOWN_HOST_MAX_BYTES - 5)}.test` + const oversizedHost = `${exactHost}x` + + rememberGlabKnownHost(exactHost) + rememberGlabKnownHost(oversizedHost) + + await expect(getGlabKnownHosts()).resolves.toEqual(['gitlab.com', exactHost]) + }) + + it('bounds aggregate retained host bytes and keeps newest values', async () => { + glabExecFileAsyncMock.mockResolvedValueOnce({ stdout: '', stderr: '' }) + await getGlabKnownHosts() + const customHosts = Array.from({ length: 40 }, (_, index) => { + const prefix = String(index).padStart(2, '0') + return `${prefix}${'x'.repeat(GLAB_KNOWN_HOST_MAX_BYTES - prefix.length)}` + }) + for (const host of customHosts) { + rememberGlabKnownHost(host) + } + + const retained = await getGlabKnownHosts() + expect(retained).toEqual(['gitlab.com', ...customHosts.slice(-31)]) + expect( + retained.reduce((total, host) => total + Buffer.byteLength(host), 0) + ).toBeLessThanOrEqual(GLAB_KNOWN_HOSTS_MAX_BYTES) + }) + + it('caps distinct in-flight probes and recovers capacity after settlement', async () => { + const resolvers: ((value: { stdout: string; stderr: string }) => void)[] = [] + glabExecFileAsyncMock.mockImplementation( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + const probes = Array.from({ length: GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT }, (_, index) => + getGlabKnownHosts(undefined, { wslDistro: `Distro-${index}` }) + ) + + expect(_getKnownHostsCacheState().inFlightContexts).toBe(GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT) + await expect(getGlabKnownHosts(undefined, { wslDistro: 'Distro-overflow' })).resolves.toEqual([ + 'gitlab.com' + ]) + expect(glabExecFileAsyncMock).toHaveBeenCalledTimes(GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT) + + for (const resolve of resolvers) { + resolve({ stdout: '', stderr: '' }) + } + await Promise.all(probes) + expect(_getKnownHostsCacheState().inFlightContexts).toBe(0) + }) }) diff --git a/src/main/gitlab/gl-utils.ts b/src/main/gitlab/gl-utils.ts index 32b25ab0717..a01182b9c8d 100644 --- a/src/main/gitlab/gl-utils.ts +++ b/src/main/gitlab/gl-utils.ts @@ -2,6 +2,7 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' import { gitExecFileAsync, glabExecFileAsync } from '../git/runner' import { parseGlabApiResponse, type GlabApiResponse } from './glab-api-response' +import { IntegrationApiConcurrencyGate } from '../integration-api-concurrency' // Why: legacy generic execFile wrapper - only used by callers that don't need // WSL-aware routing. Repo-scoped callers should use the runner exports below. @@ -10,6 +11,15 @@ export { glabExecFileAsync, gitExecFileAsync } export { classifyGlabError, classifyListIssuesError } from './glab-error-classification' export { DEFAULT_GITLAB_HOSTS, + GLAB_KNOWN_HOST_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEYS_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_KEY_MAX_BYTES, + GLAB_KNOWN_HOSTS_CONTEXT_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_BYTES, + GLAB_KNOWN_HOSTS_MAX_ENTRIES, + GLAB_KNOWN_HOSTS_MAX_IN_FLIGHT, + GLAB_KNOWN_HOSTS_OUTPUT_MAX_BYTES, + _getKnownHostsCacheState, _getProjectRefCacheSize, _resetKnownHostsCache, _resetProjectRefCache, @@ -21,6 +31,7 @@ export { glabRepoExecOptions, parseGlabAuthStatusHosts, parseGitLabProjectRef, + rememberGlabKnownHost, resolveIssueSource } from './gitlab-project-ref-resolution' export type { @@ -31,28 +42,14 @@ export type { export { parseGlabApiResponse, type GlabApiResponse } from './glab-api-response' const MAX_CONCURRENT = 4 -let running = 0 -const queue: (() => void)[] = [] +const concurrencyGate = new IntegrationApiConcurrencyGate(MAX_CONCURRENT) export function acquire(): Promise { - if (running < MAX_CONCURRENT) { - running += 1 - return Promise.resolve() - } - return new Promise((resolve) => - queue.push(() => { - running += 1 - resolve() - }) - ) + return concurrencyGate.acquire() } export function release(): void { - running -= 1 - const next = queue.shift() - if (next) { - next() - } + concurrencyGate.release() } export async function glabApiWithHeaders( diff --git a/src/main/gitlab/merge-request-creation.ts b/src/main/gitlab/merge-request-creation.ts index 1b7d253f037..fc5a08511c3 100644 --- a/src/main/gitlab/merge-request-creation.ts +++ b/src/main/gitlab/merge-request-creation.ts @@ -1,12 +1,8 @@ -import { readFile } from 'node:fs/promises' -import { join } from 'node:path' import type { CreateHostedReviewInput, CreateHostedReviewResult } from '../../shared/hosted-review' import { normalizeHostedReviewBaseRef, normalizeHostedReviewHeadRef } from '../../shared/hosted-review-refs' -import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' -import { joinWorktreeRelativePath } from '../runtime/runtime-relative-paths' import { getHostedReviewLocalGitOptions, hasHostedReviewLocalGitOptions, @@ -21,6 +17,7 @@ import { release } from './gl-utils' import { findOpenMRByHeadBase, parseMergeRequestPayload } from './merge-request-creation-lookup' +import { readGitLabMergeRequestTemplate } from '../source-control/pull-request-template' function execErrorMessage(error: unknown): string { if (error instanceof Error) { @@ -88,39 +85,6 @@ function hostedReviewExecutionOptionArgs( return hasHostedReviewLocalGitOptions(options) ? [options] : [] } -async function readMergeRequestTemplate( - repoPath: string, - connectionId?: string | null -): Promise { - const relativeCandidates = [ - '.gitlab/merge_request_templates/Default.md', - '.gitlab/merge_request_templates/default.md', - '.gitlab/merge_request_template.md', - '.gitlab/MERGE_REQUEST_TEMPLATE.md' - ] - const remoteProvider = connectionId ? getSshFilesystemProvider(connectionId) : undefined - if (connectionId && !remoteProvider) { - return '' - } - for (const relativeCandidate of relativeCandidates) { - try { - if (remoteProvider) { - const result = await remoteProvider.readFile( - joinWorktreeRelativePath(repoPath, relativeCandidate) - ) - if (result.isBinary) { - continue - } - return result.content - } - return await readFile(join(repoPath, relativeCandidate), 'utf8') - } catch { - // Try the next conventional GitLab merge-request template path. - } - } - return '' -} - export async function createGitLabMergeRequest( repoPath: string, input: CreateHostedReviewInput, @@ -170,7 +134,7 @@ export async function createGitLabMergeRequest( try { const body = input.useTemplate && !input.body?.trim() - ? await readMergeRequestTemplate(repoPath, connectionId) + ? await readGitLabMergeRequestTemplate(repoPath, connectionId) : (input.body ?? '') const createArgs = [ 'mr', diff --git a/src/main/gitlab/project-ref-inflight.test.ts b/src/main/gitlab/project-ref-inflight.test.ts new file mode 100644 index 00000000000..afdcf7dcd04 --- /dev/null +++ b/src/main/gitlab/project-ref-inflight.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { + clearProjectRefInFlight, + PROJECT_REF_MAX_IN_FLIGHT, + runProjectRefProbeOnce +} from './project-ref-inflight' + +describe('GitLab project-ref in-flight probes', () => { + it('caps retained probes and recovers coalescing after settlement', async () => { + clearProjectRefInFlight() + const releases: (() => void)[] = [] + const probes = Array.from({ length: PROJECT_REF_MAX_IN_FLIGHT }, (_, index) => + runProjectRefProbeOnce( + `key-${index}`, + () => + new Promise((resolve) => { + releases.push(() => resolve(null)) + }) + ) + ) + let overflowCalls = 0 + await expect( + runProjectRefProbeOnce('overflow', async () => { + overflowCalls += 1 + return null + }) + ).resolves.toBeNull() + await expect( + runProjectRefProbeOnce('overflow', async () => { + overflowCalls += 1 + return null + }) + ).resolves.toBeNull() + expect(overflowCalls).toBe(2) + + releases.forEach((release) => release()) + await Promise.all(probes) + + let retainedCalls = 0 + const first = runProjectRefProbeOnce('recovered', async () => { + retainedCalls += 1 + return null + }) + const second = runProjectRefProbeOnce('recovered', async () => { + retainedCalls += 1 + return null + }) + await Promise.all([first, second]) + expect(retainedCalls).toBe(1) + }) +}) diff --git a/src/main/gitlab/project-ref-inflight.ts b/src/main/gitlab/project-ref-inflight.ts index 8df88f63a7a..ab7295184b2 100644 --- a/src/main/gitlab/project-ref-inflight.ts +++ b/src/main/gitlab/project-ref-inflight.ts @@ -1,6 +1,7 @@ import type { ProjectRef } from './gl-utils' const projectRefInFlight = new Map>() +export const PROJECT_REF_MAX_IN_FLIGHT = 32 export function clearProjectRefInFlight(): void { projectRefInFlight.clear() @@ -14,6 +15,9 @@ export async function runProjectRefProbeOnce( if (inFlight) { return inFlight } + if (projectRefInFlight.size >= PROJECT_REF_MAX_IN_FLIGHT) { + return createProbe() + } const probe = createProbe() projectRefInFlight.set(cacheKey, probe) try { diff --git a/src/main/hermes/hook-service.ts b/src/main/hermes/hook-service.ts index cd4a4b99974..f37c2eaf23b 100644 --- a/src/main/hermes/hook-service.ts +++ b/src/main/hermes/hook-service.ts @@ -4,7 +4,6 @@ import { copyFileSync, existsSync, mkdirSync, - readFileSync, renameSync, rmSync, unlinkSync, @@ -16,6 +15,14 @@ import type { SFTPWrapper } from 'ssh2' import { parse, stringify } from 'yaml' import type { AgentHookInstallState, AgentHookInstallStatus } from '../../shared/agent-hook-types' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' +import { + AGENT_HOOK_CONFIG_MAX_BYTES, + AGENT_HOOK_PLUGIN_MAX_BYTES +} from '../agent-hooks/agent-hook-file-limits' import { readTextFileRemote, writeTextFileRemoteAtomic @@ -137,7 +144,18 @@ function readConfigFile(configPath: string): ConfigParseResult { if (!existsSync(configPath)) { return { ok: true, config: {} } } - return parseHermesConfig(readFileSync(configPath, 'utf-8')) + try { + const content = readNodeFileSyncWithinLimit( + configPath, + AGENT_HOOK_CONFIG_MAX_BYTES + ).buffer.toString('utf8') + return parseHermesConfig(content) + } catch (error) { + return { + ok: false, + detail: error instanceof Error ? error.message : String(error) + } + } } function writeConfigFile(configPath: string, config: HermesConfig): void { @@ -146,10 +164,17 @@ function writeConfigFile(configPath: string, config: HermesConfig): void { const serialized = serializeHermesConfig(config) if (existsSync(configPath)) { try { - if (readFileSync(configPath, 'utf-8') === serialized) { + const existing = readNodeFileSyncWithinLimit( + configPath, + AGENT_HOOK_CONFIG_MAX_BYTES + ).buffer.toString('utf8') + if (existing === serialized) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // Fall through to the atomic write path. } } @@ -186,27 +211,56 @@ function updateConfigContent( function getPluginFilesState(pluginDir = getPluginDir()): { present: boolean managed: boolean + blocked: boolean detail: string | null } { const manifestPath = getManifestPath(pluginDir) const initPath = getInitPath(pluginDir) - if (!existsSync(manifestPath) || !existsSync(initPath)) { - return { present: false, managed: false, detail: 'Managed Hermes plugin files are missing' } + const manifest = readPluginFile(manifestPath, 'plugin.yaml') + const init = readPluginFile(initPath, '__init__.py') + if (manifest.kind === 'error') { + return { present: true, managed: false, blocked: manifest.blocked, detail: manifest.detail } + } + if (init.kind === 'error') { + return { present: true, managed: false, blocked: init.blocked, detail: init.detail } + } + if (manifest.kind === 'absent' || init.kind === 'absent') { + return { + present: false, + managed: false, + blocked: false, + detail: 'Managed Hermes plugin files are missing' + } + } + const managed = + manifest.content.includes(HERMES_PLUGIN_MARKER) && init.content.includes(HERMES_PLUGIN_MARKER) + return { + present: true, + managed, + blocked: false, + detail: managed ? null : 'Hermes orca-status plugin exists but is not Orca-managed' + } +} + +type PluginFileRead = + | { kind: 'absent' } + | { kind: 'content'; content: string } + | { kind: 'error'; blocked: boolean; detail: string } + +function readPluginFile(path: string, name: string): PluginFileRead { + if (!existsSync(path)) { + return { kind: 'absent' } } try { - const manifest = readFileSync(manifestPath, 'utf-8') - const init = readFileSync(initPath, 'utf-8') - const managed = manifest.includes(HERMES_PLUGIN_MARKER) && init.includes(HERMES_PLUGIN_MARKER) - return { - present: true, - managed, - detail: managed ? null : 'Hermes orca-status plugin exists but is not Orca-managed' - } + const content = readNodeFileSyncWithinLimit(path, AGENT_HOOK_PLUGIN_MAX_BYTES).buffer.toString( + 'utf8' + ) + return { kind: 'content', content } } catch (error) { return { - present: true, - managed: false, - detail: error instanceof Error ? error.message : String(error) + kind: 'error', + blocked: error instanceof NodeFileReadTooLargeError, + detail: `Could not read Hermes ${name}: ${error instanceof Error ? error.message : String(error)}` } } } @@ -234,8 +288,11 @@ function getConfigEnablement(config: HermesConfig): { } } -function buildStatus(configPath: string, config: HermesConfig): AgentHookInstallStatus { - const pluginFiles = getPluginFilesState() +function buildStatus( + configPath: string, + config: HermesConfig, + pluginFiles = getPluginFilesState() +): AgentHookInstallStatus { const enablement = getConfigEnablement(config) const details = [ pluginFiles.detail, @@ -245,7 +302,9 @@ function buildStatus(configPath: string, config: HermesConfig): AgentHookInstall ].filter((detail): detail is string => Boolean(detail)) let state: AgentHookInstallState - if (!pluginFiles.present && !enablement.enabled) { + if (pluginFiles.blocked) { + state = 'error' + } else if (!pluginFiles.present && !enablement.enabled) { state = 'not_installed' } else if ( pluginFiles.present && @@ -438,14 +497,43 @@ def register(ctx: Any) -> None: function writePluginFiles(pluginDir = getPluginDir()): void { mkdirSync(pluginDir, { recursive: true }) - writeFileSync(getManifestPath(pluginDir), getPluginManifest(), 'utf-8') - writeFileSync(getInitPath(pluginDir), getPluginInitSource(), 'utf-8') + writePluginFile(getManifestPath(pluginDir), getPluginManifest()) + writePluginFile(getInitPath(pluginDir), getPluginInitSource()) +} + +function writePluginFile(path: string, content: string): void { + if (existsSync(path)) { + try { + const existing = readNodeFileSyncWithinLimit( + path, + AGENT_HOOK_PLUGIN_MAX_BYTES + ).buffer.toString('utf8') + if (existing === content) { + return + } + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } + } + } + writeFileSync(path, content, 'utf-8') } function stripTrailingSlash(path: string): string { return path.replace(/\/+$/, '') } +async function guardRemotePluginFileSize(sftp: SFTPWrapper, path: string): Promise { + try { + await readTextFileRemote(sftp, path, AGENT_HOOK_PLUGIN_MAX_BYTES) + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } + } +} + export class HermesHookService { getStatus(): AgentHookInstallStatus { const configPath = getConfigPath() @@ -474,6 +562,10 @@ export class HermesHookService { detail: `Could not parse Hermes config.yaml: ${parsed.detail}` } } + const pluginFiles = getPluginFilesState() + if (pluginFiles.blocked) { + return buildStatus(configPath, parsed.config, pluginFiles) + } writePluginFiles() writeConfigFile(configPath, enablePlugin(parsed.config)) @@ -486,6 +578,12 @@ export class HermesHookService { const remotePluginDir = `${remoteRoot}/.hermes/plugins/${HERMES_PLUGIN_NAME}` try { const existing = await readTextFileRemote(sftp, remoteConfigPath) + const remoteManifestPath = `${remotePluginDir}/plugin.yaml` + const remoteInitPath = `${remotePluginDir}/__init__.py` + await Promise.all([ + guardRemotePluginFileSize(sftp, remoteManifestPath), + guardRemotePluginFileSize(sftp, remoteInitPath) + ]) const next = updateConfigContent(existing, enablePlugin) if (next.content === null) { return { @@ -496,8 +594,18 @@ export class HermesHookService { detail: `Could not parse remote Hermes config.yaml: ${next.detail ?? 'unknown error'}` } } - await writeTextFileRemoteAtomic(sftp, `${remotePluginDir}/plugin.yaml`, getPluginManifest()) - await writeTextFileRemoteAtomic(sftp, `${remotePluginDir}/__init__.py`, getPluginInitSource()) + await writeTextFileRemoteAtomic( + sftp, + remoteManifestPath, + getPluginManifest(), + AGENT_HOOK_PLUGIN_MAX_BYTES + ) + await writeTextFileRemoteAtomic( + sftp, + remoteInitPath, + getPluginInitSource(), + AGENT_HOOK_PLUGIN_MAX_BYTES + ) await writeTextFileRemoteAtomic(sftp, remoteConfigPath, next.content) return { agent: 'hermes', @@ -530,7 +638,11 @@ export class HermesHookService { } } const pluginDir = getPluginDir() - if (getPluginFilesState(pluginDir).managed) { + const pluginFiles = getPluginFilesState(pluginDir) + if (pluginFiles.blocked) { + return buildStatus(configPath, parsed.config, pluginFiles) + } + if (pluginFiles.managed) { rmSync(pluginDir, { recursive: true, force: true }) } writeConfigFile(configPath, disablePlugin(parsed.config)) diff --git a/src/main/hooks-file-bounds.test.ts b/src/main/hooks-file-bounds.test.ts new file mode 100644 index 00000000000..7a61dcab820 --- /dev/null +++ b/src/main/hooks-file-bounds.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + loadHooks, + MAX_HOOK_GITIGNORE_BYTES, + MAX_ISSUE_COMMAND_BYTES, + MAX_ORCA_YAML_BYTES, + readIssueCommand, + writeIssueCommand +} from './hooks' + +describe('hook configuration file bounds', () => { + const roots: string[] = [] + + afterEach(() => { + vi.restoreAllMocks() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + }) + + function makeRepo(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-hook-file-bounds-')) + roots.push(root) + return root + } + + function writeOversizedSparseFile(path: string, limit: number): void { + writeFileSync(path, 'x') + truncateSync(path, limit + 1) + } + + it('fails closed on an oversized sparse orca.yaml', () => { + const repoPath = makeRepo() + writeOversizedSparseFile(join(repoPath, 'orca.yaml'), MAX_ORCA_YAML_BYTES) + + expect(loadHooks(repoPath)).toBeNull() + }) + + it('ignores an oversized local issue command while retaining the shared command', () => { + const repoPath = makeRepo() + mkdirSync(join(repoPath, '.orca')) + writeOversizedSparseFile(join(repoPath, '.orca', 'issue-command'), MAX_ISSUE_COMMAND_BYTES) + writeFileSync(join(repoPath, 'orca.yaml'), 'issueCommand: shared command\n') + + expect(readIssueCommand(repoPath)).toMatchObject({ + localContent: null, + sharedContent: 'shared command', + effectiveContent: 'shared command', + source: 'shared' + }) + }) + + it('does not rewrite an oversized .gitignore when saving a local command', () => { + const repoPath = makeRepo() + const gitignorePath = join(repoPath, '.gitignore') + writeOversizedSparseFile(gitignorePath, MAX_HOOK_GITIGNORE_BYTES) + const originalSize = statSync(gitignorePath).size + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + writeIssueCommand(repoPath, 'local command') + + expect(statSync(gitignorePath).size).toBe(originalSize) + expect(readFileSync(join(repoPath, '.orca', 'issue-command'), 'utf8')).toBe('local command\n') + }) +}) diff --git a/src/main/hooks.test.ts b/src/main/hooks.test.ts index d117470bffe..69df4711a8c 100644 --- a/src/main/hooks.test.ts +++ b/src/main/hooks.test.ts @@ -16,6 +16,20 @@ vi.mock('fs', () => ({ chmodSync: vi.fn() })) +vi.mock('../shared/node-bounded-file-reader', async () => { + const fs = await import('node:fs') + return { + readNodeFileSyncWithinLimit: (path: string, maxBytes: number) => { + const value = fs.readFileSync(path) + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (buffer.length > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: { size: buffer.length } } + } + } +}) + const { execMock, execFileMock, gitExecFileSyncMock } = vi.hoisted(() => ({ execMock: vi.fn(), execFileMock: vi.fn(), diff --git a/src/main/hooks.ts b/src/main/hooks.ts index ef5aaf1e5ea..cd17d2d539e 100644 --- a/src/main/hooks.ts +++ b/src/main/hooks.ts @@ -1,5 +1,5 @@ /* eslint-disable max-lines -- Why: hook parsing, layered issue-command resolution, and cross-platform runner setup share one execution surface, so keeping them together avoids subtle drift across create/read/write paths. */ -import { readFileSync, existsSync, mkdirSync, writeFileSync, chmodSync, rmSync } from 'node:fs' +import { existsSync, mkdirSync, writeFileSync, chmodSync, rmSync } from 'node:fs' import { dirname, join } from 'node:path' import { exec, execFile } from 'node:child_process' import { getDefaultRepoHookSettings } from '../shared/constants' @@ -21,8 +21,17 @@ import type { WorktreeSetupLaunch } from '../shared/types' import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { MAX_ORCA_YAML_BYTES } from '../shared/orca-yaml-file-limit' const HOOK_TIMEOUT = 120_000 // 2 minutes +export { MAX_ORCA_YAML_BYTES } +export const MAX_ISSUE_COMMAND_BYTES = 1024 * 1024 +export const MAX_HOOK_GITIGNORE_BYTES = 4 * 1024 * 1024 + +function readHookFile(path: string, maxBytes: number): string { + return readNodeFileSyncWithinLimit(path, maxBytes).buffer.toString('utf8') +} export type HookRuntimeTarget = { wslDistro?: string | null @@ -48,7 +57,7 @@ export function loadHooks(repoPath: string): OrcaHooks | null { } try { - const content = readFileSync(yamlPath, 'utf-8') + const content = readHookFile(yamlPath, MAX_ORCA_YAML_BYTES) return parseOrcaYaml(content) } catch { return null @@ -73,7 +82,7 @@ const RECOGNIZED_ORCA_YAML_KEYS = new Set([ /** True when `orca.yaml` has a top-level key this version of Orca does not handle. */ export function hasUnrecognizedOrcaYamlKeys(repoPath: string): boolean { try { - const content = readFileSync(join(repoPath, 'orca.yaml'), 'utf-8') + const content = readHookFile(join(repoPath, 'orca.yaml'), MAX_ORCA_YAML_BYTES) for (const line of iterateLfScriptLines(content)) { // Why: match bare `key:` at end-of-line too, since a mapping with a block value on the next line is valid YAML. const m = line.match(/^([A-Za-z][A-Za-z0-9_-]*):(\s|$)/) @@ -118,7 +127,7 @@ export function readIssueCommand(repoPath: string): ResolvedIssueCommand { if (existsSync(filePath)) { try { - const content = readFileSync(filePath, 'utf-8').trim() + const content = readHookFile(filePath, MAX_ISSUE_COMMAND_BYTES).trim() localContent = content || null } catch { localContent = null @@ -169,7 +178,7 @@ function ensureOrcaDirIgnored(repoPath: string): void { const gitignorePath = join(repoPath, '.gitignore') try { if (existsSync(gitignorePath)) { - const content = readFileSync(gitignorePath, 'utf-8') + const content = readHookFile(gitignorePath, MAX_HOOK_GITIGNORE_BYTES) if (/^\.orca\/?$/m.test(content)) { return } diff --git a/src/main/index.ts b/src/main/index.ts index e98003b2051..d66fb535865 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -242,7 +242,6 @@ import { preserveAgentAuthBeforeRestart } from './agent-auth-restart-preservatio import { CliInstaller } from './cli/cli-installer' import { installLinuxBareOrcaDispatcher } from './cli/linux-bare-orca-dispatcher' import { reconcileManagedWslCliRegistrations } from './cli/wsl-cli-registration-reconciliation' -import { selfHealRuntimeEnvironmentFocus } from './runtime-environment-focus-self-heal' let mainWindow: BrowserWindow | null = null /** Whether a manual app.quit() (Cmd+Q) is in progress; lets the close handler skip the running-process confirmation and go straight to close. */ @@ -1838,7 +1837,6 @@ app.whenReady().then(async () => { `[claude-live-pty] Seeded ${persistedClaudePtyIds.length} persisted Claude session id(s) into the refresh gate` ) } - selfHealRuntimeEnvironmentFocus({ store, userDataPath: app.getPath('userData') }) applyAppIcon(store.getSettings().appIcon) if (shouldSuppressDevEducation({ isDev: is.dev })) { suppressDevEducationForStore(store) diff --git a/src/main/integration-account-persistence-limits.test.ts b/src/main/integration-account-persistence-limits.test.ts new file mode 100644 index 00000000000..e81b237e049 --- /dev/null +++ b/src/main/integration-account-persistence-limits.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { + assertIntegrationAccountCount, + assertIntegrationCredentialBytes, + assertIntegrationStringBytes, + MAX_INTEGRATION_ACCOUNT_FILE_BYTES, + MAX_INTEGRATION_ACCOUNTS, + MAX_INTEGRATION_CREDENTIAL_BYTES, + serializeIntegrationAccountFile +} from './integration-account-persistence-limits' + +describe('integration account persistence limits', () => { + it('admits the exact account boundary and rejects one more', () => { + expect(() => assertIntegrationAccountCount('Test', MAX_INTEGRATION_ACCOUNTS)).not.toThrow() + expect(() => assertIntegrationAccountCount('Test', MAX_INTEGRATION_ACCOUNTS + 1)).toThrow( + `at most ${MAX_INTEGRATION_ACCOUNTS}` + ) + }) + + it('measures credential and metadata fields as UTF-8 bytes', () => { + const exactCredential = 'é'.repeat(MAX_INTEGRATION_CREDENTIAL_BYTES / 2) + expect(() => assertIntegrationCredentialBytes('Test', exactCredential)).not.toThrow() + expect(() => assertIntegrationCredentialBytes('Test', `${exactCredential}a`)).toThrow( + `${MAX_INTEGRATION_CREDENTIAL_BYTES} UTF-8 bytes` + ) + + expect(() => assertIntegrationStringBytes('Test', 'field', 'éé', 4)).not.toThrow() + expect(() => assertIntegrationStringBytes('Test', 'field', 'ééa', 4)).toThrow('4 UTF-8 bytes') + }) + + it('serializes ordinary JSON identically through the bounded writer', () => { + const value = { version: 1, accounts: [{ id: 'alpha', name: 'Ada' }] } + expect(serializeIntegrationAccountFile(value)).toBe(JSON.stringify(value, null, 2)) + }) + + it('admits an exact-size metadata file and rejects one extra byte', () => { + const base = JSON.stringify({ value: '' }, null, 2) + const exact = { value: 'a'.repeat(MAX_INTEGRATION_ACCOUNT_FILE_BYTES - base.length) } + expect(Buffer.byteLength(JSON.stringify(exact, null, 2))).toBe( + MAX_INTEGRATION_ACCOUNT_FILE_BYTES + ) + expect(serializeIntegrationAccountFile(exact)).toBe(JSON.stringify(exact, null, 2)) + expect(() => serializeIntegrationAccountFile({ value: `${exact.value}a` })).toThrow( + `${MAX_INTEGRATION_ACCOUNT_FILE_BYTES} bytes` + ) + }) +}) diff --git a/src/main/integration-account-persistence-limits.ts b/src/main/integration-account-persistence-limits.ts new file mode 100644 index 00000000000..040a9c9a5c1 --- /dev/null +++ b/src/main/integration-account-persistence-limits.ts @@ -0,0 +1,53 @@ +import { stringifyJsonWithinByteLimit } from '../shared/node-bounded-json-stringify' +import { MAX_INTEGRATION_CREDENTIAL_FILE_BYTES } from './integration-credential-file' + +export const MAX_INTEGRATION_ACCOUNTS = 256 +export const MAX_INTEGRATION_ACCOUNT_ID_BYTES = 128 +export const MAX_INTEGRATION_ACCOUNT_URL_BYTES = 16 * 1024 +export const MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES = 4 * 1024 +export const MAX_INTEGRATION_ACCOUNT_LABEL_BYTES = 16 * 1024 +export const MAX_INTEGRATION_CREDENTIAL_BYTES = 64 * 1024 +export const MAX_INTEGRATION_ACCOUNT_FILE_BYTES = MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + +export class IntegrationAccountPersistenceLimitError extends Error { + constructor(message: string) { + super(message) + this.name = 'IntegrationAccountPersistenceLimitError' + } +} + +export function assertIntegrationAccountCount(service: string, count: number): void { + if (count > MAX_INTEGRATION_ACCOUNTS) { + throw new IntegrationAccountPersistenceLimitError( + `${service} supports at most ${MAX_INTEGRATION_ACCOUNTS} saved accounts.` + ) + } +} + +export function assertIntegrationStringBytes( + service: string, + field: string, + value: string, + maxBytes: number +): void { + if (Buffer.byteLength(value, 'utf8') > maxBytes) { + throw new IntegrationAccountPersistenceLimitError( + `${service} ${field} exceeds ${maxBytes} UTF-8 bytes.` + ) + } +} + +export function assertIntegrationCredentialBytes(service: string, value: string): void { + assertIntegrationStringBytes(service, 'credential', value, MAX_INTEGRATION_CREDENTIAL_BYTES) +} + +export function serializeIntegrationAccountFile(value: unknown): string { + return stringifyJsonWithinByteLimit(value, MAX_INTEGRATION_ACCOUNT_FILE_BYTES, 2).serialized +} + +export function unreadableIntegrationAccountFileError(service: string): Error { + return new Error( + `Saved ${service} account metadata is unreadable or exceeds supported limits; ` + + 'the existing file was left unchanged.' + ) +} diff --git a/src/main/integration-api-concurrency.test.ts b/src/main/integration-api-concurrency.test.ts new file mode 100644 index 00000000000..edb199f6987 --- /dev/null +++ b/src/main/integration-api-concurrency.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from 'vitest' +import { IntegrationApiConcurrencyGate } from './integration-api-concurrency' + +describe('IntegrationApiConcurrencyGate', () => { + it('preserves FIFO admission within the ordinary concurrency limit', async () => { + const gate = new IntegrationApiConcurrencyGate(1, 2) + const first = gate.acquire() + const secondResolved = vi.fn() + const second = gate.acquire().then(secondResolved) + + await first + expect(secondResolved).not.toHaveBeenCalled() + gate.release() + await second + + expect(secondResolved).toHaveBeenCalledTimes(1) + gate.release() + }) + + it('rejects fan-out beyond the retained waiter cap', async () => { + const gate = new IntegrationApiConcurrencyGate(1, 2) + await gate.acquire() + const queued = [gate.acquire(), gate.acquire()] + + await expect(gate.acquire()).rejects.toThrow('queue is full') + + gate.release() + await queued[0] + gate.release() + await queued[1] + gate.release() + }) +}) diff --git a/src/main/integration-api-concurrency.ts b/src/main/integration-api-concurrency.ts new file mode 100644 index 00000000000..2b44a752068 --- /dev/null +++ b/src/main/integration-api-concurrency.ts @@ -0,0 +1,34 @@ +export const INTEGRATION_API_MAX_WAITERS = 1024 + +export class IntegrationApiConcurrencyGate { + private running = 0 + private readonly waiters: (() => void)[] = [] + + constructor( + private readonly maxConcurrent: number, + private readonly maxWaiters = INTEGRATION_API_MAX_WAITERS + ) {} + + acquire(): Promise { + if (this.running < this.maxConcurrent) { + this.running += 1 + return Promise.resolve() + } + if (this.waiters.length >= this.maxWaiters) { + return Promise.reject( + new Error('Integration API request queue is full; retry after current requests finish.') + ) + } + return new Promise((resolve) => { + this.waiters.push(() => { + this.running += 1 + resolve() + }) + }) + } + + release(): void { + this.running = Math.max(0, this.running - 1) + this.waiters.shift()?.() + } +} diff --git a/src/main/integration-credential-file.test.ts b/src/main/integration-credential-file.test.ts new file mode 100644 index 00000000000..3364e0324e6 --- /dev/null +++ b/src/main/integration-credential-file.test.ts @@ -0,0 +1,46 @@ +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + MAX_INTEGRATION_CREDENTIAL_FILE_BYTES, + readIntegrationCredentialFileSync, + readIntegrationCredentialFileText +} from './integration-credential-file' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function createPath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-integration-credential-')) + roots.push(root) + return join(root, 'credential') +} + +describe('integration credential file bounds', () => { + it('accepts exact-cap credential bytes in sync and async readers', async () => { + const filePath = createPath() + writeFileSync(filePath, Buffer.alloc(MAX_INTEGRATION_CREDENTIAL_FILE_BYTES, 0x61)) + + expect(readIntegrationCredentialFileSync(filePath)).toHaveLength( + MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + ) + await expect(readIntegrationCredentialFileText(filePath)).resolves.toHaveLength( + MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + ) + }) + + it('rejects a sparse credential file beyond the cap', () => { + const filePath = createPath() + const file = openSync(filePath, 'w') + ftruncateSync(file, MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + 1) + closeSync(file) + + expect(() => readIntegrationCredentialFileSync(filePath)).toThrow('exceeds') + }) +}) diff --git a/src/main/integration-credential-file.ts b/src/main/integration-credential-file.ts index 2ad90492f1b..d55d81b1beb 100644 --- a/src/main/integration-credential-file.ts +++ b/src/main/integration-credential-file.ts @@ -1,10 +1,30 @@ import { statSync } from 'node:fs' import { safeStorage } from 'electron' +import { + readNodeFileSyncWithinLimit, + readNodeFileWithinLimit +} from '../shared/node-bounded-file-reader' import { credentialDecryptionMessage, type IntegrationCredentialService } from '../shared/integration-credential-errors' +export const MAX_INTEGRATION_CREDENTIAL_FILE_BYTES = 1024 * 1024 + +export function readIntegrationCredentialFileSync(filePath: string): Buffer { + return readNodeFileSyncWithinLimit(filePath, MAX_INTEGRATION_CREDENTIAL_FILE_BYTES).buffer +} + +export function readIntegrationCredentialFileSyncText(filePath: string): string { + return readIntegrationCredentialFileSync(filePath).toString('utf8') +} + +export async function readIntegrationCredentialFileText(filePath: string): Promise { + return ( + await readNodeFileWithinLimit(filePath, MAX_INTEGRATION_CREDENTIAL_FILE_BYTES) + ).buffer.toString('utf8') +} + // Why: connection status treats a token file as a saved credential; empty // files read as "missing", so counting them would split-brain getStatus. export function credentialFileHasContent(path: string): boolean { diff --git a/src/main/integration-error-message.test.ts b/src/main/integration-error-message.test.ts new file mode 100644 index 00000000000..a40513aaef6 --- /dev/null +++ b/src/main/integration-error-message.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest' +import { + boundedIntegrationErrorLog, + boundedIntegrationErrorMessage, + MAX_INTEGRATION_ERROR_MESSAGE_CHARS +} from './integration-error-message' + +describe('boundedIntegrationErrorMessage', () => { + it('preserves the exact character boundary and truncates character +1', () => { + const exact = 'a'.repeat(MAX_INTEGRATION_ERROR_MESSAGE_CHARS) + expect(boundedIntegrationErrorMessage(new Error(exact))).toBe(exact) + + const truncated = boundedIntegrationErrorMessage(`${exact}b`) + expect(truncated).toHaveLength(MAX_INTEGRATION_ERROR_MESSAGE_CHARS) + expect(truncated.endsWith('…')).toBe(true) + }) + + it('bounds retained stack text used by provider logs', () => { + const error = new Error('failed') + error.stack = 's'.repeat(MAX_INTEGRATION_ERROR_MESSAGE_CHARS + 1) + + expect(boundedIntegrationErrorLog(error)).toHaveLength(MAX_INTEGRATION_ERROR_MESSAGE_CHARS) + }) +}) diff --git a/src/main/integration-error-message.ts b/src/main/integration-error-message.ts new file mode 100644 index 00000000000..38c533844e8 --- /dev/null +++ b/src/main/integration-error-message.ts @@ -0,0 +1,13 @@ +export const MAX_INTEGRATION_ERROR_MESSAGE_CHARS = 16 * 1024 + +export function boundedIntegrationErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error) + if (message.length <= MAX_INTEGRATION_ERROR_MESSAGE_CHARS) { + return message + } + return `${message.slice(0, MAX_INTEGRATION_ERROR_MESSAGE_CHARS - 1)}…` +} + +export function boundedIntegrationErrorLog(error: unknown): string { + return boundedIntegrationErrorMessage(error instanceof Error && error.stack ? error.stack : error) +} diff --git a/src/main/integration-fanout.test.ts b/src/main/integration-fanout.test.ts new file mode 100644 index 00000000000..4151db0682c --- /dev/null +++ b/src/main/integration-fanout.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import { + runBoundedIntegrationFanout, + runBoundedIntegrationSettledFanout +} from './integration-fanout' + +describe('runBoundedIntegrationFanout', () => { + it('bounds concurrency and preserves input order across out-of-order completions', async () => { + let active = 0 + let peak = 0 + const result = await runBoundedIntegrationFanout( + [0, 1, 2, 3, 4], + async (entry) => { + active += 1 + peak = Math.max(peak, active) + await new Promise((resolve) => setTimeout(resolve, (4 - entry) % 3)) + active -= 1 + return [entry] + }, + (items) => items, + { + maxConcurrent: 2, + limits: { maxPages: 5, maxItems: 5, maxRetainedBytes: 100 } + } + ) + + expect(peak).toBe(2) + expect(result).toEqual({ + results: [[0], [1], [2], [3], [4]], + truncated: false, + attemptedCount: 5 + }) + }) + + it('admits the exact aggregate item boundary and stops scheduling after item +1', async () => { + const visited: number[] = [] + const result = await runBoundedIntegrationFanout( + [0, 1, 2, 3], + async (entry) => { + visited.push(entry) + return [entry] + }, + (items) => items, + { + maxConcurrent: 2, + limits: { maxPages: 4, maxItems: 2, maxRetainedBytes: 100 } + } + ) + + expect(result).toEqual({ + results: [[0], [1]], + truncated: true, + attemptedCount: 2 + }) + expect(visited).toEqual([0, 1]) + }) + + it('does not retain or schedule later batches after byte +1', async () => { + const visited: number[] = [] + const result = await runBoundedIntegrationFanout( + [0, 1, 2], + async (entry) => { + visited.push(entry) + return entry === 0 ? ['a'] : ['aa'] + }, + (items) => items, + { + maxConcurrent: 1, + limits: { maxPages: 3, maxItems: 3, maxRetainedBytes: 10 } + } + ) + + expect(result).toEqual({ + results: [['a']], + truncated: true, + attemptedCount: 2 + }) + expect(visited).toEqual([0, 1]) + }) + + it('preserves settled success and failure order without rejecting the fan-out', async () => { + const result = await runBoundedIntegrationSettledFanout( + ['ok', 'bad', 'later'], + async (entry) => { + if (entry === 'bad') { + throw new Error('failed') + } + return [entry] + }, + (items) => items + ) + + expect(result.results).toMatchObject([ + { status: 'fulfilled', value: ['ok'] }, + { status: 'rejected', reason: { message: 'failed' } }, + { status: 'fulfilled', value: ['later'] } + ]) + }) +}) diff --git a/src/main/integration-fanout.ts b/src/main/integration-fanout.ts new file mode 100644 index 00000000000..a86d0f7047a --- /dev/null +++ b/src/main/integration-fanout.ts @@ -0,0 +1,82 @@ +import { MAX_INTEGRATION_ACCOUNTS } from './integration-account-persistence-limits' +import { + INTEGRATION_PAGINATION_MAX_ITEMS, + INTEGRATION_PAGINATION_MAX_RETAINED_BYTES, + IntegrationPaginationBudget, + type IntegrationPaginationLimits +} from './integration-pagination-budget' + +export const INTEGRATION_FANOUT_MAX_CONCURRENT = 4 + +const DEFAULT_FANOUT_LIMITS: IntegrationPaginationLimits = { + maxPages: MAX_INTEGRATION_ACCOUNTS, + maxItems: INTEGRATION_PAGINATION_MAX_ITEMS, + maxRetainedBytes: INTEGRATION_PAGINATION_MAX_RETAINED_BYTES +} + +export function createIntegrationFanoutBudget(): IntegrationPaginationBudget { + return new IntegrationPaginationBudget(DEFAULT_FANOUT_LIMITS) +} + +export async function runBoundedIntegrationFanout( + entries: readonly TEntry[], + load: (entry: TEntry, index: number) => Promise, + retainedValues: (result: TResult) => readonly unknown[], + options: { + budget?: IntegrationPaginationBudget + maxConcurrent?: number + limits?: IntegrationPaginationLimits + } = {} +): Promise<{ results: TResult[]; truncated: boolean; attemptedCount: number }> { + const maxConcurrent = options.maxConcurrent ?? INTEGRATION_FANOUT_MAX_CONCURRENT + if (!Number.isSafeInteger(maxConcurrent) || maxConcurrent <= 0) { + throw new RangeError('Integration fan-out concurrency must be a positive safe integer') + } + + const budget = + options.budget ?? new IntegrationPaginationBudget(options.limits ?? DEFAULT_FANOUT_LIMITS) + const results: TResult[] = [] + let attemptedCount = 0 + for (let offset = 0; offset < entries.length; offset += maxConcurrent) { + const batch = entries.slice(offset, offset + maxConcurrent) + attemptedCount += batch.length + const loaded = await Promise.all( + batch.map((entry, batchIndex) => load(entry, offset + batchIndex)) + ) + for (const result of loaded) { + if (!budget.admitPage(retainedValues(result))) { + return { results, truncated: true, attemptedCount } + } + results.push(result) + } + if (!budget.canRequestPage && offset + batch.length < entries.length) { + return { results, truncated: true, attemptedCount } + } + } + return { results, truncated: false, attemptedCount } +} + +export function runBoundedIntegrationSettledFanout( + entries: readonly TEntry[], + load: (entry: TEntry, index: number) => Promise, + retainedValues: (result: TResult) => readonly unknown[] +): Promise<{ + results: PromiseSettledResult[] + truncated: boolean + attemptedCount: number +}> { + return runBoundedIntegrationFanout( + entries, + async (entry, index): Promise> => { + try { + return { status: 'fulfilled', value: await load(entry, index) } + } catch (reason) { + return { status: 'rejected', reason } + } + }, + (result) => + result.status === 'fulfilled' + ? retainedValues(result.value) + : [result.reason instanceof Error ? result.reason.message : String(result.reason)] + ) +} diff --git a/src/main/integration-pagination-budget.test.ts b/src/main/integration-pagination-budget.test.ts new file mode 100644 index 00000000000..fa90d052662 --- /dev/null +++ b/src/main/integration-pagination-budget.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' +import { + IntegrationPaginationBudget, + IntegrationPaginationLimitError +} from './integration-pagination-budget' + +describe('IntegrationPaginationBudget', () => { + it('admits exact page and item boundaries and rejects page +1', () => { + const budget = new IntegrationPaginationBudget({ + maxPages: 1, + maxItems: 2, + maxRetainedBytes: 100 + }) + + expect(budget.admitPage(['a', 'b'])).toBe(true) + expect(budget.canRequestPage).toBe(false) + expect(budget.admitPage([])).toBe(false) + }) + + it('admits the exact retained-byte boundary and rejects byte +1', () => { + const exact = new IntegrationPaginationBudget({ + maxPages: 2, + maxItems: 2, + maxRetainedBytes: 5 + }) + const over = new IntegrationPaginationBudget({ + maxPages: 2, + maxItems: 2, + maxRetainedBytes: 5 + }) + + expect(exact.admitPage(['a'])).toBe(true) + expect(over.admitPage(['aa'])).toBe(false) + }) + + it('does not consume capacity when a page is rejected', () => { + const budget = new IntegrationPaginationBudget({ + maxPages: 1, + maxItems: 1, + maxRetainedBytes: 5 + }) + + expect(budget.admitPage(['aa'])).toBe(false) + expect(budget.admitPage(['a'])).toBe(true) + }) + + it('bounds SDK-style cumulative pages at exact and +1 byte sizes', () => { + const exact = new IntegrationPaginationBudget({ + maxPages: 2, + maxItems: 2, + maxRetainedBytes: 5 + }) + const over = new IntegrationPaginationBudget({ + maxPages: 2, + maxItems: 2, + maxRetainedBytes: 5 + }) + + expect(() => exact.assertCumulativePage(['a'])).not.toThrow() + expect(() => over.assertCumulativePage(['aa'])).toThrow(IntegrationPaginationLimitError) + }) +}) diff --git a/src/main/integration-pagination-budget.ts b/src/main/integration-pagination-budget.ts new file mode 100644 index 00000000000..202243c6702 --- /dev/null +++ b/src/main/integration-pagination-budget.ts @@ -0,0 +1,96 @@ +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../shared/node-bounded-json-stringify' + +export const INTEGRATION_PAGINATION_MAX_PAGES = 100 +export const INTEGRATION_PAGINATION_MAX_ITEMS = 10_000 +export const INTEGRATION_PAGINATION_MAX_RETAINED_BYTES = 32 * 1024 * 1024 + +export type IntegrationPaginationLimits = { + maxPages: number + maxItems: number + maxRetainedBytes: number +} + +const DEFAULT_LIMITS: IntegrationPaginationLimits = { + maxPages: INTEGRATION_PAGINATION_MAX_PAGES, + maxItems: INTEGRATION_PAGINATION_MAX_ITEMS, + maxRetainedBytes: INTEGRATION_PAGINATION_MAX_RETAINED_BYTES +} + +export class IntegrationPaginationLimitError extends Error { + constructor() { + super('Integration pagination exceeded its retained result budget.') + this.name = 'IntegrationPaginationLimitError' + } +} + +export class IntegrationPaginationBudget { + private pages = 0 + private items = 0 + private retainedBytes = 0 + + constructor(private readonly limits: IntegrationPaginationLimits = DEFAULT_LIMITS) {} + + admitPage(pageItems: readonly unknown[]): boolean { + if ( + this.pages >= this.limits.maxPages || + pageItems.length > this.limits.maxItems - this.items + ) { + return false + } + const pageBytes = this.measureWithinRemainingBudget(pageItems) + if (pageBytes === null) { + return false + } + this.pages += 1 + this.items += pageItems.length + this.retainedBytes += pageBytes + return true + } + + assertCumulativePage(items: readonly unknown[]): void { + if (this.pages >= this.limits.maxPages || items.length > this.limits.maxItems) { + throw new IntegrationPaginationLimitError() + } + const measuredBytes = this.measureWithinTotalBudget(items) + if (measuredBytes === null) { + throw new IntegrationPaginationLimitError() + } + this.pages += 1 + this.items = items.length + this.retainedBytes = measuredBytes + } + + get canRequestPage(): boolean { + return ( + this.pages < this.limits.maxPages && + this.items < this.limits.maxItems && + this.retainedBytes <= this.limits.maxRetainedBytes - 2 + ) + } + + private measureWithinRemainingBudget(value: unknown): number | null { + const remaining = this.limits.maxRetainedBytes - this.retainedBytes + try { + return stringifyJsonWithinByteLimit(value, remaining).byteLength + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + return null + } + throw error + } + } + + private measureWithinTotalBudget(value: unknown): number | null { + try { + return stringifyJsonWithinByteLimit(value, this.limits.maxRetainedBytes).byteLength + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + return null + } + throw error + } + } +} diff --git a/src/main/ipc/ai-vault-host-scan-batches.ts b/src/main/ipc/ai-vault-host-scan-batches.ts new file mode 100644 index 00000000000..488933bc684 --- /dev/null +++ b/src/main/ipc/ai-vault-host-scan-batches.ts @@ -0,0 +1,20 @@ +import type { AiVaultListResult } from '../../shared/ai-vault-types' +import { mergeAiVaultListResults } from '../ai-vault/session-list-results' + +export const AI_VAULT_ALL_HOST_SCAN_CONCURRENCY = 4 + +export async function scanAiVaultHostsInBatches( + scans: readonly (() => Promise)[], + initialResults: readonly AiVaultListResult[], + limit: number | undefined +): Promise { + let merged = mergeAiVaultListResults(initialResults, limit) + for (let offset = 0; offset < scans.length; offset += AI_VAULT_ALL_HOST_SCAN_CONCURRENCY) { + const batch = scans.slice(offset, offset + AI_VAULT_ALL_HOST_SCAN_CONCURRENCY) + merged = mergeAiVaultListResults( + [merged, ...(await Promise.all(batch.map((scan) => scan())))], + limit + ) + } + return merged +} diff --git a/src/main/ipc/ai-vault.test.ts b/src/main/ipc/ai-vault.test.ts index 5f4153106fd..820779e05f3 100644 --- a/src/main/ipc/ai-vault.test.ts +++ b/src/main/ipc/ai-vault.test.ts @@ -4,6 +4,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { AiVaultListResult, AiVaultSession } from '../../shared/ai-vault-types' import type { IFilesystemProvider } from '../providers/types' import { getRemoteHostPlatform } from '../ssh/ssh-remote-platform' +import { + AI_VAULT_SESSION_ID_MAX_UTF8_BYTES, + AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES +} from '../ai-vault/session-list-retention' const mocks = vi.hoisted(() => ({ scanAiVaultSessions: vi.fn(), @@ -50,7 +54,8 @@ vi.mock('./ssh', () => ({ getActiveSshAiVaultHostInfos: mocks.getActiveSshAiVaultHostInfos })) -const { _internals, registerAiVaultHandlers } = await import('./ai-vault') +const { AI_VAULT_ALL_HOST_SCAN_CONCURRENCY, _internals, registerAiVaultHandlers } = + await import('./ai-vault') const provider = {} as IFilesystemProvider @@ -136,6 +141,33 @@ describe('listAiVaultSessions host routing', () => { ]) }) + it('bounds concurrent scans across a pathological runtime-host roster', async () => { + let active = 0 + let peak = 0 + mocks.getActiveSshAiVaultHostInfos.mockReturnValue([]) + mocks.scanRuntimeAiVaultSessions.mockImplementation(async (environmentId: string) => { + active += 1 + peak = Math.max(peak, active) + await Promise.resolve() + active -= 1 + return result([session(`runtime:${environmentId}`, environmentId)]) + }) + registerAiVaultHandlers({ + getActiveRuntimeAiVaultHostInfos: () => + Array.from({ length: 20 }, (_, index) => ({ + environmentId: `runtime-${index}`, + executionHostId: `runtime:runtime-${index}` as const + })), + scanRuntimeAiVaultSessions: mocks.scanRuntimeAiVaultSessions + }) + + const merged = await _internals.listAiVaultSessions({ executionHostScope: 'all' }) + + expect(mocks.scanRuntimeAiVaultSessions).toHaveBeenCalledTimes(20) + expect(peak).toBeLessThanOrEqual(AI_VAULT_ALL_HOST_SCAN_CONCURRENCY) + expect(merged.sessions).toHaveLength(21) + }) + it('keeps local and SSH results when runtime host discovery fails', async () => { registerAiVaultHandlers({ getActiveRuntimeAiVaultHostInfos: () => { @@ -205,6 +237,29 @@ describe('listAiVaultSessions host routing', () => { expect(mocks.scanAiVaultSessions).toHaveBeenCalledTimes(1) expect(mocks.scanRemoteAiVaultSessions).toHaveBeenCalledTimes(1) }) + + it('bounds a pathological multi-host result before retaining it', async () => { + const oversized = session('ssh:dev-box', 'remote-session') + oversized.sessionId = 'x'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES + 1) + mocks.scanRemoteAiVaultSessions.mockResolvedValue(result([oversized])) + + const first = await _internals.listAiVaultSessions({ executionHostScope: 'ssh:dev-box' }) + const second = await _internals.listAiVaultSessions({ executionHostScope: 'ssh:dev-box' }) + + expect(first.sessions).toEqual([]) + expect(first.issues.at(-1)?.message).toContain('AI Vault omitted') + expect(second).toBe(first) + expect(mocks.scanRemoteAiVaultSessions).toHaveBeenCalledTimes(1) + }) + + it('bypasses the multi-host cache when its key exceeds the memory limit', async () => { + const scopePaths = ['x'.repeat(AI_VAULT_SESSION_LIST_CACHE_KEY_MAX_JSON_BYTES + 1)] + + await _internals.listAiVaultSessions({ executionHostScope: 'ssh:dev-box', scopePaths }) + await _internals.listAiVaultSessions({ executionHostScope: 'ssh:dev-box', scopePaths }) + + expect(mocks.scanRemoteAiVaultSessions).toHaveBeenCalledTimes(2) + }) }) describe('prepareSessionResume IPC', () => { diff --git a/src/main/ipc/ai-vault.ts b/src/main/ipc/ai-vault.ts index ebfd52e63a7..7c64344e541 100644 --- a/src/main/ipc/ai-vault.ts +++ b/src/main/ipc/ai-vault.ts @@ -11,7 +11,8 @@ import { scanRemoteAiVaultSessions } from '../ai-vault/remote-session-scanner' import { listClaudeSubagentSessions } from '../ai-vault/session-scanner-claude-subagents' import { claudeProjectsRootDirs } from '../ai-vault/session-scanner-source-discovery' import { isPathInsideOrEqual } from '../../shared/cross-platform-path' -import { aiVaultScanIssueResult, mergeAiVaultListResults } from '../ai-vault/session-list-results' +import { aiVaultScanIssueResult } from '../ai-vault/session-list-results' +import * as aiVaultListRetention from '../ai-vault/session-list-retention' import type { AiVaultListArgs, AiVaultListResult, @@ -32,6 +33,9 @@ import { SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE } from '../providers/ssh-filesystem-dispatch' import { getActiveSshAiVaultHostInfo, getActiveSshAiVaultHostInfos } from './ssh' +import { scanAiVaultHostsInBatches } from './ai-vault-host-scan-batches' + +export { AI_VAULT_ALL_HOST_SCAN_CONCURRENCY } from './ai-vault-host-scan-batches' const AI_VAULT_CACHE_TTL_MS = 15_000 const AI_VAULT_ALL_HOST_RUNTIME_TIMEOUT_MS = 3_000 @@ -46,9 +50,7 @@ type AiVaultHandlerOptions = AiVaultSessionSources & ) => Promise } -type RuntimeAiVaultScanOptions = { - timeoutMs?: number -} +type RuntimeAiVaultScanOptions = { timeoutMs?: number } type CachedAiVaultList = { key: string @@ -70,19 +72,21 @@ async function listAiVaultSessions(args?: AiVaultListArgs): Promise { + const bounded = aiVaultListRetention.boundAiVaultListResult(result) cachedList = { key, - result, + result: bounded, expiresAt: Date.now() + AI_VAULT_CACHE_TTL_MS } - return result + return bounded }) .finally(() => { // Only clear tracking if it still refers to this request: a concurrent @@ -121,21 +126,19 @@ async function scanAiVaultSessionsByHostScope( if (executionHostScope === 'all') { const runtimeHosts = getActiveRuntimeAiVaultHostInfosResult() const runtimeResults = runtimeHosts.issue ? [runtimeHosts.issue] : [] - return mergeAiVaultListResults( - await Promise.all([ - scanLocalAiVaultSessions(args), - ...getActiveSshAiVaultHostInfos().map((hostInfo) => - scanSshAiVaultSessions(hostInfo.targetId, args) - ), - ...runtimeHosts.hostInfos.map((hostInfo) => + const scans: (() => Promise)[] = [() => scanLocalAiVaultSessions(args)] + scans.push( + ...getActiveSshAiVaultHostInfos().map( + (hostInfo) => () => scanSshAiVaultSessions(hostInfo.targetId, args) + ), + ...runtimeHosts.hostInfos.map( + (hostInfo) => () => scanRuntimeAiVaultSessions(hostInfo, args, { timeoutMs: AI_VAULT_ALL_HOST_RUNTIME_TIMEOUT_MS }) - ), - ...runtimeResults - ]), - args?.limit + ) ) + return scanAiVaultHostsInBatches(scans, runtimeResults, args?.limit) } const parsed = parseExecutionHostId(executionHostScope) diff --git a/src/main/ipc/app.test.ts b/src/main/ipc/app.test.ts index 7702dcd6349..79e044dba30 100644 --- a/src/main/ipc/app.test.ts +++ b/src/main/ipc/app.test.ts @@ -33,12 +33,13 @@ vi.mock('node:child_process', () => ({ // events, so tests drive the async command lifecycle readCommandStdout expects. function createFakeSpawnChild(options: { stdout?: string + stdoutFragmentSize?: number code?: number error?: Error pid?: number hang?: boolean }): EventEmitter & { pid: number; kill: ReturnType; stdout: EventEmitter } { - const { stdout, code = 0, error, pid = 4242, hang = false } = options + const { stdout, stdoutFragmentSize, code = 0, error, pid = 4242, hang = false } = options const child = new EventEmitter() as EventEmitter & { pid: number kill: ReturnType @@ -58,7 +59,13 @@ function createFakeSpawnChild(options: { return } if (stdout !== undefined) { - stdoutStream.emit('data', stdout) + if (stdoutFragmentSize === undefined) { + stdoutStream.emit('data', stdout) + } else { + for (let offset = 0; offset < stdout.length; offset += stdoutFragmentSize) { + stdoutStream.emit('data', stdout.slice(offset, offset + stdoutFragmentSize)) + } + } } child.emit('close', code) }) @@ -342,6 +349,44 @@ describe('registerAppHandlers', () => { expect(spawnMock).toHaveBeenCalledTimes(2) }) + it('kills an oversized selected-input probe before using the layout fallback', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }) + spawnMock + .mockImplementationOnce(() => + createFakeSpawnChild({ stdout: 'x'.repeat(1024 * 1024 + 1), pid: 4242 }) + ) + .mockImplementationOnce(() => createFakeSpawnChild({ stdout: 'com.apple.keylayout.ABC\n' })) + registerAppHandlers({} as never) + + await expect(handlers.get('app:getKeyboardInputSourceId')?.(null)).resolves.toBe( + 'com.apple.keylayout.ABC' + ) + expect(processKillSpy).toHaveBeenCalledWith(-4242, 'SIGKILL') + expect(spawnMock).toHaveBeenCalledTimes(2) + }) + + it('parses selected-input JSON after 100,000 one-byte fragments', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }) + const selected = JSON.stringify([ + { + 'Bundle ID': 'com.apple.inputmethod.SCIM', + 'Input Mode': 'com.apple.inputmethod.SCIM.ITABC', + InputSourceKind: 'Input Mode' + } + ]) + spawnMock.mockImplementationOnce(() => + createFakeSpawnChild({ + stdout: `${' '.repeat(100_000)}${selected}`, + stdoutFragmentSize: 1 + }) + ) + registerAppHandlers({} as never) + + await expect(handlers.get('app:getKeyboardInputSourceId')?.(null)).resolves.toBe( + 'com.apple.inputmethod.SCIM.ITABC' + ) + }) + it('falls back when macOS keyboard input source probes never report completion', async () => { Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }) spawnMock.mockImplementation(() => createFakeSpawnChild({ pid: 4242, hang: true })) diff --git a/src/main/ipc/app.ts b/src/main/ipc/app.ts index 7485c03b5f0..1bc47fdc3bd 100644 --- a/src/main/ipc/app.ts +++ b/src/main/ipc/app.ts @@ -6,6 +6,7 @@ import { app, BrowserWindow, dialog, ipcMain, type IpcMainInvokeEvent } from 'el import { is } from '@electron-toolkit/utils' import type { AppIdentity } from '../../shared/app-identity' import type { FloatingTerminalCwdRequest, MarkdownDocument } from '../../shared/types' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import { relaunchApp } from '../app-relaunch' import type { Store } from '../persistence' import { getDevInstanceIdentity } from '../startup/dev-instance-identity' @@ -24,6 +25,7 @@ import { isMarkdownDocumentName, markdownDocumentFromFilePath } from './markdown import { registerRendererShutdownCheckpointHandler } from './renderer-shutdown-checkpoint' const KEYBOARD_INPUT_SOURCE_TIMEOUT_MS = 500 +const KEYBOARD_INPUT_SOURCE_MAX_OUTPUT_BYTES = 1024 * 1024 const MAC_HITOOLBOX_DOMAIN = 'com.apple.HIToolbox' // Why: defaults export reads live prefs (on-disk plist lags cfprefsd); xml1 dodges plutil's json abort on macOS 15 input-source arrays; absolute paths so a minimal PATH can't shadow the tools. const MAC_SELECTED_INPUT_SOURCES_JSON_COMMAND = [ @@ -116,6 +118,7 @@ function readCommandStdout( return new Promise((resolve, reject) => { let settled = false let child: ReturnType | undefined + const stdout = new GrowingByteBuffer() // Why: killing only the shell orphans pipeline stages; detached spawn lets one negative-pid SIGKILL reap the whole group. const killTree = (): void => { @@ -146,14 +149,19 @@ function readCommandStdout( settled = true clearTimeout(timer) callback() + stdout.clear() } try { child = spawn(command, args, { detached: true, stdio: ['ignore', 'pipe', 'ignore'] }) - let stdout = '' child.stdout?.setEncoding('utf8') child.stdout?.on('data', (chunk: string) => { - stdout += chunk + const bytes = Buffer.from(chunk) + if (stdout.byteLength + bytes.byteLength > KEYBOARD_INPUT_SOURCE_MAX_OUTPUT_BYTES) { + failWith(new Error(`${command} exceeded its output limit`)) + return + } + stdout.append(bytes) }) const failWith = (error: Error): void => { killTree() @@ -165,7 +173,7 @@ function readCommandStdout( child.on('close', (code, signal) => { settle(() => code === 0 - ? resolve(stdout) + ? resolve(stdout.takeString()) : reject( new Error( `${command} exited with ${signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`}` diff --git a/src/main/ipc/custom-pet-file-reader.test.ts b/src/main/ipc/custom-pet-file-reader.test.ts new file mode 100644 index 00000000000..d09a613d448 --- /dev/null +++ b/src/main/ipc/custom-pet-file-reader.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_CUSTOM_PET_FILE_BYTES } from '../../shared/custom-pet-media-limits' + +const { openMock } = vi.hoisted(() => ({ openMock: vi.fn() })) + +vi.mock('node:fs/promises', () => ({ open: openMock })) + +import { CustomPetFileTooLargeError, readCustomPetFile } from './custom-pet-file-reader' + +function fileHandle({ + afterSize, + beforeSize, + extraByte = false +}: { + afterSize?: number + beforeSize: number + extraByte?: boolean +}) { + let statCalls = 0 + return { + close: vi.fn().mockResolvedValue(undefined), + read: vi.fn(async (target: Buffer, offset: number, length: number, position: number) => { + if (position < beforeSize) { + const bytesRead = Math.min(length, beforeSize - position) + target.fill(0x2a, offset, offset + bytesRead) + return { bytesRead, buffer: target } + } + if (extraByte) { + target[offset] = 0x2b + return { bytesRead: 1, buffer: target } + } + return { bytesRead: 0, buffer: target } + }), + stat: vi.fn(async () => { + statCalls += 1 + return { + isFile: () => true, + size: statCalls === 1 ? beforeSize : (afterSize ?? beforeSize) + } + }) + } +} + +beforeEach(() => openMock.mockReset()) + +describe('readCustomPetFile', () => { + it('reads a stable file into an exact ArrayBuffer', async () => { + const handle = fileHandle({ beforeSize: 3 }) + openMock.mockResolvedValue(handle) + + const result = await readCustomPetFile('/stored/pet.png') + + expect([...new Uint8Array(result)]).toEqual([0x2a, 0x2a, 0x2a]) + expect(handle.close).toHaveBeenCalledOnce() + }) + + it('rejects an oversized file before issuing a read allocation', async () => { + const handle = fileHandle({ beforeSize: MAX_CUSTOM_PET_FILE_BYTES + 1 }) + openMock.mockResolvedValue(handle) + + await expect(readCustomPetFile('/stored/replaced-pet.png')).rejects.toBeInstanceOf( + CustomPetFileTooLargeError + ) + expect(handle.read).not.toHaveBeenCalled() + expect(handle.close).toHaveBeenCalledOnce() + }) + + it('rejects a file that grows between its handle stat and bounded read', async () => { + const handle = fileHandle({ beforeSize: 3, afterSize: 4, extraByte: true }) + openMock.mockResolvedValue(handle) + + await expect(readCustomPetFile('/stored/growing-pet.png')).rejects.toThrow( + 'changed while it was being read' + ) + expect(handle.close).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ipc/custom-pet-file-reader.ts b/src/main/ipc/custom-pet-file-reader.ts new file mode 100644 index 00000000000..c9f03db0c7d --- /dev/null +++ b/src/main/ipc/custom-pet-file-reader.ts @@ -0,0 +1,59 @@ +import { open } from 'node:fs/promises' +import { MAX_CUSTOM_PET_FILE_BYTES } from '../../shared/custom-pet-media-limits' + +const MAX_CUSTOM_PET_READ_CALLS = 1_024 + +export class CustomPetFileTooLargeError extends Error { + constructor(maxBytes: number) { + super(`Custom pet file exceeds the ${maxBytes} byte limit.`) + this.name = 'CustomPetFileTooLargeError' + } +} + +export async function readCustomPetFile( + filePath: string, + maxBytes = MAX_CUSTOM_PET_FILE_BYTES +): Promise { + const file = await open(filePath, 'r') + try { + const beforeRead = await file.stat() + if (!beforeRead.isFile()) { + throw new Error('Custom pet path is not a file.') + } + if ( + !Number.isSafeInteger(beforeRead.size) || + beforeRead.size < 0 || + beforeRead.size > maxBytes + ) { + throw new CustomPetFileTooLargeError(maxBytes) + } + + const bytes = new ArrayBuffer(beforeRead.size) + const target = Buffer.from(bytes) + let offset = 0 + let readCalls = 0 + while (offset < target.byteLength && readCalls < MAX_CUSTOM_PET_READ_CALLS) { + const result = await file.read(target, offset, target.byteLength - offset, offset) + readCalls += 1 + if (result.bytesRead === 0) { + break + } + offset += result.bytesRead + } + + const sentinel = Buffer.allocUnsafe(1) + const extra = await file.read(sentinel, 0, 1, offset) + const afterRead = await file.stat() + if ( + offset !== beforeRead.size || + extra.bytesRead !== 0 || + afterRead.size !== beforeRead.size || + bytes.byteLength > maxBytes + ) { + throw new Error('Custom pet file changed while it was being read.') + } + return bytes + } finally { + await file.close() + } +} diff --git a/src/main/ipc/emulator-frame-stream-admission.test.ts b/src/main/ipc/emulator-frame-stream-admission.test.ts new file mode 100644 index 00000000000..c144693682c --- /dev/null +++ b/src/main/ipc/emulator-frame-stream-admission.test.ts @@ -0,0 +1,190 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +type StreamMock = { + start: ReturnType + stop: ReturnType +} + +const { handlers, streamState } = vi.hoisted(() => ({ + handlers: new Map unknown>(), + streamState: { + instances: [] as StreamMock[], + startError: null as Error | null + } +})) + +vi.mock('electron', () => ({ + ipcMain: { + handle: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + } + }, + BrowserWindow: { + fromWebContents: (owner: { isDestroyed: () => boolean }) => (owner.isDestroyed() ? null : {}) + } +})) + +vi.mock('../emulator/mjpeg-frame-stream', () => ({ + MjpegFrameStream: class implements StreamMock { + readonly start = vi.fn(() => { + if (streamState.startError) { + throw streamState.startError + } + }) + readonly stop = vi.fn() + + constructor() { + streamState.instances.push(this) + } + } +})) + +import { + EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER, + EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL, + registerEmulatorFrameStreamHandlers +} from './emulator-frame-stream' + +type Owner = EventEmitter & { + destroy: () => void + isDestroyed: () => boolean + send: ReturnType +} + +const owners: Owner[] = [] + +function makeOwner(): Owner { + let destroyed = false + const owner = new EventEmitter() as Owner + owner.isDestroyed = () => destroyed + owner.send = vi.fn() + owner.destroy = () => { + if (destroyed) { + return + } + destroyed = true + owner.emit('destroyed') + } + owners.push(owner) + return owner +} + +function handler(channel: string): (event: unknown, args: unknown) => unknown { + const registered = handlers.get(channel) + if (!registered) { + throw new Error(`Missing handler for ${channel}`) + } + return registered +} + +function startStream(owner: Owner): string { + const result = handler('emulator:frameStreamStart')( + { sender: owner }, + { streamUrl: 'http://127.0.0.1:3100/stream.mjpeg' } + ) as { streamId: string } + return result.streamId +} + +function stopStream(owner: Owner, streamId: string): void { + handler('emulator:frameStreamStop')({ sender: owner }, { streamId }) +} + +function startedStreamCount(): number { + return streamState.instances.filter((stream) => stream.start.mock.calls.length > 0).length +} + +beforeEach(() => { + handlers.clear() + streamState.instances.length = 0 + streamState.startError = null + registerEmulatorFrameStreamHandlers() +}) + +afterEach(() => { + for (const owner of owners) { + owner.destroy() + } + owners.length = 0 + streamState.startError = null +}) + +describe('emulator frame stream admission', () => { + it('caps active sessions per renderer without affecting another renderer', () => { + const firstOwner = makeOwner() + for (let index = 0; index < EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER; index += 1) { + startStream(firstOwner) + } + + expect(() => startStream(firstOwner)).toThrow(/renderer can have at most/) + expect(() => startStream(makeOwner())).not.toThrow() + expect(startedStreamCount()).toBe(EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER + 1) + }) + + it('caps active sessions across renderers', () => { + const ownerCount = + EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL / EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER + for (let ownerIndex = 0; ownerIndex < ownerCount; ownerIndex += 1) { + const owner = makeOwner() + for ( + let streamIndex = 0; + streamIndex < EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER; + streamIndex += 1 + ) { + startStream(owner) + } + } + + expect(() => startStream(makeOwner())).toThrow(/Orca can have at most/) + expect(startedStreamCount()).toBe(EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL) + }) + + it('releases capacity on an owned explicit stop and ignores another renderer', () => { + const owner = makeOwner() + const otherOwner = makeOwner() + const firstStreamId = startStream(owner) + startStream(owner) + const firstStream = streamState.instances[0] + + stopStream(otherOwner, firstStreamId) + expect(firstStream.stop).not.toHaveBeenCalled() + expect(() => startStream(owner)).toThrow(/renderer can have at most/) + + stopStream(owner, firstStreamId) + expect(firstStream.stop).toHaveBeenCalledOnce() + expect(() => startStream(owner)).not.toThrow() + }) + + it('releases every owned session when its renderer is destroyed', () => { + const owner = makeOwner() + startStream(owner) + startStream(owner) + const ownedStreams = streamState.instances.slice() + + owner.destroy() + + expect(owner.listenerCount('destroyed')).toBe(0) + for (const stream of ownedStreams) { + expect(stream.stop).toHaveBeenCalledOnce() + } + expect(() => startStream(makeOwner())).not.toThrow() + }) + + it('releases admission and the destroyed listener when start throws', () => { + const owner = makeOwner() + const startError = new Error('request setup failed') + streamState.startError = startError + + expect(() => startStream(owner)).toThrow(startError) + expect(owner.listenerCount('destroyed')).toBe(0) + expect(streamState.instances[0].stop).toHaveBeenCalledOnce() + + streamState.startError = null + for (let index = 0; index < EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER; index += 1) { + expect(() => startStream(owner)).not.toThrow() + } + }) +}) diff --git a/src/main/ipc/emulator-frame-stream-delivery.test.ts b/src/main/ipc/emulator-frame-stream-delivery.test.ts new file mode 100644 index 00000000000..66e36846d5a --- /dev/null +++ b/src/main/ipc/emulator-frame-stream-delivery.test.ts @@ -0,0 +1,144 @@ +import { EventEmitter } from 'node:events' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +type StreamCallbacks = { + onError: (message: string) => void + onFrame: (frame: Buffer) => void +} + +const { handlers, streamState } = vi.hoisted(() => ({ + handlers: new Map unknown>(), + streamState: { + callbacks: [] as StreamCallbacks[] + } +})) + +vi.mock('electron', () => ({ + ipcMain: { + handle: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + } + }, + BrowserWindow: { fromWebContents: () => ({}) } +})) + +vi.mock('../emulator/mjpeg-frame-stream', () => ({ + MjpegFrameStream: class { + readonly start = vi.fn() + readonly stop = vi.fn() + + constructor(_url: string, callbacks: StreamCallbacks) { + streamState.callbacks.push(callbacks) + } + } +})) + +import { registerEmulatorFrameStreamHandlers } from './emulator-frame-stream' + +type Owner = EventEmitter & { + isDestroyed: () => boolean + send: ReturnType +} + +function makeOwner(): Owner { + const owner = new EventEmitter() as Owner + owner.isDestroyed = () => false + owner.send = vi.fn() + return owner +} + +function handler(channel: string): (event: unknown, args: unknown) => unknown { + const registered = handlers.get(channel) + if (!registered) { + throw new Error(`Missing handler for ${channel}`) + } + return registered +} + +function startStream(owner: Owner): string { + return ( + handler('emulator:frameStreamStart')( + { sender: owner }, + { streamUrl: 'http://127.0.0.1:3100/stream.mjpeg' } + ) as { streamId: string } + ).streamId +} + +function frameMessages(owner: Owner): { + streamId: string + deliveryId: number + bytes: ArrayBuffer +}[] { + return owner.send.mock.calls + .filter(([channel]) => channel === 'emulator:frameStreamFrame') + .map(([, payload]) => payload) +} + +beforeEach(() => { + handlers.clear() + streamState.callbacks.length = 0 + registerEmulatorFrameStreamHandlers() +}) + +describe('emulator frame stream delivery', () => { + it('keeps only the latest frame while waiting for an exact renderer ack', () => { + const owner = makeOwner() + const otherOwner = makeOwner() + const streamId = startStream(owner) + const callbacks = streamState.callbacks[0] + + for (let value = 0; value < 100; value += 1) { + callbacks.onFrame(Buffer.from([value])) + } + + expect(frameMessages(owner)).toHaveLength(1) + expect([...new Uint8Array(frameMessages(owner)[0].bytes)]).toEqual([0]) + + handler('emulator:frameStreamFrameAck')({ sender: otherOwner }, { streamId, deliveryId: 1 }) + expect(frameMessages(owner)).toHaveLength(1) + + handler('emulator:frameStreamFrameAck')({ sender: owner }, { streamId, deliveryId: 1 }) + expect(frameMessages(owner)).toHaveLength(2) + expect(frameMessages(owner)[1].deliveryId).toBe(2) + expect([...new Uint8Array(frameMessages(owner)[1].bytes)]).toEqual([99]) + + handler('emulator:frameStreamFrameAck')({ sender: owner }, { streamId, deliveryId: 1 }) + expect(frameMessages(owner)).toHaveLength(2) + }) + + it('coalesces repeated errors until the renderer proves progress', () => { + const owner = makeOwner() + const streamId = startStream(owner) + const callbacks = streamState.callbacks[0] + + callbacks.onError('first') + callbacks.onError('second') + expect( + owner.send.mock.calls.filter(([channel]) => channel === 'emulator:frameStreamError') + ).toHaveLength(1) + + callbacks.onFrame(Buffer.from([1])) + handler('emulator:frameStreamFrameAck')({ sender: owner }, { streamId, deliveryId: 1 }) + callbacks.onError('after progress') + expect( + owner.send.mock.calls.filter(([channel]) => channel === 'emulator:frameStreamError') + ).toHaveLength(2) + }) + + it('drops retained delivery state when the stream stops', () => { + const owner = makeOwner() + const streamId = startStream(owner) + const callbacks = streamState.callbacks[0] + callbacks.onFrame(Buffer.from([1])) + callbacks.onFrame(Buffer.from([2])) + + handler('emulator:frameStreamStop')({ sender: owner }, { streamId }) + callbacks.onFrame(Buffer.from([3])) + handler('emulator:frameStreamFrameAck')({ sender: owner }, { streamId, deliveryId: 1 }) + + expect(frameMessages(owner)).toHaveLength(1) + }) +}) diff --git a/src/main/ipc/emulator-frame-stream.ts b/src/main/ipc/emulator-frame-stream.ts index 4cf8eee1a5b..0907b044bd4 100644 --- a/src/main/ipc/emulator-frame-stream.ts +++ b/src/main/ipc/emulator-frame-stream.ts @@ -6,20 +6,47 @@ type FrameStreamSession = { owner: WebContents stream: MjpegFrameStream onOwnerDestroyed: () => void + nextDeliveryId: number + inFlightDeliveryId: number | null + pendingFrame: ArrayBuffer | null + errorDelivered: boolean } +export const EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL = 8 +export const EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER = 2 + const sessions = new Map() -function stopFrameStream(streamId: string): void { +function assertFrameStreamCapacity(owner: WebContents): void { + let ownerSessionCount = 0 + for (const session of sessions.values()) { + if (session.owner === owner) { + ownerSessionCount += 1 + } + } + if (ownerSessionCount >= EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER) { + throw new Error( + `A renderer can have at most ${EMULATOR_FRAME_STREAM_MAX_SESSIONS_PER_RENDERER} active emulator frame streams.` + ) + } + if (sessions.size >= EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL) { + throw new Error( + `Orca can have at most ${EMULATOR_FRAME_STREAM_MAX_SESSIONS_TOTAL} active emulator frame streams.` + ) + } +} + +function stopFrameStream(streamId: string, owner: WebContents): void { const session = sessions.get(streamId) - if (!session) { + if (!session || session.owner !== owner) { return } - session.stream.stop() + sessions.delete(streamId) // Why: `.once('destroyed')` self-removes only when that event fires (window // close), so an explicit stop must drop it or each show/hide cycle leaks one. session.owner.removeListener('destroyed', session.onOwnerDestroyed) - sessions.delete(streamId) + session.pendingFrame = null + session.stream.stop() } function frameToArrayBuffer(frame: Buffer): ArrayBuffer { @@ -28,6 +55,47 @@ function frameToArrayBuffer(frame: Buffer): ArrayBuffer { return arrayBuffer } +function sendFrame(streamId: string, session: FrameStreamSession, bytes: ArrayBuffer): void { + const deliveryId = session.nextDeliveryId + session.nextDeliveryId += 1 + session.inFlightDeliveryId = deliveryId + try { + session.owner.send('emulator:frameStreamFrame', { streamId, deliveryId, bytes }) + } catch { + stopFrameStream(streamId, session.owner) + } +} + +function queueFrame(streamId: string, frame: Buffer): void { + const session = sessions.get(streamId) + if (!session || session.owner.isDestroyed()) { + return + } + const bytes = frameToArrayBuffer(frame) + if (session.inFlightDeliveryId !== null) { + session.pendingFrame = bytes + return + } + sendFrame(streamId, session, bytes) +} + +function acknowledgeFrame( + owner: WebContents, + args: { streamId: string; deliveryId: number } +): void { + const session = sessions.get(args.streamId) + if (!session || session.owner !== owner || session.inFlightDeliveryId !== args.deliveryId) { + return + } + session.inFlightDeliveryId = null + session.errorDelivered = false + const pendingFrame = session.pendingFrame + session.pendingFrame = null + if (pendingFrame) { + sendFrame(args.streamId, session, pendingFrame) + } +} + export function registerEmulatorFrameStreamHandlers(): void { ipcMain.handle( 'emulator:frameStreamStart', @@ -45,31 +113,56 @@ export function registerEmulatorFrameStreamHandlers(): void { args.streamUrl, { onError: (message) => { - if (!owner.isDestroyed()) { - owner.send('emulator:frameStreamError', { streamId, message }) + const session = sessions.get(streamId) + if (!owner.isDestroyed() && session && !session.errorDelivered) { + session.errorDelivered = true + try { + owner.send('emulator:frameStreamError', { streamId, message }) + } catch { + stopFrameStream(streamId, owner) + } } }, - onFrame: (frame) => { - if (!owner.isDestroyed()) { - owner.send('emulator:frameStreamFrame', { - streamId, - bytes: frameToArrayBuffer(frame) - }) - } - } + onFrame: (frame) => queueFrame(streamId, frame) }, args.streamKey ) - const onOwnerDestroyed = (): void => stopFrameStream(streamId) - sessions.set(streamId, { owner, stream, onOwnerDestroyed }) + assertFrameStreamCapacity(owner) + const onOwnerDestroyed = (): void => stopFrameStream(streamId, owner) + sessions.set(streamId, { + owner, + stream, + onOwnerDestroyed, + nextDeliveryId: 1, + inFlightDeliveryId: null, + pendingFrame: null, + errorDelivered: false + }) owner.once('destroyed', onOwnerDestroyed) - stream.start() + try { + stream.start() + } catch (error) { + stopFrameStream(streamId, owner) + throw error + } return { streamId } } ) - ipcMain.handle('emulator:frameStreamStop', (_event, args: { streamId: string }) => { - stopFrameStream(args.streamId) + ipcMain.handle('emulator:frameStreamStop', (event, args: { streamId: string }) => { + stopFrameStream(args.streamId, event.sender) }) + ipcMain.on( + 'emulator:frameStreamFrameAck', + (event, args: { streamId: string; deliveryId: number }) => { + if ( + typeof args?.streamId === 'string' && + Number.isSafeInteger(args.deliveryId) && + args.deliveryId > 0 + ) { + acknowledgeFrame(event.sender, args) + } + } + ) } diff --git a/src/main/ipc/emulator-stream-listener-cleanup.test.ts b/src/main/ipc/emulator-stream-listener-cleanup.test.ts index 16e332ad14a..d5cb28e9bfa 100644 --- a/src/main/ipc/emulator-stream-listener-cleanup.test.ts +++ b/src/main/ipc/emulator-stream-listener-cleanup.test.ts @@ -9,6 +9,9 @@ vi.mock('electron', () => ({ ipcMain: { handle: (channel: string, listener: (event: unknown, args: unknown) => unknown) => { handlers.set(channel, listener) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown) => { + handlers.set(channel, listener) } }, // Any non-null return marks the sender as a real BrowserWindow renderer. @@ -23,6 +26,8 @@ vi.mock('../emulator/mjpeg-frame-stream', () => ({ })) vi.mock('../emulator/scrcpy-video-registry', () => ({ + SCRCPY_VIDEO_MAX_GOP_FRAMES: 120, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE: 32 * 1024 * 1024, scrcpyVideoRegistry: { subscribe: () => () => {} } })) diff --git a/src/main/ipc/emulator-video-frame-delivery.test.ts b/src/main/ipc/emulator-video-frame-delivery.test.ts new file mode 100644 index 00000000000..ae81b2c5431 --- /dev/null +++ b/src/main/ipc/emulator-video-frame-delivery.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from 'vitest' +import { + SCRCPY_VIDEO_MAX_GOP_FRAMES, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE, + type ScrcpyVideoFrameMessage +} from '../emulator/scrcpy-video-registry' +import { EmulatorVideoFrameDelivery } from './emulator-video-frame-delivery' + +function frame( + pts: string, + byteLength = 1, + options: { config?: boolean; keyFrame?: boolean } = {} +): ScrcpyVideoFrameMessage { + return { + config: options.config === true, + keyFrame: options.keyFrame === true, + pts, + bytes: { byteLength } as ArrayBuffer + } +} + +describe('EmulatorVideoFrameDelivery', () => { + it('bounds a full cached replay behind one in-flight renderer message', () => { + const send = vi.fn() + const delivery = new EmulatorVideoFrameDelivery(send) + const replay = [ + frame('config', 1, { config: true }), + frame('key', 1, { keyFrame: true }), + ...Array.from({ length: SCRCPY_VIDEO_MAX_GOP_FRAMES - 1 }, (_, index) => + frame(`delta-${index}`) + ) + ] + + for (const replayed of replay) { + delivery.enqueue(replayed) + } + expect(send).toHaveBeenCalledTimes(1) + + for (let deliveryId = 1; deliveryId <= replay.length; deliveryId += 1) { + delivery.acknowledge(deliveryId) + } + expect(send).toHaveBeenCalledTimes(replay.length) + expect(send.mock.calls.map(([sent]) => sent.pts)).toEqual( + replay.map((replayed) => replayed.pts) + ) + }) + + it('waits for a keyframe after a stalled renderer overruns the frame bound', () => { + const send = vi.fn() + const delivery = new EmulatorVideoFrameDelivery(send) + delivery.enqueue(frame('in-flight')) + for (let index = 0; index < SCRCPY_VIDEO_MAX_GOP_FRAMES + 2; index += 1) { + delivery.enqueue(frame(`dropped-${index}`)) + } + + delivery.acknowledge(1) + delivery.enqueue(frame('still-dropped')) + expect(send).toHaveBeenCalledTimes(1) + + delivery.enqueue(frame('config', 1, { config: true })) + delivery.enqueue(frame('recovery-key', 1, { keyFrame: true })) + expect(send.mock.calls[1][0].pts).toBe('config') + delivery.acknowledge(2) + expect(send.mock.calls[2][0].pts).toBe('recovery-key') + }) + + it('recovers from a recent queued keyframe without retaining the older GOP', () => { + const send = vi.fn() + const delivery = new EmulatorVideoFrameDelivery(send) + delivery.enqueue(frame('in-flight')) + delivery.enqueue(frame('stale-config', 1, { config: true })) + for (let index = 0; index < 100; index += 1) { + delivery.enqueue(frame(`old-${index}`)) + } + delivery.enqueue(frame('new-key', 1, { keyFrame: true })) + for (let index = 0; index < 30; index += 1) { + delivery.enqueue(frame(`new-${index}`)) + } + + delivery.acknowledge(1) + + expect(send.mock.calls[1][0].pts).toBe('new-key') + }) + + it('drops an over-byte continuation and ignores stale acknowledgements', () => { + const send = vi.fn() + const delivery = new EmulatorVideoFrameDelivery(send) + delivery.enqueue(frame('in-flight-key', 1, { keyFrame: true })) + delivery.enqueue(frame('large-1', SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE / 2 + 1)) + delivery.enqueue(frame('large-2', SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE / 2 + 1)) + + delivery.acknowledge(99) + expect(send).toHaveBeenCalledTimes(1) + delivery.acknowledge(1) + expect(send).toHaveBeenCalledTimes(1) + + delivery.enqueue(frame('recovery-key', 1, { keyFrame: true })) + expect(send.mock.calls[1][0].pts).toBe('recovery-key') + }) +}) diff --git a/src/main/ipc/emulator-video-frame-delivery.ts b/src/main/ipc/emulator-video-frame-delivery.ts new file mode 100644 index 00000000000..c06e8cbb56c --- /dev/null +++ b/src/main/ipc/emulator-video-frame-delivery.ts @@ -0,0 +1,148 @@ +import { + SCRCPY_VIDEO_MAX_GOP_FRAMES, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE, + type ScrcpyVideoFrameMessage +} from '../emulator/scrcpy-video-registry' + +const MAX_PENDING_FRAMES = SCRCPY_VIDEO_MAX_GOP_FRAMES + 1 + +export class EmulatorVideoFrameDelivery { + private nextDeliveryId = 1 + private inFlightDeliveryId: number | null = null + private pendingFrames: ScrcpyVideoFrameMessage[] = [] + private pendingBytes = 0 + private waitingForKeyFrame = false + private waitingConfig: ScrcpyVideoFrameMessage | null = null + + constructor( + private readonly send: (frame: ScrcpyVideoFrameMessage, deliveryId: number) => void + ) {} + + enqueue(frame: ScrcpyVideoFrameMessage): void { + if (this.waitingForKeyFrame) { + this.enqueueWhileWaiting(frame) + return + } + this.pendingFrames.push(frame) + this.pendingBytes += frame.bytes.byteLength + this.recoverFromOverflow() + this.flushNext() + } + + acknowledge(deliveryId: number): void { + if (this.inFlightDeliveryId !== deliveryId) { + return + } + this.inFlightDeliveryId = null + this.flushNext() + } + + clear(): void { + this.inFlightDeliveryId = null + this.pendingFrames = [] + this.pendingBytes = 0 + this.waitingForKeyFrame = false + this.waitingConfig = null + } + + private enqueueWhileWaiting(frame: ScrcpyVideoFrameMessage): void { + if (frame.config) { + this.waitingConfig = + frame.bytes.byteLength <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE ? frame : null + return + } + if (!frame.keyFrame) { + return + } + const pendingFrames = this.waitingConfig ? [this.waitingConfig, frame] : [frame] + const pendingBytes = pendingFrames.reduce( + (total, pending) => total + pending.bytes.byteLength, + 0 + ) + this.waitingConfig = null + if (pendingBytes > SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE) { + return + } + this.waitingForKeyFrame = false + this.pendingFrames = pendingFrames + this.pendingBytes = pendingBytes + this.flushNext() + } + + private recoverFromOverflow(): void { + if ( + this.pendingFrames.length <= MAX_PENDING_FRAMES && + this.pendingBytes <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE + ) { + return + } + + const keyFrameIndex = this.findNewestKeyFrame() + if (keyFrameIndex >= 0) { + const startIndex = this.findConfigBeforeKeyFrame(keyFrameIndex) + const recoverableFrames = this.pendingFrames.slice(startIndex) + const recoverableBytes = recoverableFrames.reduce( + (total, pending) => total + pending.bytes.byteLength, + 0 + ) + if ( + recoverableFrames.length <= MAX_PENDING_FRAMES && + recoverableBytes <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE + ) { + this.pendingFrames = recoverableFrames + this.pendingBytes = recoverableBytes + return + } + } + + this.waitingConfig = this.findNewestConfig() + this.pendingFrames = [] + this.pendingBytes = 0 + this.waitingForKeyFrame = true + } + + private findNewestKeyFrame(): number { + for (let index = this.pendingFrames.length - 1; index >= 0; index -= 1) { + if (this.pendingFrames[index].keyFrame) { + return index + } + } + return -1 + } + + private findConfigBeforeKeyFrame(keyFrameIndex: number): number { + let startIndex = keyFrameIndex + for (let index = keyFrameIndex - 1; index >= 0; index -= 1) { + if (!this.pendingFrames[index].config) { + break + } + startIndex = index + } + return startIndex + } + + private findNewestConfig(): ScrcpyVideoFrameMessage | null { + for (let index = this.pendingFrames.length - 1; index >= 0; index -= 1) { + const frame = this.pendingFrames[index] + if (frame.config && frame.bytes.byteLength <= SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE) { + return frame + } + } + return null + } + + private flushNext(): void { + if (this.inFlightDeliveryId !== null || this.pendingFrames.length === 0) { + return + } + const frame = this.pendingFrames.shift() + if (!frame) { + return + } + this.pendingBytes -= frame.bytes.byteLength + const deliveryId = this.nextDeliveryId + this.nextDeliveryId += 1 + this.inFlightDeliveryId = deliveryId + this.send(frame, deliveryId) + } +} diff --git a/src/main/ipc/emulator-video-stream-admission.test.ts b/src/main/ipc/emulator-video-stream-admission.test.ts new file mode 100644 index 00000000000..a2220ce03a2 --- /dev/null +++ b/src/main/ipc/emulator-video-stream-admission.test.ts @@ -0,0 +1,243 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +type VideoEvent = + | { type: 'meta'; meta: { codecId: string; width: number; height: number } } + | { + type: 'frame' + frame: { + config: boolean + keyFrame: boolean + pts: string + bytes: ArrayBuffer + } + } + +const { handlers, registryState, subscribeMock } = vi.hoisted(() => { + const state = { + error: null as Error | null, + subscribers: [] as ((event: VideoEvent) => void)[], + unsubscribes: [] as ReturnType[] + } + return { + handlers: new Map unknown>(), + registryState: state, + subscribeMock: vi.fn((_deviceId: string, subscriber: (event: VideoEvent) => void) => { + if (state.error) { + throw state.error + } + const unsubscribe = vi.fn() + state.subscribers.push(subscriber) + state.unsubscribes.push(unsubscribe) + return unsubscribe + }) + } +}) + +vi.mock('electron', () => ({ + ipcMain: { + handle: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown): void => { + handlers.set(channel, listener) + } + }, + BrowserWindow: { + fromWebContents: (owner: { isDestroyed: () => boolean }) => (owner.isDestroyed() ? null : {}) + } +})) + +vi.mock('../emulator/scrcpy-video-registry', () => ({ + SCRCPY_VIDEO_MAX_GOP_FRAMES: 120, + SCRCPY_VIDEO_MAX_REPLAY_BYTES_PER_DEVICE: 32 * 1024 * 1024, + scrcpyVideoRegistry: { subscribe: subscribeMock } +})) + +vi.mock('../emulator/emulator-probe', () => ({ emulatorProbe: vi.fn() })) + +import { + EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER, + EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_TOTAL, + registerEmulatorVideoStreamHandlers +} from './emulator-video-stream' + +type Owner = EventEmitter & { + destroy: () => void + isDestroyed: () => boolean + send: ReturnType +} + +const owners: Owner[] = [] + +function makeOwner(): Owner { + let destroyed = false + const owner = new EventEmitter() as Owner + owner.isDestroyed = () => destroyed + owner.send = vi.fn() + owner.destroy = () => { + if (destroyed) { + return + } + destroyed = true + owner.emit('destroyed') + } + owners.push(owner) + return owner +} + +function handler(channel: string): (event: unknown, args: unknown) => unknown { + const registered = handlers.get(channel) + if (!registered) { + throw new Error(`Missing handler for ${channel}`) + } + return registered +} + +function startStream(owner: Owner, streamId: string): void { + handler('emulator:videoStreamStart')({ sender: owner }, { deviceId: 'emulator-5554', streamId }) +} + +function stopStream(owner: Owner, streamId: string): void { + handler('emulator:videoStreamStop')({ sender: owner }, { streamId }) +} + +beforeEach(() => { + vi.useFakeTimers() + handlers.clear() + registryState.error = null + registryState.subscribers.length = 0 + registryState.unsubscribes.length = 0 + subscribeMock.mockClear() + registerEmulatorVideoStreamHandlers() +}) + +afterEach(() => { + for (const owner of owners) { + owner.destroy() + } + owners.length = 0 + vi.runOnlyPendingTimers() + vi.useRealTimers() +}) + +describe('emulator video stream admission', () => { + it('caps subscriptions per renderer without affecting another renderer', () => { + const owner = makeOwner() + for (let index = 0; index < EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER; index += 1) { + startStream(owner, `owned-${index}`) + } + + expect(() => startStream(owner, 'overflow')).toThrow(/renderer can have at most/) + expect(() => startStream(makeOwner(), 'other-renderer')).not.toThrow() + }) + + it('caps subscriptions across renderers before registry fan-out begins', () => { + const ownerCount = + EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_TOTAL / + EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER + for (let ownerIndex = 0; ownerIndex < ownerCount; ownerIndex += 1) { + const owner = makeOwner() + for ( + let streamIndex = 0; + streamIndex < EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER; + streamIndex += 1 + ) { + startStream(owner, `${ownerIndex}-${streamIndex}`) + } + } + + expect(() => startStream(makeOwner(), 'overflow')).toThrow(/Orca can have at most/) + expect(subscribeMock).not.toHaveBeenCalled() + }) + + it('allows owned replacement and stop without cross-renderer capacity changes', () => { + const owner = makeOwner() + const otherOwner = makeOwner() + startStream(owner, 'first') + startStream(owner, 'second') + vi.runOnlyPendingTimers() + const firstUnsubscribe = registryState.unsubscribes[0] + + startStream(owner, 'first') + expect(firstUnsubscribe).toHaveBeenCalledOnce() + expect(owner.listenerCount('destroyed')).toBe(2) + + stopStream(otherOwner, 'first') + expect(() => startStream(owner, 'overflow')).toThrow(/renderer can have at most/) + + stopStream(owner, 'first') + expect(() => startStream(owner, 'replacement')).not.toThrow() + }) + + it('releases pending subscriptions when the renderer is destroyed', () => { + const owner = makeOwner() + startStream(owner, 'first') + startStream(owner, 'second') + + owner.destroy() + vi.runOnlyPendingTimers() + + expect(owner.listenerCount('destroyed')).toBe(0) + expect(subscribeMock).not.toHaveBeenCalled() + const nextOwner = makeOwner() + expect(() => startStream(nextOwner, 'next-first')).not.toThrow() + expect(() => startStream(nextOwner, 'next-second')).not.toThrow() + }) + + it('releases admission when synchronous registry replay throws', () => { + const owner = makeOwner() + registryState.error = new Error('renderer send failed') + startStream(owner, 'failing') + + vi.runOnlyPendingTimers() + + expect(owner.listenerCount('destroyed')).toBe(0) + registryState.error = null + for (let index = 0; index < EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER; index += 1) { + expect(() => startStream(owner, `recovered-${index}`)).not.toThrow() + } + }) + + it('sends one H.264 frame at a time and requires the exact delivery token', () => { + const owner = makeOwner() + startStream(owner, 'bounded') + vi.runOnlyPendingTimers() + const subscriber = registryState.subscribers[0] + + for (let index = 0; index < 20; index += 1) { + subscriber({ + type: 'frame', + frame: { + config: false, + keyFrame: index === 0, + pts: String(index), + bytes: new ArrayBuffer(1) + } + }) + } + const frameCalls = () => + owner.send.mock.calls.filter(([channel]) => channel === 'emulator:videoStreamFrame') + expect(frameCalls()).toHaveLength(1) + const firstPayload = frameCalls()[0][1] as { + deliveryToken: string + deliveryId: number + } + + handler('emulator:videoStreamFrameAck')( + { sender: owner }, + { streamId: 'bounded', deliveryToken: 'stale', deliveryId: 1 } + ) + expect(frameCalls()).toHaveLength(1) + + handler('emulator:videoStreamFrameAck')( + { sender: owner }, + { + streamId: 'bounded', + deliveryToken: firstPayload.deliveryToken, + deliveryId: firstPayload.deliveryId + } + ) + expect(frameCalls()).toHaveLength(2) + }) +}) diff --git a/src/main/ipc/emulator-video-stream.ts b/src/main/ipc/emulator-video-stream.ts index 8bfe8367103..9bf93f124bc 100644 --- a/src/main/ipc/emulator-video-stream.ts +++ b/src/main/ipc/emulator-video-stream.ts @@ -2,6 +2,10 @@ import { BrowserWindow, ipcMain, type WebContents } from 'electron' import { randomUUID } from 'node:crypto' import { scrcpyVideoRegistry } from '../emulator/scrcpy-video-registry' import { emulatorProbe } from '../emulator/emulator-probe' +import { EmulatorVideoFrameDelivery } from './emulator-video-frame-delivery' + +export const EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_TOTAL = 8 +export const EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER = 2 // Bridges the main-process scrcpy video registry to renderer subscribers. The // renderer calls emulator:videoStreamStart with a deviceId; meta + H.264 access @@ -12,19 +16,41 @@ export function registerEmulatorVideoStreamHandlers(): void { owner: WebContents unsubscribe: () => void onOwnerDestroyed: () => void + delivery: EmulatorVideoFrameDelivery + deliveryToken: string } const subscriptions = new Map() + const assertSubscriptionCapacity = (owner: WebContents): void => { + let ownerSubscriptionCount = 0 + for (const subscription of subscriptions.values()) { + if (subscription.owner === owner) { + ownerSubscriptionCount += 1 + } + } + if (ownerSubscriptionCount >= EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER) { + throw new Error( + `A renderer can have at most ${EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_PER_RENDERER} active emulator video streams.` + ) + } + if (subscriptions.size >= EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_TOTAL) { + throw new Error( + `Orca can have at most ${EMULATOR_VIDEO_STREAM_MAX_SUBSCRIPTIONS_TOTAL} active emulator video streams.` + ) + } + } + const stopSubscription = (streamId: string, owner?: WebContents): void => { const subscription = subscriptions.get(streamId) if (!subscription || (owner && subscription.owner !== owner)) { return } - subscription.unsubscribe() + subscriptions.delete(streamId) // Why: `.once('destroyed')` self-removes only when that event fires (window // close), so an explicit stop must drop it or each show/hide cycle leaks one. subscription.owner.removeListener('destroyed', subscription.onOwnerDestroyed) - subscriptions.delete(streamId) + subscription.delivery.clear() + subscription.unsubscribe() } ipcMain.handle( @@ -44,36 +70,65 @@ export function registerEmulatorVideoStreamHandlers(): void { throw new Error('Video stream id is already in use by another renderer') } stopSubscription(streamId, owner) + assertSubscriptionCapacity(owner) const onOwnerDestroyed = (): void => stopSubscription(streamId, owner) + const deliveryToken = randomUUID() + const delivery = new EmulatorVideoFrameDelivery((frame, deliveryId) => { + const current = subscriptions.get(streamId) + if (!current || current.delivery !== delivery || owner.isDestroyed()) { + return + } + try { + owner.send('emulator:videoStreamFrame', { + streamId, + deliveryToken, + deliveryId, + deviceId: args.deviceId, + ...frame + }) + } catch { + stopSubscription(streamId, owner) + } + }) const pendingSubscription: Subscription = { owner, unsubscribe: () => {}, - onOwnerDestroyed + onOwnerDestroyed, + delivery, + deliveryToken } subscriptions.set(streamId, pendingSubscription) setTimeout(() => { if (owner.isDestroyed() || subscriptions.get(streamId) !== pendingSubscription) { return } - const unsubscribe = scrcpyVideoRegistry.subscribe(args.deviceId, (videoEvent) => { - if (owner.isDestroyed()) { + try { + const unsubscribe = scrcpyVideoRegistry.subscribe(args.deviceId, (videoEvent) => { + if (owner.isDestroyed() || subscriptions.get(streamId) !== pendingSubscription) { + return + } + if (videoEvent.type === 'meta') { + try { + owner.send('emulator:videoStreamMeta', { + streamId, + deviceId: args.deviceId, + meta: videoEvent.meta + }) + } catch { + stopSubscription(streamId, owner) + } + } else { + delivery.enqueue(videoEvent.frame) + } + }) + if (subscriptions.get(streamId) !== pendingSubscription) { + unsubscribe() return } - if (videoEvent.type === 'meta') { - owner.send('emulator:videoStreamMeta', { - streamId, - deviceId: args.deviceId, - meta: videoEvent.meta - }) - } else { - owner.send('emulator:videoStreamFrame', { - streamId, - deviceId: args.deviceId, - ...videoEvent.frame - }) - } - }) - pendingSubscription.unsubscribe = unsubscribe + pendingSubscription.unsubscribe = unsubscribe + } catch { + stopSubscription(streamId, owner) + } }, 0) owner.once('destroyed', onOwnerDestroyed) return { streamId } @@ -83,4 +138,19 @@ export function registerEmulatorVideoStreamHandlers(): void { ipcMain.handle('emulator:videoStreamStop', (event, args: { streamId: string }) => { stopSubscription(args.streamId, event.sender) }) + ipcMain.on( + 'emulator:videoStreamFrameAck', + (event, args: { streamId: string; deliveryToken: string; deliveryId: number }) => { + const subscription = + typeof args?.streamId === 'string' ? subscriptions.get(args.streamId) : undefined + if ( + subscription?.owner === event.sender && + args.deliveryToken === subscription.deliveryToken && + Number.isSafeInteger(args.deliveryId) && + args.deliveryId > 0 + ) { + subscription.delivery.acknowledge(args.deliveryId) + } + } + ) } diff --git a/src/main/ipc/ephemeral-vm-runtime-handlers.ts b/src/main/ipc/ephemeral-vm-runtime-handlers.ts index c76aacd25b6..f3ce3caec94 100644 --- a/src/main/ipc/ephemeral-vm-runtime-handlers.ts +++ b/src/main/ipc/ephemeral-vm-runtime-handlers.ts @@ -14,7 +14,6 @@ import { removeEnvironment, updateEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' -import { clearActiveRuntimeEnvironmentFocusIfMatches } from '../runtime-environment-focus-self-heal' import { cleanupEphemeralVmRuntime, resumeEphemeralVmRuntime, @@ -104,7 +103,6 @@ export function registerEphemeralVmRuntimeHandlers(store: Store): void { if (result.ok && runtime.runtimeEnvironmentId) { try { removeEnvironment(userDataPath, runtime.runtimeEnvironmentId) - clearActiveRuntimeEnvironmentFocusIfMatches(store, runtime.runtimeEnvironmentId) } catch { // Cleanup of provider resources matters more than hiding a stale local // environment row; users can still remove that manually. diff --git a/src/main/ipc/ephemeral-vm.test.ts b/src/main/ipc/ephemeral-vm.test.ts index 765b1471115..45f980bad9a 100644 --- a/src/main/ipc/ephemeral-vm.test.ts +++ b/src/main/ipc/ephemeral-vm.test.ts @@ -275,10 +275,8 @@ describe('registerEphemeralVmHandlers', () => { } as never) expect(cleaned).toEqual(expect.objectContaining({ status: 'cleaned' })) expect(listEnvironments(userDataPath)).toEqual([]) - expect(store.updateSettings).toHaveBeenLastCalledWith( - { activeRuntimeEnvironmentId: null }, - { notifyListeners: true } - ) + expect(store.getSettings().activeRuntimeEnvironmentId).toBe(result.environment!.id) + expect(store.updateSettings).toHaveBeenCalledTimes(1) }) it('provisions an ssh recipe without creating a runtime environment', async () => { diff --git a/src/main/ipc/export.ts b/src/main/ipc/export.ts index 667d3529287..83eec23bcf3 100644 --- a/src/main/ipc/export.ts +++ b/src/main/ipc/export.ts @@ -1,6 +1,7 @@ import { BrowserWindow, dialog, ipcMain } from 'electron' import { writeFile } from 'node:fs/promises' import { ExportTimeoutError, htmlToPdf } from '../lib/html-to-pdf' +import { assertHtmlToPdfInputWithinMemoryLimit } from '../lib/html-to-pdf-memory-limit' export type ExportHtmlToPdfArgs = { html: string @@ -17,11 +18,12 @@ export function registerExportHandlers(): void { 'export:html-to-pdf', async (event, args: ExportHtmlToPdfArgs): Promise => { const { html, title } = args - if (!html.trim()) { - return { success: false, error: 'No content to export' } - } try { + assertHtmlToPdfInputWithinMemoryLimit(html) + if (!html.trim()) { + return { success: false, error: 'No content to export' } + } const pdfBuffer = await htmlToPdf(html) // Why: sanitize to keep the suggested filename legal on every platform. diff --git a/src/main/ipc/feedback.test.ts b/src/main/ipc/feedback.test.ts index 52de8251d1b..a901b4face9 100644 --- a/src/main/ipc/feedback.test.ts +++ b/src/main/ipc/feedback.test.ts @@ -187,20 +187,20 @@ describe('submitFeedback', () => { expect(postedBody(1)).not.toHaveProperty('diagnosticBundle') }) - it('retries a diagnostic attachment server error as report-only JSON on the fallback API', async () => { - fetchMock.mockResolvedValueOnce(errorResponse(500)).mockResolvedValueOnce(okResponse()) + it('retries a diagnostic attachment server error as report-only JSON on the website API', async () => { + fetchMock.mockResolvedValueOnce(errorResponse(502)).mockResolvedValueOnce(okResponse()) await expect(submitFeedback(diagnosticSubmitArgs())).resolves.toEqual({ ok: true, - diagnosticBundleFailure: { status: 500, error: 'status 500' } + diagnosticBundleFailure: { status: 502, error: 'status 502' } }) - expect(fetchMock.mock.calls[1]?.[0]).toBe('https://api.onorca.dev/v1/feedback') + expect(fetchMock.mock.calls[1]?.[0]).toBe('https://www.onorca.dev/v1/feedback') expect(requestInit(1).headers).toEqual({ 'Content-Type': 'application/json' }) expect(postedBody(1)).not.toHaveProperty('diagnosticBundle') }) - it('retries a diagnostic attachment network error as report-only JSON on the fallback API', async () => { + it('retries a diagnostic attachment network error as report-only JSON on the website API', async () => { fetchMock.mockRejectedValueOnce(new Error('attachment network failed')) fetchMock.mockResolvedValueOnce(okResponse()) @@ -210,7 +210,7 @@ describe('submitFeedback', () => { }) expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1]?.[0]).toBe('https://api.onorca.dev/v1/feedback') + expect(fetchMock.mock.calls[1]?.[0]).toBe('https://www.onorca.dev/v1/feedback') expect(requestInit(1).body).not.toBeInstanceOf(FormData) expect(postedBody(1)).not.toHaveProperty('diagnosticBundle') }) @@ -234,7 +234,7 @@ describe('submitFeedback', () => { diagnosticBundleFailure: { status: null, error: 'request timed out after 60 seconds' } }) expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1]?.[0]).toBe('https://api.onorca.dev/v1/feedback') + expect(fetchMock.mock.calls[1]?.[0]).toBe('https://www.onorca.dev/v1/feedback') expect(postedBody(1)).not.toHaveProperty('diagnosticBundle') }) @@ -276,16 +276,14 @@ describe('submitFeedback', () => { expect(fetchMock).toHaveBeenCalledTimes(2) }) - it('falls back when the primary feedback request stalls', async () => { + it('retries the website API when the primary feedback request stalls', async () => { vi.useFakeTimers() - fetchMock.mockImplementation((url: string, init?: RequestInit) => { - if (url.includes('www.onorca.dev')) { - return new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => reject(new Error('request aborted'))) - }) - } - return Promise.resolve(okResponse()) + fetchMock.mockImplementationOnce((_url: string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('request aborted'))) + }) }) + fetchMock.mockResolvedValueOnce(okResponse()) const result = submitFeedback({ feedback: 'stalled primary', @@ -297,21 +295,38 @@ describe('submitFeedback', () => { await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toEqual({ ok: true }) expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://www.onorca.dev/v1/feedback', + 'https://www.onorca.dev/v1/feedback' + ]) }) - it('does not retry the fallback when the fallback fails after a primary server error', async () => { + it('does not retry a non-diagnostic 404', async () => { + fetchMock.mockResolvedValueOnce(errorResponse(404)) + + await expect( + submitFeedback({ + feedback: 'missing feedback route', + submitAnonymously: true, + githubLogin: null, + githubEmail: null + }) + ).resolves.toEqual({ ok: false, status: 404, error: 'status 404' }) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock.mock.calls[0]?.[0]).toBe('https://www.onorca.dev/v1/feedback') + }) + + it('does not retry again when the website retry stalls after a primary server error', async () => { vi.useFakeTimers() - fetchMock.mockImplementation((url: string, init?: RequestInit) => { - if (url.includes('www.onorca.dev')) { - return Promise.resolve({ ok: false, status: 500 } as Response) - } + fetchMock.mockResolvedValueOnce(errorResponse(500)) + fetchMock.mockImplementationOnce((_url: string, init?: RequestInit) => { return new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => reject(new Error('fallback aborted'))) + init?.signal?.addEventListener('abort', () => reject(new Error('retry aborted'))) }) }) const result = submitFeedback({ - feedback: 'primary 500 and fallback stalled', + feedback: 'primary 500 and retry stalled', submitAnonymously: false, githubLogin: 'trusted-user', githubEmail: 'trusted@example.com' @@ -321,7 +336,29 @@ describe('submitFeedback', () => { await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toEqual({ ok: false, status: null, - error: 'request timed out after 10 seconds' + error: 'status 500; retry: request timed out after 10 seconds' + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://www.onorca.dev/v1/feedback', + 'https://www.onorca.dev/v1/feedback' + ]) + }) + + it('preserves the primary status when a same-host retry also returns a server error', async () => { + fetchMock.mockResolvedValueOnce(errorResponse(502)).mockResolvedValueOnce(errorResponse(503)) + + await expect( + submitFeedback({ + feedback: 'primary and retry both server errors', + submitAnonymously: true, + githubLogin: null, + githubEmail: null + }) + ).resolves.toEqual({ + ok: false, + status: 503, + error: 'status 502; retry: status 503' }) expect(fetchMock).toHaveBeenCalledTimes(2) }) diff --git a/src/main/ipc/feedback.ts b/src/main/ipc/feedback.ts index d9fa48e3c47..09d8b5db071 100644 --- a/src/main/ipc/feedback.ts +++ b/src/main/ipc/feedback.ts @@ -7,7 +7,6 @@ import { app, ipcMain, net } from 'electron' // subject to CORS, so we proxy the submission through IPC. This mirrors the // same pattern used by updater-changelog.ts and updater-nudge.ts. const FEEDBACK_API_URL = 'https://www.onorca.dev/v1/feedback' -const FEEDBACK_API_FALLBACK_URL = 'https://api.onorca.dev/v1/feedback' const FEEDBACK_REQUEST_TIMEOUT_MS = 10_000 const FEEDBACK_ATTACHMENT_REQUEST_TIMEOUT_MS = 60_000 const DIAGNOSTIC_BUNDLE_CONTENT_TYPE = 'application/x-ndjson' @@ -171,46 +170,49 @@ function errorFailure(error: unknown): FeedbackRequestFailure { return { status: null, error: messageFromError(error) } } -async function submitFallbackFeedback( +async function retryFeedbackOnPrimary( body: FeedbackSubmitBody, primaryError?: unknown ): Promise { try { - const fallback = await postFeedback(FEEDBACK_API_FALLBACK_URL, body) - if (fallback.ok) { + const retry = await postFeedback(FEEDBACK_API_URL, body) + if (retry.ok) { return { ok: true } } - return { ok: false, status: fallback.status, error: `status ${fallback.status}` } - } catch (fallbackError) { - const message = messageFromError(fallbackError) + const retryMessage = `status ${retry.status}` + if (primaryError === undefined) { + return { ok: false, status: retry.status, error: retryMessage } + } + // Why: keep the first failure visible so support can see 5xx → retry outcome, + // not only the last error in a same-host retry chain. + return { + ok: false, + status: retry.status, + error: `${messageFromError(primaryError)}; retry: ${retryMessage}` + } + } catch (retryError) { + const message = messageFromError(retryError) if (primaryError === undefined) { return { ok: false, status: null, error: message } } return { ok: false, status: null, - error: `${messageFromError(primaryError)}; fallback: ${message}` + error: `${messageFromError(primaryError)}; retry: ${message}` } } } -function diagnosticRetryUrl(status: number): string | null { - if (DIAGNOSTIC_BUNDLE_JSON_RETRY_STATUSES.has(status)) { - return FEEDBACK_API_URL - } - if (status === 404 || status >= 500) { - return FEEDBACK_API_FALLBACK_URL - } - return null +function shouldRetryWithoutDiagnosticBundle(status: number): boolean { + return DIAGNOSTIC_BUNDLE_JSON_RETRY_STATUSES.has(status) || status === 404 || status >= 500 } async function submitFeedbackWithoutDiagnosticBundle( - url: string, body: FeedbackSubmitBody, diagnosticBundleFailure: FeedbackRequestFailure ): Promise { try { - const response = await postFeedback(url, body) + const response = await postFeedback(FEEDBACK_API_URL, body) if (response.ok) { return { ok: true, diagnosticBundleFailure } } @@ -236,21 +238,14 @@ async function submitFeedbackWithDiagnosticBundle( return { ok: true } } const failure = responseFailure(response) - if (bodyWithoutDiagnosticBundle) { - const retryUrl = diagnosticRetryUrl(response.status) - if (retryUrl) { - return submitFeedbackWithoutDiagnosticBundle(retryUrl, bodyWithoutDiagnosticBundle, failure) - } + if (bodyWithoutDiagnosticBundle && shouldRetryWithoutDiagnosticBundle(response.status)) { + return submitFeedbackWithoutDiagnosticBundle(bodyWithoutDiagnosticBundle, failure) } return { ok: false, ...failure } } catch (error) { const failure = errorFailure(error) return bodyWithoutDiagnosticBundle - ? submitFeedbackWithoutDiagnosticBundle( - FEEDBACK_API_FALLBACK_URL, - bodyWithoutDiagnosticBundle, - failure - ) + ? submitFeedbackWithoutDiagnosticBundle(bodyWithoutDiagnosticBundle, failure) : { ok: false, ...failure } } } @@ -275,17 +270,14 @@ export async function submitFeedback( if (res.ok) { return { ok: true } } - // Why: keep api.onorca.dev as a compatibility fallback, but prefer the - // website API because it owns the Slack file/snippet crash delivery path. - if (res.status === 404 || res.status >= 500) { - return submitFallbackFeedback(body) + // Why: api.onorca.dev serves a different product, so transient failures + // retry the endpoint that owns feedback and crash delivery. + if (res.status >= 500) { + return retryFeedbackOnPrimary(body, new Error(`status ${res.status}`)) } return { ok: false, status: res.status, error: `status ${res.status}` } } catch (error) { - // Why: falling back on any network-level failure preserves the prior - // behavior where DNS/connect failures on the primary host transparently - // try the legacy API endpoint. - return submitFallbackFeedback(body, error) + return retryFeedbackOnPrimary(body, error) } } diff --git a/src/main/ipc/filesystem-directory-reader.test.ts b/src/main/ipc/filesystem-directory-reader.test.ts new file mode 100644 index 00000000000..833e572a33d --- /dev/null +++ b/src/main/ipc/filesystem-directory-reader.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' +import { FILESYSTEM_DIRECTORY_LIMIT_MESSAGE } from '../../shared/filesystem-directory-listing-limit' +import { collectLocalFilesystemDirectoryEntries } from './filesystem-directory-reader' + +type TestEntryOptions = { + name: string + directory?: boolean + symlink?: boolean + onClassify?: () => void +} + +function entry(options: TestEntryOptions) { + return { + name: options.name, + isDirectory: () => { + options.onClassify?.() + return options.directory ?? false + }, + isSymbolicLink: () => options.symlink ?? false + } +} + +describe('local filesystem directory reader', () => { + it('preserves directory-first sorting and keeps symlinks file-like', async () => { + const source = [ + entry({ name: 'z.txt' }), + entry({ name: 'linked-dir', directory: true, symlink: true }), + entry({ name: 'beta', directory: true }), + entry({ name: 'alpha', directory: true }) + ] + + await expect(collectLocalFilesystemDirectoryEntries(source)).resolves.toEqual([ + { name: 'alpha', isDirectory: true, isSymlink: false }, + { name: 'beta', isDirectory: true, isSymlink: false }, + { name: 'linked-dir', isDirectory: false, isSymlink: true }, + { name: 'z.txt', isDirectory: false, isSymlink: false } + ]) + }) + + it('stops before classifying or retaining the first over-limit entry', async () => { + let enumerated = 0 + let overLimitClassified = false + const source = { + async *[Symbol.asyncIterator]() { + for (const value of [ + entry({ name: 'one' }), + entry({ name: 'two' }), + entry({ name: 'three', onClassify: () => (overLimitClassified = true) }) + ]) { + enumerated += 1 + yield value + } + } + } + + await expect( + collectLocalFilesystemDirectoryEntries(source, { + maxEntries: 2, + maxRetainedBytes: 1024 + }) + ).rejects.toThrow(FILESYSTEM_DIRECTORY_LIMIT_MESSAGE) + expect(enumerated).toBe(3) + expect(overLimitClassified).toBe(false) + }) +}) diff --git a/src/main/ipc/filesystem-directory-reader.ts b/src/main/ipc/filesystem-directory-reader.ts new file mode 100644 index 00000000000..3977f1accba --- /dev/null +++ b/src/main/ipc/filesystem-directory-reader.ts @@ -0,0 +1,41 @@ +import { opendir } from 'node:fs/promises' +import type { DirEntry } from '../../shared/types' +import { + createFilesystemDirectoryLimitState, + trackFilesystemDirectoryEntry, + type FilesystemDirectoryListingLimits +} from '../../shared/filesystem-directory-listing-limit' + +type LocalDirectorySourceEntry = { + name: string + isDirectory(): boolean + isSymbolicLink(): boolean +} + +export async function readLocalFilesystemDirectory(dirPath: string): Promise { + return collectLocalFilesystemDirectoryEntries(await opendir(dirPath)) +} + +export async function collectLocalFilesystemDirectoryEntries( + directory: AsyncIterable | Iterable, + requestedLimits?: Partial +): Promise { + const entries: DirEntry[] = [] + const limit = createFilesystemDirectoryLimitState(requestedLimits) + for await (const entry of directory) { + trackFilesystemDirectoryEntry(limit, entry) + const isSymlink = entry.isSymbolicLink() + entries.push({ + name: entry.name, + // Why: avoid probing macOS TCC-protected symlink targets during listing. + isDirectory: !isSymlink && entry.isDirectory(), + isSymlink + }) + } + return entries.sort((left, right) => { + if (left.isDirectory !== right.isDirectory) { + return left.isDirectory ? -1 : 1 + } + return left.name.localeCompare(right.name) + }) +} diff --git a/src/main/ipc/filesystem-external-import-limits.test.ts b/src/main/ipc/filesystem-external-import-limits.test.ts new file mode 100644 index 00000000000..66079a8344d --- /dev/null +++ b/src/main/ipc/filesystem-external-import-limits.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from 'vitest' +import { + admitExternalImportTreeEntry, + assertExternalImportSourcePaths, + assertExternalImportTreeDepth, + captureRuntimeUploadRetentionCheckpoint, + createExternalImportTreeBudget, + createRuntimeUploadRetentionBudget, + EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES, + EXTERNAL_IMPORT_MAX_SOURCE_PATH_BYTES, + EXTERNAL_IMPORT_MAX_SOURCE_PATHS, + EXTERNAL_IMPORT_MAX_TREE_DEPTH, + EXTERNAL_IMPORT_MAX_TREE_ENTRIES, + ExternalImportCapacityError, + REMOTE_IMPORT_MAX_FILE_BYTES, + REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES, + REMOTE_IMPORT_MAX_TOTAL_BYTES, + restoreRuntimeUploadRetentionCheckpoint, + retainRuntimeUploadFileBytes +} from './filesystem-external-import-limits' + +describe('external filesystem import limits', () => { + it('accepts the exact source-count boundary and rejects the next path', () => { + expect(() => + assertExternalImportSourcePaths( + Array.from({ length: EXTERNAL_IMPORT_MAX_SOURCE_PATHS }, () => '') + ) + ).not.toThrow() + expect(() => + assertExternalImportSourcePaths( + Array.from({ length: EXTERNAL_IMPORT_MAX_SOURCE_PATHS + 1 }, () => '') + ) + ).toThrow('External import accepts at most 256 source paths') + }) + + it('accepts the exact source-path byte boundary and rejects one byte more', () => { + expect(() => + assertExternalImportSourcePaths(['a'.repeat(EXTERNAL_IMPORT_MAX_SOURCE_PATH_BYTES)]) + ).not.toThrow() + expect(() => + assertExternalImportSourcePaths(['a'.repeat(EXTERNAL_IMPORT_MAX_SOURCE_PATH_BYTES + 1)]) + ).toThrow('External import source paths exceed 256 KiB') + }) + + it('accepts exactly 100,000 tree entries without retaining their paths', () => { + const budget = createExternalImportTreeBudget() + for (let index = 0; index < EXTERNAL_IMPORT_MAX_TREE_ENTRIES; index += 1) { + admitExternalImportTreeEntry(budget, 'entry', false) + } + + expect(budget).toEqual({ + entries: EXTERNAL_IMPORT_MAX_TREE_ENTRIES, + retainedPathBytes: 0 + }) + expect(() => admitExternalImportTreeEntry(budget, 'overflow', false)).toThrow( + 'External import tree exceeds 100,000 entries' + ) + expect(budget.entries).toBe(EXTERNAL_IMPORT_MAX_TREE_ENTRIES) + }) + + it('accepts the exact depth and per-path boundaries', () => { + const budget = createExternalImportTreeBudget() + expect(() => assertExternalImportTreeDepth(EXTERNAL_IMPORT_MAX_TREE_DEPTH)).not.toThrow() + expect(() => assertExternalImportTreeDepth(EXTERNAL_IMPORT_MAX_TREE_DEPTH + 1)).toThrow( + 'External import tree exceeds 256 nested directory levels' + ) + expect(() => + admitExternalImportTreeEntry( + budget, + 'a'.repeat(EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES), + false + ) + ).not.toThrow() + expect(() => + admitExternalImportTreeEntry( + budget, + 'a'.repeat(EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES + 1), + false + ) + ).toThrow('External import relative path exceeds 64 KiB') + }) + + it('accepts the exact aggregate retained-path boundary', () => { + const budget = createExternalImportTreeBudget() + const path = 'a'.repeat(EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES) + const pathCount = + REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES / (EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES * 2) + for (let index = 0; index < pathCount; index += 1) { + admitExternalImportTreeEntry(budget, path, true) + } + + expect(budget.retainedPathBytes).toBe(REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES) + expect(() => admitExternalImportTreeEntry(budget, 'b', true)).toThrow( + 'Remote import retained paths exceed 16 MiB' + ) + expect(budget.retainedPathBytes).toBe(REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES) + }) + + it('applies file and total byte limits across the full runtime-upload request', () => { + const budget = createRuntimeUploadRetentionBudget() + const exactFileCount = REMOTE_IMPORT_MAX_TOTAL_BYTES / REMOTE_IMPORT_MAX_FILE_BYTES + for (let index = 0; index < exactFileCount; index += 1) { + retainRuntimeUploadFileBytes(budget, `file-${index}`, REMOTE_IMPORT_MAX_FILE_BYTES) + } + + expect(budget.fileBytes).toBe(REMOTE_IMPORT_MAX_TOTAL_BYTES) + expect(() => retainRuntimeUploadFileBytes(budget, 'overflow', 1)).toThrow( + 'Remote import is too large' + ) + expect(budget.fileBytes).toBe(REMOTE_IMPORT_MAX_TOTAL_BYTES) + }) + + it('rejects one oversized file without consuming request capacity', () => { + const budget = createRuntimeUploadRetentionBudget() + + expect(() => + retainRuntimeUploadFileBytes(budget, 'large.bin', REMOTE_IMPORT_MAX_FILE_BYTES + 1) + ).toThrow("'large.bin' is too large for remote import") + expect(budget.fileBytes).toBe(0) + }) + + it('restores capacity reserved by a failed staged source', () => { + const budget = createRuntimeUploadRetentionBudget() + const checkpoint = captureRuntimeUploadRetentionCheckpoint(budget) + admitExternalImportTreeEntry(budget.tree, 'partial.txt', true) + retainRuntimeUploadFileBytes(budget, 'partial.txt', 1024) + + restoreRuntimeUploadRetentionCheckpoint(budget, checkpoint) + + expect(budget).toEqual({ + tree: { entries: 0, retainedPathBytes: 0 }, + fileBytes: 0 + }) + }) + + it('uses a typed error for every capacity rejection', () => { + const budget = createExternalImportTreeBudget() + + expect(() => + admitExternalImportTreeEntry( + budget, + 'a'.repeat(EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES + 1), + false + ) + ).toThrow(ExternalImportCapacityError) + }) +}) diff --git a/src/main/ipc/filesystem-external-import-limits.ts b/src/main/ipc/filesystem-external-import-limits.ts new file mode 100644 index 00000000000..542cb12e04c --- /dev/null +++ b/src/main/ipc/filesystem-external-import-limits.ts @@ -0,0 +1,157 @@ +import { + NATIVE_FILE_DROP_MAX_PATH_BYTES, + NATIVE_FILE_DROP_MAX_PATHS +} from '../../shared/native-file-drop' + +export const EXTERNAL_IMPORT_MAX_SOURCE_PATHS = NATIVE_FILE_DROP_MAX_PATHS +export const EXTERNAL_IMPORT_MAX_SOURCE_PATH_BYTES = NATIVE_FILE_DROP_MAX_PATH_BYTES +export const EXTERNAL_IMPORT_MAX_TREE_ENTRIES = 100_000 +export const EXTERNAL_IMPORT_MAX_TREE_DEPTH = 256 +export const EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES = 64 * 1024 +export const REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES = 16 * 1024 * 1024 +export const REMOTE_IMPORT_MAX_FILE_BYTES = 25 * 1024 * 1024 +export const REMOTE_IMPORT_MAX_TOTAL_BYTES = 100 * 1024 * 1024 + +export const EXTERNAL_IMPORT_TREE_ENTRY_LIMIT_MESSAGE = + 'External import tree exceeds 100,000 entries' +export const EXTERNAL_IMPORT_TREE_DEPTH_LIMIT_MESSAGE = + 'External import tree exceeds 256 nested directory levels' +export const EXTERNAL_IMPORT_RELATIVE_PATH_LIMIT_MESSAGE = + 'External import relative path exceeds 64 KiB' +export const REMOTE_IMPORT_RETAINED_PATH_LIMIT_MESSAGE = + 'Remote import retained paths exceed 16 MiB' + +export type ExternalImportTreeBudget = { + entries: number + retainedPathBytes: number +} + +export type RuntimeUploadRetentionBudget = { + tree: ExternalImportTreeBudget + fileBytes: number +} + +export type RuntimeUploadRetentionCheckpoint = { + entries: number + retainedPathBytes: number + fileBytes: number +} + +export class ExternalImportCapacityError extends Error { + constructor(message: string) { + super(message) + this.name = 'ExternalImportCapacityError' + } +} + +export function assertExternalImportSourcePaths( + sourcePaths: unknown +): asserts sourcePaths is readonly string[] { + if (!Array.isArray(sourcePaths)) { + throw new TypeError('External import source paths must be an array') + } + if (sourcePaths.length > EXTERNAL_IMPORT_MAX_SOURCE_PATHS) { + throw new ExternalImportCapacityError( + `External import accepts at most ${EXTERNAL_IMPORT_MAX_SOURCE_PATHS} source paths` + ) + } + + let pathBytes = 0 + for (const sourcePath of sourcePaths) { + if (typeof sourcePath !== 'string') { + throw new TypeError('External import source paths must be strings') + } + pathBytes += Buffer.byteLength(sourcePath, 'utf8') + if (pathBytes > EXTERNAL_IMPORT_MAX_SOURCE_PATH_BYTES) { + throw new ExternalImportCapacityError('External import source paths exceed 256 KiB') + } + } +} + +export function createExternalImportTreeBudget(): ExternalImportTreeBudget { + return { entries: 0, retainedPathBytes: 0 } +} + +export function assertExternalImportTreeDepth(depth: number): void { + if (depth > EXTERNAL_IMPORT_MAX_TREE_DEPTH) { + throw new ExternalImportCapacityError(EXTERNAL_IMPORT_TREE_DEPTH_LIMIT_MESSAGE) + } +} + +export function admitExternalImportTreeEntry( + budget: ExternalImportTreeBudget, + relativePath: string, + retainPath: boolean +): void { + if (budget.entries >= EXTERNAL_IMPORT_MAX_TREE_ENTRIES) { + throw new ExternalImportCapacityError(EXTERNAL_IMPORT_TREE_ENTRY_LIMIT_MESSAGE) + } + + const pathBytes = Buffer.byteLength(relativePath, 'utf8') + if (pathBytes > EXTERNAL_IMPORT_MAX_RELATIVE_PATH_BYTES) { + throw new ExternalImportCapacityError(EXTERNAL_IMPORT_RELATIVE_PATH_LIMIT_MESSAGE) + } + const retainedPathBytes = relativePath.length * 2 + if ( + retainPath && + retainedPathBytes > REMOTE_IMPORT_MAX_RETAINED_PATH_BYTES - budget.retainedPathBytes + ) { + throw new ExternalImportCapacityError(REMOTE_IMPORT_RETAINED_PATH_LIMIT_MESSAGE) + } + + budget.entries += 1 + if (retainPath) { + budget.retainedPathBytes += retainedPathBytes + } +} + +export function createRuntimeUploadRetentionBudget(): RuntimeUploadRetentionBudget { + return { + tree: createExternalImportTreeBudget(), + fileBytes: 0 + } +} + +export function captureRuntimeUploadRetentionCheckpoint( + budget: RuntimeUploadRetentionBudget +): RuntimeUploadRetentionCheckpoint { + return { + entries: budget.tree.entries, + retainedPathBytes: budget.tree.retainedPathBytes, + fileBytes: budget.fileBytes + } +} + +export function restoreRuntimeUploadRetentionCheckpoint( + budget: RuntimeUploadRetentionBudget, + checkpoint: RuntimeUploadRetentionCheckpoint +): void { + budget.tree.entries = checkpoint.entries + budget.tree.retainedPathBytes = checkpoint.retainedPathBytes + budget.fileBytes = checkpoint.fileBytes +} + +export function assertRuntimeUploadFileFits( + budget: RuntimeUploadRetentionBudget, + relativePath: string, + fileBytes: number +): void { + if (!Number.isSafeInteger(fileBytes) || fileBytes < 0) { + throw new Error(`Could not safely measure '${relativePath}' for remote import`) + } + if (fileBytes > REMOTE_IMPORT_MAX_FILE_BYTES) { + throw new ExternalImportCapacityError(`'${relativePath}' is too large for remote import`) + } + if (fileBytes > REMOTE_IMPORT_MAX_TOTAL_BYTES - budget.fileBytes) { + throw new ExternalImportCapacityError('Remote import is too large') + } +} + +export function retainRuntimeUploadFileBytes( + budget: RuntimeUploadRetentionBudget, + relativePath: string, + fileBytes: number +): void { + assertRuntimeUploadFileFits(budget, relativePath, fileBytes) + budget.fileBytes += fileBytes +} diff --git a/src/main/ipc/filesystem-import-ssh-directory.ts b/src/main/ipc/filesystem-import-ssh-directory.ts index d4dc794a7cc..f38cf4d43d6 100644 --- a/src/main/ipc/filesystem-import-ssh-directory.ts +++ b/src/main/ipc/filesystem-import-ssh-directory.ts @@ -1,7 +1,13 @@ -import { lstat, readdir, realpath } from 'node:fs/promises' +import { lstat, opendir, realpath } from 'node:fs/promises' import { isAbsolute, join, relative, sep } from 'node:path' import type { FileUploadSession, IFilesystemProvider } from '../providers/types' import { assertSafeRemotePathSegment, type RemotePathFlavor } from '../ssh/ssh-remote-platform' +import { + admitExternalImportTreeEntry, + assertExternalImportTreeDepth, + createExternalImportTreeBudget, + type ExternalImportTreeBudget +} from './filesystem-external-import-limits' export async function captureLocalUploadRoot( sourcePath: string, @@ -23,18 +29,47 @@ export async function preScanSshImportDirectory( dirPath: string, remotePathFlavor: RemotePathFlavor ): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) - for (const entry of entries) { - assertSafeRemotePathSegment(entry.name, remotePathFlavor) - if (entry.isSymbolicLink()) { - return true - } - if (entry.isDirectory()) { - const childPath = join(dirPath, entry.name) - if (await preScanSshImportDirectory(childPath, remotePathFlavor)) { + return preScanSshImportDirectoryWithinBudget( + dirPath, + remotePathFlavor, + createExternalImportTreeBudget(), + '', + 0 + ) +} + +async function preScanSshImportDirectoryWithinBudget( + dirPath: string, + remotePathFlavor: RemotePathFlavor, + budget: ExternalImportTreeBudget, + relativeDir: string, + depth: number +): Promise { + assertExternalImportTreeDepth(depth) + const directory = await opendir(dirPath) + try { + for await (const entry of directory) { + assertSafeRemotePathSegment(entry.name, remotePathFlavor) + const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name + admitExternalImportTreeEntry(budget, relativePath, false) + if (entry.isSymbolicLink()) { + return true + } + if ( + entry.isDirectory() && + (await preScanSshImportDirectoryWithinBudget( + join(dirPath, entry.name), + remotePathFlavor, + budget, + relativePath, + depth + 1 + )) + ) { return true } } + } finally { + await directory.close().catch(() => undefined) } return false } @@ -48,37 +83,72 @@ export async function uploadSshImportDirectory( remotePathFlavor: RemotePathFlavor, assertCurrent?: () => void ): Promise { + await uploadSshImportDirectoryWithinBudget( + provider, + uploadSession, + localDir, + remoteDir, + rootRealPath, + remotePathFlavor, + createExternalImportTreeBudget(), + '', + 0, + assertCurrent + ) +} + +async function uploadSshImportDirectoryWithinBudget( + provider: IFilesystemProvider, + uploadSession: FileUploadSession, + localDir: string, + remoteDir: string, + rootRealPath: string, + remotePathFlavor: RemotePathFlavor, + budget: ExternalImportTreeBudget, + relativeDir: string, + depth: number, + assertCurrent?: () => void +): Promise { + assertExternalImportTreeDepth(depth) await assertLocalUploadPathInsideRoot(rootRealPath, localDir) - const entries = await readdir(localDir, { withFileTypes: true }) - for (const entry of entries) { - assertSafeRemotePathSegment(entry.name, remotePathFlavor) - const localPath = join(localDir, entry.name) - const remotePath = `${remoteDir}/${entry.name}` - await assertLocalUploadPathInsideRoot(rootRealPath, localPath) - const statResult = await lstat(localPath) + const directory = await opendir(localDir) + try { + for await (const entry of directory) { + assertSafeRemotePathSegment(entry.name, remotePathFlavor) + const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name + admitExternalImportTreeEntry(budget, relativePath, false) + const localPath = join(localDir, entry.name) + const remotePath = `${remoteDir}/${entry.name}` + await assertLocalUploadPathInsideRoot(rootRealPath, localPath) + const statResult = await lstat(localPath) - // Why: skip symlinks and special files even after the up-front pre-scan; - // this closes the TOCTOU gap if one is created during upload. - if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { - continue - } + // Why: the up-front scan cannot prevent a source swap during upload. + if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { + continue + } - if (statResult.isDirectory()) { + if (statResult.isDirectory()) { + assertCurrent?.() + await provider.createDirNoClobber(remotePath) + await uploadSshImportDirectoryWithinBudget( + provider, + uploadSession, + localPath, + remotePath, + rootRealPath, + remotePathFlavor, + budget, + relativePath, + depth + 1, + assertCurrent + ) + continue + } assertCurrent?.() - await provider.createDirNoClobber(remotePath) - await uploadSshImportDirectory( - provider, - uploadSession, - localPath, - remotePath, - rootRealPath, - remotePathFlavor, - assertCurrent - ) - continue + await uploadSession.uploadFile(localPath, remotePath, { exclusive: true }) } - assertCurrent?.() - await uploadSession.uploadFile(localPath, remotePath, { exclusive: true }) + } finally { + await directory.close().catch(() => undefined) } } diff --git a/src/main/ipc/filesystem-import-ssh-ops.test.ts b/src/main/ipc/filesystem-import-ssh-ops.test.ts index 2c36fd9f2bb..95b5c4f0cdd 100644 --- a/src/main/ipc/filesystem-import-ssh-ops.test.ts +++ b/src/main/ipc/filesystem-import-ssh-ops.test.ts @@ -9,6 +9,7 @@ const { mkdirMock, realpathMock, copyFileMock, + opendirMock, readdirMock, getConnMgrMock } = vi.hoisted(() => ({ @@ -17,6 +18,7 @@ const { mkdirMock: vi.fn(), realpathMock: vi.fn(), copyFileMock: vi.fn(), + opendirMock: vi.fn(), readdirMock: vi.fn(), getConnMgrMock: vi.fn() })) @@ -29,14 +31,15 @@ vi.mock('fs/promises', () => ({ writeFile: vi.fn(), realpath: realpathMock, copyFile: copyFileMock, - readdir: readdirMock + opendir: opendirMock })) vi.mock('./ssh', () => ({ getSshConnectionManager: getConnMgrMock })) import { registerFilesystemMutationHandlers } from './filesystem-mutations' import { advanceSshConnectionGeneration, - resetSshConnectionGenerations + resetSshConnectionGenerations, + setSshConnectionGeneration } from '../ssh/ssh-connection-generation' import { registerSshFilesystemProvider, @@ -82,6 +85,7 @@ function createProvider(uploadSession: FileUploadSession): IFilesystemProvider { } describe('fs:importExternalPaths — SSH operations', () => { + const sessionCounterStride = 2 ** 13 const destDir = '/home/user/project/src' const connId = 'ssh-conn-1' let provider: IFilesystemProvider @@ -130,12 +134,15 @@ describe('fs:importExternalPaths — SSH operations', () => { beforeEach(() => { handlers.clear() + resetSshConnectionGenerations() + setSshConnectionGeneration(connId, 0) ;[ handleMock, lstatMock, mkdirMock, realpathMock, copyFileMock, + opendirMock, readdirMock, getConnMgrMock ].forEach((m) => m.mockReset()) @@ -145,6 +152,12 @@ describe('fs:importExternalPaths — SSH operations', () => { realpathMock.mockImplementation(async (p: string) => p) lstatMock.mockRejectedValue(enoent()) readdirMock.mockResolvedValue([]) + opendirMock.mockImplementation(async (directoryPath: string) => ({ + async *[Symbol.asyncIterator]() { + yield* await readdirMock(directoryPath, { withFileTypes: true }) + }, + close: vi.fn().mockResolvedValue(undefined) + })) getConnMgrMock.mockReturnValue({ getConnection: () => makeConn() }) uploadSession = { uploadFile: vi.fn().mockResolvedValue(undefined), @@ -162,6 +175,7 @@ describe('fs:importExternalPaths — SSH operations', () => { it('rejects a staged upload when a restarted HUB reaches the same target counter', async () => { resetSshConnectionGenerations(71) + setSshConnectionGeneration(connId, 71 * sessionCounterStride) const stagedGeneration = advanceSshConnectionGeneration(connId) const sourcePath = path.resolve('/tmp/dropped/restart.txt') lstatMock.mockImplementation(async (candidate: string) => { @@ -169,6 +183,7 @@ describe('fs:importExternalPaths — SSH operations', () => { throw enoent() } resetSshConnectionGenerations(72) + setSshConnectionGeneration(connId, 72 * sessionCounterStride) advanceSshConnectionGeneration(connId) return { size: 12, diff --git a/src/main/ipc/filesystem-import-ssh-path-safety.test.ts b/src/main/ipc/filesystem-import-ssh-path-safety.test.ts index ec01175ad35..8d1e5de545b 100644 --- a/src/main/ipc/filesystem-import-ssh-path-safety.test.ts +++ b/src/main/ipc/filesystem-import-ssh-path-safety.test.ts @@ -2,17 +2,19 @@ import path from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { FileUploadSession, IFilesystemProvider } from '../providers/types' -const { getConnMgrMock, lstatMock, providerMock, readdirMock, realpathMock } = vi.hoisted(() => ({ - getConnMgrMock: vi.fn(), - lstatMock: vi.fn(), - providerMock: vi.fn(), - readdirMock: vi.fn(), - realpathMock: vi.fn() -})) +const { getConnMgrMock, lstatMock, opendirMock, providerMock, readdirMock, realpathMock } = + vi.hoisted(() => ({ + getConnMgrMock: vi.fn(), + lstatMock: vi.fn(), + opendirMock: vi.fn(), + providerMock: vi.fn(), + readdirMock: vi.fn(), + realpathMock: vi.fn() + })) vi.mock('node:fs/promises', () => ({ lstat: lstatMock, - readdir: readdirMock, + opendir: opendirMock, realpath: realpathMock })) vi.mock('./filesystem-auth', () => ({ @@ -25,6 +27,7 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ })) import { importExternalPathsSsh } from './filesystem-import-ssh' +import { EXTERNAL_IMPORT_MAX_TREE_ENTRIES } from './filesystem-external-import-limits' function createProvider(uploadSession: FileUploadSession): IFilesystemProvider { const missing = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) @@ -54,6 +57,12 @@ describe('SSH import remote path safety', () => { provider = createProvider(uploadSession) providerMock.mockReturnValue(provider) readdirMock.mockResolvedValue([]) + opendirMock.mockImplementation(async (directoryPath: string) => ({ + async *[Symbol.asyncIterator]() { + yield* await readdirMock(directoryPath, { withFileTypes: true }) + }, + close: vi.fn().mockResolvedValue(undefined) + })) realpathMock.mockImplementation(async (value: string) => value) }) @@ -189,6 +198,47 @@ describe('SSH import remote path safety', () => { } }) + it('rejects an oversized directory before creating the remote root', async () => { + const sourcePath = path.resolve('/tmp/generated') + let yieldedEntries = 0 + let iteratorClosed = false + lstatMock.mockResolvedValue({ + isFile: () => false, + isDirectory: () => true, + isSymbolicLink: () => false + }) + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + try { + while (yieldedEntries <= EXTERNAL_IMPORT_MAX_TREE_ENTRIES) { + const index = yieldedEntries + yieldedEntries += 1 + yield { + name: `entry-${index}`, + isFile: () => true, + isDirectory: () => false, + isSymbolicLink: () => false + } + } + } finally { + iteratorClosed = true + } + }, + close: vi.fn().mockResolvedValue(undefined) + }) + + const { results } = await importExternalPathsSsh([sourcePath], destDir, connectionId) + + expect(results[0]).toMatchObject({ + status: 'failed', + reason: 'External import tree exceeds 100,000 entries' + }) + expect(yieldedEntries).toBe(EXTERNAL_IMPORT_MAX_TREE_ENTRIES + 1) + expect(iteratorClosed).toBe(true) + expect(provider.createDirNoClobber).not.toHaveBeenCalled() + expect(uploadSession.uploadFile).not.toHaveBeenCalled() + }) + it('skips special entries without failing the directory import', async () => { const sourcePath = path.resolve('/tmp/assets') const regularPath = path.join(sourcePath, 'report.txt') diff --git a/src/main/ipc/filesystem-import-ssh.test.ts b/src/main/ipc/filesystem-import-ssh.test.ts index 6a5d502b14c..db00cbd5df4 100644 --- a/src/main/ipc/filesystem-import-ssh.test.ts +++ b/src/main/ipc/filesystem-import-ssh.test.ts @@ -37,6 +37,7 @@ vi.mock('fs/promises', () => ({ copyFile: copyFileMock, open: openMock, readdir: readdirMock, + opendir: vi.fn(), unlink: unlinkMock, rm: vi.fn() })) @@ -47,7 +48,10 @@ import { registerSshFilesystemProvider, unregisterSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' -import { resetSshConnectionGenerations } from '../ssh/ssh-connection-generation' +import { + resetSshConnectionGenerations, + setSshConnectionGeneration +} from '../ssh/ssh-connection-generation' const store = { getRepos: () => [ @@ -130,6 +134,7 @@ describe('fs:importExternalPaths — SSH routing & connection', () => { beforeEach(() => { handlers.clear() resetSshConnectionGenerations() + setSshConnectionGeneration(connId, 0) ;[ handleMock, lstatMock, diff --git a/src/main/ipc/filesystem-import.test.ts b/src/main/ipc/filesystem-import.test.ts index fcd004dcd06..79180a1cfb3 100644 --- a/src/main/ipc/filesystem-import.test.ts +++ b/src/main/ipc/filesystem-import.test.ts @@ -11,7 +11,7 @@ const { realpathMock, copyFileMock, openMock, - readFileMock, + opendirMock, readdirMock, rmMock, unlinkMock @@ -22,7 +22,7 @@ const { realpathMock: vi.fn(), copyFileMock: vi.fn(), openMock: vi.fn(), - readFileMock: vi.fn(), + opendirMock: vi.fn(), readdirMock: vi.fn(), rmMock: vi.fn(), unlinkMock: vi.fn() @@ -40,13 +40,17 @@ vi.mock('fs/promises', () => ({ writeFile: vi.fn(), realpath: realpathMock, copyFile: copyFileMock, - readFile: readFileMock, - readdir: readdirMock, + opendir: opendirMock, rm: rmMock, unlink: unlinkMock })) import { registerFilesystemMutationHandlers } from './filesystem-mutations' +import { + EXTERNAL_IMPORT_MAX_SOURCE_PATHS, + EXTERNAL_IMPORT_MAX_TREE_ENTRIES, + REMOTE_IMPORT_MAX_FILE_BYTES +} from './filesystem-external-import-limits' const REPO_PATH = path.resolve('/workspace/repo') const WORKSPACE_DIR = path.resolve('/workspace') @@ -62,6 +66,24 @@ function enoent(): Error { return Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) } +function streamDirectoryEntries(entries: unknown[]) { + return { + async *[Symbol.asyncIterator]() { + yield* entries + } + } +} + +function createFileHandleRead(content: Buffer) { + return vi.fn(async (buffer: Buffer, offset: number, length: number, position: number) => { + const bytesRead = Math.min(length, Math.max(0, content.byteLength - position)) + if (bytesRead > 0) { + content.copy(buffer, offset, position, position + bytesRead) + } + return { buffer, bytesRead } + }) +} + describe('fs:importExternalPaths', () => { const destDir = path.resolve('/workspace/repo/src') @@ -158,7 +180,7 @@ describe('fs:importExternalPaths', () => { realpathMock.mockReset() copyFileMock.mockReset() openMock.mockReset() - readFileMock.mockReset() + opendirMock.mockReset() readdirMock.mockReset() rmMock.mockReset() unlinkMock.mockReset() @@ -172,8 +194,10 @@ describe('fs:importExternalPaths', () => { mkdirMock.mockResolvedValue(undefined) copyFileMock.mockResolvedValue(undefined) mockLocalCopyOpenSuccess() - readFileMock.mockResolvedValue(Buffer.from('file-content')) readdirMock.mockResolvedValue([]) + opendirMock.mockImplementation(async (dirPath: string) => + streamDirectoryEntries(await readdirMock(dirPath)) + ) rmMock.mockResolvedValue(undefined) unlinkMock.mockResolvedValue(undefined) @@ -384,6 +408,67 @@ describe('fs:importExternalPaths', () => { expect(copyFileMock).not.toHaveBeenCalled() }) + it('stops streaming an oversized local tree before creating its destination', async () => { + const sourcePath = '/tmp/dropped/generated' + const resolvedSource = path.resolve(sourcePath) + let iteratorClosed = false + let yieldedEntries = 0 + lstatMock.mockImplementation(async (candidatePath: string) => { + if (candidatePath === resolvedSource) { + return { + isFile: () => false, + isDirectory: () => true, + isSymbolicLink: () => false + } + } + throw enoent() + }) + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + try { + while (yieldedEntries <= EXTERNAL_IMPORT_MAX_TREE_ENTRIES) { + const index = yieldedEntries + yieldedEntries += 1 + yield { + name: `entry-${index}`, + isDirectory: () => false, + isSymbolicLink: () => false, + isFile: () => true + } + } + } finally { + iteratorClosed = true + } + } + }) + + const result = (await handlers.get('fs:importExternalPaths')!(null, { + sourcePaths: [sourcePath], + destDir + })) as { results: { status: string; reason?: string }[] } + + expect(result.results[0]).toMatchObject({ + status: 'failed', + reason: 'External import tree exceeds 100,000 entries' + }) + expect(yieldedEntries).toBe(EXTERNAL_IMPORT_MAX_TREE_ENTRIES + 1) + expect(iteratorClosed).toBe(true) + expect(mkdirMock).not.toHaveBeenCalled() + }) + + it('rejects oversized source batches before touching the filesystem', async () => { + const sourcePaths = Array.from( + { length: EXTERNAL_IMPORT_MAX_SOURCE_PATHS + 1 }, + (_, index) => `/tmp/dropped/${index}` + ) + + await expect( + handlers.get('fs:stageExternalPathsForRuntimeUpload')!(null, { sourcePaths }) + ).rejects.toThrow('External import accepts at most 256 source paths') + expect(lstatMock).not.toHaveBeenCalled() + expect(openMock).not.toHaveBeenCalled() + }) + it('fails and removes output if a local directory entry becomes a symlink after pre-scan', async () => { const sourcePath = '/tmp/dropped/mixeddir' const resolvedSource = path.resolve(sourcePath) @@ -481,7 +566,7 @@ describe('fs:importExternalPaths', () => { lstatMock.mockImplementation(async (p: string) => { if (p === resolvedPath) { return { - size: 4, + size: 3, ino: 1, dev: 1, isFile: () => true, @@ -492,15 +577,15 @@ describe('fs:importExternalPaths', () => { throw enoent() }) const closeMock = vi.fn().mockResolvedValue(undefined) - const readFileHandleMock = vi.fn().mockResolvedValue(Buffer.from('png')) + const readFileHandleMock = createFileHandleRead(Buffer.from('png')) openMock.mockResolvedValue({ stat: vi.fn().mockResolvedValue({ - size: 4, + size: 3, ino: 1, dev: 1, isFile: () => true }), - readFile: readFileHandleMock, + read: readFileHandleMock, close: closeMock }) @@ -580,7 +665,7 @@ describe('fs:importExternalPaths', () => { dev: 1, isFile: () => true }), - readFile: vi.fn().mockResolvedValue(Buffer.from('icon')), + read: createFileHandleRead(Buffer.from('icon')), close: vi.fn().mockResolvedValue(undefined) }) @@ -637,16 +722,10 @@ describe('fs:importExternalPaths', () => { expect(openMock).not.toHaveBeenCalled() }) - it('checks runtime upload directory byte budget before reading a file that exceeds the total cap', async () => { + it('checks the runtime upload file budget before opening an oversized entry', async () => { const sourcePath = '/tmp/dropped/project' const resolvedPath = path.resolve(sourcePath) - const filePaths = ['one.bin', 'two.bin', 'three.bin', 'four.bin', 'overflow.bin'].map((name) => - path.join(resolvedPath, name) - ) - const mib = 1024 * 1024 - const regularSize = 25 * mib - const overflowSize = 1 * mib - const readFileMock = vi.fn().mockResolvedValue(Buffer.from('chunk')) + const oversizedPath = path.join(resolvedPath, 'oversized.bin') lstatMock.mockImplementation(async (p: string) => { if (p === resolvedPath) { @@ -659,12 +738,10 @@ describe('fs:importExternalPaths', () => { isSymbolicLink: () => false } } - const fileIndex = filePaths.indexOf(p) - if (fileIndex !== -1) { - const size = fileIndex === filePaths.length - 1 ? overflowSize : regularSize + if (p === oversizedPath) { return { - size, - ino: fileIndex + 2, + size: REMOTE_IMPORT_MAX_FILE_BYTES + 1, + ino: 2, dev: 1, isFile: () => true, isDirectory: () => false, @@ -673,30 +750,14 @@ describe('fs:importExternalPaths', () => { } throw enoent() }) - readdirMock.mockResolvedValue( - filePaths.map((filePath) => ({ - name: path.basename(filePath), + readdirMock.mockResolvedValue([ + { + name: path.basename(oversizedPath), isDirectory: () => false, isSymbolicLink: () => false, isFile: () => true - })) - ) - openMock.mockImplementation(async (p: string) => { - const fileIndex = filePaths.indexOf(p) - if (fileIndex >= 0 && fileIndex < filePaths.length - 1) { - return { - stat: vi.fn().mockResolvedValue({ - size: regularSize, - ino: fileIndex + 2, - dev: 1, - isFile: () => true - }), - readFile: readFileMock, - close: vi.fn().mockResolvedValue(undefined) - } } - throw new Error(`unexpected open: ${p}`) - }) + ]) const result = (await handlers.get('fs:stageExternalPathsForRuntimeUpload')!(null, { sourcePaths: [sourcePath] @@ -704,10 +765,9 @@ describe('fs:importExternalPaths', () => { expect(result.sources[0]).toMatchObject({ status: 'failed', - reason: 'Remote import is too large' + reason: "'oversized.bin' is too large for remote import" }) - expect(readFileMock).toHaveBeenCalledTimes(4) - expect(openMock).not.toHaveBeenCalledWith(filePaths.at(-1), expect.anything()) + expect(openMock).not.toHaveBeenCalled() }) it('fails runtime upload staging when a file changes between lstat and open', async () => { @@ -726,7 +786,7 @@ describe('fs:importExternalPaths', () => { } throw enoent() }) - const readFileHandleMock = vi.fn().mockResolvedValue(Buffer.from('png')) + const readFileHandleMock = createFileHandleRead(Buffer.from('png')) openMock.mockResolvedValue({ stat: vi.fn().mockResolvedValue({ size: 4, @@ -734,7 +794,7 @@ describe('fs:importExternalPaths', () => { dev: 1, isFile: () => true }), - readFile: readFileHandleMock, + read: readFileHandleMock, close: vi.fn().mockResolvedValue(undefined) }) diff --git a/src/main/ipc/filesystem-list-files-git-directory-expansion.test.ts b/src/main/ipc/filesystem-list-files-git-directory-expansion.test.ts index 5d5cf163746..ee0eeb3a6df 100644 --- a/src/main/ipc/filesystem-list-files-git-directory-expansion.test.ts +++ b/src/main/ipc/filesystem-list-files-git-directory-expansion.test.ts @@ -62,12 +62,13 @@ describe('main Quick Open git directory expansion', () => { const promise = listFilesWithGit(root, [], {}) revParse.emit('close', 0, null) - await vi.waitFor(() => expect(gitSpawnMock).toHaveBeenCalledTimes(3)) + await vi.waitFor(() => expect(gitSpawnMock).toHaveBeenCalledTimes(2)) ;(primary.stdout as unknown as EventEmitter).emit( 'data', `100644 ${SHA1} 0\tsrc/index.ts\0scratch/\0` ) primary.emit('close', 0, null) + await vi.waitFor(() => expect(gitSpawnMock).toHaveBeenCalledTimes(3)) ;(ignored.stdout as unknown as EventEmitter).emit('data', 'dist/\0') ignored.emit('close', 0, null) @@ -79,7 +80,7 @@ describe('main Quick Open git directory expansion', () => { expect(gitSpawnMock.mock.calls[2][0]).toContain('--directory') }) - it('cancels both local Git passes when Quick Open abandons the request', async () => { + it('cancels the active local Git pass when Quick Open abandons the request', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-main-git-cancel-')) tempDirs.push(root) const revParse = createMockProcess() @@ -93,11 +94,12 @@ describe('main Quick Open git directory expansion', () => { const controller = new AbortController() const promise = listFilesWithGit(root, [], {}, controller.signal) revParse.emit('close', 0, null) - await vi.waitFor(() => expect(gitSpawnMock).toHaveBeenCalledTimes(3)) + await vi.waitFor(() => expect(gitSpawnMock).toHaveBeenCalledTimes(2)) controller.abort() await expect(promise).rejects.toSatisfy(isFileListingCancellation) expect(primary.kill).toHaveBeenCalled() - expect(ignored.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(gitSpawnMock).toHaveBeenCalledTimes(2) }) }) diff --git a/src/main/ipc/filesystem-list-files-git-fallback.ts b/src/main/ipc/filesystem-list-files-git-fallback.ts index ff390d13caf..f86d13e879a 100644 --- a/src/main/ipc/filesystem-list-files-git-fallback.ts +++ b/src/main/ipc/filesystem-list-files-git-fallback.ts @@ -12,6 +12,13 @@ import { parseQuickOpenGitLsFilesEntry } from '../../shared/quick-open-readdir-walk' import { fileListingCancellationError } from '../../shared/file-listing-cancellation' +import { + createQuickOpenListingBudget, + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QuickOpenSubprocessPathAccumulator, + resolveQuickOpenResultLimit, + retainQuickOpenPath +} from '../../shared/quick-open-listing-limits' /** * Fallback file lister using git ls-files. Used when rg is not available. @@ -81,6 +88,13 @@ export async function listFilesWithGit( signal?: AbortSignal, maxResults?: number ): Promise { + if (signal?.aborted) { + throw fileListingCancellationError(signal) + } + const resultLimit = resolveQuickOpenResultLimit(maxResults) + if (resultLimit === 0) { + return [] + } const isGitWorkTree = await isInsideGitWorkTree(rootPath, localGitOptions, signal) if (signal?.aborted) { throw fileListingCancellationError(signal) @@ -89,7 +103,7 @@ export async function listFilesWithGit( return listQuickOpenFilesWithReaddir(rootPath, { excludePathPrefixes, budget: createQuickOpenReaddirBudget(), - maxResults, + maxResults: resultLimit, signal }) } @@ -97,6 +111,7 @@ export async function listFilesWithGit( const gitPaths = new Set() const directoryPaths = new Set() const directFileCandidates = new Set() + const listingBudget = createQuickOpenListingBudget() const { primary, ignoredPass } = buildGitLsFilesArgsForQuickOpen(excludePathPrefixes) const children: { child: ChildProcess @@ -107,7 +122,7 @@ export async function listFilesWithGit( const runGitLsFiles = (args: string[]): Promise => { return new Promise((resolve, reject) => { - let buf = '' + const paths = new QuickOpenSubprocessPathAccumulator(0) let done = false const processPath = (path: string): boolean => { @@ -115,27 +130,23 @@ export async function listFilesWithGit( return false } if (path.endsWith('/')) { - directoryPaths.add(path) + retainQuickOpenPath(directoryPaths, path, listingBudget) } else { - gitPaths.add(path) - if (maxResults !== undefined) { - // Why: this duplicate classification exists only to stop bounded - // scans; unbounded scans must not retain a second repo-sized set. - const parsed = parseQuickOpenGitLsFilesEntry(path) - const relPath = parsed.path.replace(/\/+$/, '') - if ( - !parsed.isGitlink && - !parsed.isUntrackedDir && - shouldIncludeQuickOpenPath(relPath) && - !shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes) - ) { - directFileCandidates.add(relPath) - } + retainQuickOpenPath(gitPaths, path, listingBudget) + const parsed = parseQuickOpenGitLsFilesEntry(path) + const relPath = parsed.path.replace(/\/+$/, '') + if ( + !parsed.isGitlink && + !parsed.isUntrackedDir && + shouldIncludeQuickOpenPath(relPath) && + !shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes) + ) { + retainQuickOpenPath(directFileCandidates, relPath, listingBudget) } } // Why: collapsed directories and gitlinks may be discarded during the // later filesystem classification, so they cannot consume the stop cap. - return maxResults !== undefined && directFileCandidates.size >= maxResults + return directFileCandidates.size >= resultLimit } // Why: git ls-files outputs paths relative to cwd, so we set cwd to @@ -160,7 +171,7 @@ export async function listFilesWithGit( return } done = true - buf = '' + paths.clear() cleanup() reject(err) } @@ -169,6 +180,7 @@ export async function listFilesWithGit( return } done = true + paths.clear() cleanup() resolve() } @@ -178,20 +190,23 @@ export async function listFilesWithGit( reject: rejectPass, resolve: resolvePass }) - const handleStdoutData = (chunk: string): void => { - buf += chunk - let start = 0 - let nulIdx = buf.indexOf('\0', start) - while (nulIdx !== -1) { - if (processPath(buf.substring(start, nulIdx))) { - buf = '' + const failForOutput = (error: unknown): void => { + child.kill() + rejectPass(error instanceof Error ? error : new Error(String(error))) + } + const handleStdoutData = (chunk: Buffer | string): void => { + try { + const outcome = paths.push(chunk, (path) => !processPath(path)) + if (outcome === 'stopped') { finishAtLimit() - return + } else if (outcome === 'path-too-large') { + failForOutput( + new Error(`Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes`) + ) } - start = nulIdx + 1 - nulIdx = buf.indexOf('\0', start) + } catch (error) { + failForOutput(error) } - buf = start < buf.length ? buf.substring(start) : '' } const handleStderrData = (): void => { /* drain */ @@ -207,9 +222,14 @@ export async function listFilesWithGit( rejectPass(new Error(`git ls-files killed by ${signal}`)) return } - if (buf && processPath(buf)) { - buf = '' - finishAtLimit() + try { + const trailingPath = paths.finish() + if (trailingPath && processPath(trailingPath)) { + finishAtLimit() + return + } + } catch (error) { + failForOutput(error) return } if (code === 0) { @@ -219,13 +239,12 @@ export async function listFilesWithGit( rejectPass(new Error(`git ls-files exited with code ${code}`)) } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) child.once('close', handleClose) timer = setTimeout(() => { - buf = '' + paths.clear() child.kill() rejectPass(new Error('git ls-files timed out')) }, 10000) @@ -269,15 +288,11 @@ export async function listFilesWithGit( console.warn('[quick-open] git ignored-file pass failed; keeping primary results:', err) } }) - if (maxResults === undefined) { - await Promise.all([runGitLsFiles(primary), runIgnoredPass()]) - } else { - // Why: give ordinary source files first claim on a bounded autocomplete - // inventory; a large ignored tree must not win a parallel-output race. - await runGitLsFiles(primary) - if (directFileCandidates.size < maxResults) { - await runIgnoredPass() - } + // Why: give ordinary source files first claim on a bounded autocomplete + // inventory; a large ignored tree must not win a parallel-output race. + await runGitLsFiles(primary) + if (directFileCandidates.size < resultLimit) { + await runIgnoredPass() } } catch (err) { killSurvivors() @@ -295,9 +310,9 @@ export async function listFilesWithGit( directoryPaths, excludePathPrefixes, signal, - maxResults + maxResults: resultLimit }) // Why: directory placeholders are expanded after Git exits; restore Git's // path order so empty queries and fuzzy-score ties remain stable. - return files.sort().slice(0, maxResults) + return files.sort().slice(0, resultLimit) } diff --git a/src/main/ipc/filesystem-list-files-install-rg.test.ts b/src/main/ipc/filesystem-list-files-install-rg.test.ts new file mode 100644 index 00000000000..7f4c34f5eda --- /dev/null +++ b/src/main/ipc/filesystem-list-files-install-rg.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { Store } from '../persistence' + +const { + listFilesWithGitMock, + resolveAuthorizedPathMock, + checkRgAvailableMock, + getLocalGitOptionsForRegisteredWorktreeMock +} = vi.hoisted(() => ({ + listFilesWithGitMock: vi.fn(), + resolveAuthorizedPathMock: vi.fn(), + checkRgAvailableMock: vi.fn(), + getLocalGitOptionsForRegisteredWorktreeMock: vi.fn() +})) + +vi.mock('./filesystem-list-files-git-fallback', () => ({ + listFilesWithGit: listFilesWithGitMock +})) + +vi.mock('./filesystem-auth', () => ({ + resolveAuthorizedPath: resolveAuthorizedPathMock +})) + +vi.mock('./rg-availability', () => ({ + checkRgAvailable: checkRgAvailableMock +})) + +vi.mock('./local-worktree-runtime-options', () => ({ + getLocalGitOptionsForRegisteredWorktree: getLocalGitOptionsForRegisteredWorktreeMock +})) + +import { listQuickOpenFiles } from './filesystem-list-files' + +describe('filesystem-list-files ripgrep guidance', () => { + beforeEach(() => { + vi.clearAllMocks() + resolveAuthorizedPathMock.mockImplementation(async (path) => path) + checkRgAvailableMock.mockResolvedValue(false) + getLocalGitOptionsForRegisteredWorktreeMock.mockReturnValue({}) + }) + + it('turns only a readdir budget failure into install guidance', async () => { + listFilesWithGitMock.mockRejectedValue(new Error('File listing exceeded 10000 files')) + const rejection = listQuickOpenFiles('/workspace', {} as Store) + + await expect(rejection).rejects.toThrow( + 'Quick Open scan too large (File listing exceeded 10000 files).' + ) + await rejection.catch((error: Error) => + expect(error.message).toContain('Install ripgrep on the host running the Quick Open scan') + ) + }) + + it('keeps cancellation and Git errors unchanged', async () => { + const cancellation = new Error('File listing cancelled') + listFilesWithGitMock.mockRejectedValueOnce(cancellation) + await expect(listQuickOpenFiles('/workspace', {} as Store)).rejects.toBe(cancellation) + + const gitFailure = new Error('git ls-files exited with code 128') + listFilesWithGitMock.mockRejectedValueOnce(gitFailure) + await expect(listQuickOpenFiles('/workspace', {} as Store)).rejects.toBe(gitFailure) + }) + + it.skipIf(process.platform !== 'darwin')('shows the macOS install command', async () => { + listFilesWithGitMock.mockRejectedValue(new Error('File listing timed out')) + + await expect(listQuickOpenFiles('/workspace', {} as Store)).rejects.toThrow( + 'brew install ripgrep' + ) + }) +}) diff --git a/src/main/ipc/filesystem-list-files.test.ts b/src/main/ipc/filesystem-list-files.test.ts index 946f8653dd6..1801b779ece 100644 --- a/src/main/ipc/filesystem-list-files.test.ts +++ b/src/main/ipc/filesystem-list-files.test.ts @@ -36,6 +36,11 @@ import { listQuickOpenFiles } from './filesystem-list-files' import { EventEmitter } from 'node:events' import type { Store } from '../persistence' import type { ChildProcess } from 'node:child_process' +import { + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QUICK_OPEN_LISTING_MAX_RESULTS +} from '../../shared/quick-open-listing-limits' +import { isFileListingCancellation } from '../../shared/file-listing-cancellation' const SHA1 = '0123456789abcdef0123456789abcdef01234567' @@ -95,6 +100,88 @@ describe('filesystem-list-files', () => { expect(spawnMock).toHaveBeenCalledTimes(1) }) + it('bounds an omitted rg result limit and recovers on the next scan', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + spawnMock.mockImplementation((_cmd, args: string[]) => + isIgnoredRgPass(args) ? ignored : primary + ) + const promise = listQuickOpenFiles('/mock/root', {} as unknown as Store) + const paths = Array.from( + { length: QUICK_OPEN_LISTING_MAX_RESULTS + 1 }, + (_value, index) => `src/file-${index}.ts` + ) + + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1)) + ;(primary.stdout as unknown as EventEmitter).emit('data', paths.join('\n')) + + const result = await promise + expect(result).toHaveLength(QUICK_OPEN_LISTING_MAX_RESULTS) + expect(result.at(-1)).toBe(`src/file-${QUICK_OPEN_LISTING_MAX_RESULTS - 1}.ts`) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledTimes(1) + + const recoveryPrimary = createMockProcess() + const recoveryIgnored = createMockProcess() + spawnMock.mockImplementation((_cmd, args: string[]) => + isIgnoredRgPass(args) ? recoveryIgnored : recoveryPrimary + ) + const recovery = listQuickOpenFiles('/mock/root', {} as unknown as Store) + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(2)) + ;(recoveryPrimary.stdout as unknown as EventEmitter).emit('data', 'src/recovered.ts\n') + recoveryPrimary.emit('close', 0, null) + queueMicrotask(() => recoveryIgnored.emit('close', 0, null)) + + await expect(recovery).resolves.toEqual(['src/recovered.ts']) + }) + + it('kills a local rg scan whose residual path exceeds the field limit', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + spawnMock.mockImplementation((_cmd, args: string[]) => + isIgnoredRgPass(args) ? ignored : primary + ) + const promise = listQuickOpenFiles('/mock/root', {} as unknown as Store) + + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1)) + ;(primary.stdout as unknown as EventEmitter).emit( + 'data', + Buffer.alloc(QUICK_OPEN_LISTING_MAX_PATH_BYTES + 1, 0x61) + ) + + await expect(promise).rejects.toThrow( + `Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes` + ) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledTimes(1) + }) + + it('kills the active local rg pass when the listing is cancelled', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + spawnMock.mockImplementation((_cmd, args: string[]) => + isIgnoredRgPass(args) ? ignored : primary + ) + const controller = new AbortController() + const promise = listQuickOpenFiles( + '/mock/root', + {} as unknown as Store, + undefined, + controller.signal + ) + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1)) + + controller.abort() + + await expect(promise).rejects.toSatisfy(isFileListingCancellation) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect((primary.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) + expect(primary.listenerCount('close')).toBe(0) + }) + it('merges normal files and ignored files and filters correctly', async () => { const p1 = createMockProcess() const p2 = createMockProcess() @@ -119,12 +206,13 @@ describe('filesystem-list-files', () => { ;(p1.stdout as unknown as EventEmitter).emit('data', 'file2.js\n') p1.emit('close', 0, null) - // Simulate stdout output for ignored files - ;(p2.stdout as unknown as EventEmitter).emit('data', '.env.local\n') - ;(p2.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\n') - ;(p2.stdout as unknown as EventEmitter).emit('data', 'file1.ts\n') // Duplicate - ;(p2.stdout as unknown as EventEmitter).emit('data', 'node_modules/ignored.js\n') - p2.emit('close', 0, null) + queueMicrotask(() => { + ;(p2.stdout as unknown as EventEmitter).emit('data', '.env.local\n') + ;(p2.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\n') + ;(p2.stdout as unknown as EventEmitter).emit('data', 'file1.ts\n') // Duplicate + ;(p2.stdout as unknown as EventEmitter).emit('data', 'node_modules/ignored.js\n') + p2.emit('close', 0, null) + }) }, 10) const result = await promise @@ -156,7 +244,7 @@ describe('filesystem-list-files', () => { setTimeout(() => { ;(p1.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\n') p1.emit('close', 0, null) - p2.emit('close', 0, null) + queueMicrotask(() => p2.emit('close', 0, null)) }, 10) await expect(promise).resolves.toEqual(['src/index.ts']) @@ -186,7 +274,7 @@ describe('filesystem-list-files', () => { setTimeout(() => { ;(p1.stdout as unknown as EventEmitter).emit('data', '/mnt/c/repo/src/index.ts\n') p1.emit('close', 0, null) - p2.emit('close', 0, null) + queueMicrotask(() => p2.emit('close', 0, null)) }, 10) await expect(promise).resolves.toEqual(['src/index.ts']) @@ -208,13 +296,12 @@ describe('filesystem-list-files', () => { setTimeout(() => { p1.emit('close', 2, null) - p2.emit('close', 0, null) }, 10) await expect(promise).rejects.toThrow('rg exited with code 2') }) - it('kills the sibling rg pass after one pass fails', async () => { + it('does not spawn the ignored rg pass after the primary pass fails', async () => { const p1 = createMockProcess() const p2 = createMockProcess() @@ -234,7 +321,8 @@ describe('filesystem-list-files', () => { }, 10) await expect(promise).rejects.toThrow('rg exited with code 2') - expect(p2.kill).toHaveBeenCalled() + expect(p2.kill).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledTimes(1) }) it('accepts rg code 2 when rg emitted parseable paths first', async () => { @@ -254,7 +342,7 @@ describe('filesystem-list-files', () => { setTimeout(() => { ;(p1.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\n') p1.emit('close', 2, null) - p2.emit('close', 0, null) + queueMicrotask(() => p2.emit('close', 0, null)) }, 10) await expect(promise).resolves.toEqual(['src/index.ts']) @@ -288,7 +376,8 @@ describe('filesystem-list-files', () => { await rejection expect(p1.kill).toHaveBeenCalled() - expect(p2.kill).toHaveBeenCalled() + expect(p2.kill).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledTimes(1) expect((p1.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) expect((p1.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) expect(p1.listenerCount('error')).toBe(0) @@ -321,8 +410,7 @@ describe('filesystem-list-files', () => { ;(p1.stdout as unknown as EventEmitter).emit('data', 'valid.ts\n') p1.emit('close', 0, null) - // Empty ignored result - p2.emit('close', 0, null) + queueMicrotask(() => p2.emit('close', 0, null)) }, 10) const result = await promise @@ -371,9 +459,11 @@ describe('filesystem-list-files', () => { ) gitP1.emit('close', 0, null) - ;(gitP2.stdout as unknown as EventEmitter).emit('data', '.env.local\0') - ;(gitP2.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') - gitP2.emit('close', 0, null) + queueMicrotask(() => { + ;(gitP2.stdout as unknown as EventEmitter).emit('data', '.env.local\0') + ;(gitP2.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') + gitP2.emit('close', 0, null) + }) }, 10) const result = await promise @@ -438,6 +528,37 @@ describe('filesystem-list-files', () => { expect(callIndex).toBe(1) }) + it('bounds the omitted local Git result limit before spawning the ignored pass', async () => { + checkRgAvailableMock.mockResolvedValue(false) + const revParseProc = createMockProcess() + const primary = createMockProcess() + const ignored = createMockProcess() + let gitPasses = 0 + spawnMock.mockImplementation((cmd: string, args: string[]) => { + if (cmd === 'git' && args.includes('rev-parse')) { + return revParseProc + } + gitPasses++ + return gitPasses === 1 ? primary : ignored + }) + const promise = listQuickOpenFiles('/mock/root', {} as unknown as Store) + const paths = Array.from( + { length: QUICK_OPEN_LISTING_MAX_RESULTS + 1 }, + (_value, index) => `src/file-${index}.ts` + ) + setTimeout(() => revParseProc.emit('close', 0, null), 0) + setTimeout( + () => (primary.stdout as unknown as EventEmitter).emit('data', paths.join('\0')), + 10 + ) + + const result = await promise + expect(result).toHaveLength(QUICK_OPEN_LISTING_MAX_RESULTS) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(gitPasses).toBe(1) + }) + it('does not let a discarded Git directory placeholder consume the result budget', async () => { checkRgAvailableMock.mockResolvedValue(false) const revParseProc = createMockProcess() @@ -509,7 +630,7 @@ describe('filesystem-list-files', () => { ;(gitP1.stdout as unknown as EventEmitter).emit('data', `${staged('100644', 'valid.ts')}\0`) gitP1.emit('close', 0, null) - gitP2.emit('close', 0, null) + queueMicrotask(() => gitP2.emit('close', 0, null)) }, 10) const result = await promise @@ -556,7 +677,8 @@ describe('filesystem-list-files', () => { await rejection expect(gitP1.kill).toHaveBeenCalled() - expect(gitP2.kill).toHaveBeenCalled() + expect(gitP2.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) expect((gitP1.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) expect((gitP1.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) expect(gitP1.listenerCount('error')).toBe(0) @@ -604,6 +726,7 @@ describe('filesystem-list-files', () => { `${staged('100644', 'src/index.ts')}\0` ) gitP1.emit('close', 0, null) + await Promise.resolve() // Ignored entries streamed before the timeout are kept. ;(gitP2.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') @@ -639,7 +762,7 @@ describe('filesystem-list-files', () => { setTimeout(() => { ;(p1.stdout as unknown as EventEmitter).emit('data', 'file.ts\n') p1.emit('close', 0, null) - p2.emit('close', 0, null) + queueMicrotask(() => p2.emit('close', 0, null)) }, 10) const result = await promise diff --git a/src/main/ipc/filesystem-list-files.ts b/src/main/ipc/filesystem-list-files.ts index 58b1c0c0cad..49105dc9b40 100644 --- a/src/main/ipc/filesystem-list-files.ts +++ b/src/main/ipc/filesystem-list-files.ts @@ -14,6 +14,16 @@ import { shouldExcludeQuickOpenRelPath, shouldIncludeQuickOpenPath } from '../../shared/quick-open-filter' +import { isQuickOpenReaddirBudgetError } from '../../shared/quick-open-readdir-walk' +import { buildInstallRgMessage } from '../../shared/quick-open-install-rg' +import { + createQuickOpenListingBudget, + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QuickOpenSubprocessPathAccumulator, + resolveQuickOpenResultLimit, + retainQuickOpenPath +} from '../../shared/quick-open-listing-limits' +import { fileListingCancellationError } from '../../shared/file-listing-cancellation' import { listFilesWithGit } from './filesystem-list-files-git-fallback' export async function listQuickOpenFiles( @@ -35,6 +45,13 @@ export async function listQuickOpenFiles( // every worktree instead of just the active one. The shared helper // normalizes, validates, and root-relativizes every input. const excludePathPrefixes = buildExcludePathPrefixes(authorizedRootPath, excludePaths) + if (signal?.aborted) { + throw fileListingCancellationError(signal) + } + const resultLimit = resolveQuickOpenResultLimit(maxResults) + if (resultLimit === 0) { + return [] + } // Why: checking rg availability upfront avoids a race condition where // spawn('rg') emits 'close' before 'error' on some platforms, causing @@ -42,20 +59,28 @@ export async function listQuickOpenFiles( // can run. const rgAvailable = await checkRgAvailable(authorizedRootPath, localGitOptions.wslDistro) if (!rgAvailable) { - return listFilesWithGit( - authorizedRootPath, - excludePathPrefixes, - localGitOptions, - signal, - maxResults - ) + try { + return await listFilesWithGit( + authorizedRootPath, + excludePathPrefixes, + localGitOptions, + signal, + resultLimit + ) + } catch (err) { + if (!isQuickOpenReaddirBudgetError(err)) { + throw err + } + throw new Error(await buildInstallRgMessage(err)) + } } const files = new Set() + const listingBudget = createQuickOpenListingBudget() const children: { child: ChildProcess isDone: () => boolean - finish: () => void + finish: (error?: Error) => void }[] = [] // Why: WSL-routed rg can emit Linux-native absolute paths. UNC repos carry // their distro in the path; Windows-path repos carry it in project runtime. @@ -73,8 +98,11 @@ export async function listQuickOpenFiles( }) const runRg = (args: string[]): Promise => { + if (signal?.aborted) { + return Promise.reject(fileListingCancellationError(signal)) + } return new Promise((resolve, reject) => { - let buf = '' + const paths = new QuickOpenSubprocessPathAccumulator(0x0a) let done = false let parseablePathCount = 0 @@ -97,11 +125,11 @@ export async function listQuickOpenFiles( if (shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes)) { return false } - if (maxResults !== undefined && files.size >= maxResults) { + if (files.size >= resultLimit) { return true } - files.add(relPath) - return maxResults !== undefined && files.size >= maxResults + retainQuickOpenPath(files, relPath, listingBudget) + return files.size >= resultLimit } const child = wslAwareSpawn('rg', args, { @@ -110,20 +138,24 @@ export async function listQuickOpenFiles( stdio: ['ignore', 'pipe', 'pipe'] }) let timer: ReturnType - const handleStdoutData = (chunk: string): void => { - buf += chunk - let start = 0 - let newlineIdx = buf.indexOf('\n', start) - while (newlineIdx !== -1) { - if (processLine(buf.substring(start, newlineIdx))) { - buf = '' + const failForOutput = (error: unknown): void => { + paths.clear() + child.kill() + finish(error instanceof Error ? error : new Error(String(error))) + } + const handleStdoutData = (chunk: Buffer | string): void => { + try { + const outcome = paths.push(chunk, (path) => !processLine(path)) + if (outcome === 'stopped') { finishAtLimit() - return + } else if (outcome === 'path-too-large') { + failForOutput( + new Error(`Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes`) + ) } - start = newlineIdx + 1 - newlineIdx = buf.indexOf('\n', start) + } catch (error) { + failForOutput(error) } - buf = start < buf.length ? buf.substring(start) : '' } const handleStderrData = (): void => { /* drain */ @@ -131,7 +163,7 @@ export async function listQuickOpenFiles( const handleError = (): void => { // Why: treat spawn errors like an abnormal exit — discard residual // buffer so a truncated final byte sequence cannot leak as a path. - buf = '' + paths.clear() finish(new Error('rg failed to start')) } const handleClose = (code: number | null, signal: NodeJS.Signals | null): void => { @@ -139,13 +171,18 @@ export async function listQuickOpenFiles( // Why: a signal exit means timeout/OOM/external kill. Returning the // already-streamed prefix would recreate the false-empty bug this // path is meant to avoid. - buf = '' + paths.clear() finish(new Error(`rg killed by ${signal}`)) return } - if (buf && processLine(buf)) { - buf = '' - finishAtLimit() + try { + const trailingPath = paths.finish() + if (trailingPath && processLine(trailingPath)) { + finishAtLimit() + return + } + } catch (error) { + failForOutput(error) return } if (code === 0 || code === 1) { @@ -179,7 +216,6 @@ export async function listQuickOpenFiles( children.push({ child, isDone: () => done, finish }) - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) @@ -187,7 +223,7 @@ export async function listQuickOpenFiles( timer = setTimeout(() => { // Why: on timeout, the buffer is likely truncated mid-path. Discard // it so Quick Open never displays a malformed entry. - buf = '' + paths.clear() child.kill() finish(new Error('rg list timed out')) }, 10000) @@ -221,22 +257,33 @@ export async function listQuickOpenFiles( } } - try { - if (maxResults === undefined) { - await Promise.all([runRg(primary), runRg(ignoredPass)]) - } else { - // Why: ignored-file output can be much larger and faster than the primary - // pass; let source files claim the bounded autocomplete budget first. - await runRg(primary) - if (files.size < maxResults) { - await runRg(ignoredPass) + const onAbort = (): void => { + const error = fileListingCancellationError(signal) + for (const entry of children) { + if (entry.isDone()) { + continue } + entry.finish(error) + if (entry.child.exitCode === null && entry.child.signalCode === null) { + entry.child.kill() + } + } + } + signal?.addEventListener('abort', onAbort, { once: true }) + try { + // Why: ignored-file output can be much larger and faster than the primary + // pass; let source files claim the bounded autocomplete budget first. + await runRg(primary) + if (files.size < resultLimit) { + await runRg(ignoredPass) } } catch (err) { killSurvivors() throw err + } finally { + signal?.removeEventListener('abort', onAbort) } - return Array.from(files).slice(0, maxResults) + return Array.from(files).slice(0, resultLimit) } function getQuickOpenRgOutputMode( diff --git a/src/main/ipc/filesystem-mutations.test.ts b/src/main/ipc/filesystem-mutations.test.ts index dcaeb74b624..6bc17af2af7 100644 --- a/src/main/ipc/filesystem-mutations.test.ts +++ b/src/main/ipc/filesystem-mutations.test.ts @@ -24,7 +24,7 @@ vi.mock('fs/promises', () => ({ writeFile: writeFileMock, realpath: realpathMock, copyFile: copyFileMock, - readdir: vi.fn() + opendir: vi.fn() })) import { registerFilesystemMutationHandlers } from './filesystem-mutations' @@ -77,6 +77,7 @@ describe('registerFilesystemMutationHandlers', () => { writeFileMock.mockReset() realpathMock.mockReset() resetSshConnectionGenerations() + setSshConnectionGeneration('ssh-1', 0) handleMock.mockImplementation((channel: string, handler: never) => { handlers.set(channel, handler) diff --git a/src/main/ipc/filesystem-mutations.ts b/src/main/ipc/filesystem-mutations.ts index e89ff761b63..80b30617691 100644 --- a/src/main/ipc/filesystem-mutations.ts +++ b/src/main/ipc/filesystem-mutations.ts @@ -7,7 +7,7 @@ import { lstat, mkdir, open, - readdir, + opendir, realpath, rename, rm, @@ -24,6 +24,19 @@ import { importExternalPathsSsh } from './filesystem-import-ssh' import { assertNoClobberRenameDestinationAvailable } from '../../shared/filesystem-rename-collision' import type { SshMutationExpectation } from '../../shared/ssh-types' import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation' +import { readStableRuntimeUploadFile } from './filesystem-runtime-upload-file-reader' +import { + admitExternalImportTreeEntry, + assertExternalImportSourcePaths, + assertExternalImportTreeDepth, + assertRuntimeUploadFileFits, + captureRuntimeUploadRetentionCheckpoint, + createExternalImportTreeBudget, + createRuntimeUploadRetentionBudget, + restoreRuntimeUploadRetentionCheckpoint, + retainRuntimeUploadFileBytes, + type RuntimeUploadRetentionBudget +} from './filesystem-external-import-limits' /** * Re-throw filesystem errors with user-friendly messages. @@ -201,6 +214,7 @@ export function registerFilesystemMutationHandlers(store: Store): void { args.expectedSshConnectionGeneration, args.expectedExecutionHostId ) + assertExternalImportSourcePaths(args.sourcePaths) if (args.connectionId) { return importExternalPathsSsh(args.sourcePaths, args.destDir, args.connectionId, { ensureDir: args.ensureDir, @@ -242,8 +256,10 @@ export function registerFilesystemMutationHandlers(store: Store): void { args: { sourcePaths: string[] } ): Promise<{ sources: StagedExternalImportSource[] }> => { const sources: StagedExternalImportSource[] = [] + assertExternalImportSourcePaths(args.sourcePaths) + const retentionBudget = createRuntimeUploadRetentionBudget() for (const sourcePath of args.sourcePaths) { - sources.push(await stageOneSourceForRuntimeUpload(sourcePath)) + sources.push(await stageOneSourceForRuntimeUpload(sourcePath, retentionBudget)) } return { sources } } @@ -271,6 +287,7 @@ export function registerFilesystemMutationHandlers(store: Store): void { args.expectedSshConnectionGeneration, args.expectedExecutionHostId ) + assertExternalImportSourcePaths(args.paths) // Why: `== null` (not `!args.connectionId`) so an empty string is // treated as a renderer error, not silently routed to the local branch. if (args.connectionId == null) { @@ -362,9 +379,6 @@ export type StagedExternalImportEntry = | { relativePath: string; kind: 'directory' } | { relativePath: string; kind: 'file'; contentBase64: string } -const REMOTE_IMPORT_MAX_FILE_BYTES = 25 * 1024 * 1024 -const REMOTE_IMPORT_MAX_TOTAL_BYTES = 100 * 1024 * 1024 - class RuntimeUploadSymlinkError extends Error {} // ─── External Import Implementation ───────────────────────────────── @@ -425,9 +439,17 @@ async function importOneSource( // creating any destination files. This prevents partially imported // trees when a symlink is discovered halfway through recursive copy. if (isDir) { - const hasSymlink = await preScanForSymlinks(resolvedSource) - if (hasSymlink) { - return { sourcePath, status: 'skipped', reason: 'symlink' } + try { + const hasSymlink = await preScanForSymlinks(resolvedSource) + if (hasSymlink) { + return { sourcePath, status: 'skipped', reason: 'symlink' } + } + } catch (error) { + return { + sourcePath, + status: 'failed', + reason: error instanceof Error ? error.message : String(error) + } } } @@ -462,7 +484,8 @@ async function importOneSource( } async function stageOneSourceForRuntimeUpload( - sourcePath: string + sourcePath: string, + retentionBudget: RuntimeUploadRetentionBudget ): Promise { const resolvedSource = resolve(sourcePath) @@ -498,10 +521,15 @@ async function stageOneSourceForRuntimeUpload( if (!sourceStat.isFile() && !sourceStat.isDirectory()) { return { sourcePath, status: 'skipped', reason: 'unsupported' } } + const checkpoint = captureRuntimeUploadRetentionCheckpoint(retentionBudget) try { - const entries = sourceStat.isDirectory() - ? await stageDirectoryEntries(resolvedSource) - : [(await stageFileEntry(resolvedSource, '')).entry] + let entries: StagedExternalImportEntry[] + if (sourceStat.isDirectory()) { + entries = await stageDirectoryEntries(resolvedSource, retentionBudget) + } else { + admitExternalImportTreeEntry(retentionBudget.tree, '', true) + entries = [await stageFileEntry(resolvedSource, '', { retentionBudget })] + } return { sourcePath, status: 'staged', @@ -510,6 +538,7 @@ async function stageOneSourceForRuntimeUpload( entries } } catch (error) { + restoreRuntimeUploadRetentionCheckpoint(retentionBudget, checkpoint) if (error instanceof RuntimeUploadSymlinkError) { return { sourcePath, status: 'skipped', reason: 'symlink' } } @@ -521,61 +550,61 @@ async function stageOneSourceForRuntimeUpload( } } -async function stageDirectoryEntries(rootPath: string): Promise { +async function stageDirectoryEntries( + rootPath: string, + retentionBudget: RuntimeUploadRetentionBudget +): Promise { + admitExternalImportTreeEntry(retentionBudget.tree, '', true) const entries: StagedExternalImportEntry[] = [{ relativePath: '', kind: 'directory' }] - let totalBytes = 0 const rootRealPath = await realpath(rootPath) - async function visit(dirPath: string): Promise { + async function visit(dirPath: string, depth: number): Promise { + assertExternalImportTreeDepth(depth) const dirStat = await lstat(dirPath) + const dirRelativePath = normalizeRelativeUploadPath(relative(rootPath, dirPath)) if (dirStat.isSymbolicLink()) { - throw new RuntimeUploadSymlinkError( - `Symlink not allowed in '${normalizeRelativeUploadPath(relative(rootPath, dirPath))}'` - ) + throw new RuntimeUploadSymlinkError(`Symlink not allowed in '${dirRelativePath}'`) } if (!dirStat.isDirectory()) { - throw new Error( - `Unsupported file type in '${normalizeRelativeUploadPath(relative(rootPath, dirPath))}'` - ) + throw new Error(`Unsupported file type in '${dirRelativePath}'`) } - await assertRealPathInsideRoot( - rootRealPath, - dirPath, - normalizeRelativeUploadPath(relative(rootPath, dirPath)) - ) - const dirEntries = await readdir(dirPath, { withFileTypes: true }) - for (const entry of dirEntries) { + await assertRealPathInsideRoot(rootRealPath, dirPath, dirRelativePath) + const directory = await opendir(dirPath) + for await (const entry of directory) { const childPath = join(dirPath, entry.name) const childRelativePath = normalizeRelativeUploadPath(relative(rootPath, childPath)) if (entry.isSymbolicLink()) { throw new RuntimeUploadSymlinkError(`Symlink not allowed in '${childRelativePath}'`) } if (entry.isDirectory()) { + assertExternalImportTreeDepth(depth + 1) + admitExternalImportTreeEntry(retentionBudget.tree, childRelativePath, true) entries.push({ relativePath: childRelativePath, kind: 'directory' }) - await visit(childPath) + await visit(childPath, depth + 1) continue } if (!entry.isFile()) { throw new Error(`Unsupported file type in '${childRelativePath}'`) } - const stagedFile = await stageFileEntry(childPath, childRelativePath, { - rootRealPath, - totalBytesBefore: totalBytes - }) - totalBytes += stagedFile.byteLength - entries.push(stagedFile.entry) + admitExternalImportTreeEntry(retentionBudget.tree, childRelativePath, true) + entries.push( + await stageFileEntry(childPath, childRelativePath, { + rootRealPath, + retentionBudget + }) + ) } } - await visit(rootPath) + await visit(rootPath, 0) return entries } async function stageFileEntry( filePath: string, relativePath: string, - options?: { rootRealPath?: string; totalBytesBefore?: number } -): Promise<{ entry: StagedExternalImportEntry; byteLength: number }> { + options: { retentionBudget: RuntimeUploadRetentionBudget; rootRealPath?: string } +): Promise { const statResult = await lstat(filePath) const displayPath = normalizeRelativeUploadPath(relativePath) if (statResult.isSymbolicLink()) { @@ -587,11 +616,7 @@ async function stageFileEntry( if (options?.rootRealPath) { await assertRealPathInsideRoot(options.rootRealPath, filePath, displayPath) } - const initialTotalBytes = - options?.totalBytesBefore === undefined - ? statResult.size - : options.totalBytesBefore + statResult.size - assertRemoteUploadBudget(relativePath, statResult.size, initialTotalBytes) + assertRuntimeUploadFileFits(options.retentionBudget, displayPath, statResult.size) const fileHandle = await open(filePath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)) try { const openedStat = await fileHandle.stat() @@ -605,23 +630,17 @@ async function stageFileEntry( ) { throw new Error(`File changed during upload staging: '${displayPath}'`) } - const totalBytes = - options?.totalBytesBefore === undefined - ? openedStat.size - : options.totalBytesBefore + openedStat.size - assertRemoteUploadBudget(relativePath, openedStat.size, totalBytes) - const buffer = await fileHandle.readFile() + assertRuntimeUploadFileFits(options.retentionBudget, displayPath, openedStat.size) + const buffer = await readStableRuntimeUploadFile(fileHandle, openedStat.size, displayPath) const afterReadStat = await fileHandle.stat() if (afterReadStat.size !== openedStat.size) { throw new Error(`File changed during upload staging: '${displayPath}'`) } + retainRuntimeUploadFileBytes(options.retentionBudget, displayPath, buffer.byteLength) return { - entry: { - relativePath: displayPath, - kind: 'file', - contentBase64: buffer.toString('base64') - }, - byteLength: openedStat.size + relativePath: displayPath, + kind: 'file', + contentBase64: buffer.toString('base64') } } finally { await fileHandle.close() @@ -644,19 +663,6 @@ async function assertRealPathInsideRoot( } } -function assertRemoteUploadBudget( - relativePath: string, - fileBytes: number, - totalBytes: number -): void { - if (fileBytes > REMOTE_IMPORT_MAX_FILE_BYTES) { - throw new Error(`'${relativePath}' is too large for remote import`) - } - if (totalBytes > REMOTE_IMPORT_MAX_TOTAL_BYTES) { - throw new Error('Remote import is too large') - } -} - function normalizeRelativeUploadPath(path: string): string { return path.replace(/[\\/]+/g, '/').replace(/^\/+/, '') } @@ -666,19 +672,40 @@ function normalizeRelativeUploadPath(path: string): string { * is found anywhere in the subtree. */ async function preScanForSymlinks(dirPath: string): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) - for (const entry of entries) { - if (entry.isSymbolicLink()) { + const budget = createExternalImportTreeBudget() + + async function visit( + currentDirPath: string, + relativeDirPath: string, + depth: number + ): Promise { + assertExternalImportTreeDepth(depth) + const currentStat = await lstat(currentDirPath) + if (currentStat.isSymbolicLink()) { return true } - if (entry.isDirectory()) { - const childPath = join(dirPath, entry.name) - if (await preScanForSymlinks(childPath)) { + if (!currentStat.isDirectory()) { + throw new Error(`Unsupported file type in '${relativeDirPath}'`) + } + + const directory = await opendir(currentDirPath) + for await (const entry of directory) { + const childRelativePath = join(relativeDirPath, entry.name) + admitExternalImportTreeEntry(budget, childRelativePath, false) + if (entry.isSymbolicLink()) { return true } + if (entry.isDirectory()) { + assertExternalImportTreeDepth(depth + 1) + if (await visit(join(currentDirPath, entry.name), childRelativePath, depth + 1)) { + return true + } + } } + return false } - return false + + return visit(dirPath, '', 0) } /** @@ -687,24 +714,47 @@ async function preScanForSymlinks(dirPath: string): Promise { * buffering entire files into memory. */ async function recursiveCopyDir(srcDir: string, destDir: string): Promise { - await mkdir(destDir, { recursive: false }) - const entries = await readdir(srcDir, { withFileTypes: true }) - for (const entry of entries) { - const srcPath = join(srcDir, entry.name) - const dstPath = join(destDir, entry.name) - const statResult = await lstat(srcPath) - if (statResult.isSymbolicLink()) { - throw new Error(`Symlink not allowed in '${entry.name}'`) + const budget = createExternalImportTreeBudget() + + async function copyDirectory( + currentSrcDir: string, + currentDestDir: string, + relativeDirPath: string, + depth: number + ): Promise { + assertExternalImportTreeDepth(depth) + const dirStat = await lstat(currentSrcDir) + if (dirStat.isSymbolicLink()) { + throw new Error(`Symlink not allowed in '${relativeDirPath}'`) } - if (statResult.isDirectory()) { - await recursiveCopyDir(srcPath, dstPath) - continue + if (!dirStat.isDirectory()) { + throw new Error(`Unsupported file type in '${relativeDirPath}'`) } - if (!statResult.isFile()) { - throw new Error(`Unsupported file type in '${entry.name}'`) + await mkdir(currentDestDir, { recursive: false }) + + const directory = await opendir(currentSrcDir) + for await (const entry of directory) { + const srcPath = join(currentSrcDir, entry.name) + const dstPath = join(currentDestDir, entry.name) + const childRelativePath = join(relativeDirPath, entry.name) + admitExternalImportTreeEntry(budget, childRelativePath, false) + const statResult = await lstat(srcPath) + if (statResult.isSymbolicLink()) { + throw new Error(`Symlink not allowed in '${childRelativePath}'`) + } + if (statResult.isDirectory()) { + assertExternalImportTreeDepth(depth + 1) + await copyDirectory(srcPath, dstPath, childRelativePath, depth + 1) + continue + } + if (!statResult.isFile()) { + throw new Error(`Unsupported file type in '${childRelativePath}'`) + } + await copyLocalFileNoFollow(srcPath, dstPath, statResult) } - await copyLocalFileNoFollow(srcPath, dstPath, statResult) } + + await copyDirectory(srcDir, destDir, '', 0) } async function copyLocalFileNoFollow( diff --git a/src/main/ipc/filesystem-runtime-upload-file-reader.test.ts b/src/main/ipc/filesystem-runtime-upload-file-reader.test.ts new file mode 100644 index 00000000000..d2c3d7f92be --- /dev/null +++ b/src/main/ipc/filesystem-runtime-upload-file-reader.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import { readStableRuntimeUploadFile } from './filesystem-runtime-upload-file-reader' + +function createRead(content: Buffer, maxBytesPerRead = content.byteLength) { + return vi.fn(async (buffer: Buffer, offset: number, length: number, position: number) => { + const bytesRead = Math.min(length, maxBytesPerRead, Math.max(0, content.byteLength - position)) + if (bytesRead > 0) { + content.copy(buffer, offset, position, position + bytesRead) + } + return { buffer, bytesRead } + }) +} + +describe('runtime upload file reader', () => { + it('preserves an exact-size file across partial reads', async () => { + const content = Buffer.from('bounded-content') + const read = createRead(content, 3) + + await expect( + readStableRuntimeUploadFile({ read }, content.byteLength, 'asset.bin') + ).resolves.toEqual(content) + expect(read).toHaveBeenLastCalledWith(expect.any(Buffer), 0, 1, content.byteLength) + }) + + it('rejects a file that becomes shorter without exposing uninitialized bytes', async () => { + const content = Buffer.from('short') + + await expect( + readStableRuntimeUploadFile({ read: createRead(content) }, content.byteLength + 1, '') + ).rejects.toThrow("File changed during upload staging: ''") + }) + + it('uses a one-byte probe and rejects growth beyond the admitted size', async () => { + const content = Buffer.from('growth') + const admittedBytes = content.byteLength - 1 + const read = createRead(content) + + await expect( + readStableRuntimeUploadFile({ read }, admittedBytes, 'growth.bin') + ).rejects.toThrow("File changed during upload staging: 'growth.bin'") + expect(read).toHaveBeenNthCalledWith(1, expect.any(Buffer), 0, admittedBytes, 0) + expect(read).toHaveBeenNthCalledWith(2, expect.any(Buffer), 0, 1, admittedBytes) + }) +}) diff --git a/src/main/ipc/filesystem-runtime-upload-file-reader.ts b/src/main/ipc/filesystem-runtime-upload-file-reader.ts new file mode 100644 index 00000000000..842db049fef --- /dev/null +++ b/src/main/ipc/filesystem-runtime-upload-file-reader.ts @@ -0,0 +1,33 @@ +type RuntimeUploadFileHandle = { + read( + buffer: Buffer, + offset: number, + length: number, + position: number + ): Promise<{ bytesRead: number }> +} + +export async function readStableRuntimeUploadFile( + fileHandle: RuntimeUploadFileHandle, + expectedBytes: number, + displayPath: string +): Promise { + const buffer = Buffer.allocUnsafe(expectedBytes) + let offset = 0 + while (offset < expectedBytes) { + const { bytesRead } = await fileHandle.read(buffer, offset, expectedBytes - offset, offset) + if (bytesRead === 0) { + throw fileChangedError(displayPath) + } + offset += bytesRead + } + const probe = Buffer.allocUnsafe(1) + if ((await fileHandle.read(probe, 0, 1, offset)).bytesRead !== 0) { + throw fileChangedError(displayPath) + } + return buffer +} + +function fileChangedError(displayPath: string): Error { + return new Error(`File changed during upload staging: '${displayPath}'`) +} diff --git a/src/main/ipc/filesystem-search-git.ts b/src/main/ipc/filesystem-search-git.ts index 065dbd11dad..3a71c83c1ea 100644 --- a/src/main/ipc/filesystem-search-git.ts +++ b/src/main/ipc/filesystem-search-git.ts @@ -8,6 +8,7 @@ import { SEARCH_TIMEOUT_MS } from '../../shared/text-search' import { gitSpawn } from '../git/runner' +import { SearchSubprocessLineAccumulator } from '../../shared/search-subprocess-lines' /** * Fallback text search using git grep. Used when rg is not available. @@ -26,7 +27,7 @@ export function searchWithGitGrep( const gitArgs = buildGitGrepArgs(args.query, args) const matchRegex = buildSubmatchRegex(args.query, args) const acc = createAccumulator() - let stdoutBuffer = '' + const stdoutLines = new SearchSubprocessLineAccumulator() let done = false const child = gitSpawn(gitArgs, { @@ -58,12 +59,11 @@ export function searchWithGitGrep( } } - function handleStdoutData(chunk: string): void { - stdoutBuffer += chunk - const lines = stdoutBuffer.split('\n') - stdoutBuffer = lines.pop() ?? '' - for (const l of lines) { - processLine(l) + function handleStdoutData(chunk: Buffer): void { + if (!stdoutLines.push(chunk, processLine)) { + acc.truncated = true + child.kill() + resolveOnce() } } @@ -76,13 +76,13 @@ export function searchWithGitGrep( } function handleClose(): void { - if (stdoutBuffer) { - processLine(stdoutBuffer) + const trailingLine = stdoutLines.finish() + if (trailingLine !== null) { + processLine(trailingLine) } resolveOnce() } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) diff --git a/src/main/ipc/filesystem-watcher-admission.test.ts b/src/main/ipc/filesystem-watcher-admission.test.ts new file mode 100644 index 00000000000..0851c0d0886 --- /dev/null +++ b/src/main/ipc/filesystem-watcher-admission.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' +import { + FILESYSTEM_WATCHER_MAX_CLAIMS, + FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER, + FILESYSTEM_WATCHER_MAX_PATH_BYTES, + FilesystemWatcherAdmission, + parseFilesystemWatcherIdentity +} from './filesystem-watcher-admission' + +describe('filesystem watcher admission', () => { + it('caps unique claims per sender and recovers after release', () => { + const admission = new FilesystemWatcherAdmission() + const releases = Array.from({ length: FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER }, (_, index) => + admission.claim(1, `local:/repo-${index}`, 1) + ) + + expect(() => admission.claim(1, 'local:/overflow', 1)).toThrow('capacity reached') + expect(admission.evidence().claimCount).toBe(FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER) + + releases[0]?.release() + expect(() => admission.claim(1, 'local:/recovered', 1)).not.toThrow() + }) + + it('caps aggregate claims across senders without double-counting duplicate watches', () => { + const admission = new FilesystemWatcherAdmission() + for (let index = 0; index < FILESYSTEM_WATCHER_MAX_CLAIMS; index += 1) { + admission.claim(index, `local:/repo-${index}`, 1) + } + + expect(admission.claim(0, 'local:/repo-0', 1).added).toBe(false) + expect(() => admission.claim(FILESYSTEM_WATCHER_MAX_CLAIMS, 'local:/overflow', 1)).toThrow( + 'capacity reached' + ) + + admission.releaseSender(0) + expect(() => + admission.claim(FILESYSTEM_WATCHER_MAX_CLAIMS, 'local:/recovered', 1) + ).not.toThrow() + }) + + it('bounds retained path bytes before creating a watcher key', () => { + const exact = 'é'.repeat(FILESYSTEM_WATCHER_MAX_PATH_BYTES / 2) + + expect(parseFilesystemWatcherIdentity({ worktreePath: exact }).retainedBytes).toBe( + FILESYSTEM_WATCHER_MAX_PATH_BYTES + ) + expect(() => parseFilesystemWatcherIdentity({ worktreePath: `${exact}x` })).toThrow( + `exceeds ${FILESYSTEM_WATCHER_MAX_PATH_BYTES} UTF-8 bytes` + ) + }) + + it('does not let a stale release remove a replacement claim', () => { + const admission = new FilesystemWatcherAdmission() + const old = admission.claim(1, 'local:/repo', 1) + admission.clear() + admission.claim(1, 'local:/repo', 1) + + old.release() + + expect(admission.evidence().claimCount).toBe(1) + }) +}) diff --git a/src/main/ipc/filesystem-watcher-admission.ts b/src/main/ipc/filesystem-watcher-admission.ts new file mode 100644 index 00000000000..ecece338ea1 --- /dev/null +++ b/src/main/ipc/filesystem-watcher-admission.ts @@ -0,0 +1,132 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const FILESYSTEM_WATCHER_MAX_CLAIMS = 1_024 +export const FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER = 256 +export const FILESYSTEM_WATCHER_MAX_RETAINED_IDENTITY_BYTES = 16 * 1024 * 1024 +export const FILESYSTEM_WATCHER_MAX_PATH_BYTES = 64 * 1024 +export const FILESYSTEM_WATCHER_MAX_CONNECTION_ID_BYTES = 8 * 1024 + +type WatchClaim = { retainedBytes: number } + +export type FilesystemWatcherIdentity = { + worktreePath: string + connectionId?: string + retainedBytes: number +} + +export class FilesystemWatcherAdmission { + private readonly claimsBySender = new Map>() + private claimCount = 0 + private retainedBytes = 0 + + claim( + senderId: number, + key: string, + retainedBytes: number + ): { added: boolean; release: () => void } { + let senderClaims = this.claimsBySender.get(senderId) + const existing = senderClaims?.get(key) + if (existing) { + return { added: false, release: () => undefined } + } + if ( + this.claimCount >= FILESYSTEM_WATCHER_MAX_CLAIMS || + (senderClaims?.size ?? 0) >= FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER || + this.retainedBytes + retainedBytes > FILESYSTEM_WATCHER_MAX_RETAINED_IDENTITY_BYTES + ) { + throw new Error('Filesystem watcher capacity reached; close an existing watch and retry.') + } + senderClaims ??= new Map() + this.claimsBySender.set(senderId, senderClaims) + const claim = { retainedBytes } + senderClaims.set(key, claim) + this.claimCount += 1 + this.retainedBytes += retainedBytes + return { + added: true, + release: () => this.releaseClaim(senderId, key, claim) + } + } + + release(senderId: number, key: string): void { + const claim = this.claimsBySender.get(senderId)?.get(key) + if (claim) { + this.releaseClaim(senderId, key, claim) + } + } + + releaseSender(senderId: number): void { + const senderClaims = this.claimsBySender.get(senderId) + if (!senderClaims) { + return + } + this.claimsBySender.delete(senderId) + for (const claim of senderClaims.values()) { + this.claimCount -= 1 + this.retainedBytes -= claim.retainedBytes + } + } + + clear(): void { + this.claimsBySender.clear() + this.claimCount = 0 + this.retainedBytes = 0 + } + + evidence(): { claimCount: number; retainedBytes: number; senderCount: number } { + return { + claimCount: this.claimCount, + retainedBytes: this.retainedBytes, + senderCount: this.claimsBySender.size + } + } + + private releaseClaim(senderId: number, key: string, expected: WatchClaim): void { + const senderClaims = this.claimsBySender.get(senderId) + if (senderClaims?.get(key) !== expected) { + return + } + senderClaims.delete(key) + this.claimCount -= 1 + this.retainedBytes -= expected.retainedBytes + if (senderClaims.size === 0) { + this.claimsBySender.delete(senderId) + } + } +} + +export function parseFilesystemWatcherIdentity(value: unknown): FilesystemWatcherIdentity { + if (!value || typeof value !== 'object') { + throw new TypeError('Filesystem watcher arguments are required.') + } + const record = value as Record + const worktreePath = record.worktreePath + const connectionId = record.connectionId + if (typeof worktreePath !== 'string' || worktreePath.length === 0) { + throw new TypeError('Filesystem watcher worktreePath must be a non-empty string.') + } + const pathBytes = boundedFieldBytes( + worktreePath, + FILESYSTEM_WATCHER_MAX_PATH_BYTES, + 'worktreePath' + ) + if (connectionId !== undefined && typeof connectionId !== 'string') { + throw new TypeError('Filesystem watcher connectionId must be a string.') + } + const connectionBytes = connectionId + ? boundedFieldBytes(connectionId, FILESYSTEM_WATCHER_MAX_CONNECTION_ID_BYTES, 'connectionId') + : 0 + return { + worktreePath, + ...(connectionId ? { connectionId } : {}), + retainedBytes: pathBytes + connectionBytes + } +} + +function boundedFieldBytes(value: string, limit: number, field: string): number { + const measured = measureUtf8ByteLength(value, { stopAfterBytes: limit }) + if (measured.exceededLimit) { + throw new TypeError(`Filesystem watcher ${field} exceeds ${limit} UTF-8 bytes.`) + } + return measured.byteLength +} diff --git a/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts b/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts index 9a5c5831492..54f712c18ad 100644 --- a/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts +++ b/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts @@ -142,6 +142,26 @@ describe('local filesystem watcher unsubscribe cleanup', () => { expect(shutdownResolved).toBe(true) }) + it('releases watcher admission after repeated subscription errors', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + let watcherCallback: (err: Error | null, events: []) => void = () => {} + vi.mocked(subscribeParcelWatcher).mockImplementation(async (_root, callback) => { + watcherCallback = callback as typeof watcherCallback + return { unsubscribe: vi.fn() } as never + }) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + + for (let index = 0; index < 257; index += 1) { + await expect( + handlers['fs:watchWorktree']({ sender }, { worktreePath: `/tmp/erroring-repo-${index}` }) + ).resolves.toBeUndefined() + watcherCallback(new Error('root disappeared'), []) + } + + errorSpy.mockRestore() + }) + it('unsubscribes if the sender is destroyed while the local watcher is opening', async () => { vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) const destroyedCallbacks: (() => void)[] = [] diff --git a/src/main/ipc/filesystem-watcher-wsl.test.ts b/src/main/ipc/filesystem-watcher-wsl.test.ts index 4c8333d508e..955013a86b8 100644 --- a/src/main/ipc/filesystem-watcher-wsl.test.ts +++ b/src/main/ipc/filesystem-watcher-wsl.test.ts @@ -100,6 +100,17 @@ describe('createWslWatcher', () => { ) }) + it('parses a snapshot delivered as 100,000 one-byte fragments', async () => { + const { child, promise } = startWatcher() + const frame = Buffer.from(snapshotFrame([['f', '1.0', `/home/me/repo/${'x'.repeat(99_950)}`]])) + + for (let index = 0; index < frame.byteLength; index += 1) { + child.stdout.write(frame.subarray(index, index + 1)) + } + + await expect(promise).resolves.toBeDefined() + }) + it('counts WSL watcher processes against the global physical child cap', async () => { const releases = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => reserveWatcherChild() diff --git a/src/main/ipc/filesystem-watcher-wsl.ts b/src/main/ipc/filesystem-watcher-wsl.ts index b73f58d5122..366f3068405 100644 --- a/src/main/ipc/filesystem-watcher-wsl.ts +++ b/src/main/ipc/filesystem-watcher-wsl.ts @@ -6,13 +6,14 @@ * inside the distro so shutdown kills it instead of Orca restarting WSL. */ import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' -import { StringDecoder } from 'node:string_decoder' import type { WebContents } from 'electron' import type { Event as WatcherEvent } from '@parcel/watcher' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import { queueWatcherEvents } from './filesystem-watcher-event-batch' import { parseWslUncPath } from '../../shared/wsl-paths' import { createWslWatcherProcessExit, createWslWatcherStartup } from './wsl-watcher-process-exit' import { reserveWatcherChild, WatcherChildCapacityError } from './parcel-watcher-child-registry' +import { diffWslSnapshots, parseWslSnapshotFrame, type WslSnapshot } from './wsl-watcher-snapshot' export type WatcherSubscription = { unsubscribe(): Promise @@ -41,19 +42,7 @@ export type WslWatcherDeps = { const POLL_INTERVAL_SECONDS = 2 const STARTUP_TIMEOUT_MS = 10_000 const [SNAPSHOT_START, SNAPSHOT_END] = ['\x1e', '\x1f'] -const MAX_STREAM_BUFFER_CHARS = 10 * 1024 * 1024 - -type WslSnapshotEntry = { - path: string - type: string - mtime: string -} - -type WslSnapshot = Map - -function toWslUncPath(linuxPath: string, distro: string): string { - return `\\\\wsl.localhost\\${distro}${linuxPath.replace(/\//g, '\\')}` -} +export const MAX_WSL_SNAPSHOT_FRAME_BYTES = 10 * 1024 * 1024 function quoteSafeFindName(name: string): string { if (!/^[A-Za-z0-9_.-]+$/.test(name)) { @@ -86,59 +75,6 @@ function buildSnapshotScript(ignoreDirs: readonly string[]): string { ].join('\n') } -function parseSnapshotFrame(frame: string, distro: string): WslSnapshot { - const snapshot: WslSnapshot = new Map() - for (const rawEntry of frame.split('\0')) { - if (!rawEntry) { - continue - } - const firstTab = rawEntry.indexOf('\t') - const secondTab = firstTab === -1 ? -1 : rawEntry.indexOf('\t', firstTab + 1) - if (firstTab <= 0 || secondTab <= firstTab + 1) { - continue - } - const linuxPath = rawEntry.slice(secondTab + 1) - if (!linuxPath.startsWith('/')) { - continue - } - const entry: WslSnapshotEntry = { - type: rawEntry.slice(0, firstTab), - mtime: rawEntry.slice(firstTab + 1, secondTab), - path: toWslUncPath(linuxPath, distro) - } - snapshot.set(entry.path, entry) - } - return snapshot -} - -function diffSnapshots(prev: WslSnapshot, next: WslSnapshot): WatcherEvent[] { - const events: WatcherEvent[] = [] - - for (const [entryPath, nextEntry] of next) { - const prevEntry = prev.get(entryPath) - if (!prevEntry) { - events.push({ type: 'create', path: entryPath } as WatcherEvent) - continue - } - if (prevEntry.type !== nextEntry.type) { - events.push({ type: 'delete', path: entryPath } as WatcherEvent) - events.push({ type: 'create', path: entryPath } as WatcherEvent) - continue - } - if (prevEntry.mtime !== nextEntry.mtime) { - events.push({ type: 'update', path: entryPath } as WatcherEvent) - } - } - - for (const entryPath of prev.keys()) { - if (!next.has(entryPath)) { - events.push({ type: 'delete', path: entryPath } as WatcherEvent) - } - } - - return events -} - function markOverflowWithoutUncStat(root: WatchedRoot): void { if (root.batch.timer) { clearTimeout(root.batch.timer) @@ -176,13 +112,21 @@ export async function createWslWatcher( let disposed = false let prevSnapshot: WslSnapshot | null = null let stopped = false - let streamBuffer = '' - const stdoutDecoder = new StringDecoder('utf8') - const stderrDecoder = new StringDecoder('utf8') - let stderrTail = '' + const streamBuffer = new GrowingByteBuffer() + const stderrTail = new GrowingByteBuffer() const startup = createWslWatcherStartup() + function reportSnapshotOverflow(message: string): void { + if (!startup.settled) { + startup.settle(new Error(message)) + return + } + prevSnapshot = new Map() + markOverflowWithoutUncStat(root) + deps.scheduleBatchFlush(rootKey, root) + } + function signalWatcherStopped(): void { if (stopped) { return @@ -197,13 +141,17 @@ export async function createWslWatcher( } function ingestFrame(frame: string): void { - const nextSnapshot = parseSnapshotFrame(frame, distro) + const nextSnapshot = parseWslSnapshotFrame(frame, distro) + if (!nextSnapshot) { + reportSnapshotOverflow('WSL watcher snapshot exceeded its retained entry limit') + return + } if (!prevSnapshot) { prevSnapshot = nextSnapshot startup.settle() return } - const events = diffSnapshots(prevSnapshot, nextSnapshot) + const events = diffWslSnapshots(prevSnapshot, nextSnapshot) prevSnapshot = nextSnapshot if (events.length > 0) { @@ -214,25 +162,30 @@ export async function createWslWatcher( function drainFrames(): void { while (true) { - const start = streamBuffer.indexOf(SNAPSHOT_START) + const start = streamBuffer.indexOfByte(SNAPSHOT_START.charCodeAt(0)) if (start === -1) { - streamBuffer = streamBuffer.slice(-1) + streamBuffer.retainSuffix(1) return } if (start > 0) { - streamBuffer = streamBuffer.slice(start) + streamBuffer.discardPrefix(start) } - const end = streamBuffer.indexOf(SNAPSHOT_END, 1) + const end = streamBuffer.indexOfByte(SNAPSHOT_END.charCodeAt(0), 1) if (end === -1) { - if (streamBuffer.length > MAX_STREAM_BUFFER_CHARS) { - streamBuffer = '' - markOverflowWithoutUncStat(root) - deps.scheduleBatchFlush(rootKey, root) + if (streamBuffer.byteLength > MAX_WSL_SNAPSHOT_FRAME_BYTES) { + streamBuffer.clear() + reportSnapshotOverflow('WSL watcher snapshot exceeded its frame byte limit') } return } - const frame = streamBuffer.slice(1, end) - streamBuffer = streamBuffer.slice(end + 1) + if (end - 1 > MAX_WSL_SNAPSHOT_FRAME_BYTES) { + streamBuffer.discardPrefix(end + 1) + reportSnapshotOverflow('WSL watcher snapshot exceeded its frame byte limit') + continue + } + const frameWithStart = streamBuffer.takePrefixString(end) + streamBuffer.discardPrefix(1) + const frame = frameWithStart.slice(1) ingestFrame(frame) } } @@ -279,12 +232,18 @@ export async function createWslWatcher( if (disposed) { return } - streamBuffer += stdoutDecoder.write(chunk) + streamBuffer.append(chunk) drainFrames() }) child.stderr.on('data', (chunk: Buffer) => { - stderrTail = (stderrTail + stderrDecoder.write(chunk)).slice(-4096) + if (chunk.byteLength >= 4096) { + stderrTail.clear() + stderrTail.append(chunk.subarray(chunk.byteLength - 4096)) + return + } + stderrTail.append(chunk) + stderrTail.retainSuffix(4096) }) child.stdout.on('error', (error) => { @@ -319,7 +278,8 @@ export async function createWslWatcher( child.once('close', (code, signal) => { processExit.markPhysicalExit() if (!startup.settled) { - const suffix = stderrTail.trim() ? `: ${stderrTail.trim()}` : '' + const stderr = stderrTail.toString() + const suffix = stderr.trim() ? `: ${stderr.trim()}` : '' startup.settle( new Error(`WSL watcher exited before first snapshot (${code ?? signal})${suffix}`) ) diff --git a/src/main/ipc/filesystem-watcher.test.ts b/src/main/ipc/filesystem-watcher.test.ts index 8d658b25f12..d13221acf41 100644 --- a/src/main/ipc/filesystem-watcher.test.ts +++ b/src/main/ipc/filesystem-watcher.test.ts @@ -30,6 +30,7 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ import { closeAllWatchers, closeRemoteWatcherForWorktreePath, + LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY, registerFilesystemWatcherHandlers, restoreRemoteWatcherAfterFailedRemoval } from './filesystem-watcher' @@ -43,6 +44,7 @@ import { WatcherChildCapacityError } from './parcel-watcher-child-registry' import { acquireWatcherRemovalGate } from './watcher-removal-gate' +import { FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER } from './filesystem-watcher-admission' type HandlerMap = Record Promise | unknown> @@ -94,6 +96,99 @@ describe('registerFilesystemWatcherHandlers', () => { await closeAllWatchers() }) + it('rejects pending watcher churn at the sender cap and recovers after unwatch', async () => { + vi.useFakeTimers() + getSshFilesystemProviderMock.mockReturnValue(undefined) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + for (let index = 0; index < FILESYSTEM_WATCHER_MAX_CLAIMS_PER_SENDER; index += 1) { + await handlers['fs:watchWorktree']( + { sender }, + { worktreePath: `/missing/repo-${index}`, connectionId: 'conn-1' } + ) + } + await expect( + handlers['fs:watchWorktree']( + { sender }, + { worktreePath: '/missing/overflow', connectionId: 'conn-1' } + ) + ).rejects.toThrow('Filesystem watcher capacity reached') + + handlers['fs:unwatchWorktree']( + { sender }, + { worktreePath: '/missing/repo-0', connectionId: 'conn-1' } + ) + await expect( + handlers['fs:watchWorktree']( + { sender }, + { worktreePath: '/missing/recovered', connectionId: 'conn-1' } + ) + ).resolves.toBeUndefined() + await closeAllWatchers() + warn.mockRestore() + vi.useRealTimers() + }) + + it.each([ + ['at the limit', LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY], + ['above the limit', LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY + 1] + ])('bounds event directory stats %s', async (_, count) => { + let active = 0 + let peak = 0 + let started = 0 + const releases: (() => void)[] = [] + vi.mocked(stat).mockImplementation((filePath) => { + if (filePath === '/repo') { + return Promise.resolve({ isDirectory: () => true } as never) + } + started++ + active++ + peak = Math.max(peak, active) + return new Promise((resolve) => { + releases.push(() => { + active-- + resolve({ isDirectory: () => false } as never) + }) + }) + }) + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: vi.fn() } as never) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + await handlers['fs:watchWorktree']({ sender }, { worktreePath: '/repo' }) + const onEvents = vi.mocked(subscribeParcelWatcher).mock.calls[0][1] as ( + error: Error | null, + events: { type: 'create'; path: string }[] + ) => void + + vi.useFakeTimers() + onEvents( + null, + Array.from({ length: count }, (_, index) => ({ + type: 'create', + path: `/repo/file-${index}` + })) + ) + await vi.advanceTimersByTimeAsync(150) + expect(started).toBe(Math.min(count, LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY)) + if (count > LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY) { + releases.shift()?.() + for (let turn = 0; turn < 5 && started < count; turn++) { + await Promise.resolve() + } + expect(started).toBe(count) + } + releases.splice(0).forEach((release) => release()) + + expect(peak).toBe(Math.min(count, LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY)) + await vi.waitFor(() => + expect(sender.send).toHaveBeenCalledWith( + 'fs:changed', + expect.objectContaining({ events: expect.arrayContaining([expect.any(Object)]) }) + ) + ) + vi.useRealTimers() + }) + it('automatically retries a WSL watcher when child capacity becomes available', async () => { Object.defineProperty(process, 'platform', { configurable: true, diff --git a/src/main/ipc/filesystem-watcher.ts b/src/main/ipc/filesystem-watcher.ts index 2e2a27cb3f6..6e4d0d5e783 100644 --- a/src/main/ipc/filesystem-watcher.ts +++ b/src/main/ipc/filesystem-watcher.ts @@ -8,6 +8,7 @@ import { isWindowsAbsolutePathLike, normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' import { isWslPath } from '../wsl' import { createWslWatcher } from './filesystem-watcher-wsl' import type { WatchedRoot } from './filesystem-watcher-wsl' @@ -20,6 +21,11 @@ import { WatcherChildCapacityError } from './parcel-watcher-child-registry' import { beginWatcherInstall, isWatcherRemovalInProgressError } from './watcher-removal-gate' +import { + FilesystemWatcherAdmission, + parseFilesystemWatcherIdentity, + type FilesystemWatcherIdentity +} from './filesystem-watcher-admission' // Why: suppress high-churn dirs at the watcher level (separate from the File Explorer display filter, which only hides rows). import { WATCHER_IGNORE_DIRS, buildParcelWatcherIgnoreOptions } from './filesystem-watcher-ignore' @@ -27,6 +33,8 @@ import { WATCHER_IGNORE_DIRS, buildParcelWatcherIgnoreOptions } from './filesyst const DEBOUNCE_TRAILING_MS = 150 const DEBOUNCE_MAX_WAIT_MS = 500 +export const LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY = 8 +const WATCHER_LISTENER_INSTALL_CONCURRENCY = 8 // ── Per-root watcher state ─────────────────────────────────────────── // WatchedRoot/WatcherSubscription live in filesystem-watcher-wsl.ts so native and WSL watchers share one shape. @@ -34,6 +42,7 @@ const DEBOUNCE_MAX_WAIT_MS = 500 // ── Module state ───────────────────────────────────────────────────── const watchedRoots = new Map() +const watcherAdmission = new FilesystemWatcherAdmission() // Why: cache roots that failed watcher creation (e.g. WSL UNC paths) so we don't retry every worktree switch and spam the console with errors. const UNWATCHABLE_ROOT_CACHE_MAX = 256 @@ -151,8 +160,10 @@ function scheduleLocalCapacityRetry( return } pendingLocalCapacityRetries.delete(rootKey) - await Promise.all( - [...retry.listeners.values()].map(async (listener) => { + await mapWithConcurrency( + [...retry.listeners.values()], + WATCHER_LISTENER_INSTALL_CONCURRENCY, + async (listener) => { if (listener.isDestroyed()) { return } @@ -161,7 +172,7 @@ function scheduleLocalCapacityRetry( console.error(`[filesystem-watcher] capacity retry failed for ${rootKey}:`, error) } }) - }) + } ) }) retry = { listeners: new Map(), cancelWait } @@ -199,6 +210,26 @@ function localWatcherRoot(rootPath: string): { key: string; path: string } { } } +function watcherAdmissionKey(identity: FilesystemWatcherIdentity): string { + return identity.connectionId + ? `remote:${remoteWatcherKey(identity.connectionId, identity.worktreePath)}` + : `local:${localWatcherRoot(identity.worktreePath).key}` +} + +function hasRetainedLocalWatcherListener(rootKey: string, senderId: number): boolean { + return ( + watchedRoots.get(rootKey)?.listeners.has(senderId) === true || + inFlightLocalInstalls.get(rootKey)?.listeners.has(senderId) === true || + pendingLocalCapacityRetries.get(rootKey)?.listeners.has(senderId) === true + ) +} + +function releaseLocalWatcherAdmissions(rootKey: string, listeners: Map): void { + for (const senderId of listeners.keys()) { + watcherAdmission.release(senderId, `local:${rootKey}`) + } +} + function normalizeEventPath(eventPath: string): string { let resolved = path.resolve(eventPath) if (/^[a-zA-Z]:/.test(resolved)) { @@ -303,8 +334,10 @@ async function flushBatch(rootKey: string, root: WatchedRoot): Promise { const coalesced = coalesceEvents(rawEvents) - const events: FsChangeEvent[] = await Promise.all( - coalesced.map(async (evt) => { + const events = await mapWithConcurrency( + coalesced, + LOCAL_WATCHER_DIRECTORY_STAT_CONCURRENCY, + async (evt): Promise => { // Why: a deleted path can't be stat'd; leave isDirectory undefined and let the renderer infer from dirCache. const isDirectory = evt.type === 'delete' ? undefined : await tryStatIsDirectory(evt.path) @@ -313,7 +346,7 @@ async function flushBatch(rootKey: string, root: WatchedRoot): Promise { absolutePath: evt.path, isDirectory } - }) + } ) const payload: FsChangedPayload = { @@ -397,6 +430,7 @@ async function createWatcher( retainLocalWatcherPhysicalFailure(rootKey, err) void trackLocalUnsubscribe(rootKey, root) } + releaseLocalWatcherAdmissions(rootKey, root.listeners) errorCleanedUp = true watchedRoots.delete(rootKey) return @@ -432,6 +466,7 @@ async function createWatcher( // ── Subscribe / Unsubscribe ────────────────────────────────────────── function cleanupLocalWatchersForSender(senderId: number): void { + watcherAdmission.releaseSender(senderId) for (const [rootKey, suspended] of suspendedLocalWatcherListeners) { suspended.listeners.delete(senderId) if (suspended.listeners.size === 0) { @@ -657,11 +692,13 @@ async function doInstallLocalWatcher( if (!s.isDirectory()) { console.warn(`[filesystem-watcher] not a directory: ${rootKey}`) rememberUnwatchableRoot(rootKey) + finishInFlightLocalInstall(rootKey, cancelToken) return 'unavailable' } } catch { console.warn(`[filesystem-watcher] cannot stat root: ${rootKey}`) rememberUnwatchableRoot(rootKey) + finishInFlightLocalInstall(rootKey, cancelToken) return 'unavailable' } @@ -696,9 +733,7 @@ async function doInstallLocalWatcher( rememberUnwatchableRoot(rootKey) return 'unavailable' } finally { - if (inFlightLocalInstalls.get(rootKey) === cancelToken) { - inFlightLocalInstalls.delete(rootKey) - } + finishInFlightLocalInstall(rootKey, cancelToken) } const liveListeners = new Map( @@ -720,6 +755,12 @@ async function doInstallLocalWatcher( return 'installed' } +function finishInFlightLocalInstall(rootKey: string, cancelToken: LocalWatcherInstallToken): void { + if (inFlightLocalInstalls.get(rootKey) === cancelToken) { + inFlightLocalInstalls.delete(rootKey) + } +} + function unsubscribe(worktreePath: string, senderId: number): void { const { key: rootKey } = localWatcherRoot(worktreePath) const suspended = suspendedLocalWatcherListeners.get(rootKey) @@ -865,7 +906,12 @@ export async function restoreLocalWatcherAfterFailedRemoval(worktreePath: string } export function forgetLocalWatcherRemovalSnapshot(worktreePath: string): void { - suspendedLocalWatcherListeners.delete(localWatcherRoot(worktreePath).key) + const rootKey = localWatcherRoot(worktreePath).key + const suspended = suspendedLocalWatcherListeners.get(rootKey) + for (const senderId of suspended?.listeners.keys() ?? []) { + watcherAdmission.release(senderId, `local:${rootKey}`) + } + suspendedLocalWatcherListeners.delete(rootKey) } // Remote watcher state @@ -979,7 +1025,12 @@ export function forgetRemoteWatcherRemovalSnapshot( connectionId: string, worktreePath: string ): void { - suspendedRemoteWatcherListeners.delete(remoteWatcherKey(connectionId, worktreePath)) + const key = remoteWatcherKey(connectionId, worktreePath) + const suspended = suspendedRemoteWatcherListeners.get(key) + for (const senderId of suspended?.listeners.keys() ?? []) { + watcherAdmission.release(senderId, `remote:${key}`) + } + suspendedRemoteWatcherListeners.delete(key) } function addInFlightRemoteInstallListener( @@ -1292,8 +1343,8 @@ function scheduleRemoteWatcherRetry( const listeners = Array.from(retry.listeners.values()).filter( (listener) => !listener.isDestroyed() ) - void Promise.all( - listeners.map((listener) => installRemoteWatcher(listener, connectionId, worktreePath)) + void mapWithConcurrency(listeners, WATCHER_LISTENER_INSTALL_CONCURRENCY, (listener) => + installRemoteWatcher(listener, connectionId, worktreePath) ) .then((results) => { // Why: don't re-arm on 'cancelled' (renderer stopped watching) — it would fire a stale overflow when the 60s window expires. @@ -1318,9 +1369,14 @@ function scheduleRemoteWatcherRetry( // ── Public API ─────────────────────────────────────────────────────── export function registerFilesystemWatcherHandlers(): void { - ipcMain.handle( - 'fs:watchWorktree', - async (event, args: { worktreePath: string; connectionId?: string }): Promise => { + ipcMain.handle('fs:watchWorktree', async (event, rawArgs: unknown): Promise => { + const args = parseFilesystemWatcherIdentity(rawArgs) + const admission = watcherAdmission.claim( + event.sender.id, + watcherAdmissionKey(args), + args.retainedBytes + ) + try { if (args.connectionId) { // Why: a real new watch reopens the subsystem after closeAllWatchers latched it shut (also resets tests between cases). remoteWatchersClosed = false @@ -1345,41 +1401,51 @@ export function registerFilesystemWatcherHandlers(): void { // Why: reopen the local subsystem for tests and post-shutdown reattachment; stale callers keep the prior generation. localWatchersClosed = false await subscribe(args.worktreePath, event.sender) - } - ) - - ipcMain.handle( - 'fs:unwatchWorktree', - (_event, args: { worktreePath: string; connectionId?: string }): void => { - if (args.connectionId) { - const key = remoteWatcherKey(args.connectionId, args.worktreePath) - const suspended = suspendedRemoteWatcherListeners.get(key) - suspended?.listeners.delete(_event.sender.id) - if (suspended?.listeners.size === 0) { - suspendedRemoteWatcherListeners.delete(key) + if (admission.added) { + const rootKey = localWatcherRoot(args.worktreePath).key + if (!hasRetainedLocalWatcherListener(rootKey, event.sender.id)) { + admission.release() } - const retry = pendingRemoteWatcherRetryListeners.get(key) - retry?.listeners.delete(_event.sender.id) - const retryTimer = pendingRemoteWatcherRetries.get(key) - if (retryTimer && retry?.listeners.size === 0) { - clearTimeout(retryTimer) - pendingRemoteWatcherRetries.delete(key) - pendingRemoteWatcherRetryListeners.delete(key) - } - // Why: a retry-tick provider.watch() may still be in flight; mark cancelled so its resolved unwatch handle is discarded. - const inFlight = inFlightRemoteInstalls.get(key) - if (inFlight) { - inFlight.listeners.delete(_event.sender.id) - cancelInFlightRemoteInstallIfUnowned(inFlight) - } - loggedUnavailableRemoteWatchers.delete(key) - releaseRemoteWatchListener(key, _event?.sender?.id ?? 0) - return } - const senderId = _event.sender.id - unsubscribe(args.worktreePath, senderId) + } catch (error) { + if (admission.added) { + admission.release() + } + throw error } - ) + }) + + ipcMain.handle('fs:unwatchWorktree', (_event, rawArgs: unknown): void => { + const args = parseFilesystemWatcherIdentity(rawArgs) + watcherAdmission.release(_event.sender.id, watcherAdmissionKey(args)) + if (args.connectionId) { + const key = remoteWatcherKey(args.connectionId, args.worktreePath) + const suspended = suspendedRemoteWatcherListeners.get(key) + suspended?.listeners.delete(_event.sender.id) + if (suspended?.listeners.size === 0) { + suspendedRemoteWatcherListeners.delete(key) + } + const retry = pendingRemoteWatcherRetryListeners.get(key) + retry?.listeners.delete(_event.sender.id) + const retryTimer = pendingRemoteWatcherRetries.get(key) + if (retryTimer && retry?.listeners.size === 0) { + clearTimeout(retryTimer) + pendingRemoteWatcherRetries.delete(key) + pendingRemoteWatcherRetryListeners.delete(key) + } + // Why: a retry-tick provider.watch() may still be in flight; mark cancelled so its resolved unwatch handle is discarded. + const inFlight = inFlightRemoteInstalls.get(key) + if (inFlight) { + inFlight.listeners.delete(_event.sender.id) + cancelInFlightRemoteInstallIfUnowned(inFlight) + } + loggedUnavailableRemoteWatchers.delete(key) + releaseRemoteWatchListener(key, _event?.sender?.id ?? 0) + return + } + const senderId = _event.sender.id + unsubscribe(args.worktreePath, senderId) + }) } function remoteWatcherKey(connectionId: string, worktreePath: string): string { @@ -1388,6 +1454,7 @@ function remoteWatcherKey(connectionId: string, worktreePath: string): string { /** Tear down all watchers on app shutdown. */ export async function closeAllWatchers(): Promise { + watcherAdmission.clear() senderCleanupRegistered.clear() unwatchableRoots.clear() suspendedLocalWatcherListeners.clear() diff --git a/src/main/ipc/filesystem.test.ts b/src/main/ipc/filesystem.test.ts index 2f76c80791b..92490ff1a3a 100644 --- a/src/main/ipc/filesystem.test.ts +++ b/src/main/ipc/filesystem.test.ts @@ -11,7 +11,7 @@ const { fromWebContentsMock, trashItemMock, readdirMock, - readFileMock, + opendirMock, writeFileMock, statMock, openMock, @@ -54,7 +54,7 @@ const { fromWebContentsMock: vi.fn(), trashItemMock: vi.fn(), readdirMock: vi.fn(), - readFileMock: vi.fn(), + opendirMock: vi.fn(), writeFileMock: vi.fn(), statMock: vi.fn(), openMock: vi.fn(), @@ -110,7 +110,7 @@ vi.mock('electron', () => ({ vi.mock('fs/promises', () => ({ readdir: readdirMock, - readFile: readFileMock, + opendir: opendirMock, writeFile: writeFileMock, stat: statMock, open: openMock, @@ -219,6 +219,33 @@ function dirEntry({ name, directory, file, symlink }: MockDirEntry): { } } +function mockReadableFile( + content: Buffer, + stats: { dev?: number; ino?: number; birthtimeMs?: number } = {} +) { + const handle = { + stat: vi.fn().mockResolvedValue({ size: content.byteLength, ...stats }), + read: vi.fn(async (target: Buffer, offset: number, length: number, position: number) => { + const bytesRead = Math.min(length, Math.max(0, content.byteLength - position)) + content.copy(target, offset, position, position + bytesRead) + return { bytesRead, buffer: target } + }), + close: vi.fn().mockResolvedValue(undefined) + } + openMock.mockResolvedValue(handle) + return handle +} + +function pngHeader(width = 1, height = 1): Buffer { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(height, 20) + return bytes +} + async function withPlatform(platform: NodeJS.Platform, run: () => Promise): Promise { const original = Object.getOwnPropertyDescriptor(process, 'platform') Object.defineProperty(process, 'platform', { configurable: true, value: platform }) @@ -262,7 +289,7 @@ describe('registerFilesystemHandlers', () => { fromWebContentsMock, trashItemMock, readdirMock, - readFileMock, + opendirMock, writeFileMock, statMock, openMock, @@ -340,6 +367,14 @@ describe('registerFilesystemHandlers', () => { close: vi.fn() }) lstatMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })) + opendirMock.mockImplementation(async (dirPath: string) => { + const entries = await readdirMock(dirPath, { withFileTypes: true }) + return { + async *[Symbol.asyncIterator]() { + yield* entries + } + } + }) }) it('returns an actionable reconnect error when the SSH filesystem provider is unavailable', async () => { @@ -986,7 +1021,7 @@ describe('registerFilesystemHandlers', () => { 'Access denied: path resolves outside allowed directories' ) - expect(readFileMock).not.toHaveBeenCalled() + expect(openMock).not.toHaveBeenCalled() }) it('allows readDir when a registered worktree resolves to a macOS canonical alias', async () => { @@ -1105,7 +1140,7 @@ describe('registerFilesystemHandlers', () => { 'Access denied: path resolves outside allowed directories' ) - expect(readFileMock).not.toHaveBeenCalled() + expect(openMock).not.toHaveBeenCalled() }) it('does not enumerate worktrees when filesystem handlers register', () => { @@ -1148,32 +1183,56 @@ describe('registerFilesystemHandlers', () => { ) it.each([ - { ext: 'png', mime: 'image/png', data: [0x89, 0x50, 0x4e, 0x47, 0x00] }, + { + ext: 'png', + mime: 'image/png', + data: Array.from(pngHeader()), + imageDimensions: { width: 1, height: 1 } + }, { ext: 'pdf', mime: 'application/pdf', data: [0x25, 0x50, 0x44, 0x46, 0x00] }, { ext: 'svg', mime: 'image/svg+xml', data: Array.from(Buffer.from('')) } - ])('returns base64 content for supported $ext binaries', async ({ ext, mime, data }) => { - const buf = Buffer.from(data) + ])( + 'returns base64 content for supported $ext binaries', + async ({ ext, mime, data, imageDimensions }) => { + const buf = Buffer.from(data) + statMock.mockResolvedValue({ size: buf.length, isDirectory: () => false, mtimeMs: 123 }) + mockReadableFile(buf) + registerFilesystemHandlers(store as never) + await expect( + handlers.get('fs:readFile')!(null, { + filePath: path.resolve(`/workspace/repo/file.${ext}`) + }) + ).resolves.toEqual({ + content: buf.toString('base64'), + isBinary: true, + isImage: true, + mimeType: mime, + ...(imageDimensions ? { imageDimensions } : {}) + }) + } + ) + + it('rejects a raster dimension bomb before returning base64 to the renderer', async () => { + const buf = pngHeader(32_769, 1) statMock.mockResolvedValue({ size: buf.length, isDirectory: () => false, mtimeMs: 123 }) - readFileMock.mockResolvedValue(buf) + mockReadableFile(buf) registerFilesystemHandlers(store as never) + await expect( - handlers.get('fs:readFile')!(null, { filePath: path.resolve(`/workspace/repo/file.${ext}`) }) - ).resolves.toEqual({ - content: buf.toString('base64'), - isBinary: true, - isImage: true, - mimeType: mime - }) + handlers.get('fs:readFile')!(null, { + filePath: path.resolve('/workspace/repo/bomb.png') + }) + ).rejects.toThrow('Image dimensions exceed the preview safety limit') }) it('opens text files larger than the old 5MB guard', async () => { const content = 'a'.repeat(6 * 1024 * 1024) statMock.mockResolvedValue({ size: content.length, isDirectory: () => false, mtimeMs: 123 }) - readFileMock.mockResolvedValue(Buffer.from(content)) + mockReadableFile(Buffer.from(content)) registerFilesystemHandlers(store as never) @@ -1187,17 +1246,7 @@ describe('registerFilesystemHandlers', () => { it('returns stable byte metadata only for opted-in local log snapshots', async () => { const content = Buffer.from('first\npartial') - const close = vi.fn() - openMock.mockResolvedValue({ - stat: vi.fn().mockResolvedValue({ - size: content.byteLength, - dev: 1, - ino: 2, - birthtimeMs: 3 - }), - readFile: vi.fn().mockResolvedValue(content), - close - }) + const handle = mockReadableFile(content, { dev: 1, ino: 2, birthtimeMs: 3 }) registerFilesystemHandlers(store as never) await expect( @@ -1210,8 +1259,7 @@ describe('registerFilesystemHandlers', () => { isBinary: false, fileIdentity: '1:2:3' }) - expect(close).toHaveBeenCalledTimes(1) - expect(readFileMock).not.toHaveBeenCalled() + expect(handle.close).toHaveBeenCalledTimes(1) }) it('rejects text files beyond the editor read budget', async () => { @@ -1223,7 +1271,7 @@ describe('registerFilesystemHandlers', () => { handlers.get('fs:readFile')!(null, { filePath: path.resolve('/workspace/repo/huge.json') }) ).rejects.toThrow('exceeds 50MB limit') - expect(readFileMock).not.toHaveBeenCalled() + expect(openMock).not.toHaveBeenCalled() }) it('probes large unknown binaries without reading the full file', async () => { @@ -1245,7 +1293,7 @@ describe('registerFilesystemHandlers', () => { isBinary: true }) - expect(readFileMock).not.toHaveBeenCalled() + expect(openMock).toHaveBeenCalledTimes(1) }) it('moves files to trash', async () => { @@ -1316,7 +1364,7 @@ describe('registerFilesystemHandlers', () => { it('keeps non-image binaries hidden from the editor payload', async () => { statMock.mockResolvedValue({ size: 4, isDirectory: () => false, mtimeMs: 123 }) - readFileMock.mockResolvedValue(Buffer.from([0x00, 0x01, 0x02])) + mockReadableFile(Buffer.from([0x00, 0x01, 0x02])) registerFilesystemHandlers(store as never) @@ -1777,7 +1825,7 @@ describe('registerFilesystemHandlers', () => { it('lists remote markdown documents through the SSH filesystem provider', async () => { const provider = { - listFiles: vi + listMarkdownDocuments: vi .fn() .mockResolvedValue(['README.md', 'docs/guide.mdx', '../outside.md', 'src/app.ts']) } @@ -1804,6 +1852,7 @@ describe('registerFilesystemHandlers', () => { name: 'README' } ]) + expect(provider.listMarkdownDocuments).toHaveBeenCalledWith('/home/user/project') }) it('routes branch compare queries through the git compare helper', async () => { diff --git a/src/main/ipc/filesystem.ts b/src/main/ipc/filesystem.ts index 414b67516f1..0d74fa796f2 100644 --- a/src/main/ipc/filesystem.ts +++ b/src/main/ipc/filesystem.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines */ import { BrowserWindow, dialog, ipcMain, shell } from 'electron' -import { readdir, readFile, writeFile, stat, lstat, open, rename, rm } from 'node:fs/promises' +import { writeFile, stat, lstat, open, rename, rm } from 'node:fs/promises' import type { FileHandle } from 'node:fs/promises' import { randomUUID } from 'node:crypto' import { dirname, extname, join, resolve } from 'node:path' @@ -39,6 +39,7 @@ import { ingestRgJsonLine, SEARCH_TIMEOUT_MS } from '../../shared/text-search' +import { SearchSubprocessLineAccumulator } from '../../shared/search-subprocess-lines' import { getStatus, getSubmoduleStatus, @@ -119,6 +120,7 @@ import { import { listRepoWorktrees } from '../repo-worktrees' import { recordCrashBreadcrumb } from '../crash-reporting/crash-breadcrumb-store' import { buildReadDirErrorBreadcrumb, type ReadDirThrowSite } from './readdir-error-diagnostics' +import { readLocalFilesystemDirectory } from './filesystem-directory-reader' import { splitWorktreeId } from '../../shared/worktree-id' import { getRuntimePathBasename } from '../../shared/cross-platform-path' import type { LocalProjectWorktreeGitOptions } from '../project-runtime-git-options' @@ -127,6 +129,8 @@ import { localLogFileIdentity } from '../ai-vault/local-log-tail-reader' import { sanitizeLocalDownloadFilename } from '../local-download-filename' import { registerFilesystemDownloadFolderHandlers } from './filesystem-download-folder' import { createSenderScopedRequestCancellations } from './sender-scoped-request-cancellation' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' +import { assertRasterImagePreviewWithinLimits } from '../../shared/raster-image-preview-limits' // Why: Monaco degrades features on large files like VS Code, so a 5MB block would needlessly lock out ordinary JSON/log files. const MAX_TEXT_FILE_SIZE = 50 * 1024 * 1024 // 50MB @@ -150,30 +154,14 @@ async function readLocalLogSnapshot(filePath: string): Promise<{ isBinary: boolean fileIdentity?: string }> { - const handle = await open(filePath, 'r') - try { - const stats = await handle.stat() - if (stats.size > MAX_TEXT_FILE_SIZE) { - throw new Error( - `File too large: ${(stats.size / 1024 / 1024).toFixed(1)}MB exceeds ${MAX_TEXT_FILE_SIZE / 1024 / 1024}MB limit` - ) - } - const buffer = await handle.readFile() - if (buffer.byteLength > MAX_TEXT_FILE_SIZE) { - throw new Error( - `File too large: ${(buffer.byteLength / 1024 / 1024).toFixed(1)}MB exceeds ${MAX_TEXT_FILE_SIZE / 1024 / 1024}MB limit` - ) - } - if (isBinaryBuffer(buffer)) { - return { content: '', isBinary: true } - } - return { - content: buffer.toString('utf8'), - isBinary: false, - fileIdentity: localLogFileIdentity(stats) - } - } finally { - await handle.close() + const { buffer, stats } = await readNodeFileWithinLimit(filePath, MAX_TEXT_FILE_SIZE) + if (isBinaryBuffer(buffer)) { + return { content: '', isBinary: true } + } + return { + content: buffer.toString('utf8'), + isBinary: false, + fileIdentity: localLogFileIdentity(stats) } } @@ -446,23 +434,6 @@ async function isBinaryFilePrefix(filePath: string): Promise { } } -async function isDirectoryEntry( - dirPath: string, - entry: { name: string; isDirectory(): boolean; isSymbolicLink(): boolean }, - _resolveEntryPath: (entryPath: string) => Promise -): Promise { - // Why: following a symlink in readDir can touch macOS TCC-protected containers; treat links as file-like until explicitly opened. - void _resolveEntryPath - if (entry.isSymbolicLink()) { - void dirPath - return false - } - if (entry.isDirectory()) { - return true - } - return false -} - export function registerFilesystemHandlers( store: Store, commitMessageAgentEnv?: CommitMessageAgentEnvironmentResolvers @@ -510,22 +481,7 @@ export function registerFilesystemHandlers( throwSite = 'authorize' const dirPath = await resolveAuthorizedPath(args.dirPath, store) throwSite = 'readdir' - const entries = await readdir(dirPath, { withFileTypes: true }) - const mapped = await Promise.all( - entries.map(async (entry) => ({ - name: entry.name, - isDirectory: await isDirectoryEntry(dirPath, entry, (entryPath) => - resolveAuthorizedPath(entryPath, store) - ), - isSymlink: entry.isSymbolicLink() - })) - ) - return mapped.sort((a, b) => { - if (a.isDirectory !== b.isDirectory) { - return a.isDirectory ? -1 : 1 - } - return a.name.localeCompare(b.name) - }) + return await readLocalFilesystemDirectory(dirPath) } catch (error: unknown) { recordCrashBreadcrumb( 'fs_readdir_error', @@ -551,6 +507,7 @@ export function registerFilesystemHandlers( isBinary: boolean isImage?: boolean mimeType?: string + imageDimensions?: { width: number; height: number } fileIdentity?: string }> => { if (args.connectionId) { @@ -571,13 +528,15 @@ export function registerFilesystemHandlers( } if (mimeType) { - const buffer = await readFile(filePath) + const { buffer } = await readNodeFileWithinLimit(filePath, sizeLimit) + const imageDimensions = assertRasterImagePreviewWithinLimits(buffer, mimeType) return { content: buffer.toString('base64'), isBinary: true, // Why: the renderer keys previewable-binary rendering off `isImage`, so set it for PDFs too to stay compatible. isImage: true, - mimeType + mimeType, + ...(imageDimensions ? { imageDimensions } : {}) } } @@ -586,7 +545,7 @@ export function registerFilesystemHandlers( return { content: '', isBinary: true } } - const buffer = await readFile(filePath) + const { buffer } = await readNodeFileWithinLimit(filePath, sizeLimit) if (isBinaryBuffer(buffer)) { return { content: '', isBinary: true } } @@ -796,7 +755,12 @@ export function registerFilesystemHandlers( ): Promise => { if (args.connectionId) { const provider = requireSshFilesystemProvider(args.connectionId) - const relativePaths = await provider.listFiles(args.rootPath) + if (!provider.listMarkdownDocuments) { + throw new Error( + 'Remote Markdown link discovery is unavailable. Reconnect the SSH target and retry.' + ) + } + const relativePaths = await provider.listMarkdownDocuments(args.rootPath) return markdownDocumentsFromRelativePaths(args.rootPath, relativePaths) } @@ -961,7 +925,7 @@ export function registerFilesystemHandlers( activeTextSearches.get(searchKey)?.kill() const acc = createAccumulator() - let stdoutBuffer = '' + const stdoutLines = new SearchSubprocessLineAccumulator() let resolved = false let child: ChildProcess | null = null let killTimeout: ReturnType @@ -1004,12 +968,11 @@ export function registerFilesystemHandlers( child = nextChild activeTextSearches.set(searchKey, nextChild) - const handleStdoutData = (chunk: string): void => { - stdoutBuffer += chunk - const lines = stdoutBuffer.split('\n') - stdoutBuffer = lines.pop() ?? '' - for (const line of lines) { - processLine(line) + const handleStdoutData = (chunk: Buffer): void => { + if (!stdoutLines.push(chunk, processLine)) { + acc.truncated = true + child?.kill() + resolveOnce() } } const handleStderrData = (): void => { @@ -1019,13 +982,13 @@ export function registerFilesystemHandlers( resolveOnce() } const handleClose = (): void => { - if (stdoutBuffer) { - processLine(stdoutBuffer) + const trailingLine = stdoutLines.finish() + if (trailingLine !== null) { + processLine(trailingLine) } resolveOnce() } - nextChild.stdout!.setEncoding('utf-8') nextChild.stdout!.on('data', handleStdoutData) nextChild.stderr!.on('data', handleStderrData) nextChild.once('error', handleError) diff --git a/src/main/ipc/github.ts b/src/main/ipc/github.ts index a01225bfccf..531c87a026e 100644 --- a/src/main/ipc/github.ts +++ b/src/main/ipc/github.ts @@ -16,6 +16,7 @@ import type { PRRefreshOutcome } from '../../shared/types' import { getRepoExecutionHostId } from '../../shared/execution-host' +import { PR_REFRESH_VISIBLE_CANDIDATE_LIMIT } from '../../shared/pr-refresh-memory-limits' import type { TaskSourceContext } from '../../shared/task-source-context' import type { Store } from '../persistence' import type { StatsCollector } from '../stats/collector' @@ -360,6 +361,9 @@ export function registerGitHubHandlers(store: Store, stats: StatsCollector): voi const candidates: GitHubPRRefreshCandidate[] = [] const repos = store.getRepos() for (const candidate of args.candidates) { + if (candidates.length >= PR_REFRESH_VISIBLE_CANDIDATE_LIMIT) { + break + } const validation = validateAutomaticPRRefreshCandidate(candidate, store, repos) if (validation.kind === 'ok') { candidates.push(validation.candidate) diff --git a/src/main/ipc/local-log-tail-operation-admission.ts b/src/main/ipc/local-log-tail-operation-admission.ts new file mode 100644 index 00000000000..adaf41b8db1 --- /dev/null +++ b/src/main/ipc/local-log-tail-operation-admission.ts @@ -0,0 +1,88 @@ +export const MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER = 32 +export const MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE = 128 +export const MAX_LOCAL_LOG_TAIL_READS_PER_SENDER = 16 +export const MAX_LOCAL_LOG_TAIL_READS_PROCESS_WIDE = 64 + +class ScopedLocalLogTailClaims { + private readonly claims = new Set() + private readonly claimsBySender = new Map>() + + constructor( + private readonly maxPerSender: number, + private readonly maxProcessWide: number, + private readonly label: string + ) {} + + claim(senderId: number, errorMessage: string): symbol { + const senderClaims = this.claimsBySender.get(senderId) + if ((senderClaims?.size ?? 0) >= this.maxPerSender || this.claims.size >= this.maxProcessWide) { + throw new Error(errorMessage) + } + const token = Symbol(this.label) + const nextSenderClaims = senderClaims ?? new Set() + nextSenderClaims.add(token) + this.claimsBySender.set(senderId, nextSenderClaims) + this.claims.add(token) + return token + } + + release(senderId: number, token: symbol): void { + this.claims.delete(token) + const senderClaims = this.claimsBySender.get(senderId) + senderClaims?.delete(token) + if (senderClaims?.size === 0) { + this.claimsBySender.delete(senderId) + } + } + + reset(): void { + this.claims.clear() + this.claimsBySender.clear() + } + + get size(): number { + return this.claims.size + } +} + +export class LocalLogTailOperationAdmission { + private readonly starts = new ScopedLocalLogTailClaims( + MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER, + MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE, + 'local-log-tail-start' + ) + private readonly reads = new ScopedLocalLogTailClaims( + MAX_LOCAL_LOG_TAIL_READS_PER_SENDER, + MAX_LOCAL_LOG_TAIL_READS_PROCESS_WIDE, + 'local-log-tail-read' + ) + + claimStart(senderId: number): symbol { + return this.starts.claim(senderId, 'Too many local log tail starts') + } + + releaseStart(senderId: number, token: symbol): void { + this.starts.release(senderId, token) + } + + claimRead(senderId: number): symbol { + return this.reads.claim(senderId, 'Too many concurrent local log tail reads') + } + + releaseRead(senderId: number, token: symbol): void { + this.reads.release(senderId, token) + } + + reset(): void { + this.starts.reset() + this.reads.reset() + } + + get pendingStartCount(): number { + return this.starts.size + } + + get pendingReadCount(): number { + return this.reads.size + } +} diff --git a/src/main/ipc/local-log-tail.test.ts b/src/main/ipc/local-log-tail.test.ts index beac464f4a4..ffa07812ab9 100644 --- a/src/main/ipc/local-log-tail.test.ts +++ b/src/main/ipc/local-log-tail.test.ts @@ -26,8 +26,18 @@ vi.mock('../ai-vault/local-log-tail-reader', () => ({ import { closeAllLocalLogTailWatchers, getActiveLocalLogTailWatcherCount, + getLocalLogTailSenderCleanupCountForTest, + getPendingLocalLogTailReadCountForTest, + getPendingLocalLogTailStartCountForTest, + MAX_LOCAL_LOG_TAIL_FILE_IDENTITY_BYTES, + MAX_LOCAL_LOG_TAIL_SUBSCRIPTION_ID_BYTES, registerLocalLogTailHandlers } from './local-log-tail' +import { + MAX_LOCAL_LOG_TAIL_READS_PER_SENDER, + MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER, + MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE +} from './local-log-tail-operation-admission' type FakeWatcher = { close: ReturnType @@ -50,19 +60,40 @@ function makeWatcher(): FakeWatcher { function makeSender(id: number) { let destroyedListener: (() => void) | undefined + let destroyed = false return { id, send: vi.fn(), - isDestroyed: vi.fn(() => false), + isDestroyed: vi.fn(() => destroyed), once: vi.fn((event: string, listener: () => void) => { if (event === 'destroyed') { destroyedListener = listener } }), - destroy: () => destroyedListener?.() + removeListener: vi.fn((event: string, listener: () => void) => { + if (event === 'destroyed' && destroyedListener === listener) { + destroyedListener = undefined + } + }), + destroy: () => { + destroyed = true + destroyedListener?.() + } } } +async function startWatch( + sender: ReturnType, + subscriptionId: string, + filePath = `/logs/${subscriptionId}.jsonl` +): Promise { + await handlers.get('fs:startLocalLogTail')?.({ sender }, { filePath, subscriptionId }) +} + +function stopWatch(sender: ReturnType, subscriptionId: string): void { + handlers.get('fs:stopLocalLogTail')?.({ sender }, { subscriptionId }) +} + beforeEach(() => { handlers.clear() watchMock.mockReset() @@ -102,6 +133,8 @@ describe('local log tail IPC', () => { handlers.get('fs:stopLocalLogTail')?.({ sender }, { subscriptionId: 'tail-1' }) expect(watcher.close).toHaveBeenCalledTimes(1) expect(getActiveLocalLogTailWatcherCount()).toBe(0) + expect(getLocalLogTailSenderCleanupCountForTest()).toBe(0) + expect(sender.removeListener).toHaveBeenCalledWith('destroyed', expect.any(Function)) }) it('closes every watcher owned by a destroyed renderer', async () => { @@ -123,5 +156,219 @@ describe('local log tail IPC', () => { expect(first.close).toHaveBeenCalledTimes(1) expect(second.close).toHaveBeenCalledTimes(1) expect(getActiveLocalLogTailWatcherCount()).toBe(0) + expect(getLocalLogTailSenderCleanupCountForTest()).toBe(0) + }) + + it('caps one renderer at the exact watcher limit and recovers after stop', async () => { + watchMock.mockImplementation(() => makeWatcher()) + const sender = makeSender(10) + for (let index = 0; index < MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER; index++) { + await startWatch(sender, `tail-${index}`) + } + + expect(getActiveLocalLogTailWatcherCount()).toBe(MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER) + await expect(startWatch(sender, 'one-over')).rejects.toThrow('Too many local log tail watchers') + expect(watchMock).toHaveBeenCalledTimes(MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER) + + stopWatch(sender, 'tail-0') + await startWatch(sender, 'after-stop') + expect(getActiveLocalLogTailWatcherCount()).toBe(MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER) + expect(getLocalLogTailSenderCleanupCountForTest()).toBe(1) + }) + + it('allows same-key replacement at process capacity while rejecting a new key', async () => { + watchMock.mockImplementation(() => makeWatcher()) + const senderCount = Math.ceil( + MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE / MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER + ) + const senders = Array.from({ length: senderCount + 1 }, (_, index) => makeSender(20 + index)) + for (let index = 0; index < MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE; index++) { + const sender = senders[Math.floor(index / MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER)] + await startWatch(sender, `global-${index}`) + } + + await expect(startWatch(senders.at(-1)!, 'global-one-over')).rejects.toThrow( + 'Too many local log tail watchers' + ) + await startWatch(senders[0], 'global-0', '/logs/replacement.jsonl') + + expect(getActiveLocalLogTailWatcherCount()).toBe(MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE) + expect(watchMock).toHaveBeenCalledTimes(MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE + 1) + }) + + it('bounds pending authorization attempts and rechecks destruction before watching', async () => { + let resolvePaths = (): void => {} + const pathGate = new Promise((resolve) => { + resolvePaths = resolve + }) + resolveAuthorizedPathMock.mockImplementation(async (path: string) => { + await pathGate + return path + }) + watchMock.mockImplementation(() => makeWatcher()) + const sender = makeSender(30) + const starts = Array.from({ length: MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER }, (_, index) => + startWatch(sender, `pending-${index}`) + ) + + await vi.waitFor(() => + expect(getPendingLocalLogTailStartCountForTest()).toBe(MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER) + ) + await expect(startWatch(sender, 'pending-one-over')).rejects.toThrow( + 'Too many local log tail starts' + ) + sender.destroy() + resolvePaths() + await Promise.all(starts) + + expect(watchMock).not.toHaveBeenCalled() + expect(getPendingLocalLogTailStartCountForTest()).toBe(0) + }) + + it('does not install a watcher after stop wins a pending authorization race', async () => { + let authorizePath = (): void => {} + resolveAuthorizedPathMock.mockImplementation( + (path: string) => + new Promise((resolve) => { + authorizePath = () => resolve(path) + }) + ) + watchMock.mockImplementation(() => makeWatcher()) + const sender = makeSender(31) + + const pendingStart = startWatch(sender, 'pending-stop') + await vi.waitFor(() => expect(getPendingLocalLogTailStartCountForTest()).toBe(1)) + stopWatch(sender, 'pending-stop') + authorizePath() + await pendingStart + + expect(watchMock).not.toHaveBeenCalled() + expect(getActiveLocalLogTailWatcherCount()).toBe(0) + expect(getLocalLogTailSenderCleanupCountForTest()).toBe(0) + }) + + it('lets the newest same-key start win when authorizations resolve out of order', async () => { + const authorizations = new Map void>() + resolveAuthorizedPathMock.mockImplementation( + (path: string) => + new Promise((resolve) => { + authorizations.set(path, resolve) + }) + ) + watchMock.mockImplementation(() => makeWatcher()) + const sender = makeSender(32) + + const olderStart = startWatch(sender, 'same-key', '/logs/older.jsonl') + const newerStart = startWatch(sender, 'same-key', '/logs/newer.jsonl') + await vi.waitFor(() => expect(authorizations.size).toBe(2)) + authorizations.get('/logs/newer.jsonl')?.('/logs/newer.jsonl') + await newerStart + authorizations.get('/logs/older.jsonl')?.('/logs/older.jsonl') + await olderStart + + expect(watchMock).toHaveBeenCalledTimes(1) + expect(watchMock).toHaveBeenCalledWith('/logs/newer.jsonl', expect.any(Function)) + expect(getActiveLocalLogTailWatcherCount()).toBe(1) + }) + + it('bounds concurrent ranged reads and releases admission after settlement', async () => { + let resolveReads!: (value: { + contentBase64: string + nextByteOffset: number + fileSize: number + fileIdentity: string + hasMore: boolean + reset: boolean + }) => void + const readGate = new Promise<{ + contentBase64: string + nextByteOffset: number + fileSize: number + fileIdentity: string + hasMore: boolean + reset: boolean + }>((resolve) => { + resolveReads = resolve + }) + readRangeMock.mockReturnValue(readGate) + const sender = makeSender(33) + const read = handlers.get('fs:readLocalLogTail')! + const reads = Array.from({ length: MAX_LOCAL_LOG_TAIL_READS_PER_SENDER }, () => + read({ sender }, { filePath: '/logs/session.jsonl', fromByteOffset: 0 }) + ) + + await vi.waitFor(() => + expect(getPendingLocalLogTailReadCountForTest()).toBe(MAX_LOCAL_LOG_TAIL_READS_PER_SENDER) + ) + await expect( + read({ sender }, { filePath: '/logs/session.jsonl', fromByteOffset: 0 }) + ).rejects.toThrow('Too many concurrent local log tail reads') + resolveReads({ + contentBase64: '', + nextByteOffset: 0, + fileSize: 0, + fileIdentity: 'identity', + hasMore: false, + reset: false + }) + await Promise.all(reads) + + expect(readRangeMock).toHaveBeenCalledTimes(MAX_LOCAL_LOG_TAIL_READS_PER_SENDER) + expect(getPendingLocalLogTailReadCountForTest()).toBe(0) + }) + + it('rejects an oversized UTF-8 subscription id before path authorization', async () => { + const sender = makeSender(34) + const oversizedId = '😀'.repeat(Math.floor(MAX_LOCAL_LOG_TAIL_SUBSCRIPTION_ID_BYTES / 4) + 1) + + await expect(startWatch(sender, oversizedId)).rejects.toThrow( + 'Invalid local log tail subscription id' + ) + expect(resolveAuthorizedPathMock).not.toHaveBeenCalled() + expect(watchMock).not.toHaveBeenCalled() + }) + + it('validates and snapshots ranged-read fields before path authorization', async () => { + let authorizePath = (): void => {} + resolveAuthorizedPathMock.mockImplementation( + (path: string) => + new Promise((resolve) => { + authorizePath = () => resolve(path) + }) + ) + readRangeMock.mockResolvedValue({ + contentBase64: '', + nextByteOffset: 7, + fileSize: 7, + fileIdentity: 'identity-before', + hasMore: false, + reset: false + }) + const sender = makeSender(35) + const args = { + filePath: '/logs/session.jsonl', + fromByteOffset: 7, + expectedIdentity: 'identity-before' + } + + const pendingRead = handlers.get('fs:readLocalLogTail')?.({ sender }, args) + args.fromByteOffset = 99 + args.expectedIdentity = 'identity-after' + authorizePath() + await pendingRead + + expect(readRangeMock).toHaveBeenCalledWith('/logs/session.jsonl', 7, 'identity-before') + + await expect( + handlers.get('fs:readLocalLogTail')?.( + { sender }, + { + filePath: '/logs/session.jsonl', + fromByteOffset: 0, + expectedIdentity: '😀'.repeat(Math.floor(MAX_LOCAL_LOG_TAIL_FILE_IDENTITY_BYTES / 4) + 1) + } + ) + ).rejects.toThrow('Invalid local log tail file identity') + expect(resolveAuthorizedPathMock).toHaveBeenCalledTimes(1) }) }) diff --git a/src/main/ipc/local-log-tail.ts b/src/main/ipc/local-log-tail.ts index 430882b4e07..7b450aa9431 100644 --- a/src/main/ipc/local-log-tail.ts +++ b/src/main/ipc/local-log-tail.ts @@ -9,58 +9,192 @@ import type { } from '../../shared/local-log-tail-types' import { readLocalLogTailRange } from '../ai-vault/local-log-tail-reader' import { resolveAuthorizedPath } from './filesystem-auth' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' +import { + LocalLogTailOperationAdmission, + MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER, + MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE +} from './local-log-tail-operation-admission' type TailWatch = { + generation: symbol senderId: number watcher: FSWatcher } const tailWatches = new Map() -const senderCleanupRegistered = new Set() +const watchGenerations = new Map() +const senderCleanups = new Map void }>() +const operationAdmission = new LocalLogTailOperationAdmission() + +export const MAX_LOCAL_LOG_TAIL_SUBSCRIPTION_ID_BYTES = 1_024 +export const MAX_LOCAL_LOG_TAIL_PATH_BYTES = 64 * 1_024 +export const MAX_LOCAL_LOG_TAIL_FILE_IDENTITY_BYTES = 1_024 function watchKey(senderId: number, subscriptionId: string): string { return `${senderId}:${subscriptionId}` } -function closeWatch(key: string): void { +function closeWatchHandle(key: string, generation?: symbol): number | undefined { const subscription = tailWatches.get(key) - if (!subscription) { - return + if (!subscription || (generation !== undefined && subscription.generation !== generation)) { + return undefined } tailWatches.delete(key) subscription.watcher.close() + releaseSenderCleanupIfIdle(subscription.senderId) + return subscription.senderId +} + +function closeWatch(key: string): void { + const owner = watchGenerations.get(key) + watchGenerations.delete(key) + const senderId = closeWatchHandle(key) ?? owner?.senderId + if (senderId !== undefined) { + releaseSenderCleanupIfIdle(senderId) + } +} + +function closeWatchGeneration(key: string, generation: symbol): void { + const owner = watchGenerations.get(key) + if (owner?.generation === generation) { + watchGenerations.delete(key) + } + const senderId = closeWatchHandle(key, generation) ?? owner?.senderId + if (senderId !== undefined) { + releaseSenderCleanupIfIdle(senderId) + } } function closeSenderWatches(senderId: number): void { - senderCleanupRegistered.delete(senderId) - for (const [key, subscription] of tailWatches) { - if (subscription.senderId === senderId) { + releaseSenderCleanup(senderId) + for (const [key, owner] of watchGenerations) { + if (owner.senderId === senderId) { closeWatch(key) } } } function validateSubscriptionId(value: unknown): string { - if (typeof value !== 'string' || value.length === 0 || value.length > 200) { + if ( + typeof value !== 'string' || + value.length === 0 || + measureUtf8ByteLength(value, { + stopAfterBytes: MAX_LOCAL_LOG_TAIL_SUBSCRIPTION_ID_BYTES + }).exceededLimit + ) { throw new Error('Invalid local log tail subscription id') } return value } +function validateFilePath(value: unknown): string { + if ( + typeof value !== 'string' || + value.length === 0 || + measureUtf8ByteLength(value, { stopAfterBytes: MAX_LOCAL_LOG_TAIL_PATH_BYTES }).exceededLimit + ) { + throw new Error('Invalid local log tail path') + } + return value +} + +function validateExpectedIdentity(value: unknown): string | undefined { + if (value === undefined) { + return undefined + } + if ( + typeof value !== 'string' || + measureUtf8ByteLength(value, { + stopAfterBytes: MAX_LOCAL_LOG_TAIL_FILE_IDENTITY_BYTES + }).exceededLimit + ) { + throw new Error('Invalid local log tail file identity') + } + return value +} + function registerSenderCleanup(sender: WebContents): void { - if (senderCleanupRegistered.has(sender.id)) { + if (senderCleanups.has(sender.id)) { return } - senderCleanupRegistered.add(sender.id) - sender.once('destroyed', () => closeSenderWatches(sender.id)) + const onDestroyed = (): void => closeSenderWatches(sender.id) + senderCleanups.set(sender.id, { sender, onDestroyed }) + sender.once('destroyed', onDestroyed) +} + +function releaseSenderCleanup(senderId: number): void { + const cleanup = senderCleanups.get(senderId) + if (!cleanup) { + return + } + senderCleanups.delete(senderId) + cleanup.sender.removeListener('destroyed', cleanup.onDestroyed) +} + +function releaseSenderCleanupIfIdle(senderId: number): void { + for (const owner of watchGenerations.values()) { + if (owner.senderId === senderId) { + return + } + } + for (const subscription of tailWatches.values()) { + if (subscription.senderId === senderId) { + return + } + } + releaseSenderCleanup(senderId) +} + +function activeSenderWatchCount(senderId: number): number { + let count = 0 + for (const subscription of tailWatches.values()) { + if (subscription.senderId === senderId) { + count += 1 + } + } + return count +} + +function restorePreviousWatchGeneration( + key: string, + generation: symbol, + previousGeneration: symbol | undefined, + senderId: number +): void { + if (watchGenerations.get(key)?.generation !== generation) { + return + } + const previousWatch = tailWatches.get(key) + if (previousGeneration !== undefined && previousWatch?.generation === previousGeneration) { + watchGenerations.set(key, { generation: previousGeneration, senderId }) + } else { + watchGenerations.delete(key) + } +} + +function validateByteOffset(value: unknown): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error('Invalid local log tail byte offset') + } + return value as number } export function registerLocalLogTailHandlers(store: Store): void { ipcMain.handle( 'fs:readLocalLogTail', - async (_event, args: LocalLogTailReadArgs): Promise => { - const filePath = await resolveAuthorizedPath(args.filePath, store) - return readLocalLogTailRange(filePath, args.fromByteOffset, args.expectedIdentity) + async (event, args: LocalLogTailReadArgs): Promise => { + const senderId = event.sender.id + const requestedPath = validateFilePath(args.filePath) + const fromByteOffset = validateByteOffset(args.fromByteOffset) + const expectedIdentity = validateExpectedIdentity(args.expectedIdentity) + const readToken = operationAdmission.claimRead(senderId) + try { + const filePath = await resolveAuthorizedPath(requestedPath, store) + return await readLocalLogTailRange(filePath, fromByteOffset, expectedIdentity) + } finally { + operationAdmission.releaseRead(senderId, readToken) + } } ) @@ -68,26 +202,56 @@ export function registerLocalLogTailHandlers(store: Store): void { 'fs:startLocalLogTail', async (event, args: LocalLogTailWatchArgs): Promise => { const subscriptionId = validateSubscriptionId(args.subscriptionId) - const filePath = await resolveAuthorizedPath(args.filePath, store) - const key = watchKey(event.sender.id, subscriptionId) - closeWatch(key) - - const sendChange = (eventType: 'change' | 'rename'): void => { - if (!tailWatches.has(key) || event.sender.isDestroyed()) { + const requestedPath = validateFilePath(args.filePath) + const senderId = event.sender.id + const key = watchKey(senderId, subscriptionId) + const pendingToken = operationAdmission.claimStart(senderId) + const previousGeneration = watchGenerations.get(key)?.generation + const generation = Symbol(subscriptionId) + watchGenerations.set(key, { generation, senderId }) + registerSenderCleanup(event.sender) + let published = false + try { + const filePath = await resolveAuthorizedPath(requestedPath, store) + if (event.sender.isDestroyed() || watchGenerations.get(key)?.generation !== generation) { return } - const payload: LocalLogTailChangedPayload = { subscriptionId, eventType } - event.sender.send('fs:localLogTailChanged', payload) + const replacing = tailWatches.has(key) + if ( + !replacing && + (activeSenderWatchCount(senderId) >= MAX_LOCAL_LOG_TAIL_WATCHES_PER_SENDER || + tailWatches.size >= MAX_LOCAL_LOG_TAIL_WATCHES_PROCESS_WIDE) + ) { + throw new Error('Too many local log tail watchers') + } + closeWatchHandle(key) + + const sendChange = (eventType: 'change' | 'rename'): void => { + if (tailWatches.get(key)?.generation !== generation || event.sender.isDestroyed()) { + return + } + const payload: LocalLogTailChangedPayload = { subscriptionId, eventType } + event.sender.send('fs:localLogTailChanged', payload) + } + const watcher = watch(filePath, (eventType) => sendChange(eventType)) + watcher.on('error', () => { + // Why: an error commonly accompanies rotation. Signal one final drain so + // the renderer can detect identity change, then release the dead handle. + sendChange('rename') + closeWatchGeneration(key, generation) + }) + tailWatches.set(key, { generation, senderId, watcher }) + published = true + if (event.sender.isDestroyed()) { + closeWatchGeneration(key, generation) + } + } finally { + if (!published) { + restorePreviousWatchGeneration(key, generation, previousGeneration, senderId) + releaseSenderCleanupIfIdle(senderId) + } + operationAdmission.releaseStart(senderId, pendingToken) } - const watcher = watch(filePath, (eventType) => sendChange(eventType)) - watcher.on('error', () => { - // Why: an error commonly accompanies rotation. Signal one final drain so - // the renderer can detect identity change, then release the dead handle. - sendChange('rename') - closeWatch(key) - }) - tailWatches.set(key, { senderId: event.sender.id, watcher }) - registerSenderCleanup(event.sender) } ) @@ -97,13 +261,28 @@ export function registerLocalLogTailHandlers(store: Store): void { } export function closeAllLocalLogTailWatchers(): void { - for (const key of Array.from(tailWatches.keys())) { + for (const key of new Set([...watchGenerations.keys(), ...tailWatches.keys()])) { closeWatch(key) } - senderCleanupRegistered.clear() + for (const senderId of Array.from(senderCleanups.keys())) { + releaseSenderCleanup(senderId) + } + operationAdmission.reset() } /** Test-only: verifies tab/window teardown does not retain native watchers. */ export function getActiveLocalLogTailWatcherCount(): number { return tailWatches.size } + +export function getLocalLogTailSenderCleanupCountForTest(): number { + return senderCleanups.size +} + +export function getPendingLocalLogTailStartCountForTest(): number { + return operationAdmission.pendingStartCount +} + +export function getPendingLocalLogTailReadCountForTest(): number { + return operationAdmission.pendingReadCount +} diff --git a/src/main/ipc/markdown-documents.test.ts b/src/main/ipc/markdown-documents.test.ts index b23f5748cb4..b500acb3c14 100644 --- a/src/main/ipc/markdown-documents.test.ts +++ b/src/main/ipc/markdown-documents.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { markdownDocumentFromFilePath } from './markdown-documents' +import { + markdownDocumentFromFilePath, + markdownDocumentsFromRelativePaths +} from './markdown-documents' +import { MarkdownDocumentListingCapacityError } from '../../shared/markdown-document-listing-limits' describe('markdownDocumentFromFilePath', () => { it('keeps in-root path segments that merely start with parent traversal text', () => { @@ -24,3 +28,52 @@ describe('markdownDocumentFromFilePath', () => { }) }) }) + +describe('markdownDocumentsFromRelativePaths', () => { + it('preserves filtering and sorted output below every limit', () => { + expect( + markdownDocumentsFromRelativePaths('/workspace', [ + 'z-last.markdown', + 'src/app.ts', + '../outside.md', + 'docs/Guide.MDX', + 'README.md' + ]) + ).toEqual([ + { + filePath: '/workspace/docs/Guide.MDX', + relativePath: 'docs/Guide.MDX', + basename: 'Guide.MDX', + name: 'Guide' + }, + { + filePath: '/workspace/README.md', + relativePath: 'README.md', + basename: 'README.md', + name: 'README' + }, + { + filePath: '/workspace/z-last.markdown', + relativePath: 'z-last.markdown', + basename: 'z-last.markdown', + name: 'z-last' + } + ]) + }) + + it('rejects count, metadata, and UTF-8 path overflow before retaining another result', () => { + expect(() => + markdownDocumentsFromRelativePaths('/workspace', ['one.md', 'two.md'], { + maxDocuments: 1 + }) + ).toThrow(MarkdownDocumentListingCapacityError) + expect(() => + markdownDocumentsFromRelativePaths('/workspace', ['metadata.md'], { + maxMetadataBytes: 1 + }) + ).toThrow(MarkdownDocumentListingCapacityError) + expect(() => + markdownDocumentsFromRelativePaths('/workspace', [`${'é'.repeat(40_000)}.md`]) + ).toThrow(MarkdownDocumentListingCapacityError) + }) +}) diff --git a/src/main/ipc/markdown-documents.ts b/src/main/ipc/markdown-documents.ts index 713adc9e0fb..687f50e1c5e 100644 --- a/src/main/ipc/markdown-documents.ts +++ b/src/main/ipc/markdown-documents.ts @@ -1,14 +1,22 @@ -import { readdir } from 'node:fs/promises' -import { basename as pathBasename, extname, isAbsolute, join, relative, resolve } from 'node:path' +import { basename as pathBasename, extname, isAbsolute, relative, resolve } from 'node:path' import type { MarkdownDocument } from '../../shared/types' +import { + assertMarkdownDocumentPathWithinLimit, + createMarkdownDocumentListingBudget, + retainMarkdownDocument, + type MarkdownDocumentListingLimits +} from '../../shared/markdown-document-listing-limits' +import { + discoverMarkdownRelativePaths, + isMarkdownDocumentPath +} from '../../shared/node-markdown-document-discovery' function normalizeRelativePath(path: string): string { return path.replace(/[\\/]+/g, '/').replace(/^\/+/, '') } export function isMarkdownDocumentName(name: string): boolean { - const extension = extname(name).toLowerCase() - return extension === '.md' || extension === '.mdx' || extension === '.markdown' + return isMarkdownDocumentPath(name) } function basenameFromRelativePath(relativePath: string): string { @@ -39,6 +47,8 @@ export function markdownDocumentFromFilePath( filePath: string, options: { outsideRootRelativePath?: 'basename' | 'relative' } = {} ): MarkdownDocument { + assertMarkdownDocumentPathWithinLimit(rootPath) + assertMarkdownDocumentPathWithinLimit(filePath) const basename = pathBasename(filePath) const extension = extname(basename) const relativePath = @@ -46,18 +56,23 @@ export function markdownDocumentFromFilePath( (options.outsideRootRelativePath === 'basename' ? basename : normalizeRelativePath(relative(rootPath, filePath))) - return { + const document = { filePath, relativePath, basename, name: extension ? basename.slice(0, -extension.length) : basename } + const budget = createMarkdownDocumentListingBudget() + retainMarkdownDocument(budget, document) + return document } export function markdownDocumentFromRelativePath( rootPath: string, relativePath: string ): MarkdownDocument | null { + assertMarkdownDocumentPathWithinLimit(rootPath) + assertMarkdownDocumentPathWithinLimit(relativePath) const normalizedRelativePath = normalizeRelativePath(relativePath) // Why: SSH providers should return root-relative paths; reject escape // segments before building a synthetic absolute path for renderer use. @@ -70,52 +85,42 @@ export function markdownDocumentFromRelativePath( } const extension = extname(basename) const normalizedRoot = rootPath.replace(/[\\/]+$/, '') - return { + const document = { filePath: `${normalizedRoot}/${normalizedRelativePath}`, relativePath: normalizedRelativePath, basename, name: extension ? basename.slice(0, -extension.length) : basename } + const budget = createMarkdownDocumentListingBudget() + retainMarkdownDocument(budget, document) + return document } export function markdownDocumentsFromRelativePaths( rootPath: string, - relativePaths: string[] + relativePaths: readonly string[], + limits: Partial = {} ): MarkdownDocument[] { - return relativePaths - .map((relativePath) => markdownDocumentFromRelativePath(rootPath, relativePath)) - .filter((document): document is MarkdownDocument => document !== null) - .sort((a, b) => a.relativePath.localeCompare(b.relativePath)) -} - -export async function listMarkdownDocuments(rootPath: string): Promise { + const budget = createMarkdownDocumentListingBudget(limits) const documents: MarkdownDocument[] = [] - - async function visitDirectory(dirPath: string): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) - for (const entry of entries) { - if (entry.isSymbolicLink()) { - continue - } - - const entryPath = join(dirPath, entry.name) - if (entry.isDirectory()) { - if (entry.name === '.git' || entry.name === 'node_modules') { - continue - } - if (entry.name.startsWith('.') && entry.name !== '.github') { - continue - } - await visitDirectory(entryPath) - continue - } - - if (entry.isFile() && isMarkdownDocumentName(entry.name)) { - documents.push(markdownDocumentFromFilePath(rootPath, entryPath)) - } + for (const relativePath of relativePaths) { + const document = markdownDocumentFromRelativePath(rootPath, relativePath) + if (document) { + retainMarkdownDocument(budget, document) + documents.push(document) } } - - await visitDirectory(rootPath) return documents.sort((a, b) => a.relativePath.localeCompare(b.relativePath)) } + +export async function listMarkdownDocuments( + rootPath: string, + limits: Partial = {} +): Promise { + const relativePaths = await discoverMarkdownRelativePaths(rootPath, { + limits, + shouldDescend: (_relativePath, name) => + name !== '.git' && name !== 'node_modules' && (!name.startsWith('.') || name === '.github') + }) + return markdownDocumentsFromRelativePaths(rootPath, relativePaths, limits) +} diff --git a/src/main/ipc/native-chat-ipc-admission.ts b/src/main/ipc/native-chat-ipc-admission.ts new file mode 100644 index 00000000000..fd19d898895 --- /dev/null +++ b/src/main/ipc/native-chat-ipc-admission.ts @@ -0,0 +1,113 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +const DESKTOP_READ_WINDOW = 300 + +export const MAX_NATIVE_CHAT_DESKTOP_READ_LIMIT = 2_000 +export const MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER = 64 +export const MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE = 256 +export const MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER = 64 +export const MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PROCESS_WIDE = 256 +export const MAX_NATIVE_CHAT_READS_PER_SENDER = 8 +export const MAX_NATIVE_CHAT_READS_PROCESS_WIDE = 16 +export const MAX_NATIVE_CHAT_SUBSCRIPTION_ID_BYTES = 1_024 +export const MAX_NATIVE_CHAT_SESSION_ID_BYTES = 4 * 1_024 +export const MAX_NATIVE_CHAT_TRANSCRIPT_PATH_BYTES = 64 * 1_024 +export const MAX_NATIVE_CHAT_AGENT_TYPE_BYTES = 64 + +function boundedString( + value: unknown, + maxBytes: number, + options: { allowEmpty?: boolean } = {} +): value is string { + if (typeof value !== 'string' || (!options.allowEmpty && value.length === 0)) { + return false + } + return !measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }).exceededLimit +} + +export function isValidNativeChatTranscriptRequestStrings(args: { + agent: unknown + sessionId: unknown + transcriptPath?: unknown +}): boolean { + return ( + boundedString(args.agent, MAX_NATIVE_CHAT_AGENT_TYPE_BYTES) && + boundedString(args.sessionId, MAX_NATIVE_CHAT_SESSION_ID_BYTES, { allowEmpty: true }) && + (args.transcriptPath === undefined || + boundedString(args.transcriptPath, MAX_NATIVE_CHAT_TRANSCRIPT_PATH_BYTES, { + allowEmpty: true + })) + ) +} + +export function isValidNativeChatSubscriptionId(value: unknown): value is string { + return boundedString(value, MAX_NATIVE_CHAT_SUBSCRIPTION_ID_BYTES) +} + +export function normalizeNativeChatDesktopReadLimit(limit: unknown): number { + if (typeof limit !== 'number' || !Number.isFinite(limit) || limit <= 0) { + return DESKTOP_READ_WINDOW + } + return Math.min(MAX_NATIVE_CHAT_DESKTOP_READ_LIMIT, Math.floor(limit)) +} + +class NativeChatAttemptAdmission { + private readonly attempts = new Set() + private readonly attemptsBySender = new Map>() + + constructor( + private readonly maxPerSender: number, + private readonly maxProcessWide: number, + private readonly tokenLabel: string + ) {} + + claim(senderId: number): symbol | null { + const senderAttempts = this.attemptsBySender.get(senderId) + if ( + (senderAttempts?.size ?? 0) >= this.maxPerSender || + this.attempts.size >= this.maxProcessWide + ) { + return null + } + const token = Symbol(this.tokenLabel) + const nextSenderAttempts = senderAttempts ?? new Set() + nextSenderAttempts.add(token) + this.attemptsBySender.set(senderId, nextSenderAttempts) + this.attempts.add(token) + return token + } + + release(senderId: number, token: symbol): void { + this.attempts.delete(token) + const senderAttempts = this.attemptsBySender.get(senderId) + senderAttempts?.delete(token) + if (senderAttempts?.size === 0) { + this.attemptsBySender.delete(senderId) + } + } + + reset(): void { + this.attempts.clear() + this.attemptsBySender.clear() + } + + get size(): number { + return this.attempts.size + } +} + +export class NativeChatSetupAttemptAdmission extends NativeChatAttemptAdmission { + constructor() { + super( + MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER, + MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PROCESS_WIDE, + 'native-chat-setup' + ) + } +} + +export class NativeChatReadAdmission extends NativeChatAttemptAdmission { + constructor() { + super(MAX_NATIVE_CHAT_READS_PER_SENDER, MAX_NATIVE_CHAT_READS_PROCESS_WIDE, 'native-chat-read') + } +} diff --git a/src/main/ipc/native-chat-subscribe-lifecycle.test.ts b/src/main/ipc/native-chat-subscribe-lifecycle.test.ts index 683e0271086..681f0c3310d 100644 --- a/src/main/ipc/native-chat-subscribe-lifecycle.test.ts +++ b/src/main/ipc/native-chat-subscribe-lifecycle.test.ts @@ -1,9 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { NativeChatTurnLifecycle } from '../../shared/native-chat-types' -const { handlers, listeners, subscribeTranscript } = vi.hoisted(() => ({ +const { handlers, listeners, readTranscriptTail, subscribeTranscript } = vi.hoisted(() => ({ handlers: new Map unknown>(), listeners: new Map unknown>(), + readTranscriptTail: vi.fn(), subscribeTranscript: vi.fn() })) @@ -19,15 +20,28 @@ vi.mock('electron', () => ({ })) vi.mock('../native-chat/transcript-watch', () => ({ + readNativeChatTranscriptTail: readTranscriptTail, subscribeNativeChatTranscript: subscribeTranscript })) import { + _getNativeChatLiveSubscriptionCountForTest, _getNativeChatPendingSubscriptionCountForTest, + _getNativeChatReadCountForTest, _getNativeChatSenderCleanupCountForTest, + _getNativeChatSetupAttemptCountForTest, clearNativeChatSubscriptions, registerNativeChatHandlers } from './native-chat' +import { + MAX_NATIVE_CHAT_DESKTOP_READ_LIMIT, + MAX_NATIVE_CHAT_READS_PER_SENDER, + MAX_NATIVE_CHAT_SESSION_ID_BYTES, + MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER, + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE, + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER, + MAX_NATIVE_CHAT_SUBSCRIPTION_ID_BYTES +} from './native-chat-ipc-admission' type TestSubscription = { unsubscribe: ReturnType @@ -48,6 +62,7 @@ type SenderHarness = { id: number isDestroyed: () => boolean once: (event: string, callback: () => void) => void + removeListener: (event: string, callback: () => void) => void send: ReturnType } } @@ -56,6 +71,8 @@ beforeEach(() => { clearNativeChatSubscriptions() handlers.clear() listeners.clear() + readTranscriptTail.mockReset() + readTranscriptTail.mockResolvedValue({ messages: [], hasMore: false, beforeOffset: 0 }) subscribeTranscript.mockReset() registerNativeChatHandlers() }) @@ -82,7 +99,7 @@ function createSender(id: number): SenderHarness { return { destroy: () => { destroyed = true - for (const callback of destroyedCallbacks) { + for (const callback of destroyedCallbacks.slice()) { callback() } }, @@ -95,6 +112,15 @@ function createSender(id: number): SenderHarness { destroyedCallbacks.push(callback) } }, + removeListener: (event, callback) => { + if (event !== 'destroyed') { + return + } + const index = destroyedCallbacks.indexOf(callback) + if (index >= 0) { + destroyedCallbacks.splice(index, 1) + } + }, send: vi.fn() } } @@ -145,6 +171,166 @@ async function waitFor(predicate: () => boolean, timeoutMs = 1000): Promise { + it('clamps read and watch windows before transcript work starts', async () => { + const read = handlers.get('nativeChat:readSession') + if (!read) { + throw new Error('read handler not registered') + } + await read( + { sender: { id: 9 } }, + { agent: 'claude', sessionId: 'session', limit: Number.MAX_SAFE_INTEGER } + ) + expect(readTranscriptTail).toHaveBeenCalledWith( + expect.objectContaining({ limit: MAX_NATIVE_CHAT_DESKTOP_READ_LIMIT }) + ) + + const pending = deferredSubscription() + subscribeTranscript.mockReturnValueOnce(pending.promise) + const renderer = createSender(10) + const listener = listeners.get('nativeChat:subscribe')! + listener( + { sender: renderer.sender }, + { + subscriptionId: 'bounded-window', + agent: 'claude', + sessionId: 'session', + limit: Number.MAX_SAFE_INTEGER + } + ) + expect(subscribeTranscript).toHaveBeenCalledWith( + expect.objectContaining({ initialLimit: MAX_NATIVE_CHAT_DESKTOP_READ_LIMIT }) + ) + pending.resolve() + await waitFor(() => _getNativeChatSetupAttemptCountForTest() === 0) + renderer.destroy() + }) + + it('rejects oversized retained identifiers before starting transcript work', async () => { + const renderer = createSender(11) + subscribe(renderer.sender, 'x'.repeat(MAX_NATIVE_CHAT_SUBSCRIPTION_ID_BYTES + 1)) + const listener = listeners.get('nativeChat:subscribe')! + listener( + { sender: renderer.sender }, + { + subscriptionId: 'valid', + agent: 'claude', + sessionId: 'x'.repeat(MAX_NATIVE_CHAT_SESSION_ID_BYTES + 1) + } + ) + + expect(subscribeTranscript).not.toHaveBeenCalled() + expect(_getNativeChatPendingSubscriptionCountForTest()).toBe(0) + expect(_getNativeChatSetupAttemptCountForTest()).toBe(0) + }) + + it('bounds concurrent transcript pages per renderer and releases admission', async () => { + let releaseReads!: (value: { + messages: never[] + hasMore: boolean + beforeOffset: number + }) => void + const readGate = new Promise<{ + messages: never[] + hasMore: boolean + beforeOffset: number + }>((resolve) => { + releaseReads = resolve + }) + readTranscriptTail.mockReturnValue(readGate) + const read = handlers.get('nativeChat:readSession')! + const reads = Array.from({ length: MAX_NATIVE_CHAT_READS_PER_SENDER }, () => + read({ sender: { id: 20 } }, { agent: 'claude', sessionId: 'session' }) + ) + + await vi.waitFor(() => + expect(_getNativeChatReadCountForTest()).toBe(MAX_NATIVE_CHAT_READS_PER_SENDER) + ) + await expect( + read({ sender: { id: 20 } }, { agent: 'claude', sessionId: 'session' }) + ).resolves.toEqual({ error: 'Too many concurrent native chat transcript reads' }) + expect(readTranscriptTail).toHaveBeenCalledTimes(MAX_NATIVE_CHAT_READS_PER_SENDER) + + releaseReads({ messages: [], hasMore: false, beforeOffset: 0 }) + await Promise.all(reads) + expect(_getNativeChatReadCountForTest()).toBe(0) + }) + + it('caps unique logical subscriptions per renderer and recovers after unsubscribe', async () => { + subscribeTranscript.mockImplementation(async () => ({ + unsubscribe: vi.fn(), + watching: true + })) + const renderer = createSender(12) + for (let index = 0; index < MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER; index++) { + subscribe(renderer.sender, `sub-${index}`) + } + subscribe(renderer.sender, 'one-over') + await waitFor(() => _getNativeChatSetupAttemptCountForTest() === 0) + + expect(subscribeTranscript).toHaveBeenCalledTimes(MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER) + expect(_getNativeChatLiveSubscriptionCountForTest()).toBe( + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER + ) + + unsubscribe(renderer.sender, 'sub-0') + subscribe(renderer.sender, 'after-release') + await waitFor(() => _getNativeChatSetupAttemptCountForTest() === 0) + expect(subscribeTranscript).toHaveBeenCalledTimes(MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER + 1) + renderer.destroy() + }) + + it('caps logical subscriptions across renderers process-wide', async () => { + subscribeTranscript.mockImplementation(async () => ({ + unsubscribe: vi.fn(), + watching: true + })) + const rendererCount = Math.ceil( + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE / MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER + ) + const renderers = Array.from({ length: rendererCount + 1 }, (_, index) => + createSender(100 + index) + ) + for (let index = 0; index < MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE; index++) { + const renderer = renderers[Math.floor(index / MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER)] + subscribe(renderer.sender, `global-${index}`) + } + subscribe(renderers.at(-1)!.sender, 'global-one-over') + await waitFor(() => _getNativeChatSetupAttemptCountForTest() === 0) + + expect(subscribeTranscript).toHaveBeenCalledTimes(MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE) + expect(_getNativeChatLiveSubscriptionCountForTest()).toBe( + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE + ) + renderers.forEach((renderer) => renderer.destroy()) + }) + + it('caps unresolved same-id setup attempts without displacing the latest admitted setup', async () => { + const pending = Array.from( + { length: MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER }, + deferredSubscription + ) + for (const setup of pending) { + subscribeTranscript.mockReturnValueOnce(setup.promise) + } + const renderer = createSender(13) + for (let index = 0; index < pending.length; index++) { + subscribe(renderer.sender, 'same-id-storm') + } + subscribe(renderer.sender, 'same-id-storm') + + expect(subscribeTranscript).toHaveBeenCalledTimes(MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER) + expect(_getNativeChatSetupAttemptCountForTest()).toBe(MAX_NATIVE_CHAT_SETUP_ATTEMPTS_PER_SENDER) + expect(_getNativeChatPendingSubscriptionCountForTest()).toBe(1) + + pending.forEach((setup) => setup.resolve()) + await waitFor(() => _getNativeChatSetupAttemptCountForTest() === 0) + expect(pending.slice(0, -1).every((setup) => setup.unsubscribe.mock.calls.length === 1)).toBe( + true + ) + expect(pending.at(-1)?.unsubscribe).not.toHaveBeenCalled() + renderer.destroy() + }) + it('closes a watcher that resolves after renderer unsubscribe', async () => { const pending = deferredSubscription() subscribeTranscript.mockReturnValueOnce(pending.promise) @@ -161,6 +347,8 @@ describe('nativeChat subscribe lifecycle', () => { await waitFor(() => pending.unsubscribe.mock.calls.length === 1) unsubscribe(renderer.sender, 'unmount') expect(pending.unsubscribe).toHaveBeenCalledOnce() + expect(_getNativeChatSenderCleanupCountForTest()).toBe(0) + expect(renderer.registeredCleanupCount()).toBe(0) renderer.destroy() expect(_getNativeChatSenderCleanupCountForTest()).toBe(0) }) @@ -209,7 +397,8 @@ describe('nativeChat subscribe lifecycle', () => { subscribe(renderer.sender, 'retry') failed.reject(new Error('watch setup failed')) await waitFor(() => _getNativeChatPendingSubscriptionCountForTest() === 0) - expect(_getNativeChatSenderCleanupCountForTest()).toBe(1) + expect(_getNativeChatSenderCleanupCountForTest()).toBe(0) + expect(renderer.registeredCleanupCount()).toBe(0) subscribe(renderer.sender, 'retry') expect(renderer.registeredCleanupCount()).toBe(1) @@ -217,6 +406,7 @@ describe('nativeChat subscribe lifecycle', () => { await waitFor(() => _getNativeChatPendingSubscriptionCountForTest() === 0) unsubscribe(renderer.sender, 'retry') expect(retry.unsubscribe).toHaveBeenCalledOnce() + expect(renderer.registeredCleanupCount()).toBe(0) renderer.destroy() expect(_getNativeChatSenderCleanupCountForTest()).toBe(0) }) diff --git a/src/main/ipc/native-chat-subscription-registry.ts b/src/main/ipc/native-chat-subscription-registry.ts new file mode 100644 index 00000000000..0826039fd17 --- /dev/null +++ b/src/main/ipc/native-chat-subscription-registry.ts @@ -0,0 +1,178 @@ +import type { WebContents } from 'electron' +import type { NativeChatTranscriptSubscription } from '../native-chat/transcript-watch' +import { + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER, + MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE, + NativeChatSetupAttemptAdmission +} from './native-chat-ipc-admission' + +export class NativeChatSubscriptionRegistry { + private readonly live = new Map>() + private readonly pending = new Map>() + private readonly senderCleanups = new Map< + number, + { onDestroyed: () => void; sender: WebContents } + >() + private readonly setupAttempts = new NativeChatSetupAttemptAdmission() + + canAdmit(senderId: number, subscriptionId: string): boolean { + if ( + this.live.get(senderId)?.has(subscriptionId) || + this.pending.get(senderId)?.has(subscriptionId) + ) { + return true + } + return ( + this.logicalCount(senderId) < MAX_NATIVE_CHAT_SUBSCRIPTIONS_PER_SENDER && + this.logicalCount() < MAX_NATIVE_CHAT_SUBSCRIPTIONS_PROCESS_WIDE + ) + } + + claimSetup(senderId: number): symbol | null { + return this.setupAttempts.claim(senderId) + } + + releaseSetup(senderId: number, token: symbol): void { + this.setupAttempts.release(senderId, token) + } + + beginPending(senderId: number, subscriptionId: string): symbol { + this.teardown(senderId, subscriptionId) + const token = Symbol(subscriptionId) + const bySubscription = this.pending.get(senderId) ?? new Map() + bySubscription.set(subscriptionId, token) + this.pending.set(senderId, bySubscription) + return token + } + + takePending(senderId: number, subscriptionId: string, token: symbol): boolean { + const bySubscription = this.pending.get(senderId) + if (bySubscription?.get(subscriptionId) !== token) { + return false + } + bySubscription.delete(subscriptionId) + if (bySubscription.size === 0) { + this.pending.delete(senderId) + } + return true + } + + publish( + senderId: number, + subscriptionId: string, + subscription: NativeChatTranscriptSubscription + ): void { + const bySubscription = + this.live.get(senderId) ?? new Map() + bySubscription.get(subscriptionId)?.unsubscribe() + bySubscription.set(subscriptionId, subscription) + this.live.set(senderId, bySubscription) + } + + registerSenderCleanup(sender: WebContents): void { + const existing = this.senderCleanups.get(sender.id) + if (existing?.sender === sender) { + return + } + if (existing) { + this.teardownSender(sender.id) + } + const onDestroyed = (): void => { + if (this.senderCleanups.get(sender.id)?.sender === sender) { + this.teardownSender(sender.id) + } + } + this.senderCleanups.set(sender.id, { onDestroyed, sender }) + sender.once('destroyed', onDestroyed) + } + + teardown(senderId: number, subscriptionId: string): void { + const pendingBySubscription = this.pending.get(senderId) + pendingBySubscription?.delete(subscriptionId) + if (pendingBySubscription?.size === 0) { + this.pending.delete(senderId) + } + const liveBySubscription = this.live.get(senderId) + const subscription = liveBySubscription?.get(subscriptionId) + subscription?.unsubscribe() + liveBySubscription?.delete(subscriptionId) + if (liveBySubscription?.size === 0) { + this.live.delete(senderId) + } + this.releaseSenderCleanupIfIdle(senderId) + } + + releaseSenderCleanupIfIdle(senderId: number): void { + if (this.live.has(senderId) || this.pending.has(senderId)) { + return + } + this.releaseSenderCleanup(senderId) + } + + reset(): void { + const senderIds = new Set([ + ...this.live.keys(), + ...this.pending.keys(), + ...this.senderCleanups.keys() + ]) + for (const senderId of senderIds) { + this.teardownSender(senderId) + } + this.pending.clear() + this.setupAttempts.reset() + } + + get cleanupCount(): number { + return this.senderCleanups.size + } + + get pendingCount(): number { + let count = 0 + for (const bySubscription of this.pending.values()) { + count += bySubscription.size + } + return count + } + + get logicalSubscriptionCount(): number { + return this.logicalCount() + } + + get setupAttemptCount(): number { + return this.setupAttempts.size + } + + private logicalCount(senderId?: number): number { + if (senderId !== undefined) { + return (this.live.get(senderId)?.size ?? 0) + (this.pending.get(senderId)?.size ?? 0) + } + let count = 0 + const senderIds = new Set([...this.live.keys(), ...this.pending.keys()]) + for (const id of senderIds) { + count += this.logicalCount(id) + } + return count + } + + private teardownSender(senderId: number): void { + this.releaseSenderCleanup(senderId) + this.pending.delete(senderId) + const bySubscription = this.live.get(senderId) + if (!bySubscription) { + return + } + for (const subscription of bySubscription.values()) { + subscription.unsubscribe() + } + this.live.delete(senderId) + } + + private releaseSenderCleanup(senderId: number): void { + const cleanup = this.senderCleanups.get(senderId) + if (!cleanup) { + return + } + this.senderCleanups.delete(senderId) + cleanup.sender.removeListener('destroyed', cleanup.onDestroyed) + } +} diff --git a/src/main/ipc/native-chat.test.ts b/src/main/ipc/native-chat.test.ts index 4d0e1765a44..324d8422ed6 100644 --- a/src/main/ipc/native-chat.test.ts +++ b/src/main/ipc/native-chat.test.ts @@ -65,7 +65,7 @@ async function invokeReadSession(args: { if (!handler) { throw new Error('handler not registered') } - return handler({}, args) + return handler({ sender: { id: 1 } }, args) } describe('nativeChat:readSession handler', () => { @@ -197,6 +197,11 @@ describe('nativeChat:readSession handler', () => { destroyedCb = cb } }, + removeListener: (event: string, cb: () => void) => { + if (event === 'destroyed' && destroyedCb === cb) { + destroyedCb = undefined + } + }, send: (channel: string, payload: unknown) => sent.push({ channel, payload }) } @@ -283,6 +288,11 @@ describe('nativeChat:readSession handler', () => { destroyedCb = cb } }, + removeListener: (event: string, cb: () => void) => { + if (event === 'destroyed' && destroyedCb === cb) { + destroyedCb = undefined + } + }, send: vi.fn() } diff --git a/src/main/ipc/native-chat.ts b/src/main/ipc/native-chat.ts index 6521bef3964..65cc1bfac6f 100644 --- a/src/main/ipc/native-chat.ts +++ b/src/main/ipc/native-chat.ts @@ -1,4 +1,4 @@ -import { ipcMain, type IpcMainEvent, type WebContents } from 'electron' +import { ipcMain, type IpcMainEvent } from 'electron' import type { AgentType, NativeChatMessage, @@ -11,6 +11,17 @@ import { readNativeChatTranscriptTail, type NativeChatTranscriptSubscription } from '../native-chat/transcript-watch' +import { + NATIVE_CHAT_TRANSCRIPT_PAGE_RESERVATION_BYTES, + nativeChatTranscriptReadAdmission +} from '../native-chat/transcript-read-admission' +import { + isValidNativeChatSubscriptionId, + isValidNativeChatTranscriptRequestStrings, + NativeChatReadAdmission, + normalizeNativeChatDesktopReadLimit +} from './native-chat-ipc-admission' +import { NativeChatSubscriptionRegistry } from './native-chat-subscription-registry' // Re-export so existing test imports of `clearNativeChatTranscriptCache` from // this module keep working after the cache moved to transcript-read-cache.ts. @@ -27,20 +38,43 @@ export type NativeChatReadSessionArgs = { transcriptPath?: string } -// Why: render and parse only the recent window so long transcripts do not stall -// either the main process or the message list. Pagination raises this limit. -const DESKTOP_READ_WINDOW = 300 +const readAdmission = new NativeChatReadAdmission() -async function readSession(args: NativeChatReadSessionArgs): Promise { - const { agent, sessionId } = args - // Clamp to a positive window; default to the desktop window for the first page. - const limit = args.limit && args.limit > 0 ? Math.floor(args.limit) : DESKTOP_READ_WINDOW - return readNativeChatTranscriptTail({ - agent, - sessionId, - transcriptPath: args.transcriptPath, - limit - }) +async function readSession( + senderId: number, + args: NativeChatReadSessionArgs +): Promise { + if (!args || !isValidNativeChatTranscriptRequestStrings(args)) { + return { error: 'Invalid native chat transcript request' } + } + const readToken = readAdmission.claim(senderId) + if (!readToken) { + return { error: 'Too many concurrent native chat transcript reads' } + } + const { agent, sessionId, transcriptPath } = args + const limit = normalizeNativeChatDesktopReadLimit(args.limit) + let releaseMemory: (() => void) | undefined + try { + releaseMemory = await nativeChatTranscriptReadAdmission.acquire( + NATIVE_CHAT_TRANSCRIPT_PAGE_RESERVATION_BYTES + ) + return await readNativeChatTranscriptTail({ + agent, + sessionId, + transcriptPath, + limit + }) + } catch (error) { + if (!releaseMemory) { + return { + error: error instanceof Error ? error.message : 'Native chat transcript reader is busy' + } + } + throw error + } finally { + releaseMemory?.() + readAdmission.release(senderId, readToken) + } } export type NativeChatSubscribeArgs = { @@ -77,91 +111,28 @@ export type NativeChatAppendedPayload = { } } -type LiveSubscription = { - subscription: NativeChatTranscriptSubscription -} - -// Why: live subscriptions are keyed by (webContents.id, subscriptionId) so the -// same renderer can watch several panes, and a destroyed window tears down all -// of its watchers — strict teardown to avoid fd leaks (plan U4 risk). -const liveSubscriptions = new Map>() -// Why: unsubscribe and renderer destruction must invalidate async watcher setup -// before it can publish a late subscription into the live map. -const pendingSubscriptions = new Map>() -const senderCleanupRegistered = new Set() - -function teardownSubscription(senderId: number, subscriptionId: string): void { - const pendingBySubId = pendingSubscriptions.get(senderId) - pendingBySubId?.delete(subscriptionId) - if (pendingBySubId?.size === 0) { - pendingSubscriptions.delete(senderId) - } - const bySubId = liveSubscriptions.get(senderId) - const live = bySubId?.get(subscriptionId) - if (!live || !bySubId) { - return - } - live.subscription.unsubscribe() - bySubId.delete(subscriptionId) - if (bySubId.size === 0) { - liveSubscriptions.delete(senderId) - } -} - -function teardownAllForSender(senderId: number): void { - // The destroyed event can arrive before async subscription setup stores a watcher. - senderCleanupRegistered.delete(senderId) - pendingSubscriptions.delete(senderId) - const bySubId = liveSubscriptions.get(senderId) - if (!bySubId) { - return - } - for (const live of bySubId.values()) { - live.subscription.unsubscribe() - } - liveSubscriptions.delete(senderId) -} - -function registerSenderCleanup(sender: WebContents): void { - if (senderCleanupRegistered.has(sender.id)) { - return - } - senderCleanupRegistered.add(sender.id) - // Strict teardown: a closed/reloaded window releases every watcher it owns. - sender.once('destroyed', () => teardownAllForSender(sender.id)) -} - -function beginPendingSubscription(senderId: number, subscriptionId: string): symbol { - teardownSubscription(senderId, subscriptionId) - const token = Symbol(subscriptionId) - const bySubId = pendingSubscriptions.get(senderId) ?? new Map() - bySubId.set(subscriptionId, token) - pendingSubscriptions.set(senderId, bySubId) - return token -} - -function takePendingSubscription(senderId: number, subscriptionId: string, token: symbol): boolean { - const bySubId = pendingSubscriptions.get(senderId) - if (bySubId?.get(subscriptionId) !== token) { - return false - } - bySubId.delete(subscriptionId) - if (bySubId.size === 0) { - pendingSubscriptions.delete(senderId) - } - return true -} +const subscriptionRegistry = new NativeChatSubscriptionRegistry() async function handleSubscribe(event: IpcMainEvent, args: NativeChatSubscribeArgs): Promise { const sender = event.sender - if (sender.isDestroyed()) { + if ( + sender.isDestroyed() || + !args || + !isValidNativeChatSubscriptionId(args.subscriptionId) || + !isValidNativeChatTranscriptRequestStrings(args) || + !subscriptionRegistry.canAdmit(sender.id, args.subscriptionId) + ) { return } const { subscriptionId, agent, sessionId, transcriptPath } = args - const limit = args.limit && args.limit > 0 ? Math.floor(args.limit) : DESKTOP_READ_WINDOW + const setupAttempt = subscriptionRegistry.claimSetup(sender.id) + if (!setupAttempt) { + return + } + const limit = normalizeNativeChatDesktopReadLimit(args.limit) // Replace any prior subscription under the same id (session change/resubscribe). - const pendingToken = beginPendingSubscription(sender.id, subscriptionId) - registerSenderCleanup(sender) + const pendingToken = subscriptionRegistry.beginPending(sender.id, subscriptionId) + subscriptionRegistry.registerSenderCleanup(sender) let subscription: NativeChatTranscriptSubscription try { @@ -218,25 +189,21 @@ async function handleSubscribe(event: IpcMainEvent, args: NativeChatSubscribeArg } }) } catch { - takePendingSubscription(sender.id, subscriptionId, pendingToken) + subscriptionRegistry.takePending(sender.id, subscriptionId, pendingToken) + subscriptionRegistry.releaseSenderCleanupIfIdle(sender.id) return + } finally { + subscriptionRegistry.releaseSetup(sender.id, setupAttempt) } // Why: unmount, destruction, or a newer same-id subscribe can invalidate setup // while path resolution is pending; only the owning token may publish its watcher. - const stillCurrent = takePendingSubscription(sender.id, subscriptionId, pendingToken) + const stillCurrent = subscriptionRegistry.takePending(sender.id, subscriptionId, pendingToken) if (sender.isDestroyed() || !stillCurrent) { subscription.unsubscribe() return } - const bySubId = liveSubscriptions.get(sender.id) ?? new Map() - // A concurrent subscribe with the same id beat us here; honor the latest. - const existing = bySubId.get(subscriptionId) - if (existing) { - existing.subscription.unsubscribe() - } - bySubId.set(subscriptionId, { subscription }) - liveSubscriptions.set(sender.id, bySubId) + subscriptionRegistry.publish(sender.id, subscriptionId, subscription) if (!subscription.watching && !sender.isDestroyed()) { const payload: NativeChatAppendedPayload = { subscriptionId, @@ -253,34 +220,40 @@ async function handleSubscribe(event: IpcMainEvent, args: NativeChatSubscribeArg /** Test-only: drop all live and pending transcript subscriptions between runs. */ export function clearNativeChatSubscriptions(): void { - const senderIds = new Set([...liveSubscriptions.keys(), ...pendingSubscriptions.keys()]) - for (const senderId of senderIds) { - teardownAllForSender(senderId) - } - pendingSubscriptions.clear() - senderCleanupRegistered.clear() + subscriptionRegistry.reset() + readAdmission.reset() } export function _getNativeChatSenderCleanupCountForTest(): number { - return senderCleanupRegistered.size + return subscriptionRegistry.cleanupCount } export function _getNativeChatPendingSubscriptionCountForTest(): number { - let count = 0 - for (const bySubId of pendingSubscriptions.values()) { - count += bySubId.size - } - return count + return subscriptionRegistry.pendingCount +} + +export function _getNativeChatLiveSubscriptionCountForTest(): number { + return subscriptionRegistry.logicalSubscriptionCount +} + +export function _getNativeChatSetupAttemptCountForTest(): number { + return subscriptionRegistry.setupAttemptCount +} + +export function _getNativeChatReadCountForTest(): number { + return readAdmission.size } export function registerNativeChatHandlers(): void { - ipcMain.handle('nativeChat:readSession', (_event, args: NativeChatReadSessionArgs) => - readSession(args) + ipcMain.handle('nativeChat:readSession', (event, args: NativeChatReadSessionArgs) => + readSession(event.sender.id, args) ) ipcMain.on('nativeChat:subscribe', (event, args: NativeChatSubscribeArgs) => { void handleSubscribe(event, args) }) ipcMain.on('nativeChat:unsubscribe', (event, args: { subscriptionId: string }) => { - teardownSubscription(event.sender.id, args.subscriptionId) + if (args && isValidNativeChatSubscriptionId(args.subscriptionId)) { + subscriptionRegistry.teardown(event.sender.id, args.subscriptionId) + } }) } diff --git a/src/main/ipc/native-notification-retention.ts b/src/main/ipc/native-notification-retention.ts new file mode 100644 index 00000000000..13336035e25 --- /dev/null +++ b/src/main/ipc/native-notification-retention.ts @@ -0,0 +1,104 @@ +import type { Notification } from 'electron' + +const NOTIFICATION_RELEASE_FALLBACK_MS = 5 * 60 * 1000 +export const MAX_ACTIVE_NATIVE_NOTIFICATIONS = 128 + +type RetainedNotification = { + evict: () => void + release: () => void +} + +export type RetainedNativeNotificationIdEntry = { + notification: Notification + release: () => void +} + +const activeNotifications = new Map() +const activeNotificationsById = new Map() + +export function retainNativeNotification( + notification: Notification, + onRelease?: () => void, + onEvict?: () => void, + options: { fallbackMs?: number | null } = {} +): () => void { + while (activeNotifications.size >= MAX_ACTIVE_NATIVE_NOTIFICATIONS) { + const oldest = activeNotifications.values().next() + if (oldest.done) { + break + } + oldest.value.evict() + } + + let released = false + let releaseTimer: ReturnType | null = null + + function release(): void { + if (released) { + return + } + released = true + activeNotifications.delete(notification) + notification.removeListener('close', release) + if (releaseTimer) { + clearTimeout(releaseTimer) + releaseTimer = null + } + onRelease?.() + } + + function evict(): void { + try { + onEvict?.() + } finally { + try { + notification.close() + } finally { + release() + } + } + } + + activeNotifications.set(notification, { evict, release }) + notification.on('close', release) + const fallbackMs = + options.fallbackMs === undefined ? NOTIFICATION_RELEASE_FALLBACK_MS : options.fallbackMs + if (fallbackMs !== null) { + releaseTimer = setTimeout(release, fallbackMs) + releaseTimer.unref?.() + } + return release +} + +export function getRetainedNativeNotificationById( + id: string +): RetainedNativeNotificationIdEntry | undefined { + return activeNotificationsById.get(id) +} + +export function setRetainedNativeNotificationId( + id: string, + entry: RetainedNativeNotificationIdEntry +): void { + activeNotificationsById.set(id, entry) +} + +export function deleteRetainedNativeNotificationId( + id: string, + expected: RetainedNativeNotificationIdEntry +): void { + if (activeNotificationsById.get(id) === expected) { + activeNotificationsById.delete(id) + } +} + +export function getActiveNativeNotificationCountForTest(): number { + return activeNotifications.size +} + +export function clearActiveNativeNotificationsForTest(): void { + for (const entry of Array.from(activeNotifications.values())) { + entry.evict() + } + activeNotificationsById.clear() +} diff --git a/src/main/ipc/notebook.test.ts b/src/main/ipc/notebook.test.ts index d68668038bd..78f1773d9c2 100644 --- a/src/main/ipc/notebook.test.ts +++ b/src/main/ipc/notebook.test.ts @@ -89,4 +89,23 @@ describe('notebook IPC', () => { expect(processKillSpy).toHaveBeenCalledWith(-4321, 'SIGKILL') } }) + + it('preserves output delivered as 100,000 one-byte subprocess events', async () => { + const proc = createMockProcess() + spawnMock.mockReturnValue(proc) + registerNotebookHandlers({} as never) + + const resultPromise = handlers.get('notebook:runPythonCell')?.(null, { + filePath: '/repo/notebook.ipynb', + code: 'print("x")' + }) as Promise<{ stdout: string }> + await Promise.resolve() + + for (let index = 0; index < 100_000; index += 1) { + proc.stdout.emit('data', Buffer.from('x')) + } + proc.emit('close', 0) + + await expect(resultPromise).resolves.toMatchObject({ stdout: 'x'.repeat(100_000) }) + }) }) diff --git a/src/main/ipc/notebook.ts b/src/main/ipc/notebook.ts index 9255ab7383d..080a94e5ca2 100644 --- a/src/main/ipc/notebook.ts +++ b/src/main/ipc/notebook.ts @@ -4,6 +4,7 @@ import { dirname } from 'node:path' import { ipcMain } from 'electron' import type { Store } from '../persistence' import { resolveAuthorizedPath } from './filesystem-auth' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' export type NotebookRunResult = { stdout: string @@ -16,9 +17,9 @@ const PYTHON_RUN_TIMEOUT_MS = 60_000 const MAX_CAPTURE_BYTES = 2 * 1024 * 1024 type BoundedCapture = { - text: string - bytes: number + buffer: GrowingByteBuffer truncated: boolean + truncationMarker: boolean } function pythonCandidates(): { command: string; argsPrefix: string[] }[] { @@ -38,19 +39,23 @@ function appendBounded(capture: BoundedCapture, chunk: Buffer): void { if (capture.truncated) { return } - const remainingBytes = MAX_CAPTURE_BYTES - capture.bytes + const remainingBytes = MAX_CAPTURE_BYTES - capture.buffer.byteLength if (remainingBytes <= 0) { capture.truncated = true return } if (chunk.byteLength <= remainingBytes) { - capture.text += chunk.toString('utf8') - capture.bytes += chunk.byteLength + capture.buffer.append(chunk) return } - capture.text += `${chunk.subarray(0, remainingBytes).toString('utf8')}\n[output truncated]\n` - capture.bytes = MAX_CAPTURE_BYTES + capture.buffer.append(chunk.subarray(0, remainingBytes)) capture.truncated = true + capture.truncationMarker = true +} + +function boundedCaptureText(capture: BoundedCapture): string { + const text = capture.buffer.toString() + return capture.truncationMarker ? `${text}\n[output truncated]\n` : text } function terminateNotebookProcessTree( @@ -117,8 +122,16 @@ async function runPythonCandidate( cwd: string ): Promise { return new Promise((resolve) => { - const stdout: BoundedCapture = { text: '', bytes: 0, truncated: false } - const stderr: BoundedCapture = { text: '', bytes: 0, truncated: false } + const stdout: BoundedCapture = { + buffer: new GrowingByteBuffer(), + truncated: false, + truncationMarker: false + } + const stderr: BoundedCapture = { + buffer: new GrowingByteBuffer(), + truncated: false, + truncationMarker: false + } let settled = false let forceKillTimer: ReturnType | null = null let timeout: ReturnType | null = null @@ -162,8 +175,8 @@ async function runPythonCandidate( forceKillTimer = terminateNotebookProcessTree(child) finish( { - stdout: stdout.text, - stderr: stderr.text, + stdout: boundedCaptureText(stdout), + stderr: boundedCaptureText(stderr), exitCode: null, error: 'Python cell timed out.' }, @@ -178,12 +191,17 @@ async function runPythonCandidate( appendBounded(stderr, chunk) } const onError = (error: Error): void => { - finish({ stdout: stdout.text, stderr: stderr.text, exitCode: null, error: error.message }) + finish({ + stdout: boundedCaptureText(stdout), + stderr: boundedCaptureText(stderr), + exitCode: null, + error: error.message + }) } const onClose = (exitCode: number | null): void => { finish({ - stdout: stdout.text, - stderr: stderr.text, + stdout: boundedCaptureText(stdout), + stderr: boundedCaptureText(stderr), exitCode }) } diff --git a/src/main/ipc/notification-ipc-admission.test.ts b/src/main/ipc/notification-ipc-admission.test.ts new file mode 100644 index 00000000000..a4a70a6f377 --- /dev/null +++ b/src/main/ipc/notification-ipc-admission.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_NOTIFICATION_DISPATCH_INPUT_BYTES, + normalizeNotificationDispatchRequest +} from './notification-ipc-admission' + +describe('notification IPC admission', () => { + it('copies only whitelisted fields so unknown payloads are not retained', () => { + const request = normalizeNotificationDispatchRequest({ + source: 'agent-task-complete', + worktreeId: 'repo::worktree', + requireDisplayConfirmation: true, + unknownPayload: 'x'.repeat(MAX_NOTIFICATION_DISPATCH_INPUT_BYTES * 2) + }) + + expect(request).toEqual({ + source: 'agent-task-complete', + worktreeId: 'repo::worktree', + requireDisplayConfirmation: true + }) + expect(request).not.toHaveProperty('unknownPayload') + }) + + it('rejects non-boolean control fields', () => { + for (const field of [ + 'requireDisplayConfirmation', + 'hasMultipleActiveRepos', + 'isActiveWorktree', + 'agentInterrupted' + ]) { + expect( + normalizeNotificationDispatchRequest({ + source: 'agent-task-complete', + [field]: 'true' + }) + ).toBeNull() + } + }) + + it('enforces the aggregate UTF-8 budget across known string fields', () => { + expect( + normalizeNotificationDispatchRequest({ + source: 'agent-task-complete', + agentPrompt: '😀'.repeat(MAX_NOTIFICATION_DISPATCH_INPUT_BYTES / 8), + agentLastAssistantMessage: '😀'.repeat(MAX_NOTIFICATION_DISPATCH_INPUT_BYTES / 8) + }) + ).toBeNull() + }) +}) diff --git a/src/main/ipc/notification-ipc-admission.ts b/src/main/ipc/notification-ipc-admission.ts new file mode 100644 index 00000000000..ca99226daef --- /dev/null +++ b/src/main/ipc/notification-ipc-admission.ts @@ -0,0 +1,122 @@ +import type { NotificationDispatchRequest } from '../../shared/types' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MAX_NOTIFICATION_DISPATCH_INPUT_BYTES = 256 * 1024 +export const MAX_NOTIFICATION_ID_BYTES = 8 * 1024 +export const MAX_NOTIFICATION_WORKTREE_ID_BYTES = 16 * 1024 +export const MAX_NOTIFICATION_PANE_KEY_BYTES = 8 * 1024 +export const MAX_NOTIFICATION_DISMISS_IDS = 256 +export const MAX_NOTIFICATION_DISMISS_SCAN_ENTRIES = 1_024 +export const MAX_NOTIFICATION_DISMISS_TOTAL_ID_BYTES = 256 * 1024 + +const STRING_FIELDS = [ + 'notificationId', + 'worktreeId', + 'paneKey', + 'repoLabel', + 'worktreeLabel', + 'terminalTitle', + 'agentType', + 'agentState', + 'agentPrompt', + 'agentToolName', + 'agentToolInput', + 'agentLastAssistantMessage' +] as const + +const BOOLEAN_FIELDS = [ + 'requireDisplayConfirmation', + 'hasMultipleActiveRepos', + 'isActiveWorktree', + 'agentInterrupted' +] as const + +function stringLimit(field: (typeof STRING_FIELDS)[number]): number { + if (field === 'notificationId') { + return MAX_NOTIFICATION_ID_BYTES + } + if (field === 'worktreeId') { + return MAX_NOTIFICATION_WORKTREE_ID_BYTES + } + if (field === 'paneKey') { + return MAX_NOTIFICATION_PANE_KEY_BYTES + } + return MAX_NOTIFICATION_DISPATCH_INPUT_BYTES +} + +export function normalizeNotificationDispatchRequest( + value: unknown +): NotificationDispatchRequest | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return null + } + const request = value as Record + if ( + request.source !== 'agent-task-complete' && + request.source !== 'terminal-bell' && + request.source !== 'test' + ) { + return null + } + + const normalized: Record = { source: request.source } + let retainedBytes = measureUtf8ByteLength(request.source).byteLength + for (const field of STRING_FIELDS) { + const fieldValue = request[field] + if (fieldValue === undefined || fieldValue === null) { + continue + } + if (typeof fieldValue !== 'string') { + return null + } + const remaining = Math.min( + stringLimit(field), + MAX_NOTIFICATION_DISPATCH_INPUT_BYTES - retainedBytes + ) + const measured = measureUtf8ByteLength(fieldValue, { stopAfterBytes: remaining }) + if (measured.exceededLimit) { + return null + } + retainedBytes += measured.byteLength + normalized[field] = fieldValue + } + for (const field of BOOLEAN_FIELDS) { + const fieldValue = request[field] + if (fieldValue === undefined) { + continue + } + if (typeof fieldValue !== 'boolean') { + return null + } + normalized[field] = fieldValue + } + return normalized as NotificationDispatchRequest +} + +export function boundedUniqueNotificationDismissIds(value: unknown): string[] { + if (!Array.isArray(value)) { + return [] + } + const ids: string[] = [] + const unique = new Set() + let retainedBytes = 0 + const scanCount = Math.min(value.length, MAX_NOTIFICATION_DISMISS_SCAN_ENTRIES) + for (let index = 0; index < scanCount && ids.length < MAX_NOTIFICATION_DISMISS_IDS; index++) { + const id = value[index] + if (typeof id !== 'string' || id.length === 0 || unique.has(id)) { + continue + } + const remaining = Math.min( + MAX_NOTIFICATION_ID_BYTES, + MAX_NOTIFICATION_DISMISS_TOTAL_ID_BYTES - retainedBytes + ) + const measured = measureUtf8ByteLength(id, { stopAfterBytes: remaining }) + if (measured.exceededLimit) { + continue + } + unique.add(id) + ids.push(id) + retainedBytes += measured.byteLength + } + return ids +} diff --git a/src/main/ipc/notifications.test.ts b/src/main/ipc/notifications.test.ts index d7930066ad7..feb4bc798e0 100644 --- a/src/main/ipc/notifications.test.ts +++ b/src/main/ipc/notifications.test.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -90,6 +90,11 @@ vi.mock('../tray/system-tray', () => ({ })) import { + clearActiveNativeNotificationsForTest, + getActiveNativeNotificationCountForTest, + MAX_ACTIVE_NATIVE_NOTIFICATIONS, + MAX_NOTIFICATION_DISMISS_IDS, + MAX_NOTIFICATION_ID_BYTES, registerNotificationHandlers, triggerStartupNotificationRegistration } from './notifications' @@ -105,6 +110,7 @@ describe('registerNotificationHandlers', () => { beforeEach(() => { vi.useFakeTimers() + clearActiveNativeNotificationsForTest() vi.setSystemTime(new Date('2026-03-28T16:00:00Z')) tempDir = mkdtempSync(join(tmpdir(), 'orca-notification-test-')) removeHandlerMock.mockReset() @@ -1075,6 +1081,124 @@ describe('registerNotificationHandlers', () => { expect(notificationShowMock).toHaveBeenCalledTimes(2) }) + it('evicts the oldest retained notification at the process cap and keeps the newest', async () => { + registerNotificationHandlers({ + getSettings: () => ({ + notifications: { + enabled: true, + agentTaskComplete: true, + terminalBell: true, + suppressWhenFocused: false + } + }) + } as never) + const dispatch = getDispatchHandler() + for (let index = 0; index <= MAX_ACTIVE_NATIVE_NOTIFICATIONS; index++) { + await dispatch({}, { source: 'test', notificationId: `notification-${index}` }) + } + + expect(getActiveNativeNotificationCountForTest()).toBe(MAX_ACTIVE_NATIVE_NOTIFICATIONS) + expect(notificationCloseMock).toHaveBeenCalledTimes(1) + const dismiss = getDismissHandler() + expect(dismiss({}, ['notification-0'])).toEqual({ dismissed: 0 }) + expect(dismiss({}, [`notification-${MAX_ACTIVE_NATIVE_NOTIFICATIONS}`])).toEqual({ + dismissed: 1 + }) + }) + + it('settles display confirmation when retention evicts its notification', async () => { + registerNotificationHandlers({ + getSettings: () => ({ + notifications: { + enabled: true, + agentTaskComplete: true, + terminalBell: true, + suppressWhenFocused: false + } + }) + } as never) + const dispatch = getDispatchHandler() + const confirmations = Array.from({ length: MAX_ACTIVE_NATIVE_NOTIFICATIONS + 1 }, (_, index) => + Promise.resolve( + dispatch( + {}, + { + source: 'test', + notificationId: `confirmation-${index}`, + requireDisplayConfirmation: true + } + ) + ) + ) + await vi.advanceTimersByTimeAsync(0) + + await expect(confirmations[0]).resolves.toEqual({ + delivered: false, + reason: 'not-displayed' + }) + expect(getActiveNativeNotificationCountForTest()).toBe(MAX_ACTIVE_NATIVE_NOTIFICATIONS) + expect(notificationRemoveListenerMock).toHaveBeenCalledWith('show', expect.any(Function)) + expect(notificationRemoveListenerMock).toHaveBeenCalledWith('failed', expect.any(Function)) + + clearActiveNativeNotificationsForTest() + await Promise.all(confirmations) + }) + + it('rejects oversized dispatch identifiers before native or mobile delivery', async () => { + const dispatchMobileNotification = vi.fn() + registerNotificationHandlers( + { + getSettings: () => ({ + notifications: { + enabled: true, + agentTaskComplete: true, + terminalBell: true, + suppressWhenFocused: false + } + }) + } as never, + { dispatchMobileNotification } as never + ) + + const result = await getDispatchHandler()( + {}, + { + source: 'agent-task-complete', + notificationId: '😀'.repeat(Math.floor(MAX_NOTIFICATION_ID_BYTES / 4) + 1) + } + ) + + expect(result).toEqual({ delivered: false, reason: 'invalid-request' }) + expect(notificationCtorMock).not.toHaveBeenCalled() + expect(dispatchMobileNotification).not.toHaveBeenCalled() + expect(setTrayAttentionMock).not.toHaveBeenCalled() + }) + + it('bounds dismiss deduplication and mobile fanout', () => { + const dismissMobileNotification = vi.fn() + registerNotificationHandlers( + { + getSettings: () => ({ + notifications: { + enabled: true, + agentTaskComplete: true, + terminalBell: true, + suppressWhenFocused: false + } + }) + } as never, + { dismissMobileNotification } as never + ) + const ids = Array.from( + { length: MAX_NOTIFICATION_DISMISS_IDS + 1 }, + (_, index) => `id-${index}` + ) + + expect(getDismissHandler()({}, ids)).toEqual({ dismissed: 0 }) + expect(dismissMobileNotification).toHaveBeenCalledTimes(MAX_NOTIFICATION_DISMISS_IDS) + expect(dismissMobileNotification).not.toHaveBeenCalledWith(`id-${MAX_NOTIFICATION_DISMISS_IDS}`) + }) + it('silences the native notification when a custom sound is configured', async () => { registerNotificationHandlers({ getSettings: () => ({ @@ -1326,6 +1450,26 @@ describe('registerNotificationHandlers', () => { }) }) + it('rejects a sparse custom sound above the bounded read limit', async () => { + const soundPath = join(tempDir, 'oversized.ogg') + writeFileSync(soundPath, '') + truncateSync(soundPath, 10 * 1024 * 1024 + 1) + registerNotificationHandlers({ + getSettings: () => ({ + notifications: { + enabled: true, + agentTaskComplete: true, + terminalBell: true, + suppressWhenFocused: false, + customSoundPath: soundPath + } + }) + } as never) + + const handler = getLoadSoundHandler() + await expect(handler({})).resolves.toEqual({ ok: false, reason: 'too-large' }) + }) + it('rejects unsupported custom sound file types', async () => { const soundPath = join(tempDir, 'sound.txt') writeFileSync(soundPath, 'not audio') @@ -1432,6 +1576,7 @@ describe('notifications:probeDelivery', () => { beforeEach(() => { vi.useFakeTimers() + clearActiveNativeNotificationsForTest() handleMock.mockReset() removeHandlerMock.mockReset() notificationCtorMock.mockClear() @@ -1594,6 +1739,7 @@ describe('triggerStartupNotificationRegistration', () => { beforeEach(() => { vi.useFakeTimers() + clearActiveNativeNotificationsForTest() vi.clearAllTimers() notificationCtorMock.mockClear() notificationShowMock.mockClear() diff --git a/src/main/ipc/notifications.ts b/src/main/ipc/notifications.ts index 95e9e3e3a47..e2a55aa7836 100644 --- a/src/main/ipc/notifications.ts +++ b/src/main/ipc/notifications.ts @@ -1,6 +1,5 @@ /* eslint-disable max-lines -- Why: notification IPC keeps permission, dispatch, custom sound asset, and sound-loading handlers colocated so renderer/main contracts stay auditable. */ import { app, BrowserWindow, Notification, ipcMain, shell } from 'electron' -import { readFile, stat } from 'node:fs/promises' import { extname, isAbsolute, normalize } from 'node:path' import beepSoundPath from '../../../resources/notification-sounds/beep.mp3?asset' import blipSoundPath from '../../../resources/notification-sounds/blip.mp3?asset' @@ -28,11 +27,40 @@ import { readNotificationAuthorizationStatus } from './notification-authorizatio import { parsePaneKey } from '../../shared/stable-pane-id' import { setTrayAttention } from '../tray/system-tray' import { isMainWindowVisible } from '../window/main-window-visibility' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' +import { + boundedUniqueNotificationDismissIds, + normalizeNotificationDispatchRequest +} from './notification-ipc-admission' +import { + deleteRetainedNativeNotificationId, + getRetainedNativeNotificationById, + retainNativeNotification, + setRetainedNativeNotificationId, + type RetainedNativeNotificationIdEntry +} from './native-notification-retention' + +export { + MAX_NOTIFICATION_DISMISS_IDS, + MAX_NOTIFICATION_DISMISS_SCAN_ENTRIES, + MAX_NOTIFICATION_DISMISS_TOTAL_ID_BYTES, + MAX_NOTIFICATION_DISPATCH_INPUT_BYTES, + MAX_NOTIFICATION_ID_BYTES, + MAX_NOTIFICATION_PANE_KEY_BYTES, + MAX_NOTIFICATION_WORKTREE_ID_BYTES +} from './notification-ipc-admission' +export { + clearActiveNativeNotificationsForTest, + getActiveNativeNotificationCountForTest, + MAX_ACTIVE_NATIVE_NOTIFICATIONS +} from './native-notification-retention' const NOTIFICATION_COOLDOWN_MS = 5000 const MAX_RECENT_NOTIFICATION_KEYS = 50 const NOTIFICATION_DISPLAY_CONFIRMATION_TIMEOUT_MS = 2500 -const NOTIFICATION_RELEASE_FALLBACK_MS = 5 * 60 * 1000 const MAX_NOTIFICATION_SOUND_BYTES = 10 * 1024 * 1024 const MACOS_PACKAGED_BUNDLE_ID = 'com.stablyai.orca' const MACOS_NOTIFICATION_SETTINGS_URL = @@ -58,44 +86,6 @@ const BUILT_IN_NOTIFICATION_SOUNDS: ReadonlyMap = new Map([ ]) type NotificationSoundId = NotificationSettings['customSoundId'] -// Why: keep a strong reference so GC can't collect notifications (and their click handlers) before the user interacts with them. -const activeNotifications = new Set() -const activeNotificationsById = new Map< - string, - { notification: Notification; release: () => void } ->() - -function retainNotificationUntilRelease( - notification: Notification, - onRelease?: () => void -): () => void { - activeNotifications.add(notification) - let released = false - let releaseTimer: ReturnType | null = null - - function release(): void { - if (released) { - return - } - released = true - activeNotifications.delete(notification) - notification.removeListener('close', release) - if (releaseTimer) { - clearTimeout(releaseTimer) - releaseTimer = null - } - onRelease?.() - } - - notification.on('close', release) - releaseTimer = setTimeout(release, NOTIFICATION_RELEASE_FALLBACK_MS) - if (typeof releaseTimer.unref === 'function') { - releaseTimer.unref() - } - - return release -} - const NOTIFICATION_PROBE_RESULT_TIMEOUT_MS = 3000 const NOTIFICATION_PROBE_BANNER_CLOSE_DELAY_MS = 4000 @@ -123,20 +113,19 @@ function probeNotificationDelivery(): Promise { body: 'Orca will alert you when agents finish or terminals need attention.', silent: true }) - activeNotifications.add(probe) deliveryProbeInFlight = new Promise((resolve) => { let settled = false let timeoutTimer: ReturnType | null = null + let closeTimer: ReturnType | null = null + let releaseRetention = (): void => {} function releaseProbe(): void { - activeNotifications.delete(probe) - probe.removeListener('show', onShow) - probe.removeListener('failed', onFailed) + releaseRetention() probe.close() } - function settle(state: 'delivered' | 'blocked'): void { + function settle(state: 'delivered' | 'blocked', recordOutcome: boolean): void { if (settled) { return } @@ -145,14 +134,16 @@ function probeNotificationDelivery(): Promise { clearTimeout(timeoutTimer) timeoutTimer = null } - lastObservedDeliveryOutcome = state === 'delivered' ? 'delivered' : 'failed' + if (recordOutcome) { + lastObservedDeliveryOutcome = state === 'delivered' ? 'delivered' : 'failed' + } resolve({ state, authoritative: false }) } function onShow(): void { - settle('delivered') + settle('delivered', true) // Why: the probe banner doubles as the user-facing confirmation, so let it linger briefly instead of vanishing instantly. - const closeTimer = setTimeout(releaseProbe, NOTIFICATION_PROBE_BANNER_CLOSE_DELAY_MS) + closeTimer = setTimeout(releaseProbe, NOTIFICATION_PROBE_BANNER_CLOSE_DELAY_MS) if (typeof closeTimer.unref === 'function') { closeTimer.unref() } @@ -160,17 +151,34 @@ function probeNotificationDelivery(): Promise { function onFailed(_event: unknown, _error?: string): void { // Why: a rejected probe is expected (denied permission); don't log — it would spam the console on every poll. - settle('blocked') + settle('blocked', true) releaseProbe() } + releaseRetention = retainNativeNotification( + probe, + () => { + probe.removeListener('show', onShow) + probe.removeListener('failed', onFailed) + if (timeoutTimer) { + clearTimeout(timeoutTimer) + timeoutTimer = null + } + if (closeTimer) { + clearTimeout(closeTimer) + closeTimer = null + } + settle('blocked', false) + }, + () => settle('blocked', false), + { fallbackMs: null } + ) probe.once('show', onShow) probe.once('failed', onFailed) // Why: don't record 'failed' on timeout — a missing callback is ambiguous, only the 'failed' event is definitive. timeoutTimer = setTimeout(() => { if (!settled) { - settled = true - resolve({ state: 'blocked', authoritative: false }) + settle('blocked', false) releaseProbe() } }, NOTIFICATION_PROBE_RESULT_TIMEOUT_MS) @@ -228,8 +236,12 @@ function getSelectedNotificationSoundPath(settings: NotificationSettings): { return { path: normalizedPath } } -function waitForNotificationDisplay(notification: Notification): Promise { - return new Promise((resolve) => { +function waitForNotificationDisplay(notification: Notification): { + cancel: () => void + result: Promise +} { + let cancel = (): void => {} + const result = new Promise((resolve) => { let settled = false let timer: ReturnType | null = null @@ -250,6 +262,7 @@ function waitForNotificationDisplay(notification: Notification): Promise settle(false) function onShow(): void { settle(true) @@ -263,6 +276,7 @@ function waitForNotificationDisplay(notification: Notification): Promise settle(false), NOTIFICATION_DISPLAY_CONFIRMATION_TIMEOUT_MS) }) + return { cancel, result } } function logNativeNotificationFailure(context: string, error?: string): void { @@ -370,12 +384,10 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime ipcMain.removeHandler('notifications:dismiss') ipcMain.handle('notifications:dismiss', (_event, ids: string[]): NotificationDismissResult => { - const uniqueIds = Array.from( - new Set(ids.filter((id): id is string => typeof id === 'string' && id.length > 0)) - ) + const uniqueIds = boundedUniqueNotificationDismissIds(ids) let dismissed = 0 for (const id of uniqueIds) { - const entry = activeNotificationsById.get(id) + const entry = getRetainedNativeNotificationById(id) if (entry) { entry.notification.close() entry.release() @@ -391,8 +403,13 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime 'notifications:dispatch', ( _event, - args: NotificationDispatchRequest + request: unknown ): NotificationDispatchResult | Promise => { + const normalizedArgs = normalizeNotificationDispatchRequest(request) + if (!normalizedArgs) { + return { delivered: false, reason: 'invalid-request' } + } + const args: NotificationDispatchRequest = normalizedArgs // Why: light the tray attention dot before the cooldown/focus/enabled gates so they can't hold it back (clears on window show/restore; see index.ts). if (args.source === 'agent-task-complete' || args.source === 'terminal-bell') { const activeWindow = BrowserWindow.getAllWindows().find((win) => !win.isDestroyed()) ?? null @@ -465,7 +482,7 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime } const notification = new Notification(notificationOptions) if (args.notificationId) { - const previous = activeNotificationsById.get(args.notificationId) + const previous = getRetainedNativeNotificationById(args.notificationId) if (previous) { previous.notification.close() previous.release() @@ -475,9 +492,13 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime // Why: prevent GC from collecting the notification and its click handler while it's still visible. let clickHandler: (() => void) | null = null let failedHandler: ((_event: unknown, error?: string) => void) | null = null - const entryForId: { notification: Notification; release: () => void } | null = - args.notificationId ? { notification, release: () => {} } : null - const release = retainNotificationUntilRelease(notification, () => { + let cancelDisplayConfirmation: (() => void) | null = null + const entryForId: RetainedNativeNotificationIdEntry | null = args.notificationId + ? { notification, release: () => {} } + : null + const release = retainNativeNotification(notification, () => { + cancelDisplayConfirmation?.() + cancelDisplayConfirmation = null if (clickHandler) { notification.removeListener('click', clickHandler) clickHandler = null @@ -486,16 +507,13 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime notification.removeListener('failed', failedHandler) failedHandler = null } - if ( - args.notificationId && - activeNotificationsById.get(args.notificationId) === entryForId - ) { - activeNotificationsById.delete(args.notificationId) + if (args.notificationId && entryForId) { + deleteRetainedNativeNotificationId(args.notificationId, entryForId) } }) if (entryForId && args.notificationId) { entryForId.release = release - activeNotificationsById.set(args.notificationId, entryForId) + setRetainedNativeNotificationId(args.notificationId, entryForId) } failedHandler = (_event, error) => { @@ -546,10 +564,12 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime const displayConfirmation = args.requireDisplayConfirmation ? waitForNotificationDisplay(notification) : null + cancelDisplayConfirmation = displayConfirmation?.cancel ?? null notification.show() if (displayConfirmation) { - return displayConfirmation.then((displayed) => { + return displayConfirmation.result.then((displayed) => { + cancelDisplayConfirmation = null if (!displayed) { release() return { delivered: false, reason: 'not-displayed' } @@ -610,17 +630,18 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime } try { - const fileStat = await stat(normalizedPath) - if (!fileStat.isFile()) { + const { buffer, stats } = await readNodeFileWithinLimit( + normalizedPath, + MAX_NOTIFICATION_SOUND_BYTES + ) + if (!stats.isFile()) { return { ok: false, reason: 'invalid-path' } } - if (fileStat.size > MAX_NOTIFICATION_SOUND_BYTES) { + return { ok: true, data: new Uint8Array(buffer), mimeType, path: normalizedPath } + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { return { ok: false, reason: 'too-large' } } - - const data = await readFile(normalizedPath) - return { ok: true, data: new Uint8Array(data), mimeType, path: normalizedPath } - } catch { return { ok: false, reason: 'read-failed' } } }) @@ -647,12 +668,10 @@ export function triggerStartupNotificationRegistration(store: Store): void { body: 'Allow notifications so Orca can alert you when agents finish or terminals need attention.' }) - // Why: prevent GC from collecting the notification and its click handler while it's still visible. - activeNotifications.add(notification) - let handled = false let closeTimer: ReturnType | null = null let fallbackTimer: ReturnType | null = null + let releaseRetention = (): void => {} function clearStartupTimers(): void { if (closeTimer) { @@ -665,16 +684,22 @@ export function triggerStartupNotificationRegistration(store: Store): void { } } - function cleanup(): void { + function releaseState(): void { if (handled) { return } handled = true clearStartupTimers() - activeNotifications.delete(notification) notification.removeListener('click', onClick) notification.removeListener('show', onShow) notification.removeListener('failed', onFailed) + } + + function cleanup(): void { + if (handled) { + return + } + releaseRetention() notification.close() } @@ -702,6 +727,9 @@ export function triggerStartupNotificationRegistration(store: Store): void { notification.on('click', onClick) notification.on('show', onShow) notification.on('failed', onFailed) + releaseRetention = retainNativeNotification(notification, releaseState, undefined, { + fallbackMs: null + }) // Fallback in case macOS doesn't fire the 'show' event (e.g. user denies). fallbackTimer = setTimeout(cleanup, 10_000) diff --git a/src/main/ipc/pet-bundle.test.ts b/src/main/ipc/pet-bundle.test.ts index 096dd37e853..a9d25b11e77 100644 --- a/src/main/ipc/pet-bundle.test.ts +++ b/src/main/ipc/pet-bundle.test.ts @@ -4,9 +4,9 @@ import { CODEX_PET_ANIMATIONS, CODEX_PET_FRAME, CODEX_PET_SPRITESHEET_PATH, - applyCodexPetDefaults, - readWebpDimensionsFromBuffer + applyCodexPetDefaults } from './pet-bundle' +import { readRasterImageDimensions } from '../../shared/raster-image-dimensions' function u32(value: number): Buffer { const buffer = Buffer.alloc(4) @@ -129,15 +129,40 @@ describe('applyCodexPetDefaults', () => { }) }) -describe('readWebpDimensionsFromBuffer', () => { +describe('readRasterImageDimensions', () => { it('reads VP8X WebP canvas dimensions without decoding pixels', () => { - expect(readWebpDimensionsFromBuffer(webpVp8x(1536, 1872))).toEqual({ + expect(readRasterImageDimensions(webpVp8x(1536, 1872))).toEqual({ width: 1536, height: 1872 }) }) it('returns null for non-WebP data', () => { - expect(readWebpDimensionsFromBuffer(Buffer.from('not an image'))).toBeNull() + expect(readRasterImageDimensions(Buffer.from('not an image'))).toBeNull() + }) + + it('reads PNG and GIF dimensions without decoding pixels', () => { + const png = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(png) + png.writeUInt32BE(13, 8) + png.write('IHDR', 12, 'ascii') + png.writeUInt32BE(8192, 16) + png.writeUInt32BE(512, 20) + const gif = Buffer.alloc(10) + gif.write('GIF89a', 0, 'ascii') + gif.writeUInt16LE(320, 6) + gif.writeUInt16LE(240, 8) + + expect(readRasterImageDimensions(png)).toEqual({ width: 8192, height: 512 }) + expect(readRasterImageDimensions(gif)).toEqual({ width: 320, height: 240 }) + }) + + it('reads JPEG dimensions without decoding pixels', () => { + const jpeg = Buffer.from([ + 0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x10, 0x00, 0x20, 0x00, 0x03, 0x01, 0x11, 0x00, + 0x02, 0x11, 0x00, 0x03, 0x11, 0x00 + ]) + + expect(readRasterImageDimensions(jpeg)).toEqual({ width: 8192, height: 4096 }) }) }) diff --git a/src/main/ipc/pet-bundle.ts b/src/main/ipc/pet-bundle.ts index 1b701a7a105..6b39e393028 100644 --- a/src/main/ipc/pet-bundle.ts +++ b/src/main/ipc/pet-bundle.ts @@ -72,64 +72,3 @@ export function applyCodexPetDefaults( : codexAnimationsAtUniformFps(manifest.fps)) } } - -function readUInt24LE(buffer: Buffer, offset: number): number { - return buffer[offset] | (buffer[offset + 1] << 8) | (buffer[offset + 2] << 16) -} - -export function readWebpDimensionsFromBuffer( - buffer: Buffer -): { width: number; height: number } | null { - if ( - buffer.byteLength < 20 || - buffer.toString('ascii', 0, 4) !== 'RIFF' || - buffer.toString('ascii', 8, 12) !== 'WEBP' - ) { - return null - } - - let offset = 12 - while (offset + 8 <= buffer.byteLength) { - const chunkType = buffer.toString('ascii', offset, offset + 4) - const chunkSize = buffer.readUInt32LE(offset + 4) - const dataOffset = offset + 8 - const dataEnd = dataOffset + chunkSize - if (dataEnd > buffer.byteLength) { - return null - } - - if (chunkType === 'VP8X' && chunkSize >= 10) { - return { - width: readUInt24LE(buffer, dataOffset + 4) + 1, - height: readUInt24LE(buffer, dataOffset + 7) + 1 - } - } - - if (chunkType === 'VP8L' && chunkSize >= 5 && buffer[dataOffset] === 0x2f) { - const b0 = buffer[dataOffset + 1] - const b1 = buffer[dataOffset + 2] - const b2 = buffer[dataOffset + 3] - const b3 = buffer[dataOffset + 4] - return { - width: 1 + (((b1 & 0x3f) << 8) | b0), - height: 1 + (((b3 & 0x0f) << 10) | (b2 << 2) | ((b1 & 0xc0) >> 6)) - } - } - - if ( - chunkType === 'VP8 ' && - chunkSize >= 10 && - buffer[dataOffset + 3] === 0x9d && - buffer[dataOffset + 4] === 0x01 && - buffer[dataOffset + 5] === 0x2a - ) { - const width = buffer.readUInt16LE(dataOffset + 6) & 0x3fff - const height = buffer.readUInt16LE(dataOffset + 8) & 0x3fff - return width > 0 && height > 0 ? { width, height } : null - } - - offset = dataEnd + (chunkSize % 2) - } - - return null -} diff --git a/src/main/ipc/pet.test.ts b/src/main/ipc/pet.test.ts index 5e857bb1244..d78b59f9ad5 100644 --- a/src/main/ipc/pet.test.ts +++ b/src/main/ipc/pet.test.ts @@ -8,14 +8,12 @@ const { browserWindowFromWebContentsMock, browserWindowGetFocusedWindowMock, handleMock, - nativeImageCreateFromBufferMock, showOpenDialogMock } = vi.hoisted(() => ({ appGetPathMock: vi.fn(), browserWindowFromWebContentsMock: vi.fn(), browserWindowGetFocusedWindowMock: vi.fn(), handleMock: vi.fn(), - nativeImageCreateFromBufferMock: vi.fn(), showOpenDialogMock: vi.fn() })) @@ -32,9 +30,6 @@ vi.mock('electron', () => ({ }, ipcMain: { handle: handleMock - }, - nativeImage: { - createFromBuffer: nativeImageCreateFromBufferMock } })) @@ -54,7 +49,6 @@ describe('registerPetHandlers', () => { browserWindowFromWebContentsMock.mockReset() browserWindowGetFocusedWindowMock.mockReset() handleMock.mockReset() - nativeImageCreateFromBufferMock.mockReset() showOpenDialogMock.mockReset() appGetPathMock.mockReturnValue(userDataDir) @@ -63,10 +57,6 @@ describe('registerPetHandlers', () => { handleMock.mockImplementation((channel, handler) => { handlers.set(channel, handler) }) - nativeImageCreateFromBufferMock.mockReturnValue({ - isEmpty: () => true, - getSize: () => ({ width: 0, height: 0 }) - }) }) afterEach(async () => { @@ -82,9 +72,19 @@ describe('registerPetHandlers', () => { return handler } + function pngHeader(width: number, height: number): Buffer { + const png = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(png) + png.writeUInt32BE(13, 8) + png.write('IHDR', 12, 'ascii') + png.writeUInt32BE(width, 16) + png.writeUInt32BE(height, 20) + return png + } + it('imports a pet bundle whose manifest uses Windows separators', async () => { const bundleDir = join(tempDir, 'windows-export.codex-pet') - const sheetBytes = Buffer.from('not decoded without frame metadata') + const sheetBytes = pngHeader(32, 24) await mkdir(join(bundleDir, 'assets'), { recursive: true }) await writeFile( join(bundleDir, 'pet.json'), @@ -147,6 +147,45 @@ describe('registerPetHandlers', () => { return bundleDir } + it('rejects unsafe raster dimensions from a metadata-free bundle before import', async () => { + const bundleDir = join(tempDir, 'dimension-bomb.codex-pet') + await mkdir(bundleDir, { recursive: true }) + await writeFile( + join(bundleDir, 'pet.json'), + JSON.stringify({ + spritesheetPath: 'sheet.png' + }) + ) + await writeFile(join(bundleDir, 'sheet.png'), pngHeader(8_193, 512)) + showOpenDialogMock.mockResolvedValue({ canceled: false, filePaths: [bundleDir] }) + + await expect(getHandler('pet:importPetBundle')({ sender: {} })).rejects.toThrow( + 'exceed the safe limit' + ) + }) + + it('imports an under-limit legacy raster without changing its bytes', async () => { + const source = join(tempDir, 'safe-pet.png') + const sourceBytes = pngHeader(64, 48) + await writeFile(source, sourceBytes) + showOpenDialogMock.mockResolvedValue({ canceled: false, filePaths: [source] }) + + const result = (await getHandler('pet:import')({ sender: {} })) as CustomPet + + expect(result).toMatchObject({ kind: 'image', mimeType: 'image/png' }) + await expect( + readFile(join(userDataDir, 'sidekicks', 'custom', result.fileName)) + ).resolves.toEqual(sourceBytes) + }) + + it('rejects a legacy raster dimension bomb before copying it', async () => { + const source = join(tempDir, 'dimension-bomb.png') + await writeFile(source, pngHeader(32_768, 32_768)) + showOpenDialogMock.mockResolvedValue({ canceled: false, filePaths: [source] }) + + await expect(getHandler('pet:import')({ sender: {} })).rejects.toThrow('exceed the safe limit') + }) + it('imports a bundle whose animations declare per-frame durations', async () => { const bundleDir = await writeSpriteBundle({ idle: { row: 0, frames: 2, frameDurationsMs: [1680, 1920] } @@ -172,4 +211,27 @@ describe('registerPetHandlers', () => { 'declares 1 frame durations but 2 frames' ) }) + + it('reads a stored pet through the bounded file reader without changing its bytes', async () => { + const id = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + const fileName = `${id}.png` + const storedDir = join(userDataDir, 'sidekicks', 'custom') + await mkdir(storedDir, { recursive: true }) + const storedBytes = pngHeader(4, 4) + await writeFile(join(storedDir, fileName), storedBytes) + + const result = (await getHandler('pet:read')({}, id, fileName, 'image')) as ArrayBuffer + + expect(Buffer.from(result)).toEqual(storedBytes) + }) + + it('rejects a replaced legacy pet dimension bomb before renderer delivery', async () => { + const id = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + const fileName = `${id}.png` + const storedDir = join(userDataDir, 'sidekicks', 'custom') + await mkdir(storedDir, { recursive: true }) + await writeFile(join(storedDir, fileName), pngHeader(32_768, 32_768)) + + await expect(getHandler('pet:read')({}, id, fileName, 'image')).resolves.toBeNull() + }) }) diff --git a/src/main/ipc/pet.ts b/src/main/ipc/pet.ts index df48249f001..497e42b89ad 100644 --- a/src/main/ipc/pet.ts +++ b/src/main/ipc/pet.ts @@ -1,18 +1,19 @@ /* eslint-disable max-lines */ -import { app, BrowserWindow, dialog, ipcMain, nativeImage } from 'electron' -import { copyFile, mkdir, open, readFile, rename, rm, stat, lstat } from 'node:fs/promises' +import { app, BrowserWindow, dialog, ipcMain } from 'electron' +import { copyFile, mkdir, open, rename, rm, stat, lstat } from 'node:fs/promises' import { constants as fsConstants, createWriteStream } from 'node:fs' import { pipeline } from 'node:stream/promises' import { randomUUID } from 'node:crypto' import { basename, dirname, extname, isAbsolute, join, normalize, resolve, sep } from 'node:path' import { z } from 'zod' import type { CustomPet } from '../../shared/types' +import { applyCodexPetDefaults, type PetManifestLike, type ResolvedPetManifest } from './pet-bundle' import { - applyCodexPetDefaults, - readWebpDimensionsFromBuffer, - type PetManifestLike, - type ResolvedPetManifest -} from './pet-bundle' + MAX_CUSTOM_PET_FILE_BYTES, + isCustomPetSheetSizeSafe +} from '../../shared/custom-pet-media-limits' +import { readCustomPetFile } from './custom-pet-file-reader' +import { readRasterImageDimensions } from '../../shared/raster-image-dimensions' // Why: pets are image-only — render natively via (no 3D engine); main owns this format allowlist. const IMAGE_FORMATS: Record = { @@ -39,7 +40,6 @@ function getPetsDir(): string { return join(app.getPath('userData'), 'sidekicks', 'custom') } -const MAX_BYTES = 64 * 1024 * 1024 // 64 MB — generous but bounded so a user can't point at a multi-GB file and OOM the renderer when it builds a Blob URL. const MAX_MANIFEST_BYTES = 64 * 1024 // pet.json is tiny by spec; cap to defend against a malicious bundle stuffing megabytes into the manifest. function isSafeId(id: string): boolean { @@ -121,25 +121,24 @@ const PetFileRequestSchema = z.object({ kind: z.enum(['image', 'bundle']).optional() }) -async function readSheetDimensions( - buffer: Buffer -): Promise<{ width: number; height: number } | null> { - // Why: nativeImage can fail on some valid WebP that Chromium renders — read WebP dims from the header before native decode. - const webpDims = readWebpDimensionsFromBuffer(buffer) - if (webpDims) { - return webpDims - } - - // Why: nativeImage can't decode SVG (vector → no pixel grid) — pet bundles must use a raster sheet. - const image = nativeImage.createFromBuffer(buffer) - if (image.isEmpty()) { +function assertCustomPetRasterDimensions( + data: ArrayBuffer, + extension: string, + label: 'Pet image' | 'Spritesheet' +): { width: number; height: number } | null { + if (extension === '.svg') { return null } - const size = image.getSize() - if (size.width <= 0 || size.height <= 0) { - return null + const dimensions = readRasterImageDimensions(new Uint8Array(data)) + if (!dimensions) { + throw new Error(`Could not decode the ${label.toLowerCase()}.`) } - return { width: size.width, height: size.height } + if (!isCustomPetSheetSizeSafe(dimensions.width, dimensions.height)) { + throw new Error( + `${label} dimensions ${dimensions.width}×${dimensions.height} exceed the safe limit.` + ) + } + return dimensions } // Why: TOCTOU symlink-swap defense — O_NOFOLLOW makes open() fail on a symlink; Windows lacks it, so fall back to copyFile. @@ -201,11 +200,16 @@ export function registerPetHandlers(): void { if (!srcStat.isFile()) { throw new Error('Selected path is not a file') } - if (srcStat.size > MAX_BYTES) { + if (srcStat.size > MAX_CUSTOM_PET_FILE_BYTES) { throw new Error( - `File is too large (${(srcStat.size / (1024 * 1024)).toFixed(1)} MB). Max is ${MAX_BYTES / (1024 * 1024)} MB.` + `File is too large (${(srcStat.size / (1024 * 1024)).toFixed(1)} MB). Max is ${MAX_CUSTOM_PET_FILE_BYTES / (1024 * 1024)} MB.` ) } + if (classified.ext !== '.svg') { + const sourceBytes = await readCustomPetFile(src) + // Why: renderer delivery would otherwise let Chromium decode a compressed dimension bomb first. + assertCustomPetRasterDimensions(sourceBytes, classified.ext, 'Pet image') + } const dir = getPetsDir() await mkdir(dir, { recursive: true }) @@ -270,11 +274,9 @@ export function registerPetHandlers(): void { let manifest: ResolvedPetManifest try { - const raw = await readFile(manifestPath, 'utf8') - // Why: defend against TOCTOU — the file may have grown between the stat check and this read. - if (Buffer.byteLength(raw, 'utf8') > MAX_MANIFEST_BYTES) { - throw new Error('pet.json exceeded the manifest size limit.') - } + const raw = Buffer.from(await readCustomPetFile(manifestPath, MAX_MANIFEST_BYTES)).toString( + 'utf8' + ) manifest = applyCodexPetDefaults(PetManifestSchema.parse(JSON.parse(raw))) } catch (error) { throw new Error(`Invalid pet.json: ${error instanceof Error ? error.message : 'parse error'}`) @@ -318,24 +320,23 @@ export function registerPetHandlers(): void { if (!sheetStat.isFile()) { throw new Error('Spritesheet path is not a file.') } - if (sheetStat.size > MAX_BYTES) { + if (sheetStat.size > MAX_CUSTOM_PET_FILE_BYTES) { throw new Error( `Spritesheet is too large (${(sheetStat.size / (1024 * 1024)).toFixed(1)} MB).` ) } + const sheetBytes = await readCustomPetFile(sheetSrc) + // Why: metadata-free bundles also reach the renderer image decoder, so every raster needs the same pre-decode gate. + const sheetDimensions = assertCustomPetRasterDimensions( + sheetBytes, + sheetClass.ext, + 'Spritesheet' + ) + let sprite: NonNullable | undefined if (manifest.frame) { - // Why: only decode when a frame layout needs validating — nativeImage can fail on some WebP variants in headless contexts. - const sheetBuf = await readFile(sheetSrc) - // Why: defend against TOCTOU — file may have grown between stat and read. - if (sheetBuf.byteLength > MAX_BYTES) { - throw new Error('Spritesheet exceeded the size limit.') - } - const dims = await readSheetDimensions(sheetBuf) - if (!dims) { - throw new Error('Could not decode the spritesheet image.') - } + const dims = sheetDimensions! const { width: fw, height: fh } = manifest.frame if (dims.width % fw !== 0 || dims.height % fh !== 0) { throw new Error( @@ -432,8 +433,16 @@ export function registerPetHandlers(): void { return null } try { - const buf = await readFile(filePath) - return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) + // Why: stored pet files can be replaced after import; the reader keeps + // one handle and never allocates beyond the same 64 MiB import limit. + const data = await readCustomPetFile(filePath) + const classified = classifyFile(filePath) + if (!classified) { + return null + } + // Why: this gates legacy image pets and metadata-free bundles before renderer decode. + assertCustomPetRasterDimensions(data, classified.ext, 'Pet image') + return data } catch (error) { console.warn('[pet-overlay] pet:read failed', error) return null diff --git a/src/main/ipc/pty-external-renderer-delivery.test.ts b/src/main/ipc/pty-external-renderer-delivery.test.ts new file mode 100644 index 00000000000..8ed151c07fa --- /dev/null +++ b/src/main/ipc/pty-external-renderer-delivery.test.ts @@ -0,0 +1,244 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { handleMock, onMock, removeHandlerMock, removeAllListenersMock } = vi.hoisted(() => ({ + handleMock: vi.fn(), + onMock: vi.fn(), + removeHandlerMock: vi.fn(), + removeAllListenersMock: vi.fn() +})) + +vi.mock('electron', () => ({ + app: { + isPackaged: true, + getPath: () => '/tmp/orca-pty-external-delivery-test', + getVersion: () => '0.0.0-test' + }, + powerMonitor: { on: vi.fn() }, + nativeTheme: { shouldUseDarkColors: true }, + ipcMain: { + handle: handleMock, + on: onMock, + removeHandler: removeHandlerMock, + removeAllListeners: removeAllListenersMock + } +})) + +vi.mock('node-pty', () => ({ spawn: vi.fn() })) +vi.mock('../pwsh', () => ({ isPwshAvailable: vi.fn(() => false) })) + +import { LocalPtyProvider } from '../providers/local-pty-provider' +import { registerPtyHandlers, setLocalPtyProvider } from './pty' +import { + routeExternalPtyData, + routeExternalPtyExit, + routeExternalPtyReplay +} from './pty-renderer-delivery-router' + +describe('external PTY renderer delivery', () => { + const mainWindow = { + isDestroyed: () => false, + isFocused: () => true, + isVisible: () => true, + isMinimized: () => false, + webContents: { + on: vi.fn(), + send: vi.fn(), + removeListener: vi.fn(), + isLoadingMainFrame: vi.fn(() => true) + } + } + const mainWindowEvent = { sender: mainWindow.webContents } + + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + onMock.mockImplementation((channel: string, listener: (...args: unknown[]) => void) => { + if (channel === 'pty:rendererDispatcherReady') { + listener(mainWindowEvent) + vi.advanceTimersByTime(0) + } + }) + }) + + afterEach(() => { + setLocalPtyProvider(new LocalPtyProvider()) + vi.useRealTimers() + }) + + function findIpcListener(channel: string): (...args: never[]) => void { + const call = onMock.mock.calls.findLast(([registered]) => registered === channel) + if (!call) { + throw new Error(`Missing ${channel} listener`) + } + return call[1] as (...args: never[]) => void + } + + function makeRuntime(sequence = 17) { + return { + setPtyController: vi.fn(), + onPtyData: vi.fn(() => sequence), + getPtyOutputSequence: vi.fn(() => sequence), + hasRawTerminalViewSubscriber: vi.fn(() => false) + } + } + + it('returns captured SSH credit only after the renderer acknowledges parsing', () => { + const runtime = makeRuntime() + const upstreamCredit = { charCount: 9, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + mainWindow.webContents.send.mockClear() + + routeExternalPtyData({ + id: 'ssh:target@@pty-1', + data: '', + sequenceChars: 9, + transformed: true, + upstreamCredit + }) + vi.advanceTimersByTime(2) + + expect(runtime.onPtyData).toHaveBeenCalledWith( + 'ssh:target@@pty-1', + '', + expect.any(Number), + 9, + true + ) + expect(mainWindow.webContents.send).toHaveBeenCalledWith('pty:data', { + id: 'ssh:target@@pty-1', + data: '', + seq: 17, + rawLength: 9, + transformed: true + }) + expect(upstreamCredit.acknowledge).not.toHaveBeenCalled() + + findIpcListener('pty:ackData')( + mainWindowEvent as never, + { + id: 'ssh:target@@pty-1', + processedChars: 9 + } as never + ) + + expect(upstreamCredit.acknowledge).toHaveBeenCalledOnce() + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(9) + }) + + it('returns credit immediately when hidden output is intentionally dropped', () => { + const runtime = makeRuntime(42) + const upstreamCredit = { charCount: 13, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + findIpcListener('pty:setHiddenRendererPty')( + null as never, + { + id: 'ssh:target@@pty-hidden', + hidden: true + } as never + ) + mainWindow.webContents.send.mockClear() + + routeExternalPtyData({ + id: 'ssh:target@@pty-hidden', + data: 'hidden output', + upstreamCredit + }) + + expect(runtime.onPtyData).toHaveBeenCalledOnce() + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(13) + expect(mainWindow.webContents.send).toHaveBeenCalledOnce() + expect(mainWindow.webContents.send).toHaveBeenCalledWith('pty:modelRestoreNeeded', { + id: 'ssh:target@@pty-hidden', + reason: 'hidden-drop', + markerSeq: 42 + }) + }) + + it('settles queued live credit when a reconnect replay supersedes it', () => { + const runtime = makeRuntime() + const upstreamCredit = { charCount: 11, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + mainWindow.webContents.send.mockClear() + + routeExternalPtyData({ + id: 'ssh:target@@pty-replay', + data: 'queued live', + upstreamCredit + }) + routeExternalPtyReplay({ id: 'ssh:target@@pty-replay', data: 'full snapshot' }) + vi.advanceTimersByTime(10) + + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(11) + expect(mainWindow.webContents.send.mock.calls).toEqual([ + ['pty:replay', { id: 'ssh:target@@pty-replay', data: 'full snapshot' }] + ]) + }) + + it('writes off captured in-flight credit when the renderer lifecycle ends', () => { + const runtime = makeRuntime() + const upstreamCredit = { charCount: 7, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + routeExternalPtyData({ + id: 'ssh:target@@pty-reload', + data: 'inflight', + upstreamCredit + }) + vi.advanceTimersByTime(2) + expect(upstreamCredit.acknowledge).not.toHaveBeenCalled() + + const lifecycleCall = mainWindow.webContents.on.mock.calls.findLast( + ([event]) => event === 'did-start-loading' + ) + const handleLifecycleReset = lifecycleCall?.[1] as (() => void) | undefined + expect(handleLifecycleReset).toBeTypeOf('function') + handleLifecycleReset?.() + + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(7) + }) + + it('settles queued final-tail credit when exit flushes before the batch timer', () => { + const runtime = makeRuntime() + const upstreamCredit = { charCount: 10, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + mainWindow.webContents.send.mockClear() + + routeExternalPtyData({ + id: 'ssh:target@@pty-final-tail', + data: 'final tail', + upstreamCredit + }) + routeExternalPtyExit({ id: 'ssh:target@@pty-final-tail', code: 0 }) + + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(10) + expect(mainWindow.webContents.send.mock.calls).toEqual([ + [ + 'pty:data', + { + id: 'ssh:target@@pty-final-tail', + data: 'final tail', + seq: 17, + rawLength: 10 + } + ], + ['pty:exit', { id: 'ssh:target@@pty-final-tail', code: 0 }] + ]) + }) + + it('settles already-sent credit when exit arrives before renderer ACK', () => { + const runtime = makeRuntime() + const upstreamCredit = { charCount: 8, acknowledge: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + + routeExternalPtyData({ + id: 'ssh:target@@pty-inflight-exit', + data: 'inflight', + upstreamCredit + }) + vi.advanceTimersByTime(2) + expect(upstreamCredit.acknowledge).not.toHaveBeenCalled() + + routeExternalPtyExit({ id: 'ssh:target@@pty-inflight-exit', code: 0 }) + + expect(upstreamCredit.acknowledge).toHaveBeenCalledWith(8) + }) +}) diff --git a/src/main/ipc/pty-renderer-delivery-credit.test.ts b/src/main/ipc/pty-renderer-delivery-credit.test.ts new file mode 100644 index 00000000000..af1d2bb8ed7 --- /dev/null +++ b/src/main/ipc/pty-renderer-delivery-credit.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it, vi } from 'vitest' +import { + appendPtyDeliveryCredit, + MAX_PENDING_PTY_DELIVERY_CREDIT_SPANS, + MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS, + MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS_PER_PTY, + MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES, + MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES, + PtyRendererDeliveryCreditLedger, + settlePtyDeliveryCredit, + takePtyDeliveryCredit +} from './pty-renderer-delivery-credit' + +function makeCredit(charCount: number) { + return { charCount, acknowledge: vi.fn() } +} + +describe('PTY renderer delivery credit', () => { + it('splits queued provider credit without settling bytes left for a later renderer chunk', () => { + const first = makeCredit(5) + const second = makeCredit(7) + const spans = appendPtyDeliveryCredit(appendPtyDeliveryCredit(undefined, first), second) + + const taken = takePtyDeliveryCredit(spans, 8) + settlePtyDeliveryCredit(taken) + + expect(first.acknowledge).toHaveBeenCalledWith(5) + expect(second.acknowledge).toHaveBeenCalledWith(3) + expect(spans).toEqual([{ credit: second, chars: 4 }]) + }) + + it('settles explicit provider owners and legacy fallback spans in renderer order', () => { + const first = makeCredit(5) + const second = makeCredit(4) + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + + ledger.recordSent('pty-1', 5, [{ credit: first, chars: 5 }], acknowledgeFallback) + ledger.recordSent('pty-1', 3, undefined, acknowledgeFallback) + ledger.recordSent('pty-1', 4, [{ credit: second, chars: 4 }], acknowledgeFallback) + ledger.acknowledge('pty-1', 7, acknowledgeFallback) + ledger.acknowledge('pty-1', 2, acknowledgeFallback) + ledger.acknowledge('pty-1', 3, acknowledgeFallback) + + expect(first.acknowledge).toHaveBeenCalledWith(5) + expect(second.acknowledge.mock.calls).toEqual([[1], [3]]) + expect(acknowledgeFallback.mock.calls).toEqual([[2], [1]]) + }) + + it('writes off every captured owner while keeping PTY fallback routing separate', () => { + const first = makeCredit(4) + const second = makeCredit(6) + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + + ledger.recordSent('pty-1', 4, [{ credit: first, chars: 4 }], acknowledgeFallback) + ledger.recordSent('pty-2', 2, undefined, acknowledgeFallback) + ledger.recordSent('pty-3', 6, [{ credit: second, chars: 6 }], acknowledgeFallback) + ledger.writeOffAll(acknowledgeFallback) + + expect(first.acknowledge).toHaveBeenCalledWith(4) + expect(second.acknowledge).toHaveBeenCalledWith(6) + expect(acknowledgeFallback).toHaveBeenCalledOnce() + expect(acknowledgeFallback).toHaveBeenCalledWith('pty-2', 2) + }) + + it('abandons an exited PTY without crediting a replacement owner', () => { + const exited = makeCredit(4) + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + ledger.recordSent('reused-id', 4, [{ credit: exited, chars: 4 }], acknowledgeFallback) + + ledger.abandon('reused-id') + ledger.acknowledge('reused-id', 4, acknowledgeFallback) + + expect(exited.acknowledge).not.toHaveBeenCalled() + expect(acknowledgeFallback).not.toHaveBeenCalled() + }) + + it('collapses one-byte pending credit floods into already-settled metadata', () => { + const total = MAX_PENDING_PTY_DELIVERY_CREDIT_SPANS + 100 + const credits = Array.from({ length: total }, () => makeCredit(1)) + let spans + for (const credit of credits) { + spans = appendPtyDeliveryCredit(spans, credit) + } + + expect(spans).toEqual([{ chars: total, settled: true }]) + expect(credits.every((credit) => credit.acknowledge.mock.calls.length === 1)).toBe(true) + + const ledger = new PtyRendererDeliveryCreditLedger() + const acknowledgeFallback = vi.fn() + const sent = takePtyDeliveryCredit(spans, total) + ledger.recordSent('pty-tiny', total, sent, acknowledgeFallback) + ledger.acknowledge('pty-tiny', total, acknowledgeFallback) + + expect(acknowledgeFallback).not.toHaveBeenCalled() + expect(credits.every((credit) => credit.acknowledge.mock.calls.length === 1)).toBe(true) + }) + + it('bounds per-PTY in-flight credit closures and settles each exactly once', () => { + const total = MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS_PER_PTY + 100 + const credits = Array.from({ length: total }, () => makeCredit(1)) + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + + for (const credit of credits) { + ledger.recordSent('pty-tiny', 1, [{ credit, chars: 1 }], acknowledgeFallback) + } + + const state = (Reflect.get(ledger, 'inFlightByPty') as Map).get( + 'pty-tiny' + ) + expect(state?.spans.length).toBeLessThanOrEqual(MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS_PER_PTY) + ledger.acknowledge('pty-tiny', total, acknowledgeFallback) + + expect(acknowledgeFallback).not.toHaveBeenCalled() + expect(credits.every((credit) => credit.acknowledge.mock.calls.length === 1)).toBe(true) + }) + + it('bounds aggregate in-flight credit spans across PTYs', () => { + let acknowledged = 0 + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + const total = MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS + 100 + + for (let index = 0; index < total; index += 1) { + const credit = { + charCount: 1, + acknowledge: (chars: number) => { + acknowledged += chars + } + } + ledger.recordSent(`pty-${index % 8}`, 1, [{ credit, chars: 1 }], acknowledgeFallback) + } + + expect(Reflect.get(ledger, 'retainedSpanCount')).toBeLessThanOrEqual( + MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS + ) + ledger.writeOffAll(acknowledgeFallback) + expect(acknowledged).toBe(total) + expect(acknowledgeFallback).not.toHaveBeenCalled() + }) + + it('bounds unique in-flight IDs and settles evicted credit exactly once', () => { + const credits = Array.from({ length: MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES + 50 }, () => + makeCredit(1) + ) + const acknowledgeFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + + for (const [index, credit] of credits.entries()) { + ledger.recordSent(`pty-${index}`, 1, [{ credit, chars: 1 }], acknowledgeFallback) + } + + const states = Reflect.get(ledger, 'inFlightByPty') as Map + expect(states.size).toBe(MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES) + expect(states.has('pty-0')).toBe(false) + ledger.writeOffAll(acknowledgeFallback) + expect(credits.every((credit) => credit.acknowledge.mock.calls.length === 1)).toBe(true) + expect(acknowledgeFallback).not.toHaveBeenCalled() + }) + + it('bounds aggregate in-flight ID bytes and settles fallback before eviction', () => { + const id = 'x'.repeat(MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES) + const settleFallback = vi.fn() + const ledger = new PtyRendererDeliveryCreditLedger() + + ledger.recordSent(id, 3, undefined, settleFallback) + ledger.recordSent('extra', 2, undefined, settleFallback) + + const states = Reflect.get(ledger, 'inFlightByPty') as Map + expect(states.size).toBe(1) + expect(states.has(id)).toBe(false) + expect(settleFallback).toHaveBeenCalledOnce() + expect(settleFallback).toHaveBeenCalledWith(3) + ledger.acknowledge(id, 3, settleFallback) + expect(settleFallback).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ipc/pty-renderer-delivery-credit.ts b/src/main/ipc/pty-renderer-delivery-credit.ts new file mode 100644 index 00000000000..a6c0325cf64 --- /dev/null +++ b/src/main/ipc/pty-renderer-delivery-credit.ts @@ -0,0 +1,297 @@ +import type { PtyDataUpstreamCredit } from '../providers/pty-provider-events' + +export const MAX_PENDING_PTY_DELIVERY_CREDIT_SPANS = 4096 +export const MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS_PER_PTY = 4096 +export const MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS = 16_384 +export const MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES = 4096 +export const MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES = 8 * 1024 * 1024 + +export type PtyDeliveryCreditSpan = + | { + credit: PtyDataUpstreamCredit + chars: number + settled?: never + } + | { + chars: number + settled: true + credit?: never + } + +type InFlightCreditSpan = { + chars: number + credit?: PtyDataUpstreamCredit + settled?: true +} + +type InFlightCreditState = { + idBytes: number + spans: InFlightCreditSpan[] + settledPrefixChars: number + settleFallback: (chars: number) => void +} + +export function appendPtyDeliveryCredit( + spans: PtyDeliveryCreditSpan[] | undefined, + credit: PtyDataUpstreamCredit | undefined +): PtyDeliveryCreditSpan[] | undefined { + if (!credit || !Number.isFinite(credit.charCount) || credit.charCount <= 0) { + return spans + } + const chars = Math.floor(credit.charCount) + const next = spans ?? [] + const tail = next.at(-1) + if (tail?.settled === true) { + credit.acknowledge(chars) + tail.chars += chars + return next + } + if (next.length >= MAX_PENDING_PTY_DELIVERY_CREDIT_SPANS) { + let settledChars = chars + for (const span of next) { + settledChars += span.chars + span.credit?.acknowledge(span.chars) + } + credit.acknowledge(chars) + next.splice(0, next.length, { chars: settledChars, settled: true }) + return next + } + next.push({ credit, chars }) + return next +} + +export function takePtyDeliveryCredit( + spans: PtyDeliveryCreditSpan[] | undefined, + requestedChars: number +): PtyDeliveryCreditSpan[] | undefined { + if (!spans || spans.length === 0 || requestedChars <= 0) { + return undefined + } + const taken: PtyDeliveryCreditSpan[] = [] + let remaining = requestedChars + while (remaining > 0 && spans.length > 0) { + const span = spans[0] + const chars = Math.min(remaining, span.chars) + taken.push(span.settled === true ? { chars, settled: true } : { credit: span.credit, chars }) + span.chars -= chars + remaining -= chars + if (span.chars === 0) { + spans.shift() + } + } + return taken +} + +export function settlePtyDeliveryCredit(spans: PtyDeliveryCreditSpan[] | undefined): void { + if (!spans) { + return + } + for (const span of spans.splice(0)) { + span.credit?.acknowledge(span.chars) + } +} + +export class PtyRendererDeliveryCreditLedger { + private readonly inFlightByPty = new Map() + private retainedSpanCount = 0 + private retainedStateIdBytes = 0 + + recordSent( + id: string, + charCount: number, + explicitCredit: PtyDeliveryCreditSpan[] | undefined, + settleFallback: (chars: number) => void + ): void { + if (!Number.isFinite(charCount) || charCount <= 0) { + return + } + const admittedCharCount = Math.floor(charCount) + const state = this.inFlightByPty.get(id) ?? { + idBytes: Buffer.byteLength(id, 'utf8'), + spans: [], + settledPrefixChars: 0, + settleFallback + } + const isNewState = !this.inFlightByPty.has(id) + state.settleFallback = settleFallback + let recorded = 0 + for (const span of explicitCredit ?? []) { + if (recorded >= admittedCharCount || !Number.isFinite(span.chars) || span.chars <= 0) { + break + } + const chars = Math.min(Math.floor(span.chars), admittedCharCount - recorded) + this.appendSpan( + state, + span.settled === true ? { chars, settled: true } : { chars, credit: span.credit } + ) + recorded += chars + } + if (recorded < admittedCharCount) { + this.appendSpan(state, { chars: admittedCharCount - recorded }) + } + this.inFlightByPty.delete(id) + this.inFlightByPty.set(id, state) + if (isNewState) { + this.retainedStateIdBytes += state.idBytes + } + this.capStates() + } + + acknowledge(id: string, charCount: number, acknowledgeFallback: (chars: number) => void): void { + const state = this.inFlightByPty.get(id) + if (!state || !Number.isFinite(charCount) || charCount <= 0) { + return + } + let remaining = Math.floor(charCount) + const settledPrefix = Math.min(remaining, state.settledPrefixChars) + state.settledPrefixChars -= settledPrefix + remaining -= settledPrefix + let fallbackChars = 0 + while (remaining > 0 && state.spans.length > 0) { + const span = state.spans[0] + const chars = Math.min(remaining, span.chars) + if (span.credit) { + span.credit.acknowledge(chars) + } else if (span.settled !== true) { + fallbackChars += chars + } + span.chars -= chars + remaining -= chars + if (span.chars === 0) { + state.spans.shift() + this.retainedSpanCount-- + } + } + if (state.spans.length === 0 && state.settledPrefixChars === 0) { + this.deleteState(id, state) + } + if (fallbackChars > 0) { + acknowledgeFallback(fallbackChars) + } + } + + writeOff(id: string, acknowledgeFallback: (chars: number) => void): void { + const state = this.inFlightByPty.get(id) + if (!state) { + return + } + this.acknowledge( + id, + state.settledPrefixChars + state.spans.reduce((total, span) => total + span.chars, 0), + acknowledgeFallback + ) + } + + writeOffAll(acknowledgeFallback: (id: string, chars: number) => void): void { + for (const id of Array.from(this.inFlightByPty.keys())) { + this.writeOff(id, (chars) => acknowledgeFallback(id, chars)) + } + } + + abandon(id: string): void { + const state = this.inFlightByPty.get(id) + if (!state) { + return + } + this.retainedSpanCount -= state.spans.length + this.deleteState(id, state) + } + + private appendSpan(state: InFlightCreditState, span: InFlightCreditSpan): void { + if (span.chars <= 0) { + return + } + let tail = state.spans.at(-1) + if (tail && this.canMergeSpans(tail, span)) { + tail.chars += span.chars + return + } + this.ensureSpanCapacity(state) + tail = state.spans.at(-1) + if (tail && this.canMergeSpans(tail, span)) { + tail.chars += span.chars + return + } + state.spans.push(span) + this.retainedSpanCount++ + } + + private capStates(): void { + while ( + this.inFlightByPty.size > MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES || + this.retainedStateIdBytes > MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES + ) { + const oldest = this.inFlightByPty.entries().next().value as + | [string, InFlightCreditState] + | undefined + if (!oldest) { + return + } + this.collapseState(oldest[1]) + this.deleteState(oldest[0], oldest[1]) + } + } + + private ensureSpanCapacity(state: InFlightCreditState): void { + if ( + state.spans.length >= MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS_PER_PTY || + this.retainedSpanCount >= MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS + ) { + this.collapseState(state) + } + while (this.retainedSpanCount >= MAX_RENDERER_PTY_DELIVERY_CREDIT_SPANS) { + const candidate = Array.from(this.inFlightByPty.values()).find( + (entry) => entry.spans.length > 0 + ) + if (!candidate) { + break + } + this.collapseState(candidate) + } + } + + private collapseState(state: InFlightCreditState): void { + if (state.spans.length === 0) { + return + } + let settledChars = 0 + let fallbackChars = 0 + for (const span of state.spans) { + settledChars += span.chars + try { + if (span.credit) { + span.credit.acknowledge(span.chars) + } else if (span.settled !== true) { + fallbackChars += span.chars + } + } catch { + // Why: overload shedding must still release local metadata if an upstream is gone. + } + } + if (fallbackChars > 0) { + try { + state.settleFallback(fallbackChars) + } catch { + // Why: overload shedding must still release local metadata if an upstream is gone. + } + } + state.settledPrefixChars += settledChars + this.retainedSpanCount -= state.spans.length + state.spans.length = 0 + } + + private deleteState(id: string, state: InFlightCreditState): void { + if (this.inFlightByPty.get(id) !== state) { + return + } + this.inFlightByPty.delete(id) + this.retainedStateIdBytes -= state.idBytes + } + + private canMergeSpans(left: InFlightCreditSpan, right: InFlightCreditSpan): boolean { + if (left.credit || right.credit) { + return left.credit !== undefined && left.credit === right.credit + } + return left.settled === right.settled + } +} diff --git a/src/main/ipc/pty-renderer-delivery-retention.test.ts b/src/main/ipc/pty-renderer-delivery-retention.test.ts new file mode 100644 index 00000000000..3af40ac3401 --- /dev/null +++ b/src/main/ipc/pty-renderer-delivery-retention.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from 'vitest' +import { + MAX_PENDING_PTY_DATA_CHARS, + MAX_PENDING_PTY_DATA_CREDIT_SPANS, + MAX_PENDING_PTY_DATA_ID_BYTES, + MAX_PENDING_PTY_DATA_STATES, + MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_ID_BYTES, + MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES, + PendingPtyDataMap, + PtyRendererDeliveryAccountingMap, + PtyRendererDeliveryIdSet, + settleRejectedPtyRendererDelivery +} from './pty-renderer-delivery-retention' + +type Pending = { + data: string + creditSpans: number +} + +function pendingMap(onRejected = vi.fn()) { + return { + map: new PendingPtyDataMap( + (pending) => ({ chars: pending.data.length, creditSpans: pending.creditSpans }), + onRejected + ), + onRejected + } +} + +describe('PTY renderer delivery retention', () => { + it('preserves ordinary pending updates and exact map bookkeeping', () => { + const { map, onRejected } = pendingMap() + + expect(map.admit('pty-1', { data: 'one', creditSpans: 1 })).toBe(true) + expect(map.admit('pty-1', { data: 'updated', creditSpans: 2 })).toBe(true) + expect(map.delete('pty-1')).toBe(true) + + expect(map.size).toBe(0) + expect(onRejected).not.toHaveBeenCalled() + }) + + it('rejects unique pending-ID floods after the bounded state count', () => { + const { map, onRejected } = pendingMap() + for (let index = 0; index < MAX_PENDING_PTY_DATA_STATES + 50; index += 1) { + map.set(`pty-${index}`, { data: 'x', creditSpans: 1 }) + } + + expect(map.size).toBe(MAX_PENDING_PTY_DATA_STATES) + expect(map.has('pty-0')).toBe(true) + expect(map.has(`pty-${MAX_PENDING_PTY_DATA_STATES}`)).toBe(false) + expect(onRejected).toHaveBeenCalledTimes(50) + }) + + it('bounds pending ID bytes, characters, and credit records independently', () => { + const idPressure = pendingMap() + const id = 'x'.repeat(MAX_PENDING_PTY_DATA_ID_BYTES) + idPressure.map.set(id, { data: 'x', creditSpans: 0 }) + idPressure.map.set('extra', { data: 'x', creditSpans: 0 }) + expect(idPressure.map.size).toBe(1) + expect(idPressure.onRejected).toHaveBeenCalledWith('extra', { + data: 'x', + creditSpans: 0 + }) + + const charPressure = pendingMap() + charPressure.map.set('full', { + data: 'x'.repeat(MAX_PENDING_PTY_DATA_CHARS), + creditSpans: 0 + }) + charPressure.map.set('extra', { data: 'x', creditSpans: 0 }) + expect(charPressure.map.size).toBe(1) + expect(charPressure.onRejected).toHaveBeenCalledOnce() + + const creditPressure = pendingMap() + creditPressure.map.set('full', { data: 'x', creditSpans: MAX_PENDING_PTY_DATA_CREDIT_SPANS }) + creditPressure.map.set('extra', { data: 'x', creditSpans: 1 }) + expect(creditPressure.map.size).toBe(1) + expect(creditPressure.onRejected).toHaveBeenCalledOnce() + }) + + it('bounds accounting and warning IDs without evicting admitted live state', () => { + const accounting = new PtyRendererDeliveryAccountingMap() + const warned = new PtyRendererDeliveryIdSet() + for (let index = 0; index < MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES + 50; index += 1) { + accounting.set(`pty-${index}`, index) + warned.add(`pty-${index}`) + } + + expect(accounting.size).toBe(MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES) + expect(warned.size).toBe(MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES) + expect(accounting.has('pty-0')).toBe(true) + expect(accounting.has(`pty-${MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES}`)).toBe(false) + expect(warned.has(`pty-${MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES}`)).toBe(false) + + const byteBounded = new PtyRendererDeliveryAccountingMap() + byteBounded.set('x'.repeat(MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_ID_BYTES), 1) + expect(byteBounded.admit('extra', 2)).toBe(false) + expect(byteBounded.size).toBe(1) + }) + + it('settles rejected explicit, already-settled, and fallback credit exactly once', () => { + const explicit = vi.fn() + const fallback = vi.fn() + + settleRejectedPtyRendererDelivery( + 7, + [ + { chars: 2, settled: true }, + { chars: 3, credit: { charCount: 3, acknowledge: explicit } } + ], + fallback + ) + + expect(explicit).toHaveBeenCalledOnce() + expect(explicit).toHaveBeenCalledWith(3) + expect(fallback).toHaveBeenCalledOnce() + expect(fallback).toHaveBeenCalledWith(2) + }) +}) diff --git a/src/main/ipc/pty-renderer-delivery-retention.ts b/src/main/ipc/pty-renderer-delivery-retention.ts new file mode 100644 index 00000000000..71efeb1f0fa --- /dev/null +++ b/src/main/ipc/pty-renderer-delivery-retention.ts @@ -0,0 +1,188 @@ +import { + MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES, + MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES, + type PtyDeliveryCreditSpan +} from './pty-renderer-delivery-credit' + +export const MAX_PENDING_PTY_DATA_STATES = MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES +export const MAX_PENDING_PTY_DATA_ID_BYTES = MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES +export const MAX_PENDING_PTY_DATA_CHARS = 32 * 1024 * 1024 +export const MAX_PENDING_PTY_DATA_CREDIT_SPANS = 16_384 +export const MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES = MAX_RENDERER_PTY_DELIVERY_CREDIT_STATES +export const MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_ID_BYTES = + MAX_RENDERER_PTY_DELIVERY_CREDIT_STATE_ID_BYTES + +type PendingPtyDataMeasurement = { + chars: number + creditSpans: number +} + +export class PendingPtyDataMap extends Map { + private readonly measurements = new Map() + private retainedChars = 0 + private retainedCreditSpans = 0 + private retainedIdBytes = 0 + + constructor( + private readonly measure: (value: T) => PendingPtyDataMeasurement, + private readonly onRejected: (id: string, value: T) => void + ) { + super() + } + + admit(id: string, value: T): boolean { + const hadPrevious = super.has(id) + const previousMeasurement = this.measurements.get(id) ?? { chars: 0, creditSpans: 0 } + const measurement = this.measure(value) + const idBytes = Buffer.byteLength(id, 'utf8') + const nextSize = this.size + (hadPrevious ? 0 : 1) + const nextIdBytes = this.retainedIdBytes + (hadPrevious ? 0 : idBytes) + const nextChars = this.retainedChars - previousMeasurement.chars + measurement.chars + const nextCreditSpans = + this.retainedCreditSpans - previousMeasurement.creditSpans + measurement.creditSpans + + if ( + nextSize > MAX_PENDING_PTY_DATA_STATES || + nextIdBytes > MAX_PENDING_PTY_DATA_ID_BYTES || + nextChars > MAX_PENDING_PTY_DATA_CHARS || + nextCreditSpans > MAX_PENDING_PTY_DATA_CREDIT_SPANS + ) { + if (hadPrevious) { + this.delete(id) + } + this.onRejected(id, value) + return false + } + + super.set(id, value) + this.measurements.set(id, measurement) + this.retainedChars = nextChars + this.retainedCreditSpans = nextCreditSpans + this.retainedIdBytes = nextIdBytes + return true + } + + override set(id: string, value: T): this { + this.admit(id, value) + return this + } + + override delete(id: string): boolean { + const measurement = this.measurements.get(id) ?? { chars: 0, creditSpans: 0 } + if (!super.delete(id)) { + return false + } + this.measurements.delete(id) + this.retainedChars -= measurement.chars + this.retainedCreditSpans -= measurement.creditSpans + this.retainedIdBytes -= Buffer.byteLength(id, 'utf8') + return true + } + + override clear(): void { + super.clear() + this.measurements.clear() + this.retainedChars = 0 + this.retainedCreditSpans = 0 + this.retainedIdBytes = 0 + } +} + +export class PtyRendererDeliveryAccountingMap extends Map { + private retainedIdBytes = 0 + + admit(id: string, value: T): boolean { + if (super.has(id)) { + super.set(id, value) + return true + } + const idBytes = Buffer.byteLength(id, 'utf8') + if ( + this.size >= MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES || + this.retainedIdBytes + idBytes > MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_ID_BYTES + ) { + return false + } + super.set(id, value) + this.retainedIdBytes += idBytes + return true + } + + override set(id: string, value: T): this { + this.admit(id, value) + return this + } + + override delete(id: string): boolean { + if (!super.delete(id)) { + return false + } + this.retainedIdBytes -= Buffer.byteLength(id, 'utf8') + return true + } + + override clear(): void { + super.clear() + this.retainedIdBytes = 0 + } +} + +export class PtyRendererDeliveryIdSet extends Set { + private retainedIdBytes = 0 + + remember(id: string): boolean { + if (super.has(id)) { + return false + } + const idBytes = Buffer.byteLength(id, 'utf8') + if ( + this.size >= MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_STATES || + this.retainedIdBytes + idBytes > MAX_RENDERER_PTY_DELIVERY_ACCOUNTING_ID_BYTES + ) { + return false + } + super.add(id) + this.retainedIdBytes += idBytes + return true + } + + override add(id: string): this { + this.remember(id) + return this + } + + override delete(id: string): boolean { + if (!super.delete(id)) { + return false + } + this.retainedIdBytes -= Buffer.byteLength(id, 'utf8') + return true + } + + override clear(): void { + super.clear() + this.retainedIdBytes = 0 + } +} + +export function settleRejectedPtyRendererDelivery( + charCount: number, + creditSpans: PtyDeliveryCreditSpan[] | undefined, + settleFallback: (chars: number) => void +): void { + if (!Number.isFinite(charCount) || charCount <= 0) { + return + } + let remaining = Math.floor(charCount) + for (const span of creditSpans ?? []) { + if (remaining <= 0 || !Number.isFinite(span.chars) || span.chars <= 0) { + continue + } + const chars = Math.min(remaining, Math.floor(span.chars)) + remaining -= chars + span.credit?.acknowledge(chars) + } + if (remaining > 0) { + settleFallback(remaining) + } +} diff --git a/src/main/ipc/pty-renderer-delivery-router.ts b/src/main/ipc/pty-renderer-delivery-router.ts new file mode 100644 index 00000000000..1cffa26bb5f --- /dev/null +++ b/src/main/ipc/pty-renderer-delivery-router.ts @@ -0,0 +1,35 @@ +import type { PtyDataEvent } from '../providers/pty-provider-events' + +export type ExternalPtyRendererDeliveryRouter = { + data(payload: PtyDataEvent): void + replay(payload: { id: string; data: string }): void + exit(payload: { id: string; code: number }): void +} + +function returnUnroutedCredit(payload: PtyDataEvent): void { + payload.upstreamCredit?.acknowledge(payload.upstreamCredit.charCount) +} + +let router: ExternalPtyRendererDeliveryRouter = { + data: returnUnroutedCredit, + replay: () => {}, + exit: () => {} +} + +export function installExternalPtyRendererDeliveryRouter( + nextRouter: ExternalPtyRendererDeliveryRouter +): void { + router = nextRouter +} + +export function routeExternalPtyData(payload: PtyDataEvent): void { + router.data(payload) +} + +export function routeExternalPtyReplay(payload: { id: string; data: string }): void { + router.replay(payload) +} + +export function routeExternalPtyExit(payload: { id: string; code: number }): void { + router.exit(payload) +} diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 09f2f4baaaf..0645cc0ffad 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -11,6 +11,8 @@ import { redactPtyIdForDiagnostics } from '../../shared/pty-delivery-diagnostics import { FLOATING_TERMINAL_WORKTREE_ID } from '../../shared/constants' import type { TuiAgent } from '../../shared/types' import type { AgentSessionOwnerBinding } from '../../shared/agent-session-host-authority' +import { MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES } from '../../shared/claimed-agent-pty-owner' +import type * as NodeBoundedFileReader from '../../shared/node-bounded-file-reader' const isWindowsHost = process.platform === 'win32' const posixOnlyIt = isWindowsHost ? it.skip : it @@ -126,6 +128,21 @@ vi.mock('fs', () => ({ } })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => ({ + ...(await importOriginal()), + readNodeFileSyncWithinLimit: (path: string, maxBytes: number) => { + const content = readFileSyncMock(path) + if (typeof content !== 'string' && !Buffer.isBuffer(content)) { + throw new Error('File unavailable') + } + const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: { size: buffer.byteLength } } + } +})) + vi.mock('node-pty', () => ({ spawn: spawnMock })) @@ -1019,6 +1036,52 @@ describe('registerPtyHandlers', () => { clearProviderPtyState(owner.ptyId) }) + it('fails claimed ensure closed before aggregate owner listings amplify memory', async () => { + const ownersPerSession = 256 + const sessionCount = Math.floor(MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES / ownersPerSession) + 1 + const sessions = Array.from({ length: sessionCount }, (_, sessionIndex) => { + const id = `pty-owner-cap-${sessionIndex}` + return { + id, + incarnationId: 'incarnation-owner-cap', + cwd: '/tmp/recovered-worktree', + title: 'Codex', + agentSessionOwners: Array.from({ length: ownersPerSession }, (_, ownerIndex) => { + const index = sessionIndex * ownersPerSession + ownerIndex + return { + claim: { + ...recoveredAgentClaim, + identityDigest: index.toString(36).padStart(43, 'a') + }, + generation: `generation-owner-cap-${index}`, + phase: 'live' as const, + ptyId: id, + surface: recoveredAgentSurface + } + }) + } + }) + const provider = createAgentClaimProvider({ sessions }) + setLocalPtyProvider(provider as never) + const controller = registerAgentClaimController() + + await expect( + controller.spawn({ + cols: 80, + rows: 24, + cwd: '/tmp/recovered-worktree', + agentSessionEnsure: { + claim: { + ...recoveredAgentClaim, + identityDigest: '7777777777777777777777777777777777777777777' + }, + surface: recoveredAgentSurface + } + }) + ).rejects.toThrow('execution_owner_unavailable') + expect(provider.spawn).not.toHaveBeenCalled() + }) + it('releases an adopted-owner fence when that owner exits during admission', async () => { const incarnationId = 'incarnation-adopted-exit' const owner: AgentSessionOwnerBinding = { @@ -12210,7 +12273,6 @@ describe('registerPtyHandlers', () => { expect(result).toEqual({ id: expect.any(String), pid: 12345, - wslDistro: null, incarnationId: expect.any(String) }) expect(spawnMock).toHaveBeenCalledTimes(1) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 931cf8f271f..95f14c1125a 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -57,7 +57,18 @@ import { piTitlebarExtensionService } from '../pi/titlebar-extension-service' import { detectPiAgentKindFromCommand, type PiAgentKind } from '../../shared/pi-agent-kind' import { isPwshAvailable } from '../pwsh' import { LocalPtyProvider } from '../providers/local-pty-provider' -import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers/types' +import type { + IPtyProvider, + PtyProcessInfo, + PtySpawnOptions, + PtySpawnResult +} from '../providers/types' +import { inspectPtyProviderProcess } from '../providers/pty-process-inspection' +import { + PtyProcessListAdmission, + visitPtyProcessListingsInBatches +} from '../providers/pty-process-list-admission' +import type { PtyDataEvent } from '../providers/pty-provider-events' import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery' import { SSH_SESSION_EXPIRED_ERROR, @@ -168,6 +179,20 @@ import { import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { resolveLocalProjectRuntimeForWorktreeId } from '../local-project-runtime-resolution' import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import { + appendPtyDeliveryCredit, + PtyRendererDeliveryCreditLedger, + settlePtyDeliveryCredit, + takePtyDeliveryCredit, + type PtyDeliveryCreditSpan +} from './pty-renderer-delivery-credit' +import { + PendingPtyDataMap, + PtyRendererDeliveryAccountingMap, + PtyRendererDeliveryIdSet, + settleRejectedPtyRendererDelivery +} from './pty-renderer-delivery-retention' +import { installExternalPtyRendererDeliveryRouter } from './pty-renderer-delivery-router' // ─── Provider Registry ────────────────────────────────────────────── // Routes PTY operations by connectionId (null = local provider). @@ -177,6 +202,7 @@ type FreshLocalFallbackProvider = IPtyProvider & { routesFreshSpawnsToLocalProvider?: true } const sshProviders = new Map() +export const MAX_REGISTERED_SSH_PTY_PROVIDERS = 256 const SYNTHETIC_KILL_EXIT_DUPLICATE_WINDOW_MS = 30_000 // Why: kill switch — flip to disable producer flow control (pause/resume) without untangling the wiring. const PRODUCER_FLOW_CONTROL_ENABLED = true @@ -248,6 +274,18 @@ const paneSpawnReservationsByPaneKey = new Map() const agentSessionOwners = new ClaimedAgentPtyOwnerRegistry() let agentSessionOwnerReconciliation: Promise | null = null +type RegisteredPtyProvider = { + provider: IPtyProvider + connectionId: string | null +} + +function registeredPtyProviders(): RegisteredPtyProvider[] { + return [ + { provider: localProvider, connectionId: null }, + ...Array.from(sshProviders, ([connectionId, provider]) => ({ provider, connectionId })) + ] +} + function assertSpawnReplyWasLive(result: PtySpawnResult): void { if (!result.exitedBeforeSpawnReply) { return @@ -263,39 +301,35 @@ async function reconcileAgentSessionOwnerListings(): Promise { return await agentSessionOwnerReconciliation } const reconciliation = (async () => { - const providers: { provider: IPtyProvider; connectionId: string | null }[] = [ - { provider: localProvider, connectionId: null }, - ...Array.from(sshProviders, ([connectionId, provider]) => ({ provider, connectionId })) - ] - const listings = await Promise.all( - providers.map(async ({ provider, connectionId }) => ({ - connectionId, - sessions: await provider.listProcesses() - })) - ) + const admission = new PtyProcessListAdmission('execution_owner_unavailable') const advertisedOwners: AgentSessionOwnerBinding[] = [] const advertisedOwnerSessions: { id: string connectionId: string | null incarnationId: string }[] = [] - for (const { connectionId, sessions } of listings) { - for (const session of sessions) { - const incarnationId = session.incarnationId - let hasAdvertisedOwner = false - for (const owner of session.agentSessionOwners ?? []) { - if (owner.ptyId !== session.id || !isPtyIncarnationId(incarnationId)) { - // Why: a recovered claim without process-incarnation proof cannot safely reject a delayed exit. - throw new Error('agent_session_ownership_unknown') + await visitPtyProcessListingsInBatches( + registeredPtyProviders(), + ({ provider }) => provider.listProcesses(), + ({ connectionId }, sessions) => { + for (const rawSession of sessions) { + const session = admission.admit(rawSession) + const incarnationId = session.incarnationId + let hasAdvertisedOwner = false + for (const owner of session.agentSessionOwners ?? []) { + if (owner.ptyId !== session.id || !isPtyIncarnationId(incarnationId)) { + // Why: a recovered claim without process-incarnation proof cannot safely reject a delayed exit. + throw new Error('agent_session_ownership_unknown') + } + advertisedOwners.push(owner) + hasAdvertisedOwner = true + } + if (hasAdvertisedOwner && isPtyIncarnationId(incarnationId)) { + advertisedOwnerSessions.push({ id: session.id, connectionId, incarnationId }) } - advertisedOwners.push(owner) - hasAdvertisedOwner = true - } - if (hasAdvertisedOwner && isPtyIncarnationId(incarnationId)) { - advertisedOwnerSessions.push({ id: session.id, connectionId, incarnationId }) } } - } + ) agentSessionOwners.reconcileAuthoritative(advertisedOwners, { // Why: an unregistered relay can still own a live PTY during reconnect; // only providers that serialize claims may make listing absence authoritative. @@ -1192,6 +1226,9 @@ function beginPtySpawnForWorktree( /** Register an SSH PTY provider for a connection. */ export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void { + if (!sshProviders.has(connectionId) && sshProviders.size >= MAX_REGISTERED_SSH_PTY_PROVIDERS) { + throw new Error('ssh_pty_provider_capacity') + } sshProviders.set(connectionId, provider) } @@ -1586,6 +1623,7 @@ export function registerPtyHandlers( ipcMain.removeHandler('pty:hasPty') ipcMain.removeHandler('pty:hasChildProcesses') ipcMain.removeHandler('pty:getForegroundProcess') + ipcMain.removeHandler('pty:inspectProcess') ipcMain.removeHandler('pty:confirmForegroundProcess') ipcMain.removeHandler('pty:getCwd') ipcMain.removeHandler('pty:getSize') @@ -1692,6 +1730,7 @@ export function registerPtyHandlers( containsBackgroundOutput?: boolean // Why droppedOutput (not main's droppedBacklog trim): this branch's drop-to-sentinel + snapshot-restore supersedes #7630's 2MB-tail trim; both would race two cap policies over one buffer. droppedOutput?: true + upstreamCreditSpans?: PtyDeliveryCreditSpan[] } type PtyDataPayload = { @@ -1704,9 +1743,15 @@ export function registerPtyHandlers( droppedOutput?: boolean } - const pendingData = new Map() + const pendingData = new PendingPtyDataMap( + (pending) => ({ + chars: pending.data.length, + creditSpans: pending.upstreamCreditSpans?.length ?? 0 + }), + rejectPendingPtyDataForRetentionPressure + ) // Why: one restore marker per overflow episode — cleared on full drain so a later overflow re-marks exactly once. - const pendingOverflowMarkedPtys = new Set() + const pendingOverflowMarkedPtys = new PtyRendererDeliveryIdSet() // Why: TCP-style cumulative accounting — monotonic sent/acked totals self-heal on any later ACK, where relative in-flight counters would make each lost ACK a permanent debt. type RendererPtyDeliveryAccounting = { sentChars: number @@ -1714,7 +1759,9 @@ export function registerPtyHandlers( lastSendAtMs: number lastAckAtMs: number | null } - const rendererDeliveryAccountingByPty = new Map() + const rendererDeliveryAccountingByPty = + new PtyRendererDeliveryAccountingMap() + const rendererDeliveryCredit = new PtyRendererDeliveryCreditLedger() const trustedTerminalHandleEnv = new Set() let flushTimer: ReturnType | null = null let rendererInFlightTotalChars = 0 @@ -1985,12 +2032,17 @@ export function registerPtyHandlers( rendererLifecycleResetCount += 1 // Why release before clearing: pending bytes and credits belonged to the dead page; releasing producer pauses first keeps no shell wedged. producerFlowControl.releaseAll() + settleAllPendingUpstreamCredit() + rendererDeliveryCredit.writeOffAll((id, chars) => { + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + }) clearDeliveryResyncProbe() deliveryResyncUnansweredWarnLogged = false rendererDeliveryAccountingByPty.clear() rendererInFlightTotalChars = 0 pendingData.clear() pendingOverflowMarkedPtys.clear() + pendingDataDropWarnedPtys.clear() // Why hold sends: the reloading page's pty:data listener is gone until it re-registers/handshakes, so bytes would drop into a listener-less page and re-pin the gate. rendererPtyDispatcherReady = false // Why: arm the self-heal watchdog so a never-arriving handshake can't hold the gate forever; the real handshake cancels it. @@ -2081,6 +2133,9 @@ export function registerPtyHandlers( accounting.ackedChars = nextAckedChars if (acknowledged > 0) { accounting.lastAckAtMs = Date.now() + rendererDeliveryCredit.acknowledge(id, acknowledged, (chars) => { + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + }) } rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - acknowledged) return acknowledged @@ -2141,11 +2196,11 @@ export function registerPtyHandlers( if (acknowledged <= 0) { continue } - tryGetProviderForPty(id)?.acknowledgeDataEvent(id, acknowledged) // Why drop pending: everything at/before markerSeq comes from the snapshot, so flushing pre-marker bytes would double-paint the restore. const pending = pendingData.get(id) if (pending) { pendingDroppedChars += pending.data.length + settlePendingPtyDataDelivery(id, pending) pendingData.delete(id) pendingOverflowMarkedPtys.delete(id) updateProducerFlowControl(id) @@ -2174,20 +2229,30 @@ export function registerPtyHandlers( return writtenOff } - function sendPtyDataToRenderer(id: string, payload: PtyDataPayload): void { + function sendPtyDataToRenderer( + id: string, + payload: PtyDataPayload, + upstreamCredit?: PtyDeliveryCreditSpan[] + ): void { const charCount = getPtyPayloadCharCount(payload) - const accounting = rendererDeliveryAccountingByPty.get(id) - if (accounting) { - accounting.sentChars += charCount - accounting.lastSendAtMs = Date.now() - } else { - rendererDeliveryAccountingByPty.set(id, { - sentChars: charCount, + let accounting = rendererDeliveryAccountingByPty.get(id) + if (!accounting) { + accounting = { + sentChars: 0, ackedChars: 0, lastSendAtMs: Date.now(), lastAckAtMs: null - }) + } + if (!rendererDeliveryAccountingByPty.admit(id, accounting)) { + rejectSentPtyDataForRetentionPressure(id, payload, upstreamCredit) + return + } } + rendererDeliveryCredit.recordSent(id, charCount, upstreamCredit, (chars) => { + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + }) + accounting.sentChars += charCount + accounting.lastSendAtMs = Date.now() rendererInFlightTotalChars += charCount recordPtyRendererDeliveryPressure() mainWindow.webContents.send('pty:data', payload) @@ -2250,7 +2315,47 @@ export function registerPtyHandlers( return [...active, ...background] } - const pendingDataDropWarnedPtys = new Set() + const pendingDataDropWarnedPtys = new PtyRendererDeliveryIdSet() + + function markPtyRendererDeliveryDropped(id: string, droppedChars: number, detail: string): void { + pendingDroppedChars += droppedChars + if (pendingOverflowMarkedPtys.remember(id)) { + sendModelRestoreNeededMarker(id, 'pending-cap', runtime?.getPtyOutputSequence(id)) + } + if (!pendingDataDropWarnedPtys.remember(id)) { + return + } + console.error(`[pty] dropped ${droppedChars} buffered chars: ${detail}`) + recordCrashBreadcrumb('terminal_pending_output_dropped', { + droppedChars, + capChars: pendingDataCapChars() + }) + } + + function rejectPendingPtyDataForRetentionPressure(id: string, pending: PendingPtyData): void { + settlePendingPtyDataDelivery(id, pending) + producerFlowControl.release(id) + markPtyRendererDeliveryDropped( + id, + pending.data.length, + 'aggregate renderer pending-output retention limit exceeded; pane will restore from the main-owned snapshot' + ) + } + + function rejectSentPtyDataForRetentionPressure( + id: string, + payload: PtyDataPayload, + upstreamCredit: PtyDeliveryCreditSpan[] | undefined + ): void { + settleRejectedPtyRendererDelivery(getPtyPayloadCharCount(payload), upstreamCredit, (chars) => + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + ) + markPtyRendererDeliveryDropped( + id, + payload.data.length, + 'aggregate renderer delivery-accounting limit exceeded; pane will restore from the main-owned snapshot' + ) + } // Why capped: keeps O(1) memory per PTY; salvaged query bytes are tiny, so past the cap a pathological stream can degrade to the plain sentinel. const DROPPED_QUERY_SALVAGE_MAX_CHARS = 4096 @@ -2269,8 +2374,7 @@ export function registerPtyHandlers( if (pending.droppedOutput === true || pending.data.length <= capChars) { return pending } - if (!pendingDataDropWarnedPtys.has(id)) { - pendingDataDropWarnedPtys.add(id) + if (pendingDataDropWarnedPtys.remember(id)) { console.error( `[pty] dropped ${pending.data.length} buffered chars for ${id}: renderer not receiving and per-PTY pending cap exceeded; pane will restore from the main-owned snapshot` ) @@ -2281,11 +2385,11 @@ export function registerPtyHandlers( }) } // Why the marker: the snapshot can recover the dropped middle; emit it once per overflow episode so a fresh or reloaded view latches restore too. - if (isHiddenPtyDeliveryGateEnabled(getSettings?.()) && !pendingOverflowMarkedPtys.has(id)) { - pendingOverflowMarkedPtys.add(id) + if (isHiddenPtyDeliveryGateEnabled(getSettings?.()) && pendingOverflowMarkedPtys.remember(id)) { sendModelRestoreNeededMarker(id, 'pending-cap', runtime?.getPtyOutputSequence(id)) } pendingDroppedChars += pending.data.length + settlePendingPtyDataDelivery(id, pending) // Why no trimmed content tail: a mid-stream gap would corrupt the pane; the droppedOutput sentinel repaints from the snapshot and realigns by sequence (only query bytes ride along). return { data: extractDroppedPtyQueryBytes(pending.data).slice(0, DROPPED_QUERY_SALVAGE_MAX_CHARS), @@ -2301,10 +2405,16 @@ export function registerPtyHandlers( preservesSeq: boolean, containsBackgroundOutput: boolean, rawLength = data.length, - transformed = false + transformed = false, + upstreamCredit?: PtyDataEvent['upstreamCredit'] ): PendingPtyData { // Why stay dropped at O(1): once over the cap the restore sentinel supersedes interim bytes; queries still get carved out (bounded) so replies survive the whole episode. if (existing?.droppedOutput === true) { + settleRejectedPtyRendererDelivery( + rawLength, + appendPtyDeliveryCredit(undefined, upstreamCredit), + (chars) => tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + ) if (existing.data.length >= DROPPED_QUERY_SALVAGE_MAX_CHARS) { return existing } @@ -2319,16 +2429,24 @@ export function registerPtyHandlers( ...(typeof startSeq === 'number' ? { startSeq } : {}), ...(rawLength !== data.length ? { rawLength } : {}), ...(transformed ? { transformed: true } : {}), - ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}) + ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}), + ...(upstreamCredit + ? { upstreamCreditSpans: appendPtyDeliveryCredit(undefined, upstreamCredit) } + : {}) }) } const existingRawLength = existing.rawLength ?? existing.data.length + const upstreamCreditSpans = appendPtyDeliveryCredit( + existing.upstreamCreditSpans, + upstreamCredit + ) const next: PendingPtyData = { data: existing.data + data, ...(!preservesSeq || existing.transformed || transformed ? { rawLength: existingRawLength + rawLength, transformed: true as const } : {}), - ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}) + ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}), + ...(upstreamCreditSpans ? { upstreamCreditSpans } : {}) } if (typeof existing.startSeq === 'number') { next.startSeq = existing.startSeq @@ -2373,9 +2491,14 @@ export function registerPtyHandlers( if (mainWindow.isDestroyed()) { // Why release now: bookkeeping is being wiped, so no future drain can resume these producers — local shells would wedge. producerFlowControl.releaseAll() + settleAllPendingUpstreamCredit() + rendererDeliveryCredit.writeOffAll((id, chars) => { + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + }) clearDeliveryResyncProbe() pendingData.clear() pendingOverflowMarkedPtys.clear() + pendingDataDropWarnedPtys.clear() rendererDeliveryAccountingByPty.clear() rendererInFlightTotalChars = 0 clearDispatcherReadyWatchdog() @@ -2395,6 +2518,7 @@ export function registerPtyHandlers( // Why drop, never re-queue: the model already ingested hidden-gated bytes; reveal restores from the snapshot+seq machinery. if (shouldDropHiddenRendererPtyData(id, settings)) { pendingData.delete(id) + settlePendingPtyDataDelivery(id, pending) pendingOverflowMarkedPtys.delete(id) updateProducerFlowControl(id) const drop = recordHiddenRendererPtyDataDrop(id, pending.data.length) @@ -2419,6 +2543,8 @@ export function registerPtyHandlers( const indivisible = pending.transformed === true const chunk = indivisible ? data : data.slice(0, PTY_BATCH_FLUSH_CHUNK_CHARS) const remaining = indivisible ? '' : data.slice(PTY_BATCH_FLUSH_CHUNK_CHARS) + const chunkRawLength = indivisible ? (pending.rawLength ?? chunk.length) : chunk.length + const upstreamCredit = takePtyDeliveryCredit(pending.upstreamCreditSpans, chunkRawLength) if (remaining) { const nextPending: PendingPtyData = { data: remaining } if (typeof pending.startSeq === 'number') { @@ -2427,6 +2553,9 @@ export function registerPtyHandlers( if (pending.containsBackgroundOutput === true) { nextPending.containsBackgroundOutput = true } + if (pending.upstreamCreditSpans?.length) { + nextPending.upstreamCreditSpans = pending.upstreamCreditSpans + } pendingData.set(id, nextPending) } else { pendingOverflowMarkedPtys.delete(id) @@ -2441,7 +2570,8 @@ export function registerPtyHandlers( pending.containsBackgroundOutput, pending.rawLength, pending.transformed - ) + ), + upstreamCredit ) writes++ } @@ -2463,6 +2593,24 @@ export function registerPtyHandlers( flushTimer = null } + function settleAllPendingUpstreamCredit(): void { + for (const [id, pending] of pendingData) { + settlePendingPtyDataDelivery(id, pending) + } + } + + function settlePendingPtyDataDelivery(id: string, pending: PendingPtyData): void { + if (pending.droppedOutput === true) { + settlePtyDeliveryCredit(pending.upstreamCreditSpans) + return + } + settleRejectedPtyRendererDelivery( + pending.rawLength ?? pending.data.length, + pending.upstreamCreditSpans, + (chars) => tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + ) + } + const syntheticKillExitPtyIds = new Map() const reversibleStopOwnersByPtyId = new Map() @@ -2491,12 +2639,27 @@ export function registerPtyHandlers( function sendPtyExitToRenderer(payload: { id: string; code: number }): void { if (mainWindow.isDestroyed()) { + const pending = pendingData.get(payload.id) + if (pending) { + settlePendingPtyDataDelivery(payload.id, pending) + } + pendingData.delete(payload.id) + pendingDataDropWarnedPtys.delete(payload.id) + rendererDeliveryCredit.writeOff(payload.id, (chars) => { + tryGetProviderForPty(payload.id)?.acknowledgeDataEvent(payload.id, chars) + }) + rendererInFlightTotalChars = Math.max( + 0, + rendererInFlightTotalChars - getRendererInFlightCharsForPty(payload.id) + ) + rendererDeliveryAccountingByPty.delete(payload.id) return } // Why flush before exit: the renderer tears down the terminal on pty:exit, so any batched output not yet flushed would be silently lost. const remaining = pendingData.get(payload.id) if (remaining) { if (remaining.droppedOutput === true) { + settlePendingPtyDataDelivery(payload.id, remaining) // Sentinel entry: only salvaged query bytes remain; keep the flag so the renderer knows the span was dropped. sendPtyDataToRenderer(payload.id, { id: payload.id, @@ -2504,6 +2667,11 @@ export function registerPtyHandlers( droppedOutput: true }) } else { + const upstreamCredit = takePtyDeliveryCredit( + remaining.upstreamCreditSpans, + remaining.rawLength ?? remaining.data.length + ) + settlePtyDeliveryCredit(remaining.upstreamCreditSpans) sendPtyDataToRenderer( payload.id, makePtyDataPayload( @@ -2513,7 +2681,8 @@ export function registerPtyHandlers( remaining.containsBackgroundOutput, remaining.rawLength, remaining.transformed - ) + ), + upstreamCredit ) } pendingData.delete(payload.id) @@ -2521,6 +2690,7 @@ export function registerPtyHandlers( // Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id. producerFlowControl.release(payload.id) pendingOverflowMarkedPtys.delete(payload.id) + pendingDataDropWarnedPtys.delete(payload.id) lastInputAtByPty.delete(payload.id) interactiveOutputCharsByPty.delete(payload.id) rendererInFlightTotalChars = Math.max( @@ -2529,6 +2699,9 @@ export function registerPtyHandlers( ) // Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides. rendererDeliveryAccountingByPty.delete(payload.id) + rendererDeliveryCredit.writeOff(payload.id, (chars) => { + tryGetProviderForPty(payload.id)?.acknowledgeDataEvent(payload.id, chars) + }) recordPtyRendererDeliveryPressure() mainWindow.webContents.send('pty:exit', { ...payload, @@ -2536,6 +2709,20 @@ export function registerPtyHandlers( }) } + function sendPtyReplayToRenderer(payload: { id: string; data: string }): void { + const pending = pendingData.get(payload.id) + if (pending) { + settlePendingPtyDataDelivery(payload.id, pending) + pendingData.delete(payload.id) + pendingOverflowMarkedPtys.delete(payload.id) + updateProducerFlowControl(payload.id) + clearFlushTimerIfIdle() + } + if (!mainWindow.isDestroyed()) { + mainWindow.webContents.send('pty:replay', payload) + } + } + async function shutdownProviderAndDetectExit( provider: IPtyProvider, id: string, @@ -2559,6 +2746,144 @@ export function registerPtyHandlers( return providerExitObserved } + function routeProviderData(payload: PtyDataEvent, runtimeAlreadyIngested: boolean): void { + const rawLength = payload.sequenceChars ?? payload.data.length + const outputSeq = runtimeAlreadyIngested + ? runtime?.getPtyOutputSequence(payload.id) + : runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed) + const rendererData = payload.data + const preservesSeq = !payload.transformed && rawLength === payload.data.length + const startSeq = typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined + if (mainWindow.isDestroyed()) { + // Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration. + if (flushTimer) { + clearTimeout(flushTimer) + flushTimer = null + } + producerFlowControl.releaseAll() + settleRejectedPtyRendererDelivery( + rawLength, + appendPtyDeliveryCredit(undefined, payload.upstreamCredit), + (chars) => tryGetProviderForPty(payload.id)?.acknowledgeDataEvent(payload.id, chars) + ) + settleAllPendingUpstreamCredit() + rendererDeliveryCredit.writeOffAll((id, chars) => { + tryGetProviderForPty(id)?.acknowledgeDataEvent(id, chars) + }) + clearDeliveryResyncProbe() + pendingData.clear() + pendingOverflowMarkedPtys.clear() + pendingDataDropWarnedPtys.clear() + rendererDeliveryAccountingByPty.clear() + rendererInFlightTotalChars = 0 + clearDispatcherReadyWatchdog() + recordPtyRendererDeliveryPressure() + return + } + const settings = getSettings?.() + // Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot. + if (shouldDropHiddenRendererPtyData(payload.id, settings)) { + settleRejectedPtyRendererDelivery( + rawLength, + appendPtyDeliveryCredit(undefined, payload.upstreamCredit), + (chars) => tryGetProviderForPty(payload.id)?.acknowledgeDataEvent(payload.id, chars) + ) + const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length) + warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length) + if (drop.shouldEmitRestoreMarker) { + sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq) + } + return + } + if (rendererData.length === 0 && !payload.transformed) { + settleRejectedPtyRendererDelivery( + rawLength, + appendPtyDeliveryCredit(undefined, payload.upstreamCredit), + (chars) => tryGetProviderForPty(payload.id)?.acknowledgeDataEvent(payload.id, chars) + ) + return + } + const containsBackgroundOutput = + rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id) + if (containsBackgroundOutput) { + markHiddenRendererResizeOutputDelivered(payload.id) + } + const existing = pendingData.get(payload.id) + const pending = appendPendingPtyData( + payload.id, + existing, + rendererData, + startSeq, + preservesSeq, + containsBackgroundOutput, + rawLength, + payload.transformed === true, + payload.upstreamCredit + ) + const nextData = pending.data + const isInteractiveOutput = shouldSendInteractiveOutputNow( + payload.id, + nextData, + performance.now() + ) + // Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate. + if (isInteractiveOutput && rendererPtyDispatcherReady) { + // Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway. + if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) { + requestDeliveryResyncForGatedPty() + pendingData.set(payload.id, pending) + updateProducerFlowControl(payload.id) + recordPtyRendererDeliveryPressure() + return + } + pendingData.delete(payload.id) + updateProducerFlowControl(payload.id) + pendingOverflowMarkedPtys.delete(payload.id) + clearFlushTimerIfIdle() + const upstreamCredit = takePtyDeliveryCredit( + pending.upstreamCreditSpans, + pending.rawLength ?? nextData.length + ) + // Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency. + sendPtyDataToRenderer( + payload.id, + { + id: payload.id, + data: nextData, + ...(typeof pending.startSeq === 'number' + ? { + seq: pending.startSeq + (pending.rawLength ?? nextData.length), + rawLength: pending.rawLength ?? nextData.length + } + : {}), + ...(pending.transformed ? { transformed: true } : {}), + ...(pending.containsBackgroundOutput === true ? { background: true } : {}), + ...(pending.droppedOutput === true ? { droppedOutput: true } : {}) + }, + upstreamCredit + ) + return + } + pendingData.set(payload.id, pending) + updateProducerFlowControl(payload.id) + recordPtyRendererDeliveryPressure() + // Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable. + if ( + !canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) }) + ) { + requestDeliveryResyncForGatedPty() + } + if (!flushTimer) { + schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS) + } + } + + installExternalPtyRendererDeliveryRouter({ + data: (payload) => routeProviderData(payload, false), + replay: sendPtyReplayToRenderer, + exit: sendPtyExitToRenderer + }) + // Why extracted: the "Restart daemon" flow rebinds against the fresh adapter after replaceDaemonProvider, sharing this code path with startup registration. const bindProviderListeners = (): void => { localDataUnsub?.() @@ -2592,109 +2917,7 @@ export function registerPtyHandlers( // Why: daemon providers lack configure().onData, so feed the runtime here or their tail buffer (terminal.read, agent-detection, mobile stream) stays empty. const isLocalProvider = localProvider instanceof LocalPtyProvider - localDataUnsub = localProvider.onData((payload) => { - const rawLength = payload.sequenceChars ?? payload.data.length - const outputSeq = isLocalProvider - ? runtime?.getPtyOutputSequence(payload.id) - : runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed) - const rendererData = payload.data - const preservesSeq = !payload.transformed && rawLength === payload.data.length - const startSeq = - typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined - if (mainWindow.isDestroyed()) { - // Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration. - if (flushTimer) { - clearTimeout(flushTimer) - flushTimer = null - } - producerFlowControl.releaseAll() - clearDeliveryResyncProbe() - pendingData.clear() - pendingOverflowMarkedPtys.clear() - rendererDeliveryAccountingByPty.clear() - rendererInFlightTotalChars = 0 - clearDispatcherReadyWatchdog() - recordPtyRendererDeliveryPressure() - return - } - const settings = getSettings?.() - // Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot. - if (shouldDropHiddenRendererPtyData(payload.id, settings)) { - const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length) - warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length) - if (drop.shouldEmitRestoreMarker) { - sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq) - } - return - } - if (rendererData.length === 0 && !payload.transformed) { - return - } - const containsBackgroundOutput = - rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id) - if (containsBackgroundOutput) { - markHiddenRendererResizeOutputDelivered(payload.id) - } - const existing = pendingData.get(payload.id) - const pending = appendPendingPtyData( - payload.id, - existing, - rendererData, - startSeq, - preservesSeq, - containsBackgroundOutput, - rawLength, - payload.transformed === true - ) - const nextData = pending.data - const isInteractiveOutput = shouldSendInteractiveOutputNow( - payload.id, - nextData, - performance.now() - ) - // Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate. - if (isInteractiveOutput && rendererPtyDispatcherReady) { - // Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway. - if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) { - requestDeliveryResyncForGatedPty() - pendingData.set(payload.id, pending) - updateProducerFlowControl(payload.id) - recordPtyRendererDeliveryPressure() - return - } - pendingData.delete(payload.id) - updateProducerFlowControl(payload.id) - pendingOverflowMarkedPtys.delete(payload.id) - clearFlushTimerIfIdle() - // Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency. - sendPtyDataToRenderer(payload.id, { - id: payload.id, - data: nextData, - ...(typeof pending.startSeq === 'number' - ? { - seq: pending.startSeq + (pending.rawLength ?? nextData.length), - rawLength: pending.rawLength ?? nextData.length - } - : {}), - ...(pending.transformed ? { transformed: true } : {}), - ...(pending.containsBackgroundOutput === true ? { background: true } : {}), - ...(pending.droppedOutput === true ? { droppedOutput: true } : {}) - }) - return - } - pendingData.set(payload.id, pending) - updateProducerFlowControl(payload.id) - recordPtyRendererDeliveryPressure() - // Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable. - if ( - !canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) }) - ) { - requestDeliveryResyncForGatedPty() - } - if (!flushTimer) { - schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS) - } - }) + localDataUnsub = localProvider.onData((payload) => routeProviderData(payload, isLocalProvider)) localExitUnsub = localProvider.onExit((payload) => { if (!isCurrentPtyExit(payload)) { return @@ -3752,6 +3975,7 @@ export function registerPtyHandlers( return null } }, + inspectProcess: async (ptyId) => inspectPtyProviderProcess(getProviderForPty(ptyId), ptyId), confirmForegroundProcess: async (ptyId) => { try { const provider = getProviderForPty(ptyId) @@ -3793,11 +4017,18 @@ export function registerPtyHandlers( } }, listProcesses: async () => { - const providerSessions = await Promise.all([ - localProvider.listProcesses(), - ...Array.from(sshProviders.values(), (provider) => provider.listProcesses()) - ]) - return providerSessions.flat() + const processes: PtyProcessInfo[] = [] + const admission = new PtyProcessListAdmission() + await visitPtyProcessListingsInBatches( + registeredPtyProviders(), + ({ provider }) => provider.listProcesses(), + (_source, listing) => { + for (const process of listing) { + processes.push(admission.admit(process)) + } + } + ) + return processes }, serializeBuffer: (ptyId, opts) => { // Why: mobile xterm must start from the desktop's exact screen state/dimensions before live TUI chunks render correctly. @@ -4993,9 +5224,14 @@ export function registerPtyHandlers( // Why: tolerate legacy per-chunk delta payloads — dev hot-reload can pair an old renderer with a new main. const accounting = rendererDeliveryAccountingByPty.get(args.id) const delta = Number.isFinite(args.charCount) ? Math.max(0, args.charCount ?? 0) : 0 - acknowledged = accounting ? applyCumulativeAck(args.id, accounting.ackedChars + delta) : 0 + if (accounting) { + acknowledged = applyCumulativeAck(args.id, accounting.ackedChars + delta) + } + } + // Why preserve the zero callback: legacy providers observe every renderer ACK, while captured token owners still receive only their exact positive credit above. + if (acknowledged === 0) { + tryGetProviderForPty(args.id)?.acknowledgeDataEvent(args.id, 0) } - tryGetProviderForPty(args.id)?.acknowledgeDataEvent(args.id, acknowledged) recordPtyRendererDeliveryPressure() if (pendingData.size > 0 && !flushTimer) { schedulePendingDataFlush(0) @@ -5019,10 +5255,7 @@ export function registerPtyHandlers( if (typeof processedChars !== 'number' || !Number.isFinite(processedChars)) { continue } - const acknowledged = applyCumulativeAck(id, Math.max(0, processedChars)) - if (acknowledged > 0) { - tryGetProviderForPty(id)?.acknowledgeDataEvent(id, acknowledged) - } + applyCumulativeAck(id, Math.max(0, processedChars)) } recordPtyRendererDeliveryPressure() if (pendingData.size > 0 && !flushTimer) { @@ -5040,10 +5273,7 @@ export function registerPtyHandlers( if (typeof processedChars !== 'number' || !Number.isFinite(processedChars)) { continue } - const acknowledged = applyCumulativeAck(id, Math.max(0, processedChars)) - if (acknowledged > 0) { - tryGetProviderForPty(id)?.acknowledgeDataEvent(id, acknowledged) - } + applyCumulativeAck(id, Math.max(0, processedChars)) } let writtenOff: PtyDeliveryWriteOff[] = [] // Why the main-side ACK-silence check: requiring main to have also seen no ACK stops a buggy/foreign caller from writing off live delivery. @@ -5138,6 +5368,7 @@ export function registerPtyHandlers( const pending = pendingData.get(args.id) if (pending && shouldDropHiddenRendererPtyData(args.id, getSettings?.())) { pendingData.delete(args.id) + settlePendingPtyDataDelivery(args.id, pending) updateProducerFlowControl(args.id) pendingOverflowMarkedPtys.delete(args.id) const drop = recordHiddenRendererPtyDataDrop(args.id, pending.data.length) @@ -5246,24 +5477,27 @@ export function registerPtyHandlers( ipcMain.handle( 'pty:listSessions', async (): Promise<{ id: string; cwd: string; title: string }[]> => { - const providerSessions = await Promise.all([ - Promise.resolve({ - connectionId: null as string | null, - sessions: await localProvider.listProcesses() - }), - ...Array.from(sshProviders.entries(), async ([connectionId, provider]) => ({ - connectionId, - sessions: await provider.listProcesses().catch(() => []) - })) - ]) const deduped = new Map() - for (const { connectionId, sessions } of providerSessions) { - for (const session of sessions) { - // Why: kill actions only send back the PTY id, so rebuild ownership while listing to keep reconnect-discovered remote sessions routed to their provider. - ptyOwnership.set(session.id, connectionId) - deduped.set(session.id, session) + const admission = new PtyProcessListAdmission() + await visitPtyProcessListingsInBatches( + registeredPtyProviders(), + ({ provider, connectionId }) => + connectionId === null + ? provider.listProcesses() + : provider.listProcesses().catch(() => []), + ({ connectionId }, sessions) => { + for (const rawSession of sessions) { + const session = admission.admit(rawSession) + // Why: kill actions only send back the PTY id, so rebuild ownership while listing to keep reconnect-discovered remote sessions routed to their provider. + ptyOwnership.set(session.id, connectionId) + deduped.set(session.id, { + id: session.id, + cwd: session.cwd, + title: session.title + }) + } } - } + ) return Array.from(deduped.values()) } ) @@ -5337,6 +5571,10 @@ export function registerPtyHandlers( } ) + ipcMain.handle('pty:inspectProcess', async (_event, args: { id: string }) => + inspectPtyProviderProcess(getProviderForPty(args.id), args.id) + ) + ipcMain.handle( 'pty:confirmForegroundProcess', async (_event, args: { id: string }): Promise => { diff --git a/src/main/ipc/remote-workspace.test.ts b/src/main/ipc/remote-workspace.test.ts index 6da587ac86c..67b38164176 100644 --- a/src/main/ipc/remote-workspace.test.ts +++ b/src/main/ipc/remote-workspace.test.ts @@ -36,6 +36,7 @@ vi.mock('./remote-workspace-events', () => ({ import { _resetRemoteWorkspaceCachesForTests, + REMOTE_WORKSPACE_PATCH_CONCURRENCY, registerRemoteWorkspaceHandlers, remoteWorkspaceSessionMatchesSnapshot } from './remote-workspace' @@ -264,6 +265,66 @@ describe('remoteWorkspace:setForConnectedTargets', () => { expect(requestByTargetId.get('target-2')).toBeUndefined() }) + it.each([ + ['at the limit', REMOTE_WORKSPACE_PATCH_CONCURRENCY], + ['above the limit', REMOTE_WORKSPACE_PATCH_CONCURRENCY + 1] + ])('bounds per-target workspace patches %s', async (_, count) => { + const manyTargets: SshTarget[] = Array.from({ length: count }, (_, index) => ({ + id: `target-${index}`, + label: `Target ${index}`, + host: `${index}.example.com`, + port: 22, + username: 'alice' + })) + getSshConnectionStoreMock.mockReturnValue({ listTargets: () => manyTargets }) + let active = 0 + let peak = 0 + let started = 0 + const releases: (() => void)[] = [] + getActiveMultiplexerMock.mockImplementation(() => ({ + request: async (method: string) => { + if (method === 'workspace.get') { + started++ + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + return snapshot({ + activeWorktreePath: '/previous', + activeTabId: null, + tabsByWorktreePath: {}, + terminalLayoutsByTabId: {} + }) + } + return { + ok: true, + snapshot: snapshot({ + activeWorktreePath: null, + activeTabId: null, + tabsByWorktreePath: {}, + terminalLayoutsByTabId: {} + }) + } + } + })) + + const patches = callSetForConnectedTargets({ + session: baseSession, + hydratedTargetIds: manyTargets.map((target) => target.id) + }) + await vi.waitFor(() => + expect(started).toBe(Math.min(count, REMOTE_WORKSPACE_PATCH_CONCURRENCY)) + ) + if (count > REMOTE_WORKSPACE_PATCH_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(started).toBe(count)) + } + releases.splice(0).forEach((release) => release()) + + await expect(patches).resolves.toHaveLength(count) + expect(peak).toBe(Math.min(count, REMOTE_WORKSPACE_PATCH_CONCURRENCY)) + }) + it('can export from the persisted store session when no session argument is provided', async () => { getWorkspaceSessionMock.mockReturnValue({ activeRepoId: 'repo-target-1', diff --git a/src/main/ipc/remote-workspace.ts b/src/main/ipc/remote-workspace.ts index 713fe1b8a17..8d75bf838f3 100644 --- a/src/main/ipc/remote-workspace.ts +++ b/src/main/ipc/remote-workspace.ts @@ -16,6 +16,7 @@ import type { import type { SshTarget } from '../../shared/ssh-types' import type { WorkspaceSessionState } from '../../shared/types' import { getRepoIdFromWorktreeId } from '../../shared/worktree-id' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' import { getRemoteWorkspaceNamespace } from './remote-workspace-namespace' import { registerRemoteWorkspaceNotificationHandler } from './remote-workspace-events' @@ -23,6 +24,7 @@ const CLIENT_ID = randomUUID() const CLIENT_NAME = hostname() || 'This device' const SNAPSHOT_SCHEMA_VERSION = 1 export const REMOTE_WORKSPACE_SNAPSHOT_CACHE_MAX_ENTRIES = 64 +export const REMOTE_WORKSPACE_PATCH_CONCURRENCY = 4 let mainWindowGetter: (() => BrowserWindow | null) | null = null const latestSnapshotByTargetId = new Map() @@ -420,8 +422,10 @@ export function registerRemoteWorkspaceHandlers( ) ?? [] const workspaceSession = args.session ?? store.getWorkspaceSession() - const results = await Promise.all( - targets.map(async (target) => { + const results = await mapWithConcurrency( + targets, + REMOTE_WORKSPACE_PATCH_CONCURRENCY, + async (target) => { // Why: each target has its own revision stream. Keep same-target // writes queued, but do not let one slow relay block others. const session = exportSessionForTarget(store, target.id, workspaceSession) @@ -429,7 +433,7 @@ export function registerRemoteWorkspaceHandlers( patchRemoteWorkspaceSession(target, session) ) return result ? { targetId: target.id, result } : null - }) + } ) return results.filter( (entry): entry is { targetId: string; result: RemoteWorkspacePatchResult } => entry !== null diff --git a/src/main/ipc/repo-target-directory-emptiness.ts b/src/main/ipc/repo-target-directory-emptiness.ts new file mode 100644 index 00000000000..2ecd505ff3a --- /dev/null +++ b/src/main/ipc/repo-target-directory-emptiness.ts @@ -0,0 +1,12 @@ +import { opendir } from 'node:fs/promises' + +export async function isRepoTargetDirectoryEmpty(targetPath: string): Promise { + const directory = await opendir(targetPath, { bufferSize: 1 }) + try { + return (await directory.read()) === null + } finally { + await directory.close().catch(() => { + // The OS may already have closed a fully consumed directory stream. + }) + } +} diff --git a/src/main/ipc/repos-create.test.ts b/src/main/ipc/repos-create.test.ts index 040f9348250..e00fb50b8f2 100644 --- a/src/main/ipc/repos-create.test.ts +++ b/src/main/ipc/repos-create.test.ts @@ -19,7 +19,9 @@ const { mockStore, mkdirMock, accessMock, - readdirMock, + opendirMock, + directoryReadMock, + directoryCloseMock, rmMock, gitExecFileAsyncMock, homedirMock, @@ -37,7 +39,9 @@ const { }, mkdirMock: vi.fn(), accessMock: vi.fn(), - readdirMock: vi.fn(), + opendirMock: vi.fn(), + directoryReadMock: vi.fn(), + directoryCloseMock: vi.fn(), rmMock: vi.fn(), gitExecFileAsyncMock: vi.fn(), homedirMock: vi.fn(), @@ -56,7 +60,7 @@ vi.mock('electron', () => ({ vi.mock('fs/promises', () => ({ mkdir: mkdirMock, access: accessMock, - readdir: readdirMock, + opendir: opendirMock, rm: rmMock })) @@ -138,7 +142,12 @@ describe('repos:create', () => { // Default baseline: target does NOT exist yet, mkdir succeeds, git OK. accessMock.mockReset().mockRejectedValue(new Error('ENOENT')) - readdirMock.mockReset().mockResolvedValue([]) + directoryReadMock.mockReset().mockResolvedValue(null) + directoryCloseMock.mockReset().mockResolvedValue(undefined) + opendirMock.mockReset().mockResolvedValue({ + read: directoryReadMock, + close: directoryCloseMock + }) mkdirMock.mockReset().mockResolvedValue(undefined) rmMock.mockReset().mockResolvedValue(undefined) gitExecFileAsyncMock.mockReset().mockResolvedValue({ stdout: '', stderr: '' }) @@ -203,7 +212,7 @@ describe('repos:create', () => { it('rejects a non-empty existing directory without creating the target', async () => { accessMock.mockResolvedValueOnce(undefined) // exists - readdirMock.mockResolvedValueOnce(['README.md', '.DS_Store']) + directoryReadMock.mockResolvedValueOnce({ name: 'README.md' }) const result = await callCreate({ parentPath: '/tmp', name: 'busy', kind: 'git' }) @@ -211,11 +220,12 @@ describe('repos:create', () => { expect(mkdirMock).toHaveBeenCalledWith('/tmp', { recursive: true }) expect(mkdirMock).not.toHaveBeenCalledWith('/tmp/busy', expect.anything()) expect(mockStore.addRepo).not.toHaveBeenCalled() + expect(directoryReadMock).toHaveBeenCalledOnce() + expect(directoryCloseMock).toHaveBeenCalledOnce() }) it('accepts an empty existing directory and does not create the target', async () => { accessMock.mockResolvedValueOnce(undefined) // exists - readdirMock.mockResolvedValueOnce([]) const result = await callCreate({ parentPath: '/tmp', name: 'empty', kind: 'folder' }) @@ -314,7 +324,6 @@ describe('repos:create', () => { it('does NOT rm a pre-existing empty directory when git init fails', async () => { // Pretend the directory already existed (and is empty) — user pre-created it. accessMock.mockResolvedValueOnce(undefined) - readdirMock.mockResolvedValueOnce([]) gitExecFileAsyncMock.mockReset().mockRejectedValueOnce(new Error('git init blew up')) const result = await callCreate({ parentPath: '/tmp', name: 'preexisting', kind: 'git' }) @@ -357,7 +366,6 @@ describe('repos:create', () => { // The folder itself must survive (user owns it) but the half-init'd // .git/ should be removed so the folder looks untouched. accessMock.mockResolvedValueOnce(undefined) - readdirMock.mockResolvedValueOnce([]) gitExecFileAsyncMock .mockReset() .mockResolvedValueOnce({ stdout: '', stderr: '' }) diff --git a/src/main/ipc/repos-picker.test.ts b/src/main/ipc/repos-picker.test.ts index af0c5b48df4..aed95e8128f 100644 --- a/src/main/ipc/repos-picker.test.ts +++ b/src/main/ipc/repos-picker.test.ts @@ -54,6 +54,9 @@ describe('repos folder pickers', () => { getRepo: vi.fn(), updateRepo: vi.fn() } + const mockRuntime = { + notifyRepoStoreChanged: vi.fn() + } const callPickFolders = (): Promise => { const handler = handlers.get('repos:pickFolders') @@ -79,8 +82,10 @@ describe('repos folder pickers', () => { }) removeHandlerMock.mockReset() showOpenDialogMock.mockReset() + mockStore.removeProject.mockReset() + mockRuntime.notifyRepoStoreChanged.mockReset() - registerRepoHandlers(mockWindow as never, mockStore as never) + registerRepoHandlers(mockWindow as never, mockStore as never, mockRuntime as never) }) it('registers the multi-folder picker with handler cleanup', () => { @@ -109,6 +114,13 @@ describe('repos folder pickers', () => { await expect(callPickFolders()).resolves.toEqual([]) }) + it('releases runtime repo caches when local IPC removes a project', async () => { + await handlers.get('repos:remove')?.(null, { repoId: 'repo-1' }) + + expect(mockStore.removeProject).toHaveBeenCalledWith('repo-1') + expect(mockRuntime.notifyRepoStoreChanged).toHaveBeenCalledWith('repo-1') + }) + it('picks an existing directory without enabling native directory creation', async () => { const parentDir = join(sep, 'projects') showOpenDialogMock.mockResolvedValue({ diff --git a/src/main/ipc/repos.ts b/src/main/ipc/repos.ts index fd0ddcf770d..ca7289f76b1 100644 --- a/src/main/ipc/repos.ts +++ b/src/main/ipc/repos.ts @@ -39,7 +39,7 @@ import { import { isTuiAgent } from '../../shared/tui-agent-config' import { invalidateAuthorizedRootsCache } from './filesystem-auth' import type { ChildProcess } from 'node:child_process' -import { access, mkdir, readdir, rm } from 'node:fs/promises' +import { access, mkdir, rm } from 'node:fs/promises' import { gitExecFileAsync, gitSpawn, nonInteractiveGitEnv } from '../git/runner' import { isAbsolute, join, posix } from 'node:path' import { @@ -78,6 +78,7 @@ import { getSshGitUsername, resolveLocalGitUsername } from '../git/git-username' import { enrichRepoGitUsernames } from '../repo-git-username-enrichment' import { getActiveMultiplexer } from './ssh' import { normalizeSparseDirectories } from './sparse-checkout-directories' +import { isRepoTargetDirectoryEmpty } from './repo-target-directory-emptiness' import { track } from '../telemetry/client' import { scheduleCurrentWorktreeBaseDirectoryWatcherSync } from './worktree-base-directory-watcher' import { getCohortAtEmit } from '../telemetry/cohort-classifier' @@ -100,6 +101,7 @@ import { import { getGitCloneFailureMessage } from '../../shared/git-clone-failure-message' import { prepareLocalWorktreeRootForRepo } from '../worktree-root-preparation' import { runWithGitReadCacheInvalidation } from '../git/status' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' // Why: `method` is the IPC entry point the user took, not what they added (never path/URL/name); repos:create → 'folder_picker'. // Why: `isGitRepo` is a non-identifying git-vs-folder signal from the caller's detection; pass undefined when unknown, never default false. @@ -1092,7 +1094,11 @@ async function runNestedRepoScanForIpc( } } -export function registerRepoHandlers(mainWindow: BrowserWindow, store: Store): void { +export function registerRepoHandlers( + mainWindow: BrowserWindow, + store: Store, + runtime?: OrcaRuntimeService +): void { // Remove previously registered handlers so we can re-register on macOS app re-activation (new window). ipcMain.removeHandler('repos:list') ipcMain.removeHandler('repos:add') @@ -1739,8 +1745,7 @@ export function registerRepoHandlers(mainWindow: BrowserWindow, store: Store): v if (targetExists) { try { - const entries = await readdir(targetPath) - if (entries.length > 0) { + if (!(await isRepoTargetDirectoryEmpty(targetPath))) { return { error: `"${name}" already exists at this location and is not empty.` } @@ -1878,6 +1883,7 @@ export function registerRepoHandlers(mainWindow: BrowserWindow, store: Store): v ipcMain.handle('repos:remove', async (_event, args: { repoId: string }) => { store.removeProject(args.repoId) + runtime?.notifyRepoStoreChanged(args.repoId) invalidateAuthorizedRootsCache() notifyReposChanged(mainWindow) }) @@ -1891,6 +1897,7 @@ export function registerRepoHandlers(mainWindow: BrowserWindow, store: Store): v throw new Error(`Invalid host ID: ${args.hostId}`) } store.removeProjectForHost(args.repoId, hostId) + runtime?.notifyRepoStoreChanged(args.repoId) invalidateAuthorizedRootsCache() notifyReposChanged(mainWindow) } diff --git a/src/main/ipc/runtime-environment-call-queue.ts b/src/main/ipc/runtime-environment-call-queue.ts index 722f6fe8ada..a9f08e49ba6 100644 --- a/src/main/ipc/runtime-environment-call-queue.ts +++ b/src/main/ipc/runtime-environment-call-queue.ts @@ -5,7 +5,8 @@ const runtimeCallQueuePool = new RuntimeRpcCallQueuePool() export function enqueueRuntimeCall( selector: string, method: string, - run: () => Promise + run: () => Promise, + retainedBytes = 0 ): Promise { - return runtimeCallQueuePool.enqueue(selector, method, run) + return runtimeCallQueuePool.enqueue(selector, method, run, retainedBytes) } diff --git a/src/main/ipc/runtime-environment-shared-control-support.ts b/src/main/ipc/runtime-environment-shared-control-support.ts new file mode 100644 index 00000000000..28abeede78c --- /dev/null +++ b/src/main/ipc/runtime-environment-shared-control-support.ts @@ -0,0 +1,81 @@ +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { markEnvironmentUsed } from '../../shared/runtime-environment-store' +import type { + getPreferredPairingOffer, + KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { RuntimeStatus } from '../../shared/runtime-types' + +const sharedControlSupport = new Map }>() + +export function resetSharedControlSupport(): void { + sharedControlSupport.clear() +} + +export function clearSharedControlSupport(environmentId: string): void { + sharedControlSupport.delete(environmentId) +} + +export async function supportsSharedControl( + userDataPath: string, + environment: KnownRuntimeEnvironment, + pairing: ReturnType, + timeoutMs: number +): Promise { + const cacheKey = getSharedControlSupportCacheKey(environment, pairing) + const cached = sharedControlSupport.get(environment.id) + if (cached?.cacheKey === cacheKey) { + return cached.check + } + let resolvedCacheKey = cacheKey + const check = (async () => { + const response = await sendRemoteRuntimeRequest( + pairing, + 'status.get', + undefined, + timeoutMs + ) + if (response.ok === true) { + markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) + resolvedCacheKey = getSharedControlSupportCacheKey( + environment, + pairing, + response._meta.runtimeId + ) + return ( + response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) === true + ) + } + return false + })() + // Why: the same saved host can be re-paired or point at a different runtime + // binary over time; capability support belongs to that pairing/runtime identity. + sharedControlSupport.set(environment.id, { cacheKey, check }) + try { + const supported = await check + const cachedAfterCheck = sharedControlSupport.get(environment.id) + if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { + sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) + } + return supported + } catch (error) { + if (sharedControlSupport.get(environment.id)?.check === check) { + sharedControlSupport.delete(environment.id) + } + throw error + } +} + +function getSharedControlSupportCacheKey( + environment: KnownRuntimeEnvironment, + pairing: ReturnType, + runtimeId = environment.runtimeId +): string { + return [ + runtimeId ?? 'unknown-runtime', + pairing.endpoint, + pairing.deviceToken, + pairing.publicKeyB64 + ].join('\0') +} diff --git a/src/main/ipc/runtime-environment-subscription-admission.test.ts b/src/main/ipc/runtime-environment-subscription-admission.test.ts new file mode 100644 index 00000000000..2305ec33fb1 --- /dev/null +++ b/src/main/ipc/runtime-environment-subscription-admission.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { + REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES, + REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES, + REMOTE_RUNTIME_MAX_SUBSCRIPTION_PARAM_BYTES, + REMOTE_RUNTIME_MAX_SUBSCRIPTIONS +} from '../../shared/remote-runtime-memory-limits' +import { createRuntimeEnvironmentSubscriptionAdmission } from './runtime-environment-subscription-admission' + +describe('runtime environment subscription admission', () => { + it('caps pending and active subscriptions, then recovers after release', () => { + const admission = createRuntimeEnvironmentSubscriptionAdmission() + const releases = Array.from({ length: REMOTE_RUNTIME_MAX_SUBSCRIPTIONS }, (_value, index) => + admission.claim(`subscription-${index}`, undefined) + ) + + expect(admission.evidence()).toEqual({ + retainedBytes: 0, + subscriptionCount: REMOTE_RUNTIME_MAX_SUBSCRIPTIONS + }) + expect(() => admission.claim('overflow', undefined)).toThrow('subscription limit reached') + + releases[0]?.() + expect(() => admission.claim('recovered', undefined)).not.toThrow() + }) + + it('caps aggregate retained params and releases the claim exactly once', () => { + const admission = createRuntimeEnvironmentSubscriptionAdmission() + const exactParams = 'x'.repeat(REMOTE_RUNTIME_MAX_SUBSCRIPTION_PARAM_BYTES - 2) + const claimCount = + REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES / REMOTE_RUNTIME_MAX_SUBSCRIPTION_PARAM_BYTES + const releases = Array.from({ length: claimCount }, (_value, index) => + admission.claim(`large-${index}`, exactParams) + ) + + expect(admission.evidence()).toEqual({ + retainedBytes: REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES, + subscriptionCount: claimCount + }) + expect(() => admission.claim('aggregate-overflow', null)).toThrow( + 'subscription memory limit reached' + ) + + releases[0]?.() + releases[0]?.() + const recovered = admission.claim('aggregate-recovered', exactParams) + expect(admission.evidence().retainedBytes).toBe(REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES) + recovered() + }) + + it('bounds caller-supplied ids by UTF-8 bytes and reserves duplicates', () => { + const admission = createRuntimeEnvironmentSubscriptionAdmission() + const exactId = 'é'.repeat(REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES / 2) + const release = admission.claim(exactId, undefined) + + expect(() => admission.claim(exactId, undefined)).toThrow('already exists') + expect(() => admission.claim(`${exactId}x`, undefined)).toThrow( + `between 1 and ${REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES} bytes` + ) + + release() + expect(() => admission.claim(exactId, undefined)).not.toThrow() + }) +}) diff --git a/src/main/ipc/runtime-environment-subscription-admission.ts b/src/main/ipc/runtime-environment-subscription-admission.ts new file mode 100644 index 00000000000..4779ed4dc07 --- /dev/null +++ b/src/main/ipc/runtime-environment-subscription-admission.ts @@ -0,0 +1,69 @@ +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' +import { + measureRemoteRuntimeSubscriptionParams, + REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES, + REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES, + REMOTE_RUNTIME_MAX_SUBSCRIPTIONS +} from '../../shared/remote-runtime-memory-limits' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export type RuntimeEnvironmentSubscriptionAdmission = { + claim: (subscriptionId: string, params: unknown) => () => void + evidence: () => { retainedBytes: number; subscriptionCount: number } +} + +export function createRuntimeEnvironmentSubscriptionAdmission(): RuntimeEnvironmentSubscriptionAdmission { + const subscriptionIds = new Set() + let retainedBytes = 0 + + return { + claim(subscriptionId, params): () => void { + assertSubscriptionId(subscriptionId) + if (subscriptionIds.has(subscriptionId)) { + throw new RemoteRuntimeClientError( + 'invalid_argument', + 'Runtime environment subscription id already exists.' + ) + } + if (subscriptionIds.size >= REMOTE_RUNTIME_MAX_SUBSCRIPTIONS) { + throw new RemoteRuntimeClientError( + 'remote_runtime_busy', + 'Remote runtime subscription limit reached; close a subscription and retry.' + ) + } + const paramsBytes = measureRemoteRuntimeSubscriptionParams(params) + if (retainedBytes + paramsBytes > REMOTE_RUNTIME_MAX_RETAINED_SUBSCRIPTION_BYTES) { + throw new RemoteRuntimeClientError( + 'remote_runtime_busy', + 'Remote runtime subscription memory limit reached; close a subscription and retry.' + ) + } + subscriptionIds.add(subscriptionId) + retainedBytes += paramsBytes + let claimed = true + return () => { + if (!claimed) { + return + } + claimed = false + subscriptionIds.delete(subscriptionId) + retainedBytes -= paramsBytes + } + }, + evidence: () => ({ retainedBytes, subscriptionCount: subscriptionIds.size }) + } +} + +function assertSubscriptionId(subscriptionId: string): void { + if ( + subscriptionId.length === 0 || + measureUtf8ByteLength(subscriptionId, { + stopAfterBytes: REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES + }).exceededLimit + ) { + throw new RemoteRuntimeClientError( + 'invalid_argument', + `Runtime environment subscription id must be between 1 and ${REMOTE_RUNTIME_MAX_SUBSCRIPTION_ID_BYTES} bytes.` + ) + } +} diff --git a/src/main/ipc/runtime-environment-subscription-handlers.ts b/src/main/ipc/runtime-environment-subscription-handlers.ts new file mode 100644 index 00000000000..10d6c832fc6 --- /dev/null +++ b/src/main/ipc/runtime-environment-subscription-handlers.ts @@ -0,0 +1,238 @@ +import { ipcMain } from 'electron' +import { randomUUID } from 'node:crypto' +import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' +import { isRemoteRuntimeBinaryFrameWithinLimit } from '../../shared/remote-runtime-memory-limits' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { createRuntimeEnvironmentSubscriptionAdmission } from './runtime-environment-subscription-admission' +import { + retainRuntimeEnvironmentTransportGeneration, + type RuntimeEnvironmentTransportGenerationLease +} from './runtime-environment-transport-generation' +import { subscribeRuntimeEnvironment } from './runtime-environment-transport-routing' + +type RetainedRemoteRuntimeSubscription = RemoteRuntimeSubscription & { + environmentId: string + ownerWebContentsId: number + removeDestroyedListener: () => void + releaseOwnership: () => void +} + +const remoteRuntimeSubscriptions = new Map() + +export function closeAllRuntimeEnvironmentSubscriptions(): void { + for (const [subscriptionId, subscription] of Array.from(remoteRuntimeSubscriptions)) { + remoteRuntimeSubscriptions.delete(subscriptionId) + subscription.close() + } +} + +export function closeRuntimeEnvironmentSubscriptionsForEnvironment(environmentId: string): void { + for (const [subscriptionId, subscription] of remoteRuntimeSubscriptions) { + if (subscription.environmentId !== environmentId) { + continue + } + remoteRuntimeSubscriptions.delete(subscriptionId) + subscription.close() + } +} + +export function registerRuntimeEnvironmentSubscriptionHandlers( + getUserDataPath: () => string +): void { + const subscriptionAdmission = createRuntimeEnvironmentSubscriptionAdmission() + ipcMain.handle( + 'runtimeEnvironments:subscribe', + async ( + event, + args: { + selector: string + method: string + params?: unknown + timeoutMs?: number + subscriptionId?: string + expectedEnvironmentPairingRevision?: number + } + ): Promise<{ subscriptionId: string; requestId: string }> => { + const subscriptionId = + typeof args.subscriptionId === 'string' && args.subscriptionId.length > 0 + ? args.subscriptionId + : randomUUID() + const releaseAdmission = subscriptionAdmission.claim(subscriptionId, args.params) + let transportLease: RuntimeEnvironmentTransportGenerationLease | null = null + let transportLeaseReleased = false + let transportWasCurrentAtRelease = true + const transportSetupIsCurrent = (): boolean => + transportLeaseReleased + ? transportWasCurrentAtRelease + : (transportLease?.isCurrent() ?? true) + const transportIsCurrent = (): boolean => + !transportLeaseReleased && (transportLease?.isCurrent() ?? true) + const releaseTransportLease = (): void => { + if (!transportLease || transportLeaseReleased) { + return + } + transportWasCurrentAtRelease = transportLease.isCurrent() + transportLeaseReleased = true + transportLease.release() + } + const releaseOwnership = (): void => { + releaseAdmission() + releaseTransportLease() + } + let ownershipTransferred = false + try { + if (remoteRuntimeSubscriptions.has(subscriptionId)) { + throw new Error('Runtime environment subscription id already exists') + } + const environment = resolveEnvironment(getUserDataPath(), args.selector) + const pairingRevision = environment.pairingRevision ?? environment.createdAt + if ( + args.expectedEnvironmentPairingRevision !== undefined && + pairingRevision !== args.expectedEnvironmentPairingRevision + ) { + throw new Error('Runtime environment pairing changed; refresh and try again') + } + transportLease = retainRuntimeEnvironmentTransportGeneration(environment.id) + const sender = event.sender + const ownerWebContentsId = sender.id + let senderDestroyed = sender.isDestroyed() + let transportClosed = false + let subscription: RemoteRuntimeSubscription | null = null + let destroyedListenerAttached = false + const removeDestroyedListener = (): void => { + if (!destroyedListenerAttached) { + return + } + destroyedListenerAttached = false + sender.removeListener('destroyed', closeSubscription) + } + const closeSubscription = (): void => { + senderDestroyed = true + const retained = remoteRuntimeSubscriptions.get(subscriptionId) ?? null + remoteRuntimeSubscriptions.delete(subscriptionId) + if (retained) { + retained.close() + return + } + removeDestroyedListener() + subscription?.close() + releaseOwnership() + } + sender.once('destroyed', closeSubscription) + destroyedListenerAttached = true + try { + subscription = await subscribeRuntimeEnvironment( + getUserDataPath(), + environment.id, + args.method, + args.params, + args.timeoutMs, + { + onEvent: (payload) => { + if (transportIsCurrent() && !sender.isDestroyed()) { + sender.send('runtimeEnvironments:subscriptionEvent', { + subscriptionId, + ...payload + }) + } + }, + onClose: () => { + transportClosed = true + const retained = remoteRuntimeSubscriptions.get(subscriptionId) ?? null + remoteRuntimeSubscriptions.delete(subscriptionId) + if (retained) { + retained.removeDestroyedListener() + retained.releaseOwnership() + return + } + removeDestroyedListener() + releaseOwnership() + } + } + ) + } catch (error) { + removeDestroyedListener() + throw error + } + let pairingIsCurrent = false + try { + const currentEnvironment = resolveEnvironment(getUserDataPath(), environment.id) + pairingIsCurrent = + (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === pairingRevision + } catch { + pairingIsCurrent = false + } + if (!transportSetupIsCurrent() || !pairingIsCurrent) { + removeDestroyedListener() + subscription.close() + throw new Error('Runtime environment pairing changed; refresh and try again') + } + if (senderDestroyed || transportClosed || sender.isDestroyed()) { + removeDestroyedListener() + subscription.close() + return { subscriptionId, requestId: subscription.requestId } + } + remoteRuntimeSubscriptions.set(subscriptionId, { + requestId: subscription.requestId, + environmentId: environment.id, + ownerWebContentsId, + removeDestroyedListener, + releaseOwnership, + sendBinary: (bytes) => subscription?.sendBinary(bytes) ?? false, + close: () => { + removeDestroyedListener() + releaseOwnership() + subscription?.close() + } + }) + ownershipTransferred = true + return { subscriptionId, requestId: subscription.requestId } + } finally { + if (!ownershipTransferred) { + releaseOwnership() + } + } + } + ) + ipcMain.handle( + 'runtimeEnvironments:unsubscribe', + (event, args: { subscriptionId: string }): { unsubscribed: boolean } => { + const subscription = remoteRuntimeSubscriptions.get(args.subscriptionId) + if (!subscription || subscription.ownerWebContentsId !== event.sender.id) { + return { unsubscribed: false } + } + remoteRuntimeSubscriptions.delete(args.subscriptionId) + subscription.close() + return { unsubscribed: true } + } + ) + ipcMain.on( + 'runtimeEnvironments:subscriptionBinary', + (event, args: { subscriptionId?: unknown; bytes?: unknown }) => { + if (typeof args.subscriptionId !== 'string') { + return + } + const bytes = toBinaryPayload(args.bytes) + if (!bytes || !isRemoteRuntimeBinaryFrameWithinLimit(bytes)) { + return + } + const subscription = remoteRuntimeSubscriptions.get(args.subscriptionId) + if (subscription?.ownerWebContentsId === event.sender.id) { + subscription.sendBinary(bytes) + } + } + ) +} + +function toBinaryPayload(value: unknown): Uint8Array | null { + if (value instanceof Uint8Array) { + return value + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value) + } + if (ArrayBuffer.isView(value)) { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength) + } + return null +} diff --git a/src/main/ipc/runtime-environment-transport-generation.test.ts b/src/main/ipc/runtime-environment-transport-generation.test.ts new file mode 100644 index 00000000000..664ed842fd9 --- /dev/null +++ b/src/main/ipc/runtime-environment-transport-generation.test.ts @@ -0,0 +1,54 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + _internals, + advanceRuntimeEnvironmentTransportGeneration, + retainRuntimeEnvironmentTransportGeneration +} from './runtime-environment-transport-generation' + +afterEach(() => { + _internals.resetForTest() +}) + +describe('runtime environment transport generations', () => { + it('invalidates retained transports and releases their environment entry', () => { + const lease = retainRuntimeEnvironmentTransportGeneration('environment-1') + + expect(lease.isCurrent()).toBe(true) + advanceRuntimeEnvironmentTransportGeneration('environment-1') + expect(lease.isCurrent()).toBe(false) + + lease.release() + expect(_internals.trackedEnvironmentCountForTest()).toBe(0) + }) + + it('retains a shared environment entry until its final lease closes', () => { + const first = retainRuntimeEnvironmentTransportGeneration('environment-1') + const second = retainRuntimeEnvironmentTransportGeneration('environment-1') + + first.release() + expect(second.isCurrent()).toBe(true) + expect(_internals.trackedEnvironmentCountForTest()).toBe(1) + + second.release() + expect(_internals.trackedEnvironmentCountForTest()).toBe(0) + }) + + it('does not accumulate tombstones under sequential unique-environment churn', () => { + for (let index = 0; index < 10_000; index += 1) { + const environmentId = `environment-${index}` + const lease = retainRuntimeEnvironmentTransportGeneration(environmentId) + advanceRuntimeEnvironmentTransportGeneration(environmentId) + lease.release() + } + + expect(_internals.trackedEnvironmentCountForTest()).toBe(0) + }) + + it('does not allocate tombstones when invalidating environments without transports', () => { + for (let index = 0; index < 10_000; index += 1) { + advanceRuntimeEnvironmentTransportGeneration(`environment-${index}`) + } + + expect(_internals.trackedEnvironmentCountForTest()).toBe(0) + }) +}) diff --git a/src/main/ipc/runtime-environment-transport-generation.ts b/src/main/ipc/runtime-environment-transport-generation.ts index e39c86a52df..263b6dbc7a4 100644 --- a/src/main/ipc/runtime-environment-transport-generation.ts +++ b/src/main/ipc/runtime-environment-transport-generation.ts @@ -1,12 +1,71 @@ -const generationByEnvironment = new Map() +type RuntimeEnvironmentTransportGenerationEntry = { + token: number + leaseCount: number +} -export function getRuntimeEnvironmentTransportGeneration(environmentId: string): number { - return generationByEnvironment.get(environmentId) ?? 0 +export type RuntimeEnvironmentTransportGenerationLease = { + isCurrent: () => boolean + release: () => void +} + +const generationByEnvironment = new Map() +let nextGenerationToken = 1 + +function allocateGenerationToken(): number { + if (!Number.isSafeInteger(nextGenerationToken)) { + throw new Error('Runtime environment transport generation exhausted') + } + const token = nextGenerationToken + nextGenerationToken += 1 + return token +} + +export function retainRuntimeEnvironmentTransportGeneration( + environmentId: string +): RuntimeEnvironmentTransportGenerationLease { + let entry = generationByEnvironment.get(environmentId) + if (!entry) { + entry = { token: allocateGenerationToken(), leaseCount: 0 } + generationByEnvironment.set(environmentId, entry) + } + entry.leaseCount += 1 + const retainedEntry = entry + const retainedToken = entry.token + let retained = true + return { + isCurrent: () => + retained && + generationByEnvironment.get(environmentId) === retainedEntry && + retainedEntry.token === retainedToken, + release: () => { + if (!retained) { + return + } + retained = false + retainedEntry.leaseCount -= 1 + if ( + retainedEntry.leaseCount === 0 && + generationByEnvironment.get(environmentId) === retainedEntry + ) { + generationByEnvironment.delete(environmentId) + } + } + } } export function advanceRuntimeEnvironmentTransportGeneration(environmentId: string): void { - generationByEnvironment.set( - environmentId, - getRuntimeEnvironmentTransportGeneration(environmentId) + 1 - ) + const entry = generationByEnvironment.get(environmentId) + if (entry) { + entry.token = allocateGenerationToken() + } +} + +export const _internals = { + trackedEnvironmentCountForTest(): number { + return generationByEnvironment.size + }, + resetForTest(): void { + generationByEnvironment.clear() + nextGenerationToken = 1 + } } diff --git a/src/main/ipc/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environment-transport-routing.ts index 46241a82521..efde61ccd5c 100644 --- a/src/main/ipc/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environment-transport-routing.ts @@ -1,11 +1,8 @@ -import { - getPreferredPairingOffer, - type KnownRuntimeEnvironment -} from '../../shared/runtime-environments' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store' import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' +import { serializeRemoteRuntimeRpcRequest } from '../../shared/remote-runtime-memory-limits' import type { RuntimeStatus } from '../../shared/runtime-types' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' import { sendRemoteRuntimeRequest, subscribeRemoteRuntimeRequest, @@ -22,14 +19,14 @@ import { import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' +import { supportsSharedControl } from './runtime-environment-shared-control-support' + +export { + clearSharedControlSupport, + resetSharedControlSupport +} from './runtime-environment-shared-control-support' const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000 -const sharedControlSupport = new Map }>() - -export const resetSharedControlSupport = (): void => sharedControlSupport.clear() - -export const clearSharedControlSupport = (environmentId: string): void => - void sharedControlSupport.delete(environmentId) export async function getRuntimeEnvironmentStatus( userDataPath: string, @@ -84,58 +81,78 @@ export async function callRuntimeEnvironment( expectedEnvironmentPairingRevision?: number ): Promise> { const environment = resolveEnvironment(userDataPath, selector) + const initialPairing = getPreferredPairingOffer(environment) + const retainedBytes = Buffer.byteLength( + serializeRemoteRuntimeRpcRequest({ + requestId: '00000000-0000-4000-8000-000000000000', + deviceToken: initialPairing.deviceToken, + method, + params + }), + 'utf8' + ) // Why: connection failures reject (they don't resolve as ok:false), so the // Tailscale hint is applied to the thrown error here — wrapping the resolved // value would miss the in-use connect/timeout case the toast surfaces. // Track the endpoint the queued closure actually used: it re-resolves the // environment, so a re-pair between enqueue and dispatch can change it. - let endpoint = getPreferredPairingOffer(environment).endpoint + let endpoint = initialPairing.endpoint try { - return await enqueueRuntimeCall(environment.id, method, async () => { - const currentEnvironment = resolveEnvironment(userDataPath, environment.id) - const revisionFailure = runtimeEnvironmentRevisionFailure( - currentEnvironment, - expectedEnvironmentPairingRevision, - method - ) - if (revisionFailure) { - return revisionFailure - } - const pairing = getPreferredPairingOffer(currentEnvironment) - endpoint = pairing.endpoint - const effectiveTimeoutMs = timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS - if (shouldUseCachedRequestConnection(method)) { - const response = await sendRemoteRuntimeConnectionRequest( - currentEnvironment.id, - pairing, - method, - params, - effectiveTimeoutMs + return await enqueueRuntimeCall( + environment.id, + method, + async () => { + const currentEnvironment = resolveEnvironment(userDataPath, environment.id) + const revisionFailure = runtimeEnvironmentRevisionFailure( + currentEnvironment, + expectedEnvironmentPairingRevision, + method ) + if (revisionFailure) { + return revisionFailure + } + const pairing = getPreferredPairingOffer(currentEnvironment) + endpoint = pairing.endpoint + const effectiveTimeoutMs = timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS + if (shouldUseCachedRequestConnection(method)) { + const response = await sendRemoteRuntimeConnectionRequest( + currentEnvironment.id, + pairing, + method, + params, + effectiveTimeoutMs + ) + markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) + return response + } + if ( + method !== 'status.get' && + !shouldUseOneShotRequest(method) && + (await supportsSharedControl( + userDataPath, + currentEnvironment, + pairing, + effectiveTimeoutMs + )) + ) { + const response = await sendRemoteRuntimeSharedControlRequest( + currentEnvironment.id, + pairing, + method, + params, + effectiveTimeoutMs + ) + markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) + return response + } + // Why: startup/control-plane RPCs use the proven one-shot path so repo + // hydration cannot be coupled to a stale terminal-control connection. + const response = await sendRemoteRuntimeRequest(pairing, method, params, effectiveTimeoutMs) markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) return response - } - if ( - method !== 'status.get' && - !shouldUseOneShotRequest(method) && - (await supportsSharedControl(userDataPath, currentEnvironment, pairing, effectiveTimeoutMs)) - ) { - const response = await sendRemoteRuntimeSharedControlRequest( - currentEnvironment.id, - pairing, - method, - params, - effectiveTimeoutMs - ) - markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) - return response - } - // Why: startup/control-plane RPCs use the proven one-shot path so repo - // hydration cannot be coupled to a stale terminal-control connection. - const response = await sendRemoteRuntimeRequest(pairing, method, params, effectiveTimeoutMs) - markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) - return response - }) + }, + retainedBytes + ) } catch (error) { if (error instanceof Error) { error.message = withRemoteRuntimeTailscaleHint(error.message, endpoint) @@ -257,66 +274,3 @@ function shouldUseSharedControlSubscription(method: string): boolean { method === 'files.watch' ) } - -async function supportsSharedControl( - userDataPath: string, - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - timeoutMs: number -): Promise { - const cacheKey = getSharedControlSupportCacheKey(environment, pairing) - const cached = sharedControlSupport.get(environment.id) - if (cached?.cacheKey === cacheKey) { - return cached.check - } - let resolvedCacheKey = cacheKey - const check = (async () => { - const response = await sendRemoteRuntimeRequest( - pairing, - 'status.get', - undefined, - timeoutMs - ) - if (response.ok === true) { - markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) - resolvedCacheKey = getSharedControlSupportCacheKey( - environment, - pairing, - response._meta.runtimeId - ) - return ( - response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) === true - ) - } - return false - })() - // Why: the same saved host can be re-paired or point at a different runtime - // binary over time; capability support belongs to that pairing/runtime identity. - sharedControlSupport.set(environment.id, { cacheKey, check }) - try { - const supported = await check - const cachedAfterCheck = sharedControlSupport.get(environment.id) - if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { - sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) - } - return supported - } catch (error) { - if (sharedControlSupport.get(environment.id)?.check === check) { - sharedControlSupport.delete(environment.id) - } - throw error - } -} - -function getSharedControlSupportCacheKey( - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - runtimeId = environment.runtimeId -): string { - return [ - runtimeId ?? 'unknown-runtime', - pairing.endpoint, - pairing.deviceToken, - pairing.publicKeyB64 - ].join('\0') -} diff --git a/src/main/ipc/runtime-environments.test.ts b/src/main/ipc/runtime-environments.test.ts index 9b94212c76a..ee90414938a 100644 --- a/src/main/ipc/runtime-environments.test.ts +++ b/src/main/ipc/runtime-environments.test.ts @@ -168,8 +168,6 @@ describe('registerRuntimeEnvironmentHandlers', () => { const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) expect(JSON.stringify(added)).not.toContain('device-token') expect(JSON.stringify(added)).not.toContain('publicKeyB64') - activeRuntimeEnvironmentId = added.environment.id - const list = handler('runtimeEnvironments:list') expect(await list(null, undefined)).toMatchObject([{ id: added.environment.id, name: 'desk' }]) expect(JSON.stringify(await list(null, undefined))).not.toContain('device-token') @@ -190,16 +188,31 @@ describe('registerRuntimeEnvironmentHandlers', () => { expect(removed).toMatchObject({ removed: { id: added.environment.id, name: 'desk' } }) - expect(store.updateSettings).toHaveBeenCalledWith( - { activeRuntimeEnvironmentId: null }, - { notifyListeners: true } - ) expect(activeRuntimeEnvironmentId).toBeNull() expect(closeRemoteRuntimeRequestConnectionMock).toHaveBeenCalledWith(added.environment.id) expect(JSON.stringify(removed)).not.toContain('device-token') expect(await list(null, undefined)).toEqual([]) }) + it('requires an explicit Advanced selection before removing the Active Server', async () => { + registerRuntimeEnvironmentHandlers(store as never) + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + activeRuntimeEnvironmentId = added.environment.id + const remove = handler<{ selector: string }, { removed: { id: string } }>( + 'runtimeEnvironments:remove' + ) + + expect(() => remove(null, { selector: added.environment.id })).toThrow( + 'Choose another Active Server in Advanced' + ) + expect(activeRuntimeEnvironmentId).toBe(added.environment.id) + expect(store.updateSettings).not.toHaveBeenCalled() + }) + it('disconnects a saved runtime without removing it', async () => { registerRuntimeEnvironmentHandlers(store as never) @@ -1368,6 +1381,56 @@ describe('registerRuntimeEnvironmentHandlers', () => { markUsedSpy.mockRestore() }) + it('drops a queued old-peer frame after unsubscribe and transport invalidation', async () => { + registerRuntimeEnvironmentHandlers(store as never) + const close = vi.fn() + subscribeRemoteRuntimeRequestMock.mockResolvedValue({ + requestId: 'queued-old-peer', + close, + sendBinary: vi.fn() + }) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + const senderSend = vi.fn() + const subscribe = handler< + { selector: string; method: string; subscriptionId: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + const result = await subscribe( + { + sender: { + id: 1, + isDestroyed: () => false, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + }, + { + selector: added.environment.id, + method: 'terminal.subscribe', + subscriptionId: 'queued-old-peer' + } + ) + const callbacks = subscribeRemoteRuntimeRequestMock.mock.calls[0]![4] as { + onBinary: (bytes: Uint8Array) => void + } + const unsubscribe = handler<{ subscriptionId: string }, { unsubscribed: boolean }>( + 'runtimeEnvironments:unsubscribe' + ) + + expect(await unsubscribe({ sender: { id: 1 } }, result)).toEqual({ unsubscribed: true }) + invalidateRuntimeEnvironmentTransport(added.environment.id) + callbacks.onBinary(new Uint8Array([1, 2, 3])) + + expect(close).toHaveBeenCalledTimes(1) + expect(senderSend).not.toHaveBeenCalled() + }) + it('closes streaming subscriptions when their saved runtime is removed', async () => { registerRuntimeEnvironmentHandlers(store as never) const close = vi.fn() diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index f2c31beaf54..5e5cede71b2 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -1,5 +1,4 @@ import { app, ipcMain } from 'electron' -import { randomUUID } from 'node:crypto' import { addEnvironmentFromPairingCode, listEnvironments, @@ -12,21 +11,20 @@ import { } from '../../shared/runtime-environments' import type { RuntimeStatus } from '../../shared/runtime-types' import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' -import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import type { Store } from '../persistence' -import { clearActiveRuntimeEnvironmentFocusIfMatches } from '../runtime-environment-focus-self-heal' import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections' -import { - advanceRuntimeEnvironmentTransportGeneration, - getRuntimeEnvironmentTransportGeneration -} from './runtime-environment-transport-generation' +import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { callRuntimeEnvironment, clearSharedControlSupport, getRuntimeEnvironmentStatus, - resetSharedControlSupport, - subscribeRuntimeEnvironment + resetSharedControlSupport } from './runtime-environment-transport-routing' +import { + closeAllRuntimeEnvironmentSubscriptions, + closeRuntimeEnvironmentSubscriptionsForEnvironment, + registerRuntimeEnvironmentSubscriptionHandlers +} from './runtime-environment-subscription-handlers' const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:list', @@ -40,30 +38,14 @@ const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:unsubscribe' ] as const -type RetainedRemoteRuntimeSubscription = RemoteRuntimeSubscription & { - environmentId: string - ownerWebContentsId: number - removeDestroyedListener: () => void -} -const remoteRuntimeSubscriptions = new Map() const getUserDataPath = (): string => app.getPath('userData') -function closeSubscriptionsForEnvironment(environmentId: string): void { - // Why: removed runtimes must not retain terminal/browser WebSockets until renderer teardown. - for (const [subscriptionId, subscription] of remoteRuntimeSubscriptions) { - if (subscription.environmentId !== environmentId) { - continue - } - remoteRuntimeSubscriptions.delete(subscriptionId) - subscription.close() - } -} export function invalidateRuntimeEnvironmentTransport(environmentId: string): void { // Why: a same-id re-pair must retire every transport that still authenticates as the old peer. advanceRuntimeEnvironmentTransportGeneration(environmentId) closeRemoteRuntimeRequestConnection(environmentId) clearSharedControlSupport(environmentId) - closeSubscriptionsForEnvironment(environmentId) + closeRuntimeEnvironmentSubscriptionsForEnvironment(environmentId) } function listPublicRuntimeEnvironments(): PublicKnownRuntimeEnvironment[] { @@ -75,14 +57,13 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { // Why: keep direct re-registration safe even though register-core-handlers // normally guards this path; otherwise the binary send listener can stack. resetSharedControlSupport() + closeAllRuntimeEnvironmentSubscriptions() for (const channel of RUNTIME_ENVIRONMENT_HANDLER_CHANNELS) { ipcMain.removeHandler(channel) } ipcMain.removeAllListeners('runtimeEnvironments:subscriptionBinary') - ipcMain.handle('runtimeEnvironments:list', (): PublicKnownRuntimeEnvironment[] => - listPublicRuntimeEnvironments() - ) + ipcMain.handle('runtimeEnvironments:list', listPublicRuntimeEnvironments) ipcMain.handle( 'runtimeEnvironments:addFromPairingCode', ( @@ -92,21 +73,22 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { environment: redactRuntimeEnvironment(addEnvironmentFromPairingCode(getUserDataPath(), args)) }) ) - ipcMain.handle( - 'runtimeEnvironments:resolve', - (_event, args: { selector: string }): PublicKnownRuntimeEnvironment => - redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector)) + ipcMain.handle('runtimeEnvironments:resolve', (_event, args: { selector: string }) => + redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector)) ) ipcMain.handle( 'runtimeEnvironments:remove', (_event, args: { selector: string }): { removed: PublicKnownRuntimeEnvironment } => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + if (store.getSettings().activeRuntimeEnvironmentId === environment.id) { + throw new Error('Choose another Active Server in Advanced before removing this server.') + } const removed = removeEnvironment(getUserDataPath(), args.selector) invalidateRuntimeEnvironmentTransport(removed.id) if (args.selector !== removed.id) { closeRemoteRuntimeRequestConnection(args.selector) clearSharedControlSupport(args.selector) } - clearActiveRuntimeEnvironmentFocusIfMatches(store, removed.id) return { removed: redactRuntimeEnvironment(removed) } } ) @@ -155,160 +137,5 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { ) } ) - ipcMain.handle( - 'runtimeEnvironments:subscribe', - async ( - event, - args: { - selector: string - method: string - params?: unknown - timeoutMs?: number - subscriptionId?: string - expectedEnvironmentPairingRevision?: number - } - ): Promise<{ subscriptionId: string; requestId: string }> => { - const subscriptionId = - typeof args.subscriptionId === 'string' && args.subscriptionId.length > 0 - ? args.subscriptionId - : randomUUID() - if (remoteRuntimeSubscriptions.has(subscriptionId)) { - throw new Error('Runtime environment subscription id already exists') - } - const environment = resolveEnvironment(getUserDataPath(), args.selector) - const pairingRevision = environment.pairingRevision ?? environment.createdAt - if ( - args.expectedEnvironmentPairingRevision !== undefined && - pairingRevision !== args.expectedEnvironmentPairingRevision - ) { - throw new Error('Runtime environment pairing changed; refresh and try again') - } - const transportGeneration = getRuntimeEnvironmentTransportGeneration(environment.id) - const transportIsCurrent = (): boolean => - getRuntimeEnvironmentTransportGeneration(environment.id) === transportGeneration - const sender = event.sender - const ownerWebContentsId = sender.id - let senderDestroyed = sender.isDestroyed() - let subscription: RemoteRuntimeSubscription | null = null - let destroyedListenerAttached = false - const removeDestroyedListener = (): void => { - if (!destroyedListenerAttached) { - return - } - destroyedListenerAttached = false - sender.removeListener('destroyed', closeSubscription) - } - const closeSubscription = (): void => { - senderDestroyed = true - const retained = remoteRuntimeSubscriptions.get(subscriptionId) ?? null - remoteRuntimeSubscriptions.delete(subscriptionId) - if (retained) { - retained.close() - return - } - removeDestroyedListener() - subscription?.close() - } - sender.once('destroyed', closeSubscription) - destroyedListenerAttached = true - try { - subscription = await subscribeRuntimeEnvironment( - getUserDataPath(), - environment.id, - args.method, - args.params, - args.timeoutMs, - { - onEvent: (payload) => { - if (transportIsCurrent() && !sender.isDestroyed()) { - sender.send('runtimeEnvironments:subscriptionEvent', { - subscriptionId, - ...payload - }) - } - }, - onClose: () => { - const retained = remoteRuntimeSubscriptions.get(subscriptionId) ?? null - retained?.removeDestroyedListener() - remoteRuntimeSubscriptions.delete(subscriptionId) - } - } - ) - } catch (error) { - removeDestroyedListener() - throw error - } - let pairingIsCurrent = false - try { - const currentEnvironment = resolveEnvironment(getUserDataPath(), environment.id) - pairingIsCurrent = - (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === pairingRevision - } catch { - pairingIsCurrent = false - } - if (!transportIsCurrent() || !pairingIsCurrent) { - removeDestroyedListener() - subscription.close() - throw new Error('Runtime environment pairing changed; refresh and try again') - } - if (senderDestroyed || sender.isDestroyed()) { - removeDestroyedListener() - subscription.close() - return { subscriptionId, requestId: subscription.requestId } - } - remoteRuntimeSubscriptions.set(subscriptionId, { - requestId: subscription.requestId, - environmentId: environment.id, - ownerWebContentsId, - removeDestroyedListener, - sendBinary: (bytes) => subscription?.sendBinary(bytes) ?? false, - close: () => { - removeDestroyedListener() - subscription?.close() - } - }) - return { subscriptionId, requestId: subscription.requestId } - } - ) - ipcMain.handle( - 'runtimeEnvironments:unsubscribe', - (event, args: { subscriptionId: string }): { unsubscribed: boolean } => { - const subscription = remoteRuntimeSubscriptions.get(args.subscriptionId) - if (!subscription || subscription.ownerWebContentsId !== event.sender.id) { - return { unsubscribed: false } - } - remoteRuntimeSubscriptions.delete(args.subscriptionId) - subscription.close() - return { unsubscribed: true } - } - ) - ipcMain.on( - 'runtimeEnvironments:subscriptionBinary', - (event, args: { subscriptionId?: unknown; bytes?: unknown }) => { - if (typeof args.subscriptionId !== 'string') { - return - } - const bytes = toBinaryPayload(args.bytes) - if (!bytes) { - return - } - const subscription = remoteRuntimeSubscriptions.get(args.subscriptionId) - if (subscription?.ownerWebContentsId === event.sender.id) { - subscription.sendBinary(bytes) - } - } - ) -} - -function toBinaryPayload(value: unknown): Uint8Array | null { - if (value instanceof Uint8Array) { - return value - } - if (value instanceof ArrayBuffer) { - return new Uint8Array(value) - } - if (ArrayBuffer.isView(value)) { - return new Uint8Array(value.buffer, value.byteOffset, value.byteLength) - } - return null + registerRuntimeEnvironmentSubscriptionHandlers(getUserDataPath) } diff --git a/src/main/ipc/settings.test.ts b/src/main/ipc/settings.test.ts index 08b0abbe98d..01bbcbd12b4 100644 --- a/src/main/ipc/settings.test.ts +++ b/src/main/ipc/settings.test.ts @@ -9,6 +9,7 @@ const { previewGhosttyImportMock, previewWarpThemeImportMock, prepareLocalWorktreeRootsForReposMock, + resolveEnvironmentMock, rebuildAppMenuMock } = vi.hoisted(() => ({ applyAppIconMock: vi.fn(), @@ -19,10 +20,12 @@ const { previewGhosttyImportMock: vi.fn(), previewWarpThemeImportMock: vi.fn(), prepareLocalWorktreeRootsForReposMock: vi.fn(), + resolveEnvironmentMock: vi.fn(), rebuildAppMenuMock: vi.fn() })) vi.mock('electron', () => ({ + app: { getPath: vi.fn(() => '/test/user-data') }, BrowserWindow: { getAllWindows: browserWindowGetAllWindowsMock }, ipcMain: { handle: handleMock, on: onMock }, nativeTheme: { themeSource: 'system' } @@ -52,6 +55,10 @@ vi.mock('../menu/register-app-menu', () => ({ rebuildAppMenu: rebuildAppMenuMock })) +vi.mock('../../shared/runtime-environment-store', () => ({ + resolveEnvironment: resolveEnvironmentMock +})) + import { registerSettingsHandlers } from './settings' const settingsInvokeEvent = { sender: { id: 1 } } @@ -79,6 +86,12 @@ describe('registerSettingsHandlers', () => { previewGhosttyImportMock.mockClear() previewWarpThemeImportMock.mockClear() prepareLocalWorktreeRootsForReposMock.mockReset().mockResolvedValue(undefined) + resolveEnvironmentMock.mockReset().mockImplementation((_userDataPath, selector) => { + if (selector !== 'windows-2' && selector !== 'Windows 2') { + throw new Error('Runtime environment not found') + } + return { id: 'windows-2' } + }) rebuildAppMenuMock.mockClear() browserWindowGetAllWindowsMock.mockReset() store.getSettings.mockReset() @@ -108,6 +121,52 @@ describe('registerSettingsHandlers', () => { expect(event.returnValue).toEqual({ terminalMainSideEffectAuthority: false }) }) + it('rejects durable Active Server writes through generic settings:set', async () => { + store.getSettings.mockReturnValue({ activeRuntimeEnvironmentId: null }) + store.updateSettings.mockReturnValue({ activeRuntimeEnvironmentId: null }) + registerSettingsHandlers(store as never) + const handler = handleMock.mock.calls.find((call) => call[0] === 'settings:set')?.[1] as ( + event: typeof settingsInvokeEvent, + args: { activeRuntimeEnvironmentId: string } + ) => Promise + + await handler(settingsInvokeEvent, { activeRuntimeEnvironmentId: 'windows-2' }) + + expect(store.updateSettings).toHaveBeenCalledWith( + {}, + expect.objectContaining({ originWebContentsId: 1 }) + ) + }) + + it('persists Active Server only through the dedicated preference channel', () => { + store.updateSettings.mockReturnValue({ activeRuntimeEnvironmentId: 'windows-2' }) + registerSettingsHandlers(store as never) + const handler = handleMock.mock.calls.find( + (call) => call[0] === 'settings:set-active-runtime-environment-preference' + )?.[1] as (event: typeof settingsInvokeEvent, args: { environmentId: string | null }) => unknown + + expect(handler(settingsInvokeEvent, { environmentId: ' windows-2 ' })).toEqual({ + activeRuntimeEnvironmentId: 'windows-2' + }) + expect(store.updateSettings).toHaveBeenCalledWith( + { activeRuntimeEnvironmentId: 'windows-2' }, + { notifyListeners: true, originWebContentsId: 1 } + ) + handler(settingsInvokeEvent, { environmentId: 'Windows 2' }) + expect(store.updateSettings).toHaveBeenLastCalledWith( + { activeRuntimeEnvironmentId: 'windows-2' }, + { notifyListeners: true, originWebContentsId: 1 } + ) + + expect(() => handler(settingsInvokeEvent, { environmentId: 42 as never })).toThrow( + 'Invalid Active Server preference' + ) + expect(() => handler(settingsInvokeEvent, { environmentId: 'does-not-exist' })).toThrow( + 'Runtime environment not found' + ) + expect(store.updateSettings).toHaveBeenCalledTimes(2) + }) + it('applies bot-author deltas against the authoritative settings snapshot', () => { store.getSettings .mockReturnValueOnce({ prBotAuthorOverrides: ['alice'] }) diff --git a/src/main/ipc/settings.ts b/src/main/ipc/settings.ts index 1ba9a015b66..c4e37b9c01c 100644 --- a/src/main/ipc/settings.ts +++ b/src/main/ipc/settings.ts @@ -1,4 +1,4 @@ -import { BrowserWindow, ipcMain, nativeTheme } from 'electron' +import { app, BrowserWindow, ipcMain, nativeTheme } from 'electron' import type { Store } from '../persistence' import type { GlobalSettings, PersistedState } from '../../shared/types' import { listSystemFontFamilies } from '../system-fonts' @@ -22,6 +22,7 @@ import { normalizeTerminalLineHeight } from '../../shared/terminal-line-height-s import { prepareLocalWorktreeRootsForRepos } from '../worktree-root-preparation' import { scheduleCurrentWorktreeBaseDirectoryWatcherSync } from './worktree-base-directory-watcher' import { applyPRBotAuthorOverride } from '../../shared/pr-bot-author-overrides' +import { resolveEnvironment } from '../../shared/runtime-environment-store' // Why: the whitelist is the source-of-truth for which keys we emit on. Casting // to a Set once at module load lets the IPC handler's per-key membership @@ -92,6 +93,9 @@ export function registerSettingsHandlers( ipcMain.handle('settings:set', async (event, args: Partial) => { const sanitizedArgs = sanitizeRendererSettingsUpdate(args) + // Why: connection/navigation code receives the generic settings writer; the + // durable server preference has a dedicated Advanced-control boundary. + delete sanitizedArgs.activeRuntimeEnvironmentId // Why: Floating Workspace grants are trusted only when written by the // main-process directory picker, never by renderer-provided settings IPC. delete sanitizedArgs.floatingTerminalTrustedCwds @@ -205,6 +209,23 @@ export function registerSettingsHandlers( return result }) + ipcMain.handle( + 'settings:set-active-runtime-environment-preference', + (event, args: { environmentId?: unknown }): GlobalSettings => { + const requestedEnvironmentId = args?.environmentId + if (requestedEnvironmentId !== null && typeof requestedEnvironmentId !== 'string') { + throw new Error('Invalid Active Server preference') + } + const requestedId = requestedEnvironmentId?.trim() || null + const environmentId = + requestedId === null ? null : resolveEnvironment(app.getPath('userData'), requestedId).id + return store.updateSettings( + { activeRuntimeEnvironmentId: environmentId }, + { notifyListeners: true, originWebContentsId: event.sender.id } + ) + } + ) + ipcMain.handle('settings:listFonts', () => { return listSystemFontFamilies() }) diff --git a/src/main/ipc/shell-repo-icon-picker.ts b/src/main/ipc/shell-repo-icon-picker.ts new file mode 100644 index 00000000000..3d376f1ab85 --- /dev/null +++ b/src/main/ipc/shell-repo-icon-picker.ts @@ -0,0 +1,52 @@ +import { dialog } from 'electron' +import { stat } from 'node:fs/promises' +import { basename, extname } from 'node:path' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' +import { assertRasterImagePreviewWithinLimits } from '../../shared/raster-image-preview-limits' +import { MAX_REPO_ICON_UPLOAD_BYTES } from '../../shared/repo-icon' + +const REPO_ICON_IMAGE_MIME_TYPES: Record = { + '.png': 'image/png' +} + +export async function pickRepoIconImage(): Promise<{ + dataUrl: string + fileName: string +} | null> { + const result = await dialog.showOpenDialog({ + properties: ['openFile'], + filters: [{ name: 'Repo icon images', extensions: ['png'] }] + }) + if (result.canceled || result.filePaths.length === 0) { + return null + } + + const filePath = result.filePaths[0] + const mimeType = REPO_ICON_IMAGE_MIME_TYPES[extname(filePath).toLowerCase()] + if (!mimeType) { + throw new Error('Repo icons must be PNG files.') + } + + const stats = await stat(filePath) + if (stats.size > MAX_REPO_ICON_UPLOAD_BYTES) { + throw new Error('Repo icon image must be 256KB or smaller.') + } + + let buffer: Buffer + try { + buffer = (await readNodeFileWithinLimit(filePath, MAX_REPO_ICON_UPLOAD_BYTES)).buffer + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw new Error('Repo icon image must be 256KB or smaller.') + } + throw error + } + assertRasterImagePreviewWithinLimits(buffer, mimeType) + return { + dataUrl: `data:${mimeType};base64,${buffer.toString('base64')}`, + fileName: basename(filePath) + } +} diff --git a/src/main/ipc/shell.test.ts b/src/main/ipc/shell.test.ts index 7193da4beaa..9656fbf18d2 100644 --- a/src/main/ipc/shell.test.ts +++ b/src/main/ipc/shell.test.ts @@ -1,12 +1,15 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { normalize, resolve } from 'node:path' import { pathToFileURL } from 'node:url' +import * as NodeBoundedFileReader from '../../shared/node-bounded-file-reader' const { getSpawnArgsForWindowsMock, handleMock, openPathMock, + readNodeFileWithinLimitMock, resolveCliCommandMock, + readFileMock, showItemInFolderMock, showOpenDialogMock, spawnMock, @@ -15,7 +18,9 @@ const { getSpawnArgsForWindowsMock: vi.fn(), handleMock: vi.fn(), openPathMock: vi.fn(), + readNodeFileWithinLimitMock: vi.fn(), resolveCliCommandMock: vi.fn(), + readFileMock: vi.fn(), showItemInFolderMock: vi.fn(), showOpenDialogMock: vi.fn(), spawnMock: vi.fn(), @@ -42,6 +47,11 @@ vi.mock('node:fs/promises', () => ({ stat: statMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, readNodeFileWithinLimit: readNodeFileWithinLimitMock } +}) + vi.mock('node:child_process', () => ({ spawn: spawnMock })) @@ -105,6 +115,11 @@ describe('registerShellHandlers', () => { getSpawnArgsForWindowsMock.mockReset() openPathMock.mockReset() resolveCliCommandMock.mockReset() + readFileMock.mockReset() + readNodeFileWithinLimitMock.mockReset() + readNodeFileWithinLimitMock.mockImplementation(async (path: string) => ({ + buffer: await readFileMock(path) + })) showItemInFolderMock.mockReset() showOpenDialogMock.mockReset() spawnMock.mockReset() @@ -154,6 +169,32 @@ describe('registerShellHandlers', () => { await expect(handler({})).resolves.toBeNull() }) + it('rejects a repo icon raster dimension bomb before returning a data URL', async () => { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(32_769, 16) + bytes.writeUInt32BE(1, 20) + showOpenDialogMock.mockResolvedValue({ canceled: false, filePaths: ['/tmp/bomb.png'] }) + statMock.mockResolvedValue({ size: bytes.length }) + readFileMock.mockResolvedValue(bytes) + + const handler = getHandler('shell:pickRepoIconImage') + await expect(handler({})).rejects.toThrow('Image dimensions exceed the preview safety limit') + }) + + it('rejects a repo icon that grows past the byte cap after its size check', async () => { + showOpenDialogMock.mockResolvedValue({ canceled: false, filePaths: ['/tmp/growing.png'] }) + statMock.mockResolvedValue({ size: 1 }) + readNodeFileWithinLimitMock.mockRejectedValue( + new NodeBoundedFileReader.NodeFileReadTooLargeError(256 * 1024 + 1, 256 * 1024) + ) + + const handler = getHandler('shell:pickRepoIconImage') + await expect(handler({})).rejects.toThrow('Repo icon image must be 256KB or smaller.') + }) + it('picks an existing directory without enabling native directory creation', async () => { showOpenDialogMock.mockResolvedValue({ canceled: false, diff --git a/src/main/ipc/shell.ts b/src/main/ipc/shell.ts index 760b32eed2a..60a04205447 100644 --- a/src/main/ipc/shell.ts +++ b/src/main/ipc/shell.ts @@ -1,14 +1,13 @@ import { ipcMain, shell, dialog } from 'electron' import { spawn } from 'node:child_process' -import { constants, copyFile, readFile, stat } from 'node:fs/promises' -import { basename, extname, isAbsolute, normalize, posix, win32 } from 'node:path' +import { constants, copyFile, stat } from 'node:fs/promises' +import { isAbsolute, normalize, posix, win32 } from 'node:path' import { fileURLToPath } from 'node:url' import type { ShellOpenExternalEditorRequest, ShellOpenExternalEditorResult, ShellOpenLocalPathResult } from '../../shared/shell-open-types' -import { MAX_REPO_ICON_UPLOAD_BYTES } from '../../shared/repo-icon' import type { Store } from '../persistence' import { getSpawnArgsForWindows } from '../win32-utils' import { @@ -18,13 +17,10 @@ import { type ExternalEditorLaunchSpec } from '../external-editor-launch' import { resolveVsCodeSshAuthority } from '../ssh/vscode-ssh-authority' +import { pickRepoIconImage } from './shell-repo-icon-picker' export { EXTERNAL_EDITOR_CLI_COMMAND } -const REPO_ICON_IMAGE_MIME_TYPES: Record = { - '.png': 'image/png' -} - async function pathExists(pathValue: string): Promise { try { await stat(pathValue) @@ -296,36 +292,7 @@ export function registerShellHandlers(store: Store): void { return result.filePaths[0] }) - ipcMain.handle( - 'shell:pickRepoIconImage', - async (): Promise<{ dataUrl: string; fileName: string } | null> => { - const result = await dialog.showOpenDialog({ - properties: ['openFile'], - filters: [{ name: 'Repo icon images', extensions: ['png'] }] - }) - if (result.canceled || result.filePaths.length === 0) { - return null - } - - const filePath = result.filePaths[0] - const extension = extname(filePath).toLowerCase() - const mimeType = REPO_ICON_IMAGE_MIME_TYPES[extension] - if (!mimeType) { - throw new Error('Repo icons must be PNG files.') - } - - const stats = await stat(filePath) - if (stats.size > MAX_REPO_ICON_UPLOAD_BYTES) { - throw new Error('Repo icon image must be 256KB or smaller.') - } - - const buffer = await readFile(filePath) - return { - dataUrl: `data:${mimeType};base64,${buffer.toString('base64')}`, - fileName: basename(filePath) - } - } - ) + ipcMain.handle('shell:pickRepoIconImage', pickRepoIconImage) ipcMain.handle('shell:pickAudio', async (): Promise => { const result = await dialog.showOpenDialog({ diff --git a/src/main/ipc/speech-ipc-admission.ts b/src/main/ipc/speech-ipc-admission.ts new file mode 100644 index 00000000000..d9e0f380e83 --- /dev/null +++ b/src/main/ipc/speech-ipc-admission.ts @@ -0,0 +1,152 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MAX_SPEECH_AUDIO_CHUNK_BYTES = 1024 * 1024 +export const MAX_SPEECH_HOTWORDS = 256 +export const MAX_SPEECH_HOTWORD_BYTES = 4 * 1024 +export const MAX_SPEECH_HOTWORDS_TOTAL_BYTES = 256 * 1024 +export const MAX_SPEECH_SESSION_ID_BYTES = 1024 +export const MAX_SPEECH_MODEL_ID_BYTES = 1024 +export const MAX_SPEECH_OPENAI_API_KEY_BYTES = 64 * 1024 +export const MAX_PENDING_DESKTOP_DICTATION_STARTS = 16 +export const MAX_ACTIVE_DESKTOP_DICTATION_LISTENERS = 16 + +const MIN_SPEECH_SAMPLE_RATE = 8_000 +const MAX_SPEECH_SAMPLE_RATE = 384_000 +const HOTWORD_LINE_SUFFIX = ' :2.0\n' + +export type DesktopDictationListener = { release: () => void } + +function requireBoundedString( + value: unknown, + name: string, + maxBytes: number, + options: { allowEmpty?: boolean } = {} +): string { + if ( + typeof value !== 'string' || + (!options.allowEmpty && value.length === 0) || + measureUtf8ByteLength(value, { stopAfterBytes: maxBytes }).exceededLimit + ) { + throw new Error(`Invalid ${name}`) + } + return value +} + +export function validateSpeechSessionId(value: unknown): string { + return requireBoundedString( + value === undefined ? 'desktop' : value, + 'speech session id', + MAX_SPEECH_SESSION_ID_BYTES + ) +} + +export function validateSpeechModelId(value: unknown): string { + return requireBoundedString(value, 'speech model id', MAX_SPEECH_MODEL_ID_BYTES) +} + +export function validateOpenAiSpeechApiKey(value: unknown): string { + return requireBoundedString(value, 'OpenAI speech API key', MAX_SPEECH_OPENAI_API_KEY_BYTES) +} + +export function buildSpeechHotwordsContent(value: unknown): string | undefined { + if (value === undefined) { + return undefined + } + if (!Array.isArray(value) || value.length > MAX_SPEECH_HOTWORDS) { + throw new Error('Invalid speech hotwords') + } + const lines: string[] = [] + let retainedBytes = 0 + for (const hotword of value) { + const text = requireBoundedString(hotword, 'speech hotword', MAX_SPEECH_HOTWORD_BYTES, { + allowEmpty: true + }) + const lineBytes = measureUtf8ByteLength(text).byteLength + HOTWORD_LINE_SUFFIX.length + if (lineBytes > MAX_SPEECH_HOTWORDS_TOTAL_BYTES - retainedBytes) { + throw new Error('Speech hotwords are too large') + } + retainedBytes += lineBytes + lines.push(`${text}${HOTWORD_LINE_SUFFIX}`) + } + return lines.length > 0 ? lines.join('') : undefined +} + +export function decodeSpeechAudioChunk( + value: unknown, + sampleRate: unknown +): { samples: Float32Array; sampleRate: number } { + if ( + !(value instanceof Uint8Array) || + value.byteLength === 0 || + value.byteLength > MAX_SPEECH_AUDIO_CHUNK_BYTES || + value.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0 + ) { + throw new Error('Invalid speech audio chunk') + } + if ( + typeof sampleRate !== 'number' || + !Number.isFinite(sampleRate) || + sampleRate < MIN_SPEECH_SAMPLE_RATE || + sampleRate > MAX_SPEECH_SAMPLE_RATE + ) { + throw new Error('Invalid speech sample rate') + } + // Why: worker transfer moves the whole backing buffer; own only the admitted view. + const ownedBytes = new Uint8Array(value.byteLength) + ownedBytes.set(value) + return { samples: new Float32Array(ownedBytes.buffer), sampleRate } +} + +export class SpeechIpcAdmission { + private readonly pendingStarts = new Set() + private readonly activeListeners = new Map() + + claimStart(owner: string): void { + if ( + this.pendingStarts.has(owner) || + this.pendingStarts.size >= MAX_PENDING_DESKTOP_DICTATION_STARTS + ) { + throw new Error('Too many pending speech dictation starts') + } + this.pendingStarts.add(owner) + } + + releaseStart(owner: string): void { + this.pendingStarts.delete(owner) + } + + commitListener(owner: string, listener: DesktopDictationListener): void { + const previous = this.activeListeners.get(owner) + if (!previous && this.activeListeners.size >= MAX_ACTIVE_DESKTOP_DICTATION_LISTENERS) { + this.activeListeners.values().next().value?.release() + } + this.activeListeners.set(owner, listener) + previous?.release() + } + + deleteListenerIfCurrent(owner: string, listener: DesktopDictationListener): void { + if (this.activeListeners.get(owner) === listener) { + this.activeListeners.delete(owner) + } + } + + releaseListener(owner: string): void { + this.activeListeners.get(owner)?.release() + } + + reset(): void { + this.pendingStarts.clear() + for (const listener of Array.from(this.activeListeners.values())) { + listener.release() + } + this.activeListeners.clear() + } + + get pendingStartCount(): number { + return this.pendingStarts.size + } + + get activeListenerCount(): number { + return this.activeListeners.size + } +} diff --git a/src/main/ipc/speech.test.ts b/src/main/ipc/speech.test.ts index 2e84124b8e9..f3587b947e5 100644 --- a/src/main/ipc/speech.test.ts +++ b/src/main/ipc/speech.test.ts @@ -43,18 +43,46 @@ vi.mock('../speech/speech-model-deletion', () => ({ deleteLocalSpeechModel: deleteLocalSpeechModelMock })) -import { registerSpeechHandlers } from './speech' +import { + clearSpeechIpcAdmissionForTests, + getActiveDesktopDictationListenerCountForTest, + getPendingDesktopDictationStartCountForTest, + registerSpeechHandlers +} from './speech' +import { + MAX_PENDING_DESKTOP_DICTATION_STARTS, + MAX_SPEECH_AUDIO_CHUNK_BYTES, + MAX_SPEECH_HOTWORD_BYTES, + MAX_SPEECH_HOTWORDS, + MAX_SPEECH_SESSION_ID_BYTES +} from './speech-ipc-admission' type SpeechDownloadHandler = (event: { sender: { id: number } }, modelId: string) => Promise -function getHandler(channel: string): SpeechDownloadHandler { +function getHandler(channel: string): T { const call = handleMock.mock.calls.find((entry) => entry[0] === channel) if (!call) { throw new Error(`${channel} handler not registered`) } - return call[1] as SpeechDownloadHandler + return call[1] as T } +type SpeechFeedHandler = ( + event: { sender: { id: number } }, + buffer: Uint8Array, + sampleRate: number, + sessionId?: string +) => Promise + +type SpeechStartHandler = ( + event: { sender: { id: number } }, + modelId: string, + hotwords?: string[], + sessionId?: string +) => Promise + +type SpeechStopHandler = (event: { sender: { id: number } }, sessionId?: string) => Promise + describe('registerSpeechHandlers', () => { beforeEach(() => { handleMock.mockReset() @@ -62,6 +90,7 @@ describe('registerSpeechHandlers', () => { getSpeechModelManagerMock.mockReset() getSpeechSttServiceMock.mockReset() deleteLocalSpeechModelMock.mockReset() + clearSpeechIpcAdmissionForTests() }) it('clears the model download progress callback after completion', async () => { @@ -156,4 +185,151 @@ describe('registerSpeechHandlers', () => { modelId: 'model-1' }) }) + + it('rejects oversized or malformed audio chunks before reaching the STT service', async () => { + const feedAudio = vi.fn() + getSpeechSttServiceMock.mockReturnValue({ feedAudio }) + registerSpeechHandlers({} as never) + const handler = getHandler('speech:feedAudio') + + await expect( + handler( + { sender: { id: 7 } }, + new Uint8Array(MAX_SPEECH_AUDIO_CHUNK_BYTES + 4), + 48_000, + 'session' + ) + ).rejects.toThrow('Invalid speech audio chunk') + await expect( + handler({ sender: { id: 7 } }, new Uint8Array(3), 48_000, 'session') + ).rejects.toThrow('Invalid speech audio chunk') + expect(feedAudio).not.toHaveBeenCalled() + }) + + it('copies an admitted audio view into an exact-size transferable buffer', async () => { + const feedAudio = vi.fn() + getSpeechSttServiceMock.mockReturnValue({ feedAudio }) + registerSpeechHandlers({} as never) + const handler = getHandler('speech:feedAudio') + const oversizedBacking = new Uint8Array(MAX_SPEECH_AUDIO_CHUNK_BYTES + 128) + const admittedView = oversizedBacking.subarray(64, 64 + MAX_SPEECH_AUDIO_CHUNK_BYTES) + + await handler({ sender: { id: 8 } }, admittedView, 48_000, 'session') + + const samples = feedAudio.mock.calls[0]?.[0] as Float32Array + expect(samples.byteLength).toBe(MAX_SPEECH_AUDIO_CHUNK_BYTES) + expect(samples.buffer.byteLength).toBe(MAX_SPEECH_AUDIO_CHUNK_BYTES) + expect(feedAudio).toHaveBeenCalledWith(samples, 48_000, 'desktop:8:session') + }) + + it.each([ + ['count', Array.from({ length: MAX_SPEECH_HOTWORDS + 1 }, () => 'word')], + ['per-word bytes', ['😀'.repeat(Math.floor(MAX_SPEECH_HOTWORD_BYTES / 4) + 1)]], + ['aggregate bytes', Array.from({ length: 64 }, () => 'x'.repeat(MAX_SPEECH_HOTWORD_BYTES))] + ])('rejects hotword %s overflow before window or service retention', async (_label, hotwords) => { + registerSpeechHandlers({} as never) + const handler = getHandler('speech:startDictation') + + await expect(handler({ sender: { id: 9 } }, 'model-1', hotwords, 'session')).rejects.toThrow( + /hotword/i + ) + expect(fromWebContentsMock).not.toHaveBeenCalled() + expect(getSpeechSttServiceMock).not.toHaveBeenCalled() + }) + + it('rejects oversized session ids for start, feed, and stop before service work', async () => { + registerSpeechHandlers({} as never) + const oversizedSession = '😀'.repeat(Math.floor(MAX_SPEECH_SESSION_ID_BYTES / 4) + 1) + + await expect( + getHandler('speech:startDictation')( + { sender: { id: 10 } }, + 'model-1', + undefined, + oversizedSession + ) + ).rejects.toThrow('Invalid speech session id') + await expect( + getHandler('speech:feedAudio')( + { sender: { id: 10 } }, + new Uint8Array(4), + 16_000, + oversizedSession + ) + ).rejects.toThrow('Invalid speech session id') + await expect( + getHandler('speech:stopDictation')( + { sender: { id: 10 } }, + oversizedSession + ) + ).rejects.toThrow('Invalid speech session id') + expect(getSpeechSttServiceMock).not.toHaveBeenCalled() + }) + + it('caps concurrent desktop dictation starts and releases admission after settlement', async () => { + let resolveStarts = (): void => {} + const startGate = new Promise((resolve) => { + resolveStarts = resolve + }) + const startDictation = vi.fn(() => startGate) + getSpeechSttServiceMock.mockReturnValue({ startDictation, stopDictation: vi.fn() }) + fromWebContentsMock.mockImplementation((sender: { id: number }) => ({ + isDestroyed: vi.fn(() => false), + webContents: { send: vi.fn() }, + once: vi.fn(), + off: vi.fn(), + sender + })) + registerSpeechHandlers({} as never) + const handler = getHandler('speech:startDictation') + const starts = Array.from({ length: MAX_PENDING_DESKTOP_DICTATION_STARTS }, (_, index) => + handler({ sender: { id: 100 + index } }, 'model-1', undefined, 'session') + ) + + await vi.waitFor(() => + expect(getPendingDesktopDictationStartCountForTest()).toBe( + MAX_PENDING_DESKTOP_DICTATION_STARTS + ) + ) + await expect(handler({ sender: { id: 999 } }, 'model-1', undefined, 'session')).rejects.toThrow( + 'Too many pending speech dictation starts' + ) + resolveStarts() + await Promise.all(starts) + + expect(startDictation).toHaveBeenCalledTimes(MAX_PENDING_DESKTOP_DICTATION_STARTS) + expect(getPendingDesktopDictationStartCountForTest()).toBe(0) + }) + + it('replaces the retained window listener for a sequential same-owner restart', async () => { + const stopDictation = vi.fn().mockResolvedValue(undefined) + getSpeechSttServiceMock.mockReturnValue({ + startDictation: vi.fn().mockResolvedValue(undefined), + stopDictation + }) + const firstWindow = { + isDestroyed: vi.fn(() => false), + webContents: { send: vi.fn() }, + once: vi.fn(), + off: vi.fn() + } + const secondWindow = { + isDestroyed: vi.fn(() => false), + webContents: { send: vi.fn() }, + once: vi.fn(), + off: vi.fn() + } + fromWebContentsMock.mockReturnValueOnce(firstWindow).mockReturnValueOnce(secondWindow) + registerSpeechHandlers({} as never) + const start = getHandler('speech:startDictation') + + await start({ sender: { id: 200 } }, 'model-1', undefined, 'session') + await start({ sender: { id: 200 } }, 'model-1', undefined, 'session') + + expect(getActiveDesktopDictationListenerCountForTest()).toBe(1) + expect(firstWindow.off).toHaveBeenCalledWith('closed', expect.any(Function)) + await getHandler('speech:stopDictation')({ sender: { id: 200 } }, 'session') + expect(secondWindow.off).toHaveBeenCalledWith('closed', expect.any(Function)) + expect(getActiveDesktopDictationListenerCountForTest()).toBe(0) + }) }) diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index f89a2ab8b70..572493c9deb 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -11,6 +11,28 @@ import { saveOpenAiSpeechApiKey } from '../speech/openai-api-key-store' import type { Store } from '../persistence' +import { + buildSpeechHotwordsContent, + decodeSpeechAudioChunk, + SpeechIpcAdmission, + validateOpenAiSpeechApiKey, + validateSpeechModelId, + validateSpeechSessionId +} from './speech-ipc-admission' + +const speechAdmission = new SpeechIpcAdmission() + +export function clearSpeechIpcAdmissionForTests(): void { + speechAdmission.reset() +} + +export function getPendingDesktopDictationStartCountForTest(): number { + return speechAdmission.pendingStartCount +} + +export function getActiveDesktopDictationListenerCountForTest(): number { + return speechAdmission.activeListenerCount +} export function registerSpeechHandlers(store: Store): void { ipcMain.handle('speech:getCatalog', () => { @@ -26,7 +48,7 @@ export function registerSpeechHandlers(store: Store): void { }) ipcMain.handle('speech:saveOpenAiApiKey', async (_event, apiKey: string) => { - saveOpenAiSpeechApiKey(apiKey) + saveOpenAiSpeechApiKey(validateOpenAiSpeechApiKey(apiKey)) return { configured: true } }) @@ -36,6 +58,7 @@ export function registerSpeechHandlers(store: Store): void { }) ipcMain.handle('speech:downloadModel', async (event, modelId: string) => { + const validatedModelId = validateSpeechModelId(modelId) const manager = getSpeechModelManager(store) const window = BrowserWindow.fromWebContents(event.sender) if (!window) { @@ -59,22 +82,23 @@ export function registerSpeechHandlers(store: Store): void { } window.once('closed', cleanupProgressCallback) try { - await manager.downloadModel(modelId) + await manager.downloadModel(validatedModelId) } finally { cleanupProgressCallback() } }) ipcMain.handle('speech:cancelDownload', async (_event, modelId: string) => { - getSpeechModelManager(store).cancelDownload(modelId) + getSpeechModelManager(store).cancelDownload(validateSpeechModelId(modelId)) }) ipcMain.handle('speech:deleteModel', async (_event, modelId: string) => { + const validatedModelId = validateSpeechModelId(modelId) await deleteLocalSpeechModel({ store, modelManager: getSpeechModelManager(store), sttService: getSpeechSttService(store), - modelId + modelId: validatedModelId }) }) @@ -91,15 +115,21 @@ export function registerSpeechHandlers(store: Store): void { ipcMain.handle( 'speech:startDictation', async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => { + const validatedModelId = validateSpeechModelId(modelId) + const validatedSessionId = validateSpeechSessionId(sessionId) + const hotwordsContent = buildSpeechHotwordsContent(hotwords) const window = BrowserWindow.fromWebContents(event.sender) if (!window) { return } let resolvedHotwordsPath: string | undefined let windowClosed = false - const owner = getDesktopOwner(event.sender.id, sessionId) + let sessionListenerReleased = false + const owner = getDesktopOwner(event.sender.id, validatedSessionId) + speechAdmission.claimStart(owner) const cleanupOnWindowClosed = (): void => { windowClosed = true + cleanupSessionListener() void getSpeechSttService(store) .stopDictation(owner) .finally(() => { @@ -110,8 +140,14 @@ export function registerSpeechHandlers(store: Store): void { .catch(() => {}) } const cleanupSessionListener = (): void => { + if (sessionListenerReleased) { + return + } + sessionListenerReleased = true window.off('closed', cleanupOnWindowClosed) + speechAdmission.deleteListenerIfCurrent(owner, sessionListener) } + const sessionListener = { release: cleanupSessionListener } window.once('closed', cleanupOnWindowClosed) try { @@ -133,10 +169,9 @@ export function registerSpeechHandlers(store: Store): void { } } - if (hotwords && hotwords.length > 0) { - const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n` - const hotwordsFilePath = getHotwordsFilePath(content) - await writeFile(hotwordsFilePath, content, 'utf-8') + if (hotwordsContent) { + const hotwordsFilePath = getHotwordsFilePath(hotwordsContent) + await writeFile(hotwordsFilePath, hotwordsContent, 'utf-8') resolvedHotwordsPath = hotwordsFilePath } @@ -149,27 +184,36 @@ export function registerSpeechHandlers(store: Store): void { } await getSpeechSttService(store).startDictation( - modelId, + validatedModelId, (msg) => { if (window.isDestroyed()) { return } switch (msg.type) { case 'ready': - window.webContents.send('speech:ready', { sessionId }) + window.webContents.send('speech:ready', { sessionId: validatedSessionId }) break case 'partial': - window.webContents.send('speech:partial', { text: msg.text ?? '', sessionId }) + window.webContents.send('speech:partial', { + text: msg.text ?? '', + sessionId: validatedSessionId + }) break case 'final': - window.webContents.send('speech:final', { text: msg.text ?? '', sessionId }) + window.webContents.send('speech:final', { + text: msg.text ?? '', + sessionId: validatedSessionId + }) break case 'stopped': cleanupSessionListener() - window.webContents.send('speech:stopped', { sessionId }) + window.webContents.send('speech:stopped', { sessionId: validatedSessionId }) break case 'error': - window.webContents.send('speech:error', { error: msg.error ?? '', sessionId }) + window.webContents.send('speech:error', { + error: msg.error ?? '', + sessionId: validatedSessionId + }) void getSpeechSttService(store) .stopDictation(owner) .catch(() => undefined) @@ -180,6 +224,14 @@ export function registerSpeechHandlers(store: Store): void { resolvedHotwordsPath, owner ) + if (windowClosed || window.isDestroyed() || sessionListenerReleased) { + cleanupSessionListener() + if (resolvedHotwordsPath) { + unlink(resolvedHotwordsPath).catch(() => {}) + } + return + } + speechAdmission.commitListener(owner, sessionListener) if (resolvedHotwordsPath) { unlink(resolvedHotwordsPath).catch(() => {}) } @@ -189,25 +241,33 @@ export function registerSpeechHandlers(store: Store): void { unlink(resolvedHotwordsPath).catch(() => {}) } throw err + } finally { + speechAdmission.releaseStart(owner) } } ) ipcMain.handle( 'speech:feedAudio', - async (_event, buffer: Buffer, sampleRate: number, sessionId = 'desktop') => { + async (_event, buffer: Uint8Array, sampleRate: number, sessionId = 'desktop') => { // Why: the preload sends audio as a Buffer to avoid Float32Array data // being zeroed out during contextBridge + IPC serialization. - const samples = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + const audio = decodeSpeechAudioChunk(buffer, sampleRate) + const validatedSessionId = validateSpeechSessionId(sessionId) getSpeechSttService(store).feedAudio( - samples, - sampleRate, - getDesktopOwner(_event.sender.id, sessionId) + audio.samples, + audio.sampleRate, + getDesktopOwner(_event.sender.id, validatedSessionId) ) } ) ipcMain.handle('speech:stopDictation', async (_event, sessionId = 'desktop') => { - await getSpeechSttService(store).stopDictation(getDesktopOwner(_event.sender.id, sessionId)) + const owner = getDesktopOwner(_event.sender.id, validateSpeechSessionId(sessionId)) + try { + await getSpeechSttService(store).stopDictation(owner) + } finally { + speechAdmission.releaseListener(owner) + } }) } diff --git a/src/main/ipc/ssh-browse.test.ts b/src/main/ipc/ssh-browse.test.ts index 6d04e8c7b0f..64f4e81c94c 100644 --- a/src/main/ipc/ssh-browse.test.ts +++ b/src/main/ipc/ssh-browse.test.ts @@ -1,6 +1,10 @@ import { EventEmitter } from 'node:events' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { registerSshBrowseHandler } from './ssh-browse' +import { registerSshBrowseHandler, type RemoteDirEntry } from './ssh-browse' +import { + FILESYSTEM_DIRECTORY_LIMIT_MESSAGE, + FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES +} from '../../shared/filesystem-directory-listing-limit' const { handleMock, removeHandlerMock } = vi.hoisted(() => ({ handleMock: vi.fn(), @@ -77,6 +81,68 @@ describe('registerSshBrowseHandler', () => { expect(channel.stderr.listenerCount('error')).toBe(0) }) + it('closes and rejects a remote listing that exceeds the shared byte limit', async () => { + const channel = Object.assign(createMockChannel(), { close: vi.fn() }) + const exec = vi.fn().mockResolvedValue(channel) + const getConnectionManager = () => ({ getConnection: () => ({ exec }) }) + registerSshBrowseHandler(getConnectionManager as never) + + const resultPromise = handler(null, { targetId: 'ssh-1', dirPath: '~' }) + await Promise.resolve() + channel.emit('data', Buffer.alloc(FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES + 1, 0x78)) + + await expect(resultPromise).rejects.toThrow(FILESYSTEM_DIRECTORY_LIMIT_MESSAGE) + expect(channel.close).toHaveBeenCalledOnce() + expect(channel.listenerCount('data')).toBe(0) + expect(channel.stderr.listenerCount('data')).toBe(0) + }) + + it('retains tens of thousands of tiny stdout chunks without quadratic copying', async () => { + const channel = createMockChannel() + const exec = vi.fn().mockResolvedValue(channel) + const getConnectionManager = () => ({ getConnection: () => ({ exec }) }) + registerSshBrowseHandler(getConnectionManager as never) + + const resultPromise = handler(null, { targetId: 'ssh-1', dirPath: '~' }) + await Promise.resolve() + channel.emit('data', Buffer.from('/home/user\n')) + for (let index = 0; index < 65_000; index += 1) { + channel.emit('data', Buffer.from(`f${String(index).padStart(5, '0')}\n`)) + } + channel.emit('exit', 0) + channel.emit('close') + + const result = (await resultPromise) as { + resolvedPath: string + entries: RemoteDirEntry[] + } + expect(result.resolvedPath).toBe('/home/user') + expect(result.entries).toHaveLength(65_000) + expect(result.entries[0]).toEqual({ name: 'f00000', isDirectory: false }) + expect(result.entries.at(-1)).toEqual({ name: 'f64999', isDirectory: false }) + }) + + it('decodes a filename whose UTF-8 bytes span stdout chunks', async () => { + const channel = createMockChannel() + const exec = vi.fn().mockResolvedValue(channel) + const getConnectionManager = () => ({ getConnection: () => ({ exec }) }) + registerSshBrowseHandler(getConnectionManager as never) + + const resultPromise = handler(null, { targetId: 'ssh-1', dirPath: '~' }) + await Promise.resolve() + const output = Buffer.from('/home/user\nemoji-🙂.txt\n') + const emojiOffset = output.indexOf(Buffer.from('🙂')) + channel.emit('data', output.subarray(0, emojiOffset + 1)) + channel.emit('data', output.subarray(emojiOffset + 1)) + channel.emit('exit', 0) + channel.emit('close') + + await expect(resultPromise).resolves.toEqual({ + resolvedPath: '/home/user', + entries: [{ name: 'emoji-🙂.txt', isDirectory: false }] + }) + }) + it('escapes remote browse paths before invoking command ls', async () => { const channel = createMockChannel() const exec = vi.fn().mockResolvedValue(channel) diff --git a/src/main/ipc/ssh-browse.ts b/src/main/ipc/ssh-browse.ts index 964c877a557..f8539f63183 100644 --- a/src/main/ipc/ssh-browse.ts +++ b/src/main/ipc/ssh-browse.ts @@ -2,6 +2,14 @@ import { ipcMain } from 'electron' import type { SshConnectionManager } from '../ssh/ssh-connection' import type { SshExecOptions } from '../ssh/ssh-connection-utils' import { powerShellCommand, powerShellLiteral } from '../ssh/ssh-remote-powershell' +import { + createFilesystemDirectoryLimitState, + FILESYSTEM_DIRECTORY_LIMIT_MESSAGE, + FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES, + trackFilesystemDirectoryEntry +} from '../../shared/filesystem-directory-listing-limit' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' +import { SystemSshOutputTail } from '../ssh/system-ssh-output-tail' export type RemoteDirEntry = { name: string @@ -102,8 +110,8 @@ async function runBrowseCommand( const channel = options ? await conn.exec(command, options) : await conn.exec(command) return new Promise((resolve, reject) => { - let stdout = '' - let stderr = '' + const stdout = new GrowingByteBuffer() + const stderr = new SystemSshOutputTail() let exitCode: number | null = null let settled = false let timeout: ReturnType | null = null @@ -155,10 +163,16 @@ async function runBrowseCommand( } const onStdoutData = (data: Buffer): void => { - stdout += data.toString() + if (data.byteLength > FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES - stdout.byteLength) { + stdout.clear() + rejectOnce(new Error(FILESYSTEM_DIRECTORY_LIMIT_MESSAGE)) + closeChannel() + return + } + stdout.append(data) } const onStderrData = (data: Buffer): void => { - stderr += data.toString() + stderr.push(data) } // `exit` fires before `close`; capture the code to tell a failed `ls` (that still printed `pwd`) from an empty listing. const onExit = (code: number | null): void => { @@ -170,21 +184,24 @@ async function runBrowseCommand( const onClose = (): void => { // Why: a null exitCode (channel closed without exit status) isn't success; don't treat empty stdout as an empty dir. if (exitCode !== 0) { + const stderrText = stderr.toString() const msg = - stderr.trim() || + stderrText.trim() || (exitCode === null ? 'Remote listing failed (channel closed without exit status)' : `Remote listing failed (exit ${exitCode})`) rejectOnce(new RemoteBrowseError(msg, exitCode)) return } - if (stderr.trim() && !stdout.trim()) { - rejectOnce(new Error(stderr.trim())) + const stderrText = stderr.toString() + const stdoutText = stdout.toString('utf8') + if (stderrText.trim() && !stdoutText.trim()) { + rejectOnce(new Error(stderrText.trim())) return } // Why: Windows OpenSSH exec emits CRLF; split on \r?\n so a trailing \r doesn't defeat the endsWith('/') dir check or leave a stray CR in names. - const lines = stdout.trim().split(/\r?\n/) + const lines = stdoutText.trim().split(/\r?\n/) if (lines.length === 0) { rejectOnce(new Error('Empty response from remote')) return @@ -192,16 +209,24 @@ async function runBrowseCommand( const resolvedPath = lines[0] const entries: RemoteDirEntry[] = [] + const listingLimit = createFilesystemDirectoryLimitState() for (let i = 1; i < lines.length; i++) { const line = lines[i] if (!line || line === './' || line === '../') { continue } + const name = line.endsWith('/') ? line.slice(0, -1) : line + try { + trackFilesystemDirectoryEntry(listingLimit, { name }) + } catch (error) { + rejectOnce(error instanceof Error ? error : new Error(String(error))) + return + } if (line.endsWith('/')) { - entries.push({ name: line.slice(0, -1), isDirectory: true }) + entries.push({ name, isDirectory: true }) } else { - entries.push({ name: line, isDirectory: false }) + entries.push({ name, isDirectory: false }) } } diff --git a/src/main/ipc/ssh-passphrase.test.ts b/src/main/ipc/ssh-passphrase.test.ts new file mode 100644 index 00000000000..538a648c45d --- /dev/null +++ b/src/main/ipc/ssh-passphrase.test.ts @@ -0,0 +1,179 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getUtf8ByteLength } from '../../shared/utf8-byte-limits' +import { + SSH_CREDENTIAL_DETAIL_MAX_UTF8_BYTES, + SSH_RETAINED_IDENTIFIER_MAX_UTF8_BYTES +} from '../../shared/ssh-retained-payload-admission' + +const electronMocks = vi.hoisted(() => { + const handlers = new Map unknown>() + return { + handlers, + ipcMain: { + removeHandler: vi.fn((channel: string) => handlers.delete(channel)), + handle: vi.fn((channel: string, handler: (event: unknown, args: unknown) => unknown) => { + handlers.set(channel, handler) + }) + } + } +}) + +vi.mock('electron', () => ({ ipcMain: electronMocks.ipcMain })) + +import { + getPendingCredentialRequestCountForTests, + registerCredentialHandler, + requestCredential, + resetPendingCredentialRequestsForTests, + SSH_CREDENTIAL_VALUE_MAX_UTF8_BYTES, + SSH_MAX_PENDING_CREDENTIAL_REQUESTS +} from './ssh-passphrase' + +function createWindow() { + return { + isDestroyed: vi.fn(() => false), + webContents: { send: vi.fn() } + } +} + +describe('SSH credential request admission', () => { + beforeEach(() => { + vi.useFakeTimers() + electronMocks.handlers.clear() + resetPendingCredentialRequestsForTests() + }) + + afterEach(() => { + resetPendingCredentialRequestsForTests() + vi.useRealTimers() + }) + + it('forwards ordinary metadata and resolves the submitted credential', async () => { + const win = createWindow() + registerCredentialHandler(() => win as never) + const result = requestCredential(() => win as never, 'ssh-a', 'password', 'example.test') + const request = win.webContents.send.mock.calls[0][1] as { requestId: string } + + electronMocks.handlers.get('ssh:submitCredential')?.( + {}, + { + requestId: request.requestId, + value: 'secret' + } + ) + + await expect(result).resolves.toBe('secret') + expect(win.webContents.send).toHaveBeenNthCalledWith(1, 'ssh:credential-request', { + requestId: request.requestId, + targetId: 'ssh-a', + kind: 'password', + detail: 'example.test' + }) + expect(win.webContents.send).toHaveBeenNthCalledWith(2, 'ssh:credential-resolved', { + requestId: request.requestId + }) + }) + + it('caps detail before sending it to the renderer', () => { + const win = createWindow() + + void requestCredential( + () => win as never, + 'ssh-a', + 'passphrase', + '🙂'.repeat(SSH_CREDENTIAL_DETAIL_MAX_UTF8_BYTES) + ) + + const sent = win.webContents.send.mock.calls[0][1] as { detail: string } + expect(getUtf8ByteLength(sent.detail)).toBeLessThanOrEqual(SSH_CREDENTIAL_DETAIL_MAX_UTF8_BYTES) + }) + + it('rejects an oversized submitted credential without retaining the request', async () => { + const win = createWindow() + registerCredentialHandler(() => win as never) + const result = requestCredential(() => win as never, 'ssh-a', 'password', 'example.test') + const request = win.webContents.send.mock.calls[0][1] as { requestId: string } + + electronMocks.handlers.get('ssh:submitCredential')?.( + {}, + { + requestId: request.requestId, + value: '🙂'.repeat(SSH_CREDENTIAL_VALUE_MAX_UTF8_BYTES) + } + ) + + await expect(result).resolves.toBeNull() + expect(getPendingCredentialRequestCountForTests()).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('settles a submitted credential when its resolution notification throws', async () => { + const win = createWindow() + registerCredentialHandler(() => win as never) + const result = requestCredential(() => win as never, 'ssh-a', 'password', 'example.test') + const request = win.webContents.send.mock.calls[0][1] as { requestId: string } + win.webContents.send.mockImplementation(() => { + throw new Error('renderer gone') + }) + + electronMocks.handlers.get('ssh:submitCredential')?.( + {}, + { + requestId: request.requestId, + value: 'secret' + } + ) + + await expect(result).resolves.toBe('secret') + expect(getPendingCredentialRequestCountForTests()).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('settles a timed-out credential when its resolution notification throws', async () => { + const win = createWindow() + const result = requestCredential(() => win as never, 'ssh-a', 'password', 'example.test') + win.webContents.send.mockImplementation(() => { + throw new Error('renderer gone') + }) + + await vi.runAllTimersAsync() + + await expect(result).resolves.toBeNull() + expect(getPendingCredentialRequestCountForTests()).toBe(0) + }) + + it('rejects an oversized target without allocating request state', async () => { + const win = createWindow() + const result = requestCredential( + () => win as never, + 'x'.repeat(SSH_RETAINED_IDENTIFIER_MAX_UTF8_BYTES + 1), + 'password', + 'host' + ) + + await expect(result).resolves.toBeNull() + expect(getPendingCredentialRequestCountForTests()).toBe(0) + expect(vi.getTimerCount()).toBe(0) + expect(win.webContents.send).not.toHaveBeenCalled() + }) + + it('fails closed at the pending-request cap without adding a timer or map row', async () => { + const win = createWindow() + const pending = Array.from({ length: SSH_MAX_PENDING_CREDENTIAL_REQUESTS }, (_, index) => + requestCredential(() => win as never, `ssh-${index}`, 'password', 'host') + ) + expect(getPendingCredentialRequestCountForTests()).toBe(SSH_MAX_PENDING_CREDENTIAL_REQUESTS) + expect(vi.getTimerCount()).toBe(SSH_MAX_PENDING_CREDENTIAL_REQUESTS) + + const overflow = requestCredential(() => win as never, 'ssh-overflow', 'password', 'host') + + await expect(overflow).resolves.toBeNull() + expect(getPendingCredentialRequestCountForTests()).toBe(SSH_MAX_PENDING_CREDENTIAL_REQUESTS) + expect(vi.getTimerCount()).toBe(SSH_MAX_PENDING_CREDENTIAL_REQUESTS) + expect(win.webContents.send).toHaveBeenCalledTimes(SSH_MAX_PENDING_CREDENTIAL_REQUESTS) + resetPendingCredentialRequestsForTests() + await expect(Promise.all(pending)).resolves.toEqual( + Array.from({ length: SSH_MAX_PENDING_CREDENTIAL_REQUESTS }, () => null) + ) + }) +}) diff --git a/src/main/ipc/ssh-passphrase.ts b/src/main/ipc/ssh-passphrase.ts index e1e21045373..f89fc60a247 100644 --- a/src/main/ipc/ssh-passphrase.ts +++ b/src/main/ipc/ssh-passphrase.ts @@ -1,8 +1,15 @@ import { ipcMain, type BrowserWindow } from 'electron' import { randomUUID } from 'node:crypto' import type { SshCredentialKind } from '../ssh/ssh-connection-utils' +import { + isSshRetainedIdentifier, + SSH_CREDENTIAL_DETAIL_MAX_UTF8_BYTES +} from '../../shared/ssh-retained-payload-admission' +import { clampUtf8TextPrefix, measureUtf8ByteLength } from '../../shared/utf8-byte-limits' const CREDENTIAL_TIMEOUT_MS = 120_000 +export const SSH_MAX_PENDING_CREDENTIAL_REQUESTS = 64 +export const SSH_CREDENTIAL_VALUE_MAX_UTF8_BYTES = 64 * 1024 const pendingRequests = new Map void }>() function notifyCredentialResolved( @@ -11,7 +18,11 @@ function notifyCredentialResolved( ): void { const win = getMainWindow() if (win && !win.isDestroyed()) { - win.webContents.send('ssh:credential-resolved', { requestId }) + try { + win.webContents.send('ssh:credential-resolved', { requestId }) + } catch { + // The SSH caller must still settle if its renderer disappears between checks. + } } } @@ -21,6 +32,16 @@ export function requestCredential( kind: SshCredentialKind, detail: string ): Promise { + const win = getMainWindow() + if ( + !isSshRetainedIdentifier(targetId) || + pendingRequests.size >= SSH_MAX_PENDING_CREDENTIAL_REQUESTS || + !win || + win.isDestroyed() + ) { + return Promise.resolve(null) + } + const retainedDetail = clampUtf8TextPrefix(detail, SSH_CREDENTIAL_DETAIL_MAX_UTF8_BYTES) const requestId = randomUUID() return new Promise((resolve) => { const timer = setTimeout(() => { @@ -37,13 +58,16 @@ export function requestCredential( } }) - const win = getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('ssh:credential-request', { requestId, targetId, kind, detail }) - } else { + try { + win.webContents.send('ssh:credential-request', { + requestId, + targetId, + kind, + detail: retainedDetail + }) + } catch { pendingRequests.delete(requestId) clearTimeout(timer) - notifyCredentialResolved(getMainWindow, requestId) resolve(null) } }) @@ -53,13 +77,36 @@ export function registerCredentialHandler(getMainWindow: () => BrowserWindow | n ipcMain.removeHandler('ssh:submitCredential') ipcMain.handle( 'ssh:submitCredential', - (_event, args: { requestId: string; value: string | null }) => { + (_event, args: { requestId?: unknown; value?: unknown }) => { + if (!args || typeof args !== 'object' || typeof args.requestId !== 'string') { + return + } const pending = pendingRequests.get(args.requestId) if (pending) { pendingRequests.delete(args.requestId) notifyCredentialResolved(getMainWindow, args.requestId) - pending.resolve(args.value) + const value = + args.value === null || + (typeof args.value === 'string' && + !measureUtf8ByteLength(args.value, { + stopAfterBytes: SSH_CREDENTIAL_VALUE_MAX_UTF8_BYTES + }).exceededLimit) + ? args.value + : null + pending.resolve(value) } } ) } + +export function resetPendingCredentialRequestsForTests(): void { + const pending = Array.from(pendingRequests.values()) + pendingRequests.clear() + for (const request of pending) { + request.resolve(null) + } +} + +export function getPendingCredentialRequestCountForTests(): number { + return pendingRequests.size +} diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index 909498882ee..45abd3a7bfb 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -18,7 +18,10 @@ const { mockPortForwardManager, mockPortScannerCallbacks, mockNextConnectionManagers, - mockNextPortForwardManagers + mockNextPortForwardManagers, + routeExternalPtyDataMock, + routeExternalPtyReplayMock, + routeExternalPtyExitMock } = vi.hoisted(() => ({ handleMock: vi.fn(), powerMonitorOffMock: vi.fn(), @@ -81,7 +84,10 @@ const { }, mockPortScannerCallbacks: new Map(), mockNextConnectionManagers: [] as unknown[], - mockNextPortForwardManagers: [] as unknown[] + mockNextPortForwardManagers: [] as unknown[], + routeExternalPtyDataMock: vi.fn(), + routeExternalPtyReplayMock: vi.fn(), + routeExternalPtyExitMock: vi.fn() })) vi.mock('electron', () => ({ @@ -167,6 +173,12 @@ vi.mock('./pty', () => ({ isRendererPtyOutputPaused: vi.fn().mockReturnValue(false) })) +vi.mock('./pty-renderer-delivery-router', () => ({ + routeExternalPtyData: routeExternalPtyDataMock, + routeExternalPtyReplay: routeExternalPtyReplayMock, + routeExternalPtyExit: routeExternalPtyExitMock +})) + vi.mock('../providers/ssh-filesystem-dispatch', () => ({ registerSshFilesystemProvider: vi.fn(), unregisterSshFilesystemProvider: vi.fn(), @@ -225,7 +237,11 @@ import { getSshPtyProvider, getPtyIdsForConnection } from './pty' -import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation' +import { + MAX_TRACKED_SSH_CONNECTION_GENERATIONS, + assertSshMutationExpectation, + retainSshConnectionGeneration +} from '../ssh/ssh-connection-generation' describe('SSH IPC handlers', () => { const handlers = new Map unknown>() @@ -334,6 +350,9 @@ describe('SSH IPC handlers', () => { mockPtyProvider.onReplay.mockReset() mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({}) mockPtyProvider.shutdown.mockReset() + routeExternalPtyDataMock.mockReset() + routeExternalPtyReplayMock.mockReset() + routeExternalPtyExitMock.mockReset() mockPortForwardManager.addForward.mockReset() mockPortForwardManager.updateForward.mockReset() mockPortForwardManager.removeForward.mockReset() @@ -481,6 +500,32 @@ describe('SSH IPC handlers', () => { ) }) + it('rejects the 4,097th retained SSH target before starting connection work', async () => { + const target: SshTarget = { + id: 'ssh-overflow', + label: 'Overflow', + host: 'example.com', + port: 22, + username: 'deploy' + } + const leases = Array.from({ length: MAX_TRACKED_SSH_CONNECTION_GENERATIONS }, (_, index) => + retainSshConnectionGeneration(`ssh-cap-${index}`) + ) + mockSshStore.getTarget.mockReturnValue(target) + + try { + await expect(handlers.get('ssh:connect')!(null, { targetId: target.id })).rejects.toThrow( + 'SSH connection generation target capacity exhausted' + ) + expect(mockConnectionManager.connect).not.toHaveBeenCalled() + expect(mockDeployAndLaunchRelay).not.toHaveBeenCalled() + } finally { + for (const lease of leases) { + lease.release() + } + } + }) + it('ssh:connect calls connection manager', async () => { const target: SshTarget = { id: 'ssh-1', @@ -848,7 +893,7 @@ describe('SSH IPC handlers', () => { } }) - it('forwards remote PTY events into the runtime', async () => { + it('forwards remote PTY events through shared renderer delivery', async () => { const runtime = { onPtyData: vi.fn(), onPtyExit: vi.fn() @@ -881,13 +926,12 @@ describe('SSH IPC handlers', () => { onData?.({ id: 'remote-pty', data: 'hello' }) onExit?.({ id: 'remote-pty', code: 7 }) - expect(runtime.onPtyData).toHaveBeenCalledWith( - 'remote-pty', - 'hello', - expect.any(Number), - 'hello'.length, - undefined - ) + expect(routeExternalPtyDataMock).toHaveBeenCalledWith({ + id: 'remote-pty', + data: 'hello' + }) + expect(routeExternalPtyExitMock).toHaveBeenCalledWith({ id: 'remote-pty', code: 7 }) + expect(runtime.onPtyData).not.toHaveBeenCalled() expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 7, undefined) }) @@ -1250,11 +1294,11 @@ describe('SSH IPC handlers', () => { connectionGeneration: 1 } }) - expect(secondWindow.webContents.send).toHaveBeenCalledWith( - 'pty:data', - expect.objectContaining({ id: 'remote-pty', data: 'hello' }) - ) - expect(secondWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { + expect(routeExternalPtyDataMock).toHaveBeenCalledWith({ + id: 'remote-pty', + data: 'hello' + }) + expect(routeExternalPtyExitMock).toHaveBeenCalledWith({ id: 'remote-pty', code: 9 }) @@ -1262,13 +1306,7 @@ describe('SSH IPC handlers', () => { targetId: 'ssh-1', ports: expect.arrayContaining([expect.objectContaining({ port: 3000 })]) }) - expect(secondRuntime.onPtyData).toHaveBeenCalledWith( - 'remote-pty', - 'hello', - expect.any(Number), - 'hello'.length, - undefined - ) + expect(secondRuntime.onPtyData).not.toHaveBeenCalled() expect(secondRuntime.onPtyExit).toHaveBeenCalledWith('remote-pty', 9, undefined) expect(firstRuntime.onPtyData).not.toHaveBeenCalled() expect(firstRuntime.onPtyExit).not.toHaveBeenCalled() diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index 16029e22a6a..d088aff842b 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -18,6 +18,7 @@ import type { } from '../../shared/ssh-types' import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' import { isRuntimeOwnedSshTargetId } from '../../shared/execution-host' +import { clampSshConnectionError } from '../../shared/ssh-retained-payload-admission' import { isAuthError } from '../ssh/ssh-connection-utils' import { forceStopRelayForTarget } from '../ssh/ssh-relay-reset' import { isSshPtyNotFoundError } from '../providers/ssh-pty-errors' @@ -42,7 +43,9 @@ import { advanceSshConnectionGeneration, getSshConnectionGeneration, initializeSshConnectionGenerationSession, - resetSshConnectionGenerations + resetSshConnectionGenerations, + retainSshConnectionGeneration, + type SshConnectionGenerationLease } from '../ssh/ssh-connection-generation' let sshStore: SshConnectionStore | null = null @@ -135,6 +138,24 @@ export async function removeRegisteredSshTarget(targetId: string): Promise // One session per SSH target owns the whole relay lifecycle (mux, providers, abort controller, state machine). const activeSessions = new Map() +const activeSessionGenerationLeases = new Map() + +function retainActiveSessionGeneration(targetId: string): boolean { + if (activeSessionGenerationLeases.has(targetId)) { + return false + } + activeSessionGenerationLeases.set(targetId, retainSshConnectionGeneration(targetId)) + return true +} + +function releaseActiveSessionGeneration(targetId: string): void { + const lease = activeSessionGenerationLeases.get(targetId) + if (!lease) { + return + } + activeSessionGenerationLeases.delete(targetId) + lease.release() +} export function getActiveSshAiVaultHostInfo(targetId: string): SshRelayAiVaultHostInfo | null { if (isRuntimeOwnedSshTargetId(targetId)) { @@ -173,6 +194,7 @@ async function teardownActiveSshSession( await portForwardManager?.removeAllForwards(targetId) teardown(session) activeSessions.delete(targetId) + releaseActiveSessionGeneration(targetId) clearRelayLostBackoff(targetId) clearRelayStateOverride(targetId) } @@ -184,7 +206,6 @@ function relayGracePeriodForTarget(target: SshTarget | null | undefined): number // Why: tabs must share one connect, while a disconnect must invalidate that // attempt so its late continuation cannot clobber a replacement. type ConnectAttempt = { - generation: number promise: Promise } @@ -263,6 +284,7 @@ function withSshRemotePlatform(targetId: string, state: SshConnectionState): Ssh const remotePlatform = activeSessions.get(targetId)?.getHostPlatform()?.os return { ...state, + error: clampSshConnectionError(state.error), connectionGeneration: currentConnectGeneration(targetId), ...(remotePlatform ? { remotePlatform } : {}) } @@ -804,31 +826,41 @@ export function registerSshHandlers( appendFileSync(e2eProbePath, `${JSON.stringify(targetId)}\n`) throw new Error('e2e_forbidden_local_ssh_connect') } - const observedGeneration = currentConnectGeneration(targetId) - const reset = resetRelayInFlight.get(targetId) - if (reset) { - await reset + const target = sshStore!.getTarget(targetId) + if (!target) { + throw new Error(`SSH target "${targetId}" not found`) } - - // Why: serialize concurrent ssh:connect for the same target; interleaved connects otherwise leak the first session. - const existing = connectInFlight.get(targetId) - if (existing) { - return existing.promise - } - if (currentConnectGeneration(targetId) !== observedGeneration) { - throw connectCancelledError() - } - - pendingTransportReconnects.delete(targetId) - const promise = doConnect(targetId) - const attempt = { generation: currentConnectGeneration(targetId), promise } - connectInFlight.set(targetId, attempt) + const generationLease = retainSshConnectionGeneration(targetId) + let observedGeneration = currentConnectGeneration(targetId) try { - return await promise - } finally { - if (connectInFlight.get(targetId) === attempt) { - connectInFlight.delete(targetId) + const reset = resetRelayInFlight.get(targetId) + if (reset) { + await reset + observedGeneration = currentConnectGeneration(targetId) } + + // Why: serialize concurrent ssh:connect for the same target; interleaved connects otherwise leak the first session. + const existing = connectInFlight.get(targetId) + if (existing) { + return existing.promise + } + if (currentConnectGeneration(targetId) !== observedGeneration) { + throw connectCancelledError() + } + + pendingTransportReconnects.delete(targetId) + const promise = doConnect(targetId) + const attempt = { promise } + connectInFlight.set(targetId, attempt) + try { + return await promise + } finally { + if (connectInFlight.get(targetId) === attempt) { + connectInFlight.delete(targetId) + } + } + } finally { + generationLease.release() } } @@ -862,7 +894,14 @@ export function registerSshHandlers( return getPublicSshState(targetId)! } - const generation = advanceSshConnectionGeneration(targetId) + const createdGenerationOwnership = retainActiveSessionGeneration(targetId) + const generation = createdGenerationOwnership + ? currentConnectGeneration(targetId) + : advanceSshConnectionGeneration(targetId) + if (generation === null) { + releaseActiveSessionGeneration(targetId) + throw connectCancelledError() + } clearRelayStateOverride(targetId) let conn // Why: tear down any existing session first to avoid leaking its multiplexer, providers, and timers (double-connect / reconnect-after-error). @@ -907,6 +946,7 @@ export function registerSshHandlers( // Why: clear this failed connect's flag so a later non-prompting connect isn't deferred. credentialRequestedForTarget.delete(targetId) activeSessions.delete(targetId) + releaseActiveSessionGeneration(targetId) clearRelayLostBackoff(targetId) clearRelayStateOverride(targetId) broadcastSshState(getCurrentMainWindow, targetId, { @@ -945,6 +985,7 @@ export function registerSshHandlers( throw connectCancelledError() } activeSessions.delete(targetId) + releaseActiveSessionGeneration(targetId) clearRelayLostBackoff(targetId) await connectionManager!.disconnect(targetId) throw err @@ -1020,6 +1061,7 @@ export function registerSshHandlers( await portForwardManager!.removeAllForwards(args.targetId) session.dispose() activeSessions.delete(args.targetId) + releaseActiveSessionGeneration(args.targetId) clearRelayLostBackoff(args.targetId) clearRelayStateOverride(args.targetId) } @@ -1027,46 +1069,53 @@ export function registerSshHandlers( }) async function doResetRelay(targetId: string, target: SshTarget): Promise { - const inFlightConnect = connectInFlight.get(targetId) - if (inFlightConnect) { - try { - // Why: resetting activeSessions mid-deploy would dispose the session doConnect will use. - await inFlightConnect.promise - } catch { - // The reset can still recover a stale remote relay after a failed connect. - } - } - - const session = activeSessions.get(targetId) - if (session) { - await portForwardManager!.removeAllForwards(targetId) - // Why: detach() not dispose() — reset has its own stale-lease semantics below that dispose()'s clean-termination recording would hide. - session.detach() - activeSessions.delete(targetId) - clearRelayLostBackoff(targetId) - } - - const existingConn = connectionManager!.getConnection(targetId) - const conn = existingConn ?? (await connectionManager!.connect(target)) + const generationLease = retainSshConnectionGeneration(targetId) try { - await forceStopRelayForTarget(conn, targetId) - } finally { - const ptyIds = new Set(getPtyIdsForConnection(targetId)) - for (const lease of persistedStore!.getSshRemotePtyLeases(targetId)) { - if (lease.state !== 'terminated' && lease.state !== 'expired') { - ptyIds.add(lease.ptyId) - persistedStore!.markSshRemotePtyLease(targetId, lease.ptyId, 'expired') + const inFlightConnect = connectInFlight.get(targetId) + if (inFlightConnect) { + try { + // Why: resetting activeSessions mid-deploy would dispose the session doConnect will use. + await inFlightConnect.promise + } catch { + // The reset can still recover a stale remote relay after a failed connect. } } - // Why: reset force-kills the remote relay, so every local PTY handle it owned is stale even if the reset command failed after SIGTERM. - for (const ptyId of ptyIds) { - const appPtyId = toAppSshPtyId(targetId, ptyId) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) + advanceSshConnectionGeneration(targetId) + + const session = activeSessions.get(targetId) + if (session) { + await portForwardManager!.removeAllForwards(targetId) + // Why: detach() not dispose() — reset has its own stale-lease semantics below that dispose()'s clean-termination recording would hide. + session.detach() + activeSessions.delete(targetId) + releaseActiveSessionGeneration(targetId) + clearRelayLostBackoff(targetId) } - // Why: reset's connect() may trip onCredentialRequest; clear so a later non-prompting doConnect doesn't persist lastRequiredPassphrase=true. - credentialRequestedForTarget.delete(targetId) - await connectionManager!.disconnect(targetId) + + const existingConn = connectionManager!.getConnection(targetId) + const conn = existingConn ?? (await connectionManager!.connect(target)) + try { + await forceStopRelayForTarget(conn, targetId) + } finally { + const ptyIds = new Set(getPtyIdsForConnection(targetId)) + for (const lease of persistedStore!.getSshRemotePtyLeases(targetId)) { + if (lease.state !== 'terminated' && lease.state !== 'expired') { + ptyIds.add(lease.ptyId) + persistedStore!.markSshRemotePtyLease(targetId, lease.ptyId, 'expired') + } + } + // Why: reset force-kills the remote relay, so every local PTY handle it owned is stale even if the reset command failed after SIGTERM. + for (const ptyId of ptyIds) { + const appPtyId = toAppSshPtyId(targetId, ptyId) + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + } + // Why: reset's connect() may trip onCredentialRequest; clear so a later non-prompting doConnect doesn't persist lastRequiredPassphrase=true. + credentialRequestedForTarget.delete(targetId) + await connectionManager!.disconnect(targetId) + } + } finally { + generationLease.release() } } @@ -1139,7 +1188,9 @@ export function registerSshHandlers( } testingTargets.add(args.targetId) + let generationLease: SshConnectionGenerationLease | null = null try { + generationLease = retainSshConnectionGeneration(args.targetId) const conn = await connectionManager!.connect(target) const state = conn.getState() await connectionManager!.disconnect(args.targetId) @@ -1150,6 +1201,7 @@ export function registerSshHandlers( error: err instanceof Error ? err.message : String(err) } } finally { + generationLease?.release() testingTargets.delete(args.targetId) // Why: clear so a test's credential prompt doesn't leave lastRequiredPassphrase=true and defer this target at startup. credentialRequestedForTarget.delete(args.targetId) @@ -1271,6 +1323,10 @@ export async function resetSshHandlerStateForTests(): Promise { session.dispose() } activeSessions.clear() + for (const lease of activeSessionGenerationLeases.values()) { + lease.release() + } + activeSessionGenerationLeases.clear() for (const targetId of relayLostBackoff.keys()) { clearRelayLostBackoff(targetId) } diff --git a/src/main/ipc/terminal-preview-admission.test.ts b/src/main/ipc/terminal-preview-admission.test.ts new file mode 100644 index 00000000000..50f3b750179 --- /dev/null +++ b/src/main/ipc/terminal-preview-admission.test.ts @@ -0,0 +1,109 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { handlers, ipcMainMock } = vi.hoisted(() => { + const map = new Map unknown>() + return { + handlers: map, + ipcMainMock: { + removeHandler: vi.fn(), + handle: (channel: string, fn: (...args: unknown[]) => unknown) => map.set(channel, fn) + } + } +}) + +vi.mock('electron', () => ({ ipcMain: ipcMainMock })) +vi.mock('../window/dashboard-popout-window', () => ({ + isDashboardPopoutRenderer: () => true +})) + +import { + registerTerminalPreviewHandlers, + TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS, + TERMINAL_PREVIEW_MAX_ENTRIES_TOTAL +} from './terminal-preview' + +function makeRuntime() { + return { + serializeTerminalBuffer: vi.fn(async () => ({ data: '', cols: 80, rows: 24, seq: 0 })), + subscribeToTerminalData: vi.fn(() => vi.fn()), + subscribeToTerminalResize: vi.fn(() => vi.fn()), + registerRawTerminalViewSubscriber: vi.fn(() => vi.fn()), + writeTerminalPreviewInput: vi.fn(async () => true), + updateRemoteDesktopViewer: vi.fn(async () => true), + unregisterRemoteDesktopViewer: vi.fn(async () => true), + getTerminalSize: vi.fn(() => ({ cols: 80, rows: 24 })) + } +} + +function makeSender(id: number) { + return { + id, + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn() + } +} + +function eventFor(sender: ReturnType) { + return { sender } as never +} + +describe('terminal preview aggregate admission', () => { + beforeEach(() => handlers.clear()) + + it('bounds output streams per renderer without affecting replacements', async () => { + const runtime = makeRuntime() + registerTerminalPreviewHandlers(runtime as never) + const sender = makeSender(1) + const connect = handlers.get('terminalPreview:connect')! + + for (let index = 0; index < TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS; index += 1) { + await connect(eventFor(sender), { ptyId: `pty-${index}` }) + } + await expect(connect(eventFor(sender), { ptyId: 'overflow' })).resolves.toEqual({ + snapshot: null, + replay: [] + }) + await expect(connect(eventFor(sender), { ptyId: 'pty-0' })).resolves.toMatchObject({ + snapshot: { cols: 80, rows: 24 } + }) + expect(runtime.subscribeToTerminalData).toHaveBeenCalledTimes( + TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS + 1 + ) + }) + + it('bounds output streams across renderers', async () => { + const runtime = makeRuntime() + registerTerminalPreviewHandlers(runtime as never) + const connect = handlers.get('terminalPreview:connect')! + + for (let index = 0; index < TERMINAL_PREVIEW_MAX_ENTRIES_TOTAL; index += 1) { + const sender = makeSender(Math.floor(index / TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS) + 1) + await connect(eventFor(sender), { ptyId: `pty-${index}` }) + } + await expect(connect(eventFor(makeSender(99)), { ptyId: 'overflow' })).resolves.toEqual({ + snapshot: null, + replay: [] + }) + expect(runtime.subscribeToTerminalData).toHaveBeenCalledTimes( + TERMINAL_PREVIEW_MAX_ENTRIES_TOTAL + ) + }) + + it('bounds independent fit claims per renderer', async () => { + const runtime = makeRuntime() + registerTerminalPreviewHandlers(runtime as never) + const sender = makeSender(1) + const fit = handlers.get('terminalPreview:fit')! + + for (let index = 0; index < TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS; index += 1) { + await fit(eventFor(sender), { ptyId: `pty-${index}`, cols: 80, rows: 24 }) + } + await expect( + fit(eventFor(sender), { ptyId: 'overflow', cols: 80, rows: 24 }) + ).resolves.toBeNull() + expect(runtime.updateRemoteDesktopViewer).toHaveBeenCalledTimes( + TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS + ) + }) +}) diff --git a/src/main/ipc/terminal-preview-output-stream-memory.test.ts b/src/main/ipc/terminal-preview-output-stream-memory.test.ts new file mode 100644 index 00000000000..271f0eb87d1 --- /dev/null +++ b/src/main/ipc/terminal-preview-output-stream-memory.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RETAINED_STRING_CHUNK_LIMIT } from '../../shared/string-chunk-compaction' +import { + TERMINAL_PREVIEW_OUTPUT_BATCH_MAX_BYTES, + TERMINAL_PREVIEW_PENDING_MAX_RECORDS, + TerminalPreviewOutputStream +} from './terminal-preview-output-stream' + +function makeStream() { + const contents = { + isDestroyed: () => false, + send: vi.fn() + } + const stream = new TerminalPreviewOutputStream(contents as never, 'pty-1', vi.fn(), vi.fn()) + return { contents, stream } +} + +describe('TerminalPreviewOutputStream memory bounds', () => { + afterEach(() => vi.useRealTimers()) + + it('ignores empty snapshot records and resyncs when tiny records reach the count cap', () => { + const { stream } = makeStream() + + for (let index = 0; index < 100_000; index += 1) { + stream.append('', { seq: index, rawLength: 0 }) + } + expect(stream.consumeInitialOverflow()).toBe(false) + + for (let index = 0; index <= TERMINAL_PREVIEW_PENDING_MAX_RECORDS; index += 1) { + stream.append('x', { seq: index + 1, rawLength: 1 }) + } + expect(stream.consumeInitialOverflow()).toBe(true) + }) + + it('compacts 50,000 live fragments without changing the emitted output', () => { + vi.useFakeTimers() + const { contents, stream } = makeStream() + stream.completeSnapshot() + + for (let index = 0; index < 50_000; index += 1) { + stream.append(String.fromCharCode(97 + (index % 26))) + } + const retained = stream as unknown as { batchChunks: string[] } + expect(retained.batchChunks.length).toBeLessThanOrEqual(RETAINED_STRING_CHUNK_LIMIT) + + vi.advanceTimersByTime(5) + expect(contents.send).toHaveBeenCalledWith( + 'terminalPreview:data', + expect.objectContaining({ data: expect.stringMatching(/^[a-z]{50000}$/), bytes: 50_000 }) + ) + }) + + it('resyncs when a stalled renderer accumulates too many tiny pending batches', () => { + vi.useFakeTimers() + const { contents, stream } = makeStream() + stream.completeSnapshot() + const fullBatch = 'x'.repeat(TERMINAL_PREVIEW_OUTPUT_BATCH_MAX_BYTES) + + for (let index = 0; index < 8; index += 1) { + stream.append(fullBatch) + } + for (let index = 0; index <= TERMINAL_PREVIEW_PENDING_MAX_RECORDS; index += 1) { + stream.append('x') + vi.advanceTimersByTime(5) + } + + stream.acknowledge(8 * TERMINAL_PREVIEW_OUTPUT_BATCH_MAX_BYTES) + expect(contents.send).toHaveBeenLastCalledWith('terminalPreview:data', { + type: 'resync', + ptyId: 'pty-1' + }) + }) +}) diff --git a/src/main/ipc/terminal-preview-output-stream.ts b/src/main/ipc/terminal-preview-output-stream.ts index af14424cbe2..03382b40f3a 100644 --- a/src/main/ipc/terminal-preview-output-stream.ts +++ b/src/main/ipc/terminal-preview-output-stream.ts @@ -1,4 +1,5 @@ import type { WebContents } from 'electron' +import { appendCompactedStringChunk } from '../../shared/string-chunk-compaction' import { iterateTerminalInputChunks } from '../../shared/terminal-input' import type { TerminalPreviewDataPayload } from '../../shared/terminal-preview' @@ -7,6 +8,7 @@ export const TERMINAL_PREVIEW_OUTPUT_BATCH_MAX_BYTES = 64 * 1024 const OUTPUT_IN_FLIGHT_MAX_BYTES = 512 * 1024 const OUTPUT_PENDING_MAX_BYTES = 256 * 1024 const INITIAL_PENDING_MAX_BYTES = 256 * 1024 +export const TERMINAL_PREVIEW_PENDING_MAX_RECORDS = 4_096 export type TerminalPreviewOutputMeta = { seq?: number @@ -81,7 +83,7 @@ export class TerminalPreviewOutputStream { } append(data: string, meta?: TerminalPreviewOutputMeta): void { - if (this.isDisposed || this.awaitingReconnect || this.resyncPending) { + if (data.length === 0 || this.isDisposed || this.awaitingReconnect || this.resyncPending) { return } if (this.bufferingSnapshot) { @@ -205,7 +207,10 @@ export class TerminalPreviewOutputStream { } this.pendingBatches.push({ data, bytes }) this.pendingBatchBytes += bytes - if (this.pendingBatchBytes > OUTPUT_PENDING_MAX_BYTES) { + if ( + this.pendingBatchBytes > OUTPUT_PENDING_MAX_BYTES || + this.pendingBatches.length > TERMINAL_PREVIEW_PENDING_MAX_RECORDS + ) { // Why: a stuck renderer heals from a fresh authoritative snapshot instead of retaining output without bound. this.pendingBatches = [] this.pendingBatchBytes = 0 @@ -237,7 +242,7 @@ export class TerminalPreviewOutputStream { ) { this.flushBatch() } - this.batchChunks.push(chunk) + appendCompactedStringChunk(this.batchChunks, chunk) this.batchBytes += bytes if (this.batchBytes >= TERMINAL_PREVIEW_OUTPUT_BATCH_MAX_BYTES) { this.flushBatch() @@ -249,11 +254,18 @@ export class TerminalPreviewOutputStream { } private appendInitial(data: string, meta?: TerminalPreviewOutputMeta): void { + if (this.initialPendingOverflowed) { + return + } const bytes = Buffer.byteLength(data, 'utf8') this.initialPending.push({ data, bytes, meta }) this.initialPendingBytes += bytes - while (this.initialPendingBytes > INITIAL_PENDING_MAX_BYTES && this.initialPending.length > 0) { - this.initialPendingBytes -= this.initialPending.shift()!.bytes + if ( + this.initialPendingBytes > INITIAL_PENDING_MAX_BYTES || + this.initialPending.length > TERMINAL_PREVIEW_PENDING_MAX_RECORDS + ) { + this.initialPending = [] + this.initialPendingBytes = 0 this.initialPendingOverflowed = true } } diff --git a/src/main/ipc/terminal-preview.ts b/src/main/ipc/terminal-preview.ts index 59193df6894..39e75390737 100644 --- a/src/main/ipc/terminal-preview.ts +++ b/src/main/ipc/terminal-preview.ts @@ -11,11 +11,32 @@ import { } from './terminal-preview-output-stream' const PREVIEW_ID_MAX_LENGTH = 4096 +export const TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS = 64 +export const TERMINAL_PREVIEW_MAX_ENTRIES_TOTAL = 256 function isValidPtyId(value: unknown): value is string { return typeof value === 'string' && value.length > 0 && value.length <= PREVIEW_ID_MAX_LENGTH } +function canRetainPreviewEntry( + registry: Map>, + contentsId: number, + ptyId: string +): boolean { + const retained = registry.get(contentsId) + if (retained?.has(ptyId)) { + return true + } + if ((retained?.size ?? 0) >= TERMINAL_PREVIEW_MAX_ENTRIES_PER_CONTENTS) { + return false + } + let total = 0 + for (const perContents of registry.values()) { + total += perContents.size + } + return total < TERMINAL_PREVIEW_MAX_ENTRIES_TOTAL +} + /** Pop-out terminal transport with an atomic snapshot/live boundary. */ export function registerTerminalPreviewHandlers(runtime: OrcaRuntimeService): void { ipcMain.removeHandler('terminalPreview:connect') @@ -88,6 +109,9 @@ export function registerTerminalPreviewHandlers(runtime: OrcaRuntimeService): vo } const ptyId = args.ptyId const perPty = subscriptionsFor(event.sender) + if (!canRetainPreviewEntry(subscriptionsByContents, event.sender.id, ptyId)) { + return { snapshot: null, replay: [] } + } perPty.get(ptyId)?.dispose() const subscription = new TerminalPreviewOutputStream( @@ -210,6 +234,9 @@ export function registerTerminalPreviewHandlers(runtime: OrcaRuntimeService): vo // Why: guarantees the destroyed hook exists even if this claim outlives // the current output stream across a resync reconnect. subscriptionsFor(event.sender) + if (!canRetainPreviewEntry(fitClaimsByContents, event.sender.id, ptyId)) { + return null + } let claimed = fitClaimsByContents.get(event.sender.id) if (!claimed) { claimed = new Map() diff --git a/src/main/ipc/terminal-render-desync-evidence.ts b/src/main/ipc/terminal-render-desync-evidence.ts index 9ea82246539..6952fb5815f 100644 --- a/src/main/ipc/terminal-render-desync-evidence.ts +++ b/src/main/ipc/terminal-render-desync-evidence.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { mkdir, opendir, rm, stat, writeFile } from 'node:fs/promises' import path from 'node:path' import { app, ipcMain } from 'electron' import type { @@ -79,29 +79,23 @@ export async function writeTerminalRenderDesyncEvidence( } async function pruneRenderDesyncEvidence(root: string, currentDirectory: string): Promise { - const entries = await readdir(root, { withFileTypes: true }) - const captures = await Promise.all( - entries - .filter((entry) => entry.isDirectory()) - .map(async (entry) => { - const directory = path.join(root, entry.name) - const [directoryStat, files] = await Promise.all([ - stat(directory), - readdir(directory, { withFileTypes: true }) - ]) - const sizes = await Promise.all( - files.filter((file) => file.isFile()).map((file) => stat(path.join(directory, file.name))) - ) - return { - directory, - bytes: sizes.reduce((total, file) => total + file.size, 0), - modifiedAt: directoryStat.mtimeMs - } - }) - ) + const captures: EvidenceCapture[] = [] + const entries = await opendir(root) + for await (const entry of entries) { + if (!entry.isDirectory()) { + continue + } + captures.push(await inspectEvidenceCapture(path.join(root, entry.name))) + captures.sort((a, b) => a.modifiedAt - b.modifiedAt) + if (captures.length > MAX_CAPTURE_DIRECTORIES) { + const removableIndex = captures.findIndex((capture) => capture.directory !== currentDirectory) + const [capture] = captures.splice(Math.max(removableIndex, 0), 1) + await rm(capture.directory, { recursive: true, force: true }) + } + } captures.sort((a, b) => a.modifiedAt - b.modifiedAt) let totalBytes = captures.reduce((total, capture) => total + capture.bytes, 0) - while (captures.length > MAX_CAPTURE_DIRECTORIES || totalBytes > MAX_EVIDENCE_BYTES) { + while (totalBytes > MAX_EVIDENCE_BYTES) { const removableIndex = captures.findIndex((capture) => capture.directory !== currentDirectory) const index = Math.max(removableIndex, 0) const [capture] = captures.splice(index, 1) @@ -112,3 +106,23 @@ async function pruneRenderDesyncEvidence(root: string, currentDirectory: string) } } } + +type EvidenceCapture = { + directory: string + bytes: number + modifiedAt: number +} + +async function inspectEvidenceCapture(directory: string): Promise { + const directoryStat = await stat(directory) + const files = await opendir(directory) + let bytes = 0 + for await (const file of files) { + if (!file.isFile()) { + continue + } + const fileStat = await stat(path.join(directory, file.name)) + bytes = Math.min(MAX_EVIDENCE_BYTES + 1, bytes + fileStat.size) + } + return { directory, bytes, modifiedAt: directoryStat.mtimeMs } +} diff --git a/src/main/ipc/workspace-cleanup-activity.ts b/src/main/ipc/workspace-cleanup-activity.ts index 9026b4464d9..259bf73f99e 100644 --- a/src/main/ipc/workspace-cleanup-activity.ts +++ b/src/main/ipc/workspace-cleanup-activity.ts @@ -1,8 +1,10 @@ -import { lstat, readFile } from 'node:fs/promises' +import { lstat } from 'node:fs/promises' import path from 'node:path' import type { Repo, Worktree } from '../../shared/types' import { parseWslUncPath } from '../../shared/wsl-paths' import { toWindowsWslPath } from '../wsl' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' +import { MAX_WORKTREE_GIT_POINTER_BYTES } from '../worktree-orphan-gitdir-proof' type StatPath = (targetPath: string) => Promise<{ mtimeMs: number }> type ReadTextFile = (targetPath: string) => Promise @@ -42,7 +44,9 @@ async function statLocalPath(targetPath: string): Promise<{ mtimeMs: number }> { } async function readLocalTextFile(targetPath: string): Promise { - return readFile(targetPath, 'utf8') + return ( + await readNodeFileWithinLimit(targetPath, MAX_WORKTREE_GIT_POINTER_BYTES) + ).buffer.toString('utf8') } async function resolveWorkspaceCleanupActivityAt( diff --git a/src/main/ipc/worktree-base-directory-poller.ts b/src/main/ipc/worktree-base-directory-poller.ts index 31a068146c8..8ec685fbff6 100644 --- a/src/main/ipc/worktree-base-directory-poller.ts +++ b/src/main/ipc/worktree-base-directory-poller.ts @@ -1,12 +1,17 @@ -import { readdir, stat } from 'node:fs/promises' import { join } from 'node:path' -import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' import { isMainWindowVisible, onMainWindowBecameVisible } from '../window/main-window-visibility' import type { WorktreeBaseRepoWatchConfig, WorktreeBaseWatchTarget } from './worktree-base-directory-event-filter' import { startGitCommonWatch } from './worktree-git-common-watch' +import type { WorktreePollingScanLimits } from './worktree-polling-scan-budget' +import { + collectWorktreeBaseDirectorySignatures, + hasWorktreeGitMarker, + takeWorktreeBaseDirectorySnapshot, + type WorktreeBaseDirectorySnapshot +} from './worktree-base-directory-snapshot' export type WorktreeBasePollEvent = { type: 'create' | 'update' | 'delete'; path: string } @@ -61,6 +66,7 @@ export type WorktreeBasePollerOptions = { visibility?: WorktreePollerWindowVisibility /** Test hook: called whenever a full snapshot scan runs (vs. a gated skip). */ onFullScan?: () => void + scanLimits?: Partial } // Why: these targets used to be recursive FSEvents subscriptions spanning the @@ -84,93 +90,10 @@ export const WORKTREE_BASE_BACKSTOP_TICKS = 15 // backstop scan cover the pathological case. const PENDING_MARKER_MAX_TICKS = 300 -function statSignature(s: { mtimeMs: number; ctimeMs: number; ino: number }): string { - return `${s.mtimeMs}:${s.ctimeMs}:${s.ino}` -} - -async function dirSignature(path: string): Promise { - try { - return statSignature(await stat(path)) - } catch { - return 'missing' - } -} - -async function hasGitMarker(dir: string): Promise { - try { - await stat(join(dir, '.git')) - return true - } catch { - return false - } -} - -type BaseSnapshot = { - // worktree-candidate dir → whether its `.git` completion marker exists - markers: Map - // dirs whose listing determines the candidate set: the root plus any - // nested repo containers. Their stat signatures gate the next full scan. - gateDirs: string[] -} - -// Depth-1 worktree dirs (flat layout), plus depth-2 dirs under each nested -// repo's container, mirroring what worktree-base-directory-event-filter -// matches: `/.git` completion markers and `` deletions. -async function snapshotBase( - rootPath: string, - repos: ReadonlyMap -): Promise { - const markers = new Map() - const gateDirs = [rootPath] - const configs = [...repos.values()] - const includeFlat = configs.some((config) => !config.nestWorkspaces) - const nestedRepoNames = new Set( - configs - .filter((config) => config.nestWorkspaces) - .map((config) => normalizeRuntimePathForComparison(config.repoName)) - ) - - let rootEntries - try { - rootEntries = await readdir(rootPath, { withFileTypes: true }) - } catch { - // Root vanished: an empty snapshot diffs into delete events for every - // previously-known worktree dir, matching the old watcher's error path. - return { markers, gateDirs } - } - - const candidates: string[] = [] - for (const entry of rootEntries) { - if (!entry.isDirectory() && !entry.isSymbolicLink()) { - continue - } - const entryPath = join(rootPath, entry.name) - if (includeFlat) { - candidates.push(entryPath) - } - if (nestedRepoNames.has(normalizeRuntimePathForComparison(entry.name))) { - gateDirs.push(entryPath) - let subEntries - try { - subEntries = await readdir(entryPath, { withFileTypes: true }) - } catch { - subEntries = [] - } - for (const sub of subEntries) { - if (sub.isDirectory() || sub.isSymbolicLink()) { - candidates.push(join(entryPath, sub.name)) - } - } - } - } - - for (const dir of candidates) { - markers.set(dir, await hasGitMarker(dir)) - } - return { markers, gateDirs } -} - -function diffBase(prev: BaseSnapshot, next: BaseSnapshot): WorktreeBasePollEvent[] { +function diffBase( + prev: WorktreeBaseDirectorySnapshot, + next: WorktreeBaseDirectorySnapshot +): WorktreeBasePollEvent[] { const events: WorktreeBasePollEvent[] = [] for (const [dir, marker] of next.markers) { if (marker && prev.markers.get(dir) !== true) { @@ -191,13 +114,14 @@ async function startBasePoller( onEvents: (events: WorktreeBasePollEvent[]) => void, pollIntervalMs: number, visibility: WorktreePollerWindowVisibility, - onFullScan?: () => void + onFullScan: (() => void) | undefined, + scanLimits: Partial ): Promise { let disposed = false let ticking = false let tickCount = 0 - let snapshot = await snapshotBase(target.path, getRepos()) - let gateSignatures = await Promise.all(snapshot.gateDirs.map(dirSignature)) + let snapshot = await takeWorktreeBaseDirectorySnapshot(target.path, getRepos(), scanLimits) + let gateSignatures = await collectWorktreeBaseDirectorySignatures(snapshot.gateDirs) let timer: ReturnType | null = null let parkedWhileHidden = false // dir → tick when first seen without a `.git` marker @@ -210,8 +134,8 @@ async function startBasePoller( const fullScan = async (): Promise => { onFullScan?.() - const next = await snapshotBase(target.path, getRepos()) - const nextSignatures = await Promise.all(next.gateDirs.map(dirSignature)) + const next = await takeWorktreeBaseDirectorySnapshot(target.path, getRepos(), scanLimits) + const nextSignatures = await collectWorktreeBaseDirectorySignatures(next.gateDirs) if (disposed) { return } @@ -238,7 +162,7 @@ async function startBasePoller( const checkPendingMarkers = async (): Promise => { const events: WorktreeBasePollEvent[] = [] for (const dir of pendingMarkers.keys()) { - if (await hasGitMarker(dir)) { + if (await hasWorktreeGitMarker(dir)) { pendingMarkers.delete(dir) snapshot.markers.set(dir, true) events.push({ type: 'create', path: join(dir, '.git') }) @@ -257,7 +181,7 @@ async function startBasePoller( } // Idle fast path: when the dirs whose listings define the candidate set // are untouched, skip the readdir + per-candidate stat fan-out entirely. - const signatures = await Promise.all(snapshot.gateDirs.map(dirSignature)) + const signatures = await collectWorktreeBaseDirectorySignatures(snapshot.gateDirs) const gateChanged = signatures.length !== gateSignatures.length || signatures.some((sig, index) => sig !== gateSignatures[index]) @@ -349,8 +273,17 @@ export async function startWorktreeBaseDirectoryPoller( pollIntervalMs, platform, visibility, - options.onFullScan + options.onFullScan, + options.scanLimits ) } - return startBasePoller(target, getRepos, onEvents, pollIntervalMs, visibility, options.onFullScan) + return startBasePoller( + target, + getRepos, + onEvents, + pollIntervalMs, + visibility, + options.onFullScan, + options.scanLimits ?? {} + ) } diff --git a/src/main/ipc/worktree-base-directory-snapshot.ts b/src/main/ipc/worktree-base-directory-snapshot.ts new file mode 100644 index 00000000000..197c0540009 --- /dev/null +++ b/src/main/ipc/worktree-base-directory-snapshot.ts @@ -0,0 +1,109 @@ +import { opendir, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' +import type { WorktreeBaseRepoWatchConfig } from './worktree-base-directory-event-filter' +import { + WorktreePollingScanBudget, + type WorktreePollingScanLimits +} from './worktree-polling-scan-budget' + +export type WorktreeBaseDirectorySnapshot = { + markers: Map + gateDirs: string[] +} + +function statSignature(value: { mtimeMs: number; ctimeMs: number; ino: number }): string { + return `${value.mtimeMs}:${value.ctimeMs}:${value.ino}` +} + +async function directorySignature(path: string): Promise { + try { + return statSignature(await stat(path)) + } catch { + return 'missing' + } +} + +export async function hasWorktreeGitMarker(dir: string): Promise { + try { + await stat(join(dir, '.git')) + return true + } catch { + return false + } +} + +export async function takeWorktreeBaseDirectorySnapshot( + rootPath: string, + repos: ReadonlyMap, + scanLimits: Partial +): Promise { + const markers = new Map() + const gateDirs = [rootPath] + const budget = new WorktreePollingScanBudget(scanLimits) + budget.claimRetainedPath(rootPath) + let includeFlat = false + const nestedRepoNames = new Set() + for (const config of repos.values()) { + budget.claimRepoConfig(config.repoId, config.repoName) + if (config.nestWorkspaces) { + nestedRepoNames.add(normalizeRuntimePathForComparison(config.repoName)) + } else { + includeFlat = true + } + } + + let rootDirectory + try { + rootDirectory = await opendir(rootPath, { bufferSize: 32 }) + } catch { + return { markers, gateDirs } + } + + const candidates: string[] = [] + for await (const entry of rootDirectory) { + budget.claimEntry() + if (!entry.isDirectory() && !entry.isSymbolicLink()) { + continue + } + const entryPath = join(rootPath, entry.name) + if (includeFlat) { + budget.claimRetainedPath(entryPath) + candidates.push(entryPath) + } + if (!nestedRepoNames.has(normalizeRuntimePathForComparison(entry.name))) { + continue + } + budget.claimRetainedPath(entryPath) + gateDirs.push(entryPath) + let subDirectory + try { + subDirectory = await opendir(entryPath, { bufferSize: 32 }) + } catch { + continue + } + for await (const sub of subDirectory) { + budget.claimEntry() + if (sub.isDirectory() || sub.isSymbolicLink()) { + const candidatePath = join(entryPath, sub.name) + budget.claimRetainedPath(candidatePath) + candidates.push(candidatePath) + } + } + } + + for (const dir of candidates) { + markers.set(dir, await hasWorktreeGitMarker(dir)) + } + return { markers, gateDirs } +} + +export async function collectWorktreeBaseDirectorySignatures( + paths: readonly string[] +): Promise { + const signatures: string[] = [] + for (const path of paths) { + signatures.push(await directorySignature(path)) + } + return signatures +} diff --git a/src/main/ipc/worktree-base-directory-watch-targets-bounds.test.ts b/src/main/ipc/worktree-base-directory-watch-targets-bounds.test.ts new file mode 100644 index 00000000000..be4151e0c64 --- /dev/null +++ b/src/main/ipc/worktree-base-directory-watch-targets-bounds.test.ts @@ -0,0 +1,81 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { GlobalSettings, Repo } from '../../shared/types' +import { + buildWorktreeBaseDirectoryWatchTargets, + clearWorktreeBaseDirectoryWatchTargetWarnings +} from './worktree-base-directory-watch-targets' +import { WORKTREE_POLLING_MAX_REPO_CONFIGS } from './worktree-polling-scan-budget' + +const settings = { + workspaceDir: '/worktrees', + nestWorkspaces: false +} as GlobalSettings + +function folderRepo(index: number): Repo { + return { + id: `repo-${index}`, + path: `/repos/${index}`, + displayName: `Repo ${index}`, + badgeColor: '#000000', + addedAt: index, + kind: 'folder' + } +} + +describe('worktree watcher target repo bounds', () => { + afterEach(() => { + clearWorktreeBaseDirectoryWatchTargetWarnings() + vi.restoreAllMocks() + }) + + it('accepts the exact repo count before hydrating repos', async () => { + const repos = Array.from({ length: WORKTREE_POLLING_MAX_REPO_CONFIGS }, (_, index) => + folderRepo(index) + ) + const getRepos = vi.fn(() => repos) + + await expect( + buildWorktreeBaseDirectoryWatchTargets({ + getRepoCount: () => repos.length, + getRepos, + getSettings: () => settings + } as never) + ).resolves.toEqual(new Map()) + expect(getRepos).toHaveBeenCalledOnce() + }) + + it('rejects one repo over the cap without hydrating the repo array', async () => { + const getRepos = vi.fn(() => { + throw new Error('must not hydrate') + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await expect( + buildWorktreeBaseDirectoryWatchTargets({ + getRepoCount: () => WORKTREE_POLLING_MAX_REPO_CONFIGS + 1, + getRepos, + getSettings: () => settings + } as never) + ).resolves.toEqual(new Map()) + expect(getRepos).not.toHaveBeenCalled() + expect(warn).toHaveBeenCalledOnce() + }) + + it('resumes target construction after the repo count returns below the cap', async () => { + let repoCount = WORKTREE_POLLING_MAX_REPO_CONFIGS + 1 + const getRepos = vi.fn(() => [folderRepo(1)]) + const store = { + getRepoCount: () => repoCount, + getRepos, + getSettings: () => settings + } + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await buildWorktreeBaseDirectoryWatchTargets(store as never) + expect(getRepos).not.toHaveBeenCalled() + + repoCount = 1 + await buildWorktreeBaseDirectoryWatchTargets(store as never) + expect(getRepos).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ipc/worktree-base-directory-watch-targets.ts b/src/main/ipc/worktree-base-directory-watch-targets.ts index 1283333ee95..b98594d0db9 100644 --- a/src/main/ipc/worktree-base-directory-watch-targets.ts +++ b/src/main/ipc/worktree-base-directory-watch-targets.ts @@ -27,9 +27,11 @@ import type { WorktreeBaseWatchKind, WorktreeBaseWatchTarget } from './worktree-base-directory-event-filter' +import { WORKTREE_POLLING_MAX_REPO_CONFIGS } from './worktree-polling-scan-budget' const missingRootWarnings = new Set() const skippedWslWarnings = new Set() +let repoCapacityWarningEmitted = false function normalizeWatchKey(pathValue: string): string { return normalizeRuntimePathForComparison(normalize(pathValue)) @@ -179,8 +181,23 @@ async function maybeAddBaseTarget( export async function buildWorktreeBaseDirectoryWatchTargets( store: Store ): Promise> { - const settings = store.getSettings() const targets = new Map() + const repoCount = store.getRepoCount() + if ( + !Number.isSafeInteger(repoCount) || + repoCount < 0 || + repoCount > WORKTREE_POLLING_MAX_REPO_CONFIGS + ) { + if (!repoCapacityWarningEmitted) { + console.warn( + `[worktree-base-watcher] skipping background watchers for ${repoCount} repos; limit is ${WORKTREE_POLLING_MAX_REPO_CONFIGS}` + ) + repoCapacityWarningEmitted = true + } + return targets + } + repoCapacityWarningEmitted = false + const settings = store.getSettings() for (const repo of store.getRepos()) { if (isFolderRepo(repo)) { continue @@ -198,4 +215,5 @@ export async function buildWorktreeBaseDirectoryWatchTargets( export function clearWorktreeBaseDirectoryWatchTargetWarnings(): void { missingRootWarnings.clear() skippedWslWarnings.clear() + repoCapacityWarningEmitted = false } diff --git a/src/main/ipc/worktree-base-directory-watcher.test.ts b/src/main/ipc/worktree-base-directory-watcher.test.ts index 0a5773b13a0..aafa690ea5f 100644 --- a/src/main/ipc/worktree-base-directory-watcher.test.ts +++ b/src/main/ipc/worktree-base-directory-watcher.test.ts @@ -73,6 +73,7 @@ function makeRepo(overrides: Partial = {}): Repo { function makeStore(repos: Repo[]) { return { getSettings: () => settings, + getRepoCount: () => repos.length, getRepos: () => repos } } diff --git a/src/main/ipc/worktree-base-directory-watcher.ts b/src/main/ipc/worktree-base-directory-watcher.ts index 85cd501a2eb..ab0a19d8540 100644 --- a/src/main/ipc/worktree-base-directory-watcher.ts +++ b/src/main/ipc/worktree-base-directory-watcher.ts @@ -1,5 +1,6 @@ import type { BrowserWindow } from 'electron' import type { Store } from '../persistence' +import { forEachWithConcurrency } from '../../shared/map-with-concurrency' import { notifyWorktreeGitStatusMetadataChanged, notifyWorktreesChanged } from './worktree-remote' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { @@ -34,6 +35,7 @@ type ActiveWatch = WorktreeBaseWatchTarget & { } const WATCH_DEBOUNCE_MS = 250 +export const WORKTREE_WATCH_CLOSE_LIMIT = 8 const activeWatches = new Map() let syncGeneration = 0 let scheduledSync: ReturnType | null = null @@ -327,10 +329,8 @@ export function scheduleCurrentWorktreeBaseDirectoryWatcherSync(): void { export async function disposeWorktreeBaseDirectoryWatchers(): Promise { syncGeneration++ latestSyncContext = null - if (scheduledSync) { - clearTimeout(scheduledSync) - scheduledSync = null - } - await Promise.all([...activeWatches.keys()].map((key) => removeWatch(key))) + clearTimeout(scheduledSync ?? undefined) + scheduledSync = null + await forEachWithConcurrency([...activeWatches.keys()], WORKTREE_WATCH_CLOSE_LIMIT, removeWatch) clearWorktreeBaseDirectoryWatchTargetWarnings() } diff --git a/src/main/ipc/worktree-common-git-directory.test.ts b/src/main/ipc/worktree-common-git-directory.test.ts new file mode 100644 index 00000000000..73c4fd76a07 --- /dev/null +++ b/src/main/ipc/worktree-common-git-directory.test.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import type { Repo } from '../../shared/types' +import { + MAX_GIT_DIRECTORY_POINTER_BYTES, + resolveWorktreeCommonGitDirectory +} from './worktree-common-git-directory' + +describe('resolveWorktreeCommonGitDirectory', () => { + const roots: string[] = [] + + afterEach(async () => { + vi.restoreAllMocks() + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + async function makeRepo(): Promise<{ repo: Repo; root: string }> { + const root = await mkdtemp(join(tmpdir(), 'orca-common-git-dir-')) + roots.push(root) + const repoPath = join(root, 'checkout') + await mkdir(repoPath) + return { repo: { id: 'repo-1', path: repoPath } as Repo, root } + } + + it('resolves a normal linked-worktree pointer to its common directory', async () => { + const { repo, root } = await makeRepo() + await writeFile(join(repo.path, '.git'), 'gitdir: ../common/.git/worktrees/checkout\n') + + await expect(resolveWorktreeCommonGitDirectory(repo)).resolves.toBe( + resolve(root, 'common', '.git') + ) + }) + + it('rejects an oversized sparse local pointer without loading it', async () => { + const { repo } = await makeRepo() + const dotGitPath = join(repo.path, '.git') + await writeFile(dotGitPath, 'x') + await truncate(dotGitPath, MAX_GIT_DIRECTORY_POINTER_BYTES + 1) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(resolveWorktreeCommonGitDirectory(repo)).resolves.toBeNull() + }) + + it('rejects oversized provider content at the same boundary', async () => { + const { repo } = await makeRepo() + const readFile = vi.fn(async () => `gitdir: ${'x'.repeat(MAX_GIT_DIRECTORY_POINTER_BYTES)}\n`) + + await expect( + resolveWorktreeCommonGitDirectory(repo, { + stat: async () => ({ type: 'file', size: 1, mtime: 0 }), + readFile + }) + ).resolves.toBeNull() + expect(readFile).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ipc/worktree-common-git-directory.ts b/src/main/ipc/worktree-common-git-directory.ts index a2ec6b0e990..42941c4c41b 100644 --- a/src/main/ipc/worktree-common-git-directory.ts +++ b/src/main/ipc/worktree-common-git-directory.ts @@ -1,5 +1,5 @@ import type { Stats } from 'node:fs' -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import type { Repo } from '../../shared/types' import { getRuntimePathBasename, @@ -7,6 +7,9 @@ import { resolveRuntimePath } from '../../shared/cross-platform-path' import type { FileStat } from '../providers/types' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' + +export const MAX_GIT_DIRECTORY_POINTER_BYTES = 64 * 1024 type GitDirectoryStat = Stats | FileStat @@ -41,7 +44,12 @@ export async function resolveWorktreeCommonGitDirectory( ): Promise { const dotGitPath = resolveRuntimePath(repo.path, '.git') const statPath = access.stat ?? stat - const readText = access.readFile ?? ((path: string) => readFile(path, 'utf8')) + const readText = + access.readFile ?? + (async (path: string) => + (await readNodeFileWithinLimit(path, MAX_GIT_DIRECTORY_POINTER_BYTES)).buffer.toString( + 'utf8' + )) try { const dotGitStat = await statPath(dotGitPath) if (isDirectoryStat(dotGitStat)) { @@ -51,6 +59,9 @@ export async function resolveWorktreeCommonGitDirectory( return null } const content = await readText(dotGitPath) + if (Buffer.byteLength(content, 'utf8') > MAX_GIT_DIRECTORY_POINTER_BYTES) { + return null + } const gitDir = content.match(/^gitdir:\s*(.+)\s*$/m)?.[1]?.trim() if (!gitDir) { return null diff --git a/src/main/ipc/worktree-git-common-polling.ts b/src/main/ipc/worktree-git-common-polling.ts index ed0043b3c5b..3c16440636d 100644 --- a/src/main/ipc/worktree-git-common-polling.ts +++ b/src/main/ipc/worktree-git-common-polling.ts @@ -1,270 +1,41 @@ -import { readdir, stat } from 'node:fs/promises' -import { join } from 'node:path' import type { WorktreeBasePollEvent, WorktreeBaseSubscription, WorktreePollerWindowVisibility } from './worktree-base-directory-poller' +import type { WorktreePollingScanLimits } from './worktree-polling-scan-budget' +import { + diffGitCommon, + PRIMARY_CHECKOUT_METADATA_FILES, + snapshotGitCommon +} from './worktree-git-common-snapshot' + +export { PRIMARY_CHECKOUT_METADATA_FILES } -// Shared with the darwin primary-metadata poll so platforms cannot drift. -// `logs/HEAD` catches head moves; `config.worktree` carries the sparse flag. -export const PRIMARY_CHECKOUT_METADATA_FILES = [ - 'HEAD', - 'packed-refs', - 'index', - 'config.worktree', - 'logs/HEAD' -] -const LINKED_WORKTREE_STRUCTURAL_METADATA_FILES = ['HEAD', 'gitdir', 'locked', 'config.worktree'] -const LINKED_WORKTREE_INDEX_FILE = 'index' -const LINKED_WORKTREE_HEAD_LOG_FILE = join('logs', 'HEAD') // Why: the entry-dir signature gate can miss same-granule index rewrites on // coarse-mtime filesystems; a periodic ungated re-stat bounds that miss the // same way the base poller's backstop rescan does. const INDEX_BACKSTOP_TICKS = 15 -function statSignature(s: { mtimeMs: number; ctimeMs: number; ino: number }): string { - return `${s.mtimeMs}:${s.ctimeMs}:${s.ino}` -} - -async function dirSignature(path: string): Promise { - try { - // Why: keep `size` — on a coarse-timestamp filesystem a same-granule directory - // allocation change would otherwise slip the readdir gate to the backstop. - const s = await stat(path) - return `${statSignature(s)}:${s.size}` - } catch { - return 'missing' - } -} - -async function fileSignature(path: string): Promise { - try { - const s = await stat(path) - return s.isFile() ? `${statSignature(s)}:${s.size}` : null - } catch { - return null - } -} - -type GitCommonEntrySnapshot = { - dirSignature: string - structuralSignatures: Map - indexSignature: string | null - headLogSignature: string | null -} - -type GitCommonSnapshot = { - worktreesDirSignature: string - entries: Map - primarySignatures: Map - didFullScan: boolean -} - -async function snapshotGitCommonEntry( - entryPath: string, - previous: GitCommonEntrySnapshot | undefined, - forceFullScan: boolean -): Promise { - // Why: HEAD, gitdir, locked, config.worktree and logs/HEAD are rewritten in place without bumping - // the entry-dir mtime, so — like the pre-idle-gate poller — they are re-stat'd EVERY tick, never - // gated behind the dir signature (else a raw HEAD/structural rewrite would slip to the ~30s - // backstop). Only `index` rides the entry-dir signature (its same-dir rewrites are index-backstop-bounded). - const structuralSignatures = new Map() - const [nextDirSignature, headLogSignature] = await Promise.all([ - dirSignature(entryPath), - fileSignature(join(entryPath, LINKED_WORKTREE_HEAD_LOG_FILE)), - Promise.all( - LINKED_WORKTREE_STRUCTURAL_METADATA_FILES.map(async (name) => { - const signature = await fileSignature(join(entryPath, name)) - if (signature !== null) { - structuralSignatures.set(name, signature) - } - }) - ) - ]) - if (nextDirSignature === 'missing') { - // A transient stat failure must not masquerade as a removal; the parent listing is authoritative. - return ( - previous ?? { - dirSignature: nextDirSignature, - structuralSignatures, - indexSignature: null, - headLogSignature - } - ) - } - const shouldReadIndex = forceFullScan || !previous || previous.dirSignature !== nextDirSignature - const indexSignature = shouldReadIndex - ? await fileSignature(join(entryPath, LINKED_WORKTREE_INDEX_FILE)) - : previous.indexSignature - return { - dirSignature: nextDirSignature, - structuralSignatures, - indexSignature, - headLogSignature - } -} - -async function snapshotPrimaryCheckoutSignatures( - commonDirPath: string -): Promise> { - const signatures = new Map() - await Promise.all( - PRIMARY_CHECKOUT_METADATA_FILES.map(async (name) => { - const signature = await fileSignature(join(commonDirPath, name)) - if (signature !== null) { - signatures.set(name, signature) - } - }) - ) - return signatures -} - -async function snapshotGitCommon( - commonDirPath: string, - previous?: GitCommonSnapshot, - includePrimary = true, - forceFullScan = false -): Promise { - const worktreesDir = join(commonDirPath, 'worktrees') - const [worktreesDirSignature, primarySignatures] = await Promise.all([ - dirSignature(worktreesDir), - includePrimary ? snapshotPrimaryCheckoutSignatures(commonDirPath) : new Map() - ]) - // Why: enumerate the worktrees dir EVERY tick rather than gating the readdir on its stat signature. - // A single readdir of a small dir is negligible next to the per-entry structural stats that already - // run each tick, and the signature gate could miss a same-granule add+remove on a coarse-mtime/FAT - // filesystem (its size/mtime/ino/ctime all collide), leaving a linked worktree add/remove undetected - // until the ~30s index backstop (#9882 review). The listing is the authoritative add/remove signal. - let entryPaths: string[] - try { - const entries = await readdir(worktreesDir, { withFileTypes: true }) - entryPaths = entries - .filter((entry) => entry.isDirectory()) - .map((entry) => join(worktreesDir, entry.name)) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - // Dir genuinely absent (no linked worktrees, or all removed) → authoritative empty listing. - entryPaths = [] - } else { - // Why: a TRANSIENT readdir failure (EIO/ESTALE/EMFILE, network/SSH hiccup) must not masquerade as - // "every worktree removed" — that would emit false delete events (and false creates next tick). - // Reuse the known entries so per-entry stats still run; a real removal surfaces as that entry's own - // stat miss (handled in snapshotGitCommonEntry), and the next successful readdir catches any add. - entryPaths = previous ? [...previous.entries.keys()] : [] - } - } - - const entries = new Map() - await Promise.all( - entryPaths.map(async (entryPath) => { - const previousEntry = previous?.entries.get(entryPath) - entries.set(entryPath, await snapshotGitCommonEntry(entryPath, previousEntry, forceFullScan)) - }) - ) - // Why: the expensive per-entry `index` read stays gated on each entry's own dir signature; onFullScan - // now reflects an ungated index-metadata backstop fan-out (forceFullScan) — the real periodic cost — - // rather than the always-run worktrees-dir readdir. - return { - worktreesDirSignature, - entries, - primarySignatures, - didFullScan: forceFullScan - } -} - -function classifySignatureDiff( - prevSignature: string | null | undefined, - nextSignature: string | null | undefined -): 'create' | 'update' | 'delete' | null { - if (prevSignature == null && nextSignature == null) { - return null - } - if (prevSignature == null) { - return 'create' - } - if (nextSignature == null) { - return 'delete' - } - return prevSignature === nextSignature ? null : 'update' -} - -function diffSignatureMaps( - prev: Map, - next: Map, - resolvePath: (name: string) => string -): WorktreeBasePollEvent[] { - const events: WorktreeBasePollEvent[] = [] - const names = new Set([...prev.keys(), ...next.keys()]) - for (const name of names) { - const type = classifySignatureDiff(prev.get(name), next.get(name)) - if (type) { - events.push({ type, path: resolvePath(name) }) - } - } - return events -} - -function diffGitCommon( - commonDirPath: string, - prev: GitCommonSnapshot, - next: GitCommonSnapshot -): WorktreeBasePollEvent[] { - const events: WorktreeBasePollEvent[] = [] - const worktreesDir = join(commonDirPath, 'worktrees') - const worktreesDirDiff = classifySignatureDiff( - prev.worktreesDirSignature, - next.worktreesDirSignature - ) - if (worktreesDirDiff) { - events.push({ type: worktreesDirDiff, path: worktreesDir }) - } - for (const [entryPath, entry] of next.entries) { - const prevEntry = prev.entries.get(entryPath) - if (!prevEntry) { - events.push({ type: 'create', path: entryPath }) - continue - } - events.push( - ...diffSignatureMaps(prevEntry.structuralSignatures, entry.structuralSignatures, (name) => - join(entryPath, name) - ) - ) - const indexDiff = classifySignatureDiff(prevEntry.indexSignature, entry.indexSignature) - if (indexDiff) { - events.push({ type: indexDiff, path: join(entryPath, LINKED_WORKTREE_INDEX_FILE) }) - } - const headLogDiff = classifySignatureDiff(prevEntry.headLogSignature, entry.headLogSignature) - if (headLogDiff) { - events.push({ type: headLogDiff, path: join(entryPath, LINKED_WORKTREE_HEAD_LOG_FILE) }) - } - } - for (const entryPath of prev.entries.keys()) { - if (!next.entries.has(entryPath)) { - events.push({ type: 'delete', path: entryPath }) - } - } - events.push( - ...diffSignatureMaps(prev.primarySignatures, next.primarySignatures, (name) => - join(commonDirPath, name) - ) - ) - return events -} - export async function startGitCommonPolling( commonDirPath: string, onEvents: (events: WorktreeBasePollEvent[]) => void, pollIntervalMs: number, visibility: WorktreePollerWindowVisibility, onFullScan?: () => void, - includePrimary = true + includePrimary = true, + scanLimits: Partial = {} ): Promise { let disposed = false let ticking = false let tickCount = 0 - let snapshot = await snapshotGitCommon(commonDirPath, undefined, includePrimary) + let snapshot = await snapshotGitCommon( + commonDirPath, + undefined, + includePrimary, + false, + scanLimits + ) let timer: ReturnType | null = null let parkedWhileHidden = false @@ -291,7 +62,8 @@ export async function startGitCommonPolling( commonDirPath, snapshot, includePrimary, - shouldForceFullScan + shouldForceFullScan, + scanLimits ) if (disposed) { return diff --git a/src/main/ipc/worktree-git-common-snapshot.ts b/src/main/ipc/worktree-git-common-snapshot.ts new file mode 100644 index 00000000000..229c36e1ae2 --- /dev/null +++ b/src/main/ipc/worktree-git-common-snapshot.ts @@ -0,0 +1,258 @@ +import { opendir, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' +import type { WorktreeBasePollEvent } from './worktree-base-directory-poller' +import { + WorktreePollingScanBudget, + type WorktreePollingScanLimits +} from './worktree-polling-scan-budget' + +export const PRIMARY_CHECKOUT_METADATA_FILES = [ + 'HEAD', + 'packed-refs', + 'index', + 'config.worktree', + 'logs/HEAD' +] + +const LINKED_WORKTREE_STRUCTURAL_METADATA_FILES = ['HEAD', 'gitdir', 'locked', 'config.worktree'] +const LINKED_WORKTREE_INDEX_FILE = 'index' +const LINKED_WORKTREE_HEAD_LOG_FILE = join('logs', 'HEAD') +const GIT_COMMON_ENTRY_STAT_CONCURRENCY = 16 + +function statSignature(s: { mtimeMs: number; ctimeMs: number; ino: number }): string { + return `${s.mtimeMs}:${s.ctimeMs}:${s.ino}` +} + +async function dirSignature(path: string): Promise { + try { + const stats = await stat(path) + return `${statSignature(stats)}:${stats.size}` + } catch { + return 'missing' + } +} + +async function fileSignature(path: string): Promise { + try { + const stats = await stat(path) + return stats.isFile() ? `${statSignature(stats)}:${stats.size}` : null + } catch { + return null + } +} + +type GitCommonEntrySnapshot = { + dirSignature: string + structuralSignatures: Map + indexSignature: string | null + headLogSignature: string | null +} + +export type GitCommonSnapshot = { + worktreesDirSignature: string + entries: Map + primarySignatures: Map + didFullScan: boolean +} + +async function snapshotGitCommonEntry( + entryPath: string, + previous: GitCommonEntrySnapshot | undefined, + forceFullScan: boolean +): Promise { + // Why: structural leaves can change without the entry directory changing; only the index uses the gate. + const structuralSignatures = new Map() + const [nextDirSignature, headLogSignature] = await Promise.all([ + dirSignature(entryPath), + fileSignature(join(entryPath, LINKED_WORKTREE_HEAD_LOG_FILE)), + Promise.all( + LINKED_WORKTREE_STRUCTURAL_METADATA_FILES.map(async (name) => { + const signature = await fileSignature(join(entryPath, name)) + if (signature !== null) { + structuralSignatures.set(name, signature) + } + }) + ) + ]) + if (nextDirSignature === 'missing') { + // A transient stat failure must not masquerade as a removal; the parent listing is authoritative. + return ( + previous ?? { + dirSignature: nextDirSignature, + structuralSignatures, + indexSignature: null, + headLogSignature + } + ) + } + const shouldReadIndex = forceFullScan || !previous || previous.dirSignature !== nextDirSignature + const indexSignature = shouldReadIndex + ? await fileSignature(join(entryPath, LINKED_WORKTREE_INDEX_FILE)) + : previous.indexSignature + return { + dirSignature: nextDirSignature, + structuralSignatures, + indexSignature, + headLogSignature + } +} + +async function snapshotPrimaryCheckoutSignatures( + commonDirPath: string +): Promise> { + const signatures = new Map() + await Promise.all( + PRIMARY_CHECKOUT_METADATA_FILES.map(async (name) => { + const signature = await fileSignature(join(commonDirPath, name)) + if (signature !== null) { + signatures.set(name, signature) + } + }) + ) + return signatures +} + +export async function snapshotGitCommon( + commonDirPath: string, + previous?: GitCommonSnapshot, + includePrimary = true, + forceFullScan = false, + scanLimits: Partial = {} +): Promise { + const worktreesDir = join(commonDirPath, 'worktrees') + const [worktreesDirSignature, primarySignatures] = await Promise.all([ + dirSignature(worktreesDir), + includePrimary ? snapshotPrimaryCheckoutSignatures(commonDirPath) : new Map() + ]) + // Why: listing is authoritative for add/remove; directory timestamps can collide on coarse filesystems. + const budget = new WorktreePollingScanBudget(scanLimits) + let directory: Awaited> | undefined + let entryPaths: string[] = [] + try { + directory = await opendir(worktreesDir, { bufferSize: 32 }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + entryPaths = previous ? [...previous.entries.keys()] : [] + } + } + if (directory) { + for await (const entry of directory) { + budget.claimEntry() + if (!entry.isDirectory()) { + continue + } + const entryPath = join(worktreesDir, entry.name) + budget.claimRetainedPath(entryPath) + entryPaths.push(entryPath) + } + } else { + for (const entryPath of entryPaths) { + budget.claimRetainedPath(entryPath) + } + } + const entrySnapshots = await mapWithConcurrency( + entryPaths, + GIT_COMMON_ENTRY_STAT_CONCURRENCY, + async (entryPath) => ({ + entryPath, + snapshot: await snapshotGitCommonEntry( + entryPath, + previous?.entries.get(entryPath), + forceFullScan + ) + }) + ) + const entries = new Map() + for (const { entryPath, snapshot } of entrySnapshots) { + entries.set(entryPath, snapshot) + } + return { + worktreesDirSignature, + entries, + primarySignatures, + didFullScan: forceFullScan + } +} + +function classifySignatureDiff( + previous: string | null | undefined, + next: string | null | undefined +): 'create' | 'update' | 'delete' | null { + if (previous == null && next == null) { + return null + } + if (previous == null) { + return 'create' + } + if (next == null) { + return 'delete' + } + return previous === next ? null : 'update' +} + +function diffSignatureMaps( + previous: Map, + next: Map, + resolvePath: (name: string) => string +): WorktreeBasePollEvent[] { + const events: WorktreeBasePollEvent[] = [] + const names = new Set([...previous.keys(), ...next.keys()]) + for (const name of names) { + const type = classifySignatureDiff(previous.get(name), next.get(name)) + if (type) { + events.push({ type, path: resolvePath(name) }) + } + } + return events +} + +export function diffGitCommon( + commonDirPath: string, + previous: GitCommonSnapshot, + next: GitCommonSnapshot +): WorktreeBasePollEvent[] { + const events: WorktreeBasePollEvent[] = [] + const worktreesDir = join(commonDirPath, 'worktrees') + const worktreesDirDiff = classifySignatureDiff( + previous.worktreesDirSignature, + next.worktreesDirSignature + ) + if (worktreesDirDiff) { + events.push({ type: worktreesDirDiff, path: worktreesDir }) + } + for (const [entryPath, entry] of next.entries) { + const previousEntry = previous.entries.get(entryPath) + if (!previousEntry) { + events.push({ type: 'create', path: entryPath }) + continue + } + events.push( + ...diffSignatureMaps(previousEntry.structuralSignatures, entry.structuralSignatures, (name) => + join(entryPath, name) + ) + ) + const indexDiff = classifySignatureDiff(previousEntry.indexSignature, entry.indexSignature) + if (indexDiff) { + events.push({ type: indexDiff, path: join(entryPath, LINKED_WORKTREE_INDEX_FILE) }) + } + const headLogDiff = classifySignatureDiff( + previousEntry.headLogSignature, + entry.headLogSignature + ) + if (headLogDiff) { + events.push({ type: headLogDiff, path: join(entryPath, LINKED_WORKTREE_HEAD_LOG_FILE) }) + } + } + for (const entryPath of previous.entries.keys()) { + if (!next.entries.has(entryPath)) { + events.push({ type: 'delete', path: entryPath }) + } + } + events.push( + ...diffSignatureMaps(previous.primarySignatures, next.primarySignatures, (name) => + join(commonDirPath, name) + ) + ) + return events +} diff --git a/src/main/ipc/worktree-git-common-watch.test.ts b/src/main/ipc/worktree-git-common-watch.test.ts index e4c508e45ff..f067e3225b1 100644 --- a/src/main/ipc/worktree-git-common-watch.test.ts +++ b/src/main/ipc/worktree-git-common-watch.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { appendFile, mkdtemp, mkdir, realpath, rm, writeFile } from 'node:fs/promises' +import { rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { subscribeViaWatcherProcess } from './parcel-watcher-process' @@ -362,8 +363,11 @@ describe('worktree git-common polling gate (non-darwin)', () => { await new Promise((resolve) => setTimeout(resolve, POLL_MS * 2)) const worktreesDir = join(commonDir, 'worktrees') - await rm(worktreesDir, { recursive: true, force: true }) - await writeFile(worktreesDir, 'not-a-dir') + // Swap dir -> file synchronously so no event-loop turn (and thus no poll + // tick) observes the transient ENOENT window, which would legitimately emit + // a delete and defeat the non-ENOENT assertion below. + rmSync(worktreesDir, { recursive: true, force: true }) + writeFileSync(worktreesDir, 'not-a-dir') await new Promise((resolve) => setTimeout(resolve, POLL_MS * 4)) expect(received.flat()).not.toContainEqual({ type: 'delete', path: entry }) diff --git a/src/main/ipc/worktree-git-common-watch.ts b/src/main/ipc/worktree-git-common-watch.ts index a24adf8e357..b8745dd129e 100644 --- a/src/main/ipc/worktree-git-common-watch.ts +++ b/src/main/ipc/worktree-git-common-watch.ts @@ -11,6 +11,7 @@ import { PRIMARY_CHECKOUT_METADATA_FILES, startGitCommonPolling } from './worktree-git-common-polling' +import type { WorktreePollingScanLimits } from './worktree-polling-scan-budget' // Watches a repo's `/.git/worktrees` metadata plus the primary // checkout's shallow branch/index files — the only paths the git-common event @@ -284,7 +285,8 @@ export async function startGitCommonWatch( pollIntervalMs: number, platform: NodeJS.Platform, visibility: WorktreePollerWindowVisibility, - onFullScan?: () => void + onFullScan?: () => void, + scanLimits: Partial = {} ): Promise { if (platform === 'darwin') { const [narrowWatch, primaryMetadataPoll] = await Promise.all([ @@ -303,5 +305,13 @@ export async function startGitCommonWatch( } } } - return startGitCommonPolling(target.path, onEvents, pollIntervalMs, visibility, onFullScan) + return startGitCommonPolling( + target.path, + onEvents, + pollIntervalMs, + visibility, + onFullScan, + true, + scanLimits + ) } diff --git a/src/main/ipc/worktree-head-identity-reader.test.ts b/src/main/ipc/worktree-head-identity-reader.test.ts index c722d171ee0..e4055e8a0bf 100644 --- a/src/main/ipc/worktree-head-identity-reader.test.ts +++ b/src/main/ipc/worktree-head-identity-reader.test.ts @@ -1,8 +1,13 @@ import { afterEach, describe, expect, it } from 'vitest' -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' -import { readGitCommonHeadIdentities } from './worktree-head-identity-reader' +import { + MAX_GIT_HEAD_METADATA_BYTES, + MAX_LINKED_WORKTREE_ENTRIES, + MAX_PACKED_REFS_BYTES, + readGitCommonHeadIdentities +} from './worktree-head-identity-reader' const OID_A = 'a'.repeat(40) const OID_B = 'b'.repeat(40) @@ -149,4 +154,32 @@ describe('readGitCommonHeadIdentities', () => { expect(await readGitCommonHeadIdentities(commonDir)).toEqual([]) }) + + it('rejects metadata that grows beyond the exact file limits', async () => { + const commonDir = await makeCommonDir() + const headPath = join(commonDir, 'HEAD') + await writeFile(headPath, 'x') + await truncate(headPath, MAX_GIT_HEAD_METADATA_BYTES + 1) + expect(await readGitCommonHeadIdentities(commonDir)).toEqual([]) + + await writeFile(headPath, 'ref: refs/heads/main\n') + const packedRefsPath = join(commonDir, 'packed-refs') + await writeFile(packedRefsPath, 'x') + await truncate(packedRefsPath, MAX_PACKED_REFS_BYTES + 1) + expect(await readGitCommonHeadIdentities(commonDir)).toEqual([]) + }) + + it('fails closed instead of retaining a partial oversized worktree listing', async () => { + const commonDir = await makeCommonDir() + await writeFile(join(commonDir, 'HEAD'), `${OID_A}\n`) + const worktreesDir = join(commonDir, 'worktrees') + await mkdir(worktreesDir) + await Promise.all( + Array.from({ length: MAX_LINKED_WORKTREE_ENTRIES + 1 }, (_, index) => + writeFile(join(worktreesDir, `entry-${index}`), '') + ) + ) + + expect(await readGitCommonHeadIdentities(commonDir)).toEqual([]) + }) }) diff --git a/src/main/ipc/worktree-head-identity-reader.ts b/src/main/ipc/worktree-head-identity-reader.ts index e99de115480..8310bc78b75 100644 --- a/src/main/ipc/worktree-head-identity-reader.ts +++ b/src/main/ipc/worktree-head-identity-reader.ts @@ -1,16 +1,26 @@ -import { readdir, readFile } from 'node:fs/promises' +import { opendir } from 'node:fs/promises' import { basename, dirname, isAbsolute, join } from 'node:path' import type { WorktreeHeadIdentity } from '../../shared/types' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' // Why: the whole point of this reader is replacing `git worktree list` fanout // with bounded metadata-file reads, so head freshness never re-creates the // spawn pressure that stalled terminal input. Keep it spawn-free. const MAX_SYMREF_DEPTH = 5 +export const MAX_GIT_HEAD_METADATA_BYTES = 64 * 1024 +export const MAX_PACKED_REFS_BYTES = 16 * 1024 * 1024 +export const MAX_PACKED_REFS_ENTRIES = 100_000 +export const MAX_LINKED_WORKTREE_ENTRIES = 1_024 +const MAX_PACKED_REFS_RETAINED_BYTES = 16 * 1024 * 1024 +const MAX_IDENTITY_RETAINED_BYTES = 4 * 1024 * 1024 -async function readTrimmedFile(path: string): Promise { +async function readTrimmedFile( + path: string, + maxBytes = MAX_GIT_HEAD_METADATA_BYTES +): Promise { try { - return (await readFile(path, 'utf8')).trim() + return (await readNodeFileWithinLimit(path, maxBytes)).buffer.toString('utf8').trim() } catch { return null } @@ -19,19 +29,39 @@ async function readTrimmedFile(path: string): Promise { // packed-refs lines are ` `; `#` headers and `^` peel lines skipped. async function readPackedRefs(commonDirPath: string): Promise> { const refs = new Map() - const content = await readTrimmedFile(join(commonDirPath, 'packed-refs')) + const content = await readTrimmedFile(join(commonDirPath, 'packed-refs'), MAX_PACKED_REFS_BYTES) if (content === null) { return refs } - for (const line of content.split('\n')) { + let retainedBytes = 0 + for (let start = 0; start <= content.length; ) { + const newline = content.indexOf('\n', start) + const end = newline === -1 ? content.length : newline + if (end - start > MAX_GIT_HEAD_METADATA_BYTES) { + return new Map() + } + const line = content.slice(start, end) if (!line || line.startsWith('#') || line.startsWith('^')) { - continue + // Headers and peeled-object lines never resolve a branch head. + } else { + const separator = line.indexOf(' ') + if (separator > 0) { + const ref = line.slice(separator + 1).trim() + const oid = line.slice(0, separator) + retainedBytes += (ref.length + oid.length) * 2 + if ( + refs.size >= MAX_PACKED_REFS_ENTRIES || + retainedBytes > MAX_PACKED_REFS_RETAINED_BYTES + ) { + return new Map() + } + refs.set(ref, oid) + } } - const separator = line.indexOf(' ') - if (separator <= 0) { - continue + if (newline === -1) { + break } - refs.set(line.slice(separator + 1).trim(), line.slice(0, separator)) + start = newline + 1 } return refs } @@ -113,6 +143,16 @@ export async function readGitCommonHeadIdentities( (packedRefsPromise ??= readPackedRefs(commonDirPath)) const identities: WorktreeHeadIdentity[] = [] + let retainedIdentityBytes = 0 + const retainIdentity = (identity: WorktreeHeadIdentity): boolean => { + retainedIdentityBytes += + (identity.worktreePath.length + (identity.branch?.length ?? 0) + identity.head.length) * 2 + if (retainedIdentityBytes > MAX_IDENTITY_RETAINED_BYTES) { + return false + } + identities.push(identity) + return true + } // Only the standard `/.git` layout maps a common dir back to its // primary checkout path; bare/custom GIT_DIR layouts have no primary row. if (basename(commonDirPath) === '.git') { @@ -122,40 +162,51 @@ export async function readGitCommonHeadIdentities( dirname(commonDirPath), packedRefs ) - if (primary) { - identities.push(primary) + if (primary && !retainIdentity(primary)) { + return [] } } - let entries + let directory: Awaited> try { - entries = await readdir(join(commonDirPath, 'worktrees'), { withFileTypes: true }) + directory = await opendir(join(commonDirPath, 'worktrees')) } catch { return identities } - for (const entry of entries) { - if (!entry.isDirectory()) { - continue - } - const entryPath = join(commonDirPath, 'worktrees', entry.name) - const gitdirContent = await readTrimmedFile(join(entryPath, 'gitdir')) - if (!gitdirContent) { - continue - } - // `gitdir` holds `/.git`, absolute or (with relative-path - // worktrees) relative to the entry dir. - const gitdirAbsolute = isAbsolute(gitdirContent) - ? gitdirContent - : join(entryPath, gitdirContent) - const identity = await readHeadIdentity( - commonDirPath, - join(entryPath, 'HEAD'), - dirname(gitdirAbsolute), - packedRefs - ) - if (identity) { - identities.push(identity) + let entriesSeen = 0 + try { + for await (const entry of directory) { + entriesSeen += 1 + if (entriesSeen > MAX_LINKED_WORKTREE_ENTRIES) { + return [] + } + if (!entry.isDirectory()) { + continue + } + const entryPath = join(commonDirPath, 'worktrees', entry.name) + const gitdirContent = await readTrimmedFile(join(entryPath, 'gitdir')) + if (!gitdirContent) { + continue + } + // `gitdir` holds `/.git`, absolute or (with relative-path + // worktrees) relative to the entry dir. + const gitdirAbsolute = isAbsolute(gitdirContent) + ? gitdirContent + : join(entryPath, gitdirContent) + const identity = await readHeadIdentity( + commonDirPath, + join(entryPath, 'HEAD'), + dirname(gitdirAbsolute), + packedRefs + ) + if (identity && !retainIdentity(identity)) { + return [] + } } + } catch { + return identities + } finally { + await directory.close().catch(() => undefined) } return identities } diff --git a/src/main/ipc/worktree-polling-scan-budget.test.ts b/src/main/ipc/worktree-polling-scan-budget.test.ts new file mode 100644 index 00000000000..02aca191ade --- /dev/null +++ b/src/main/ipc/worktree-polling-scan-budget.test.ts @@ -0,0 +1,287 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { WorktreeBaseWatchTarget } from './worktree-base-directory-event-filter' +import { startWorktreeBaseDirectoryPoller } from './worktree-base-directory-poller' +import { + WorktreePollingScanBudget, + type WorktreePollingCapacityError +} from './worktree-polling-scan-budget' + +describe('worktree polling scan budget', () => { + const cleanupPaths: string[] = [] + + afterEach(async () => { + await Promise.all( + cleanupPaths.splice(0).map((path) => rm(path, { recursive: true, force: true })) + ) + }) + + async function createRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-worktree-poll-bounds-')) + cleanupPaths.push(root) + return root + } + + function makeTarget(kind: 'base' | 'git-common', path: string): WorktreeBaseWatchTarget { + return { + key: `${kind}:local:${path}`, + kind, + path, + repos: new Map([ + [ + 'repo-1', + { + repoId: 'repo-1', + repoName: 'project', + nestWorkspaces: false + } + ] + ]) + } + } + + it('accepts exact entry, path-count, and path-byte limits', () => { + const budget = new WorktreePollingScanBudget({ + maxScannedEntries: 2, + maxRetainedPaths: 2, + maxRetainedPathBytes: 6 + }) + + budget.claimEntry() + budget.claimEntry() + budget.claimRetainedPath('one') + expect(() => budget.claimRetainedPath('two')).not.toThrow() + }) + + it('rejects the first scanned entry over the limit', () => { + const budget = new WorktreePollingScanBudget({ maxScannedEntries: 1 }) + budget.claimEntry() + + expect(() => budget.claimEntry()).toThrow( + expect.objectContaining>({ + resource: 'scanned entries', + observed: 2, + limit: 1 + }) + ) + }) + + it('rejects the first retained path byte over the limit', () => { + const budget = new WorktreePollingScanBudget({ + maxRetainedPaths: 2, + maxRetainedPathBytes: 5 + }) + budget.claimRetainedPath('one') + + expect(() => budget.claimRetainedPath('two')).toThrow( + expect.objectContaining>({ + resource: 'retained path bytes', + observed: 6, + limit: 5 + }) + ) + }) + + it('accepts the exact retained repo budget and rejects one byte less', () => { + const exact = new WorktreePollingScanBudget({ + maxRepoConfigs: 1, + maxRetainedRepoBytes: 130 + }) + expect(() => exact.claimRepoConfig('a', 'b')).not.toThrow() + + const overflow = new WorktreePollingScanBudget({ + maxRepoConfigs: 1, + maxRetainedRepoBytes: 129 + }) + expect(() => overflow.claimRepoConfig('a', 'b')).toThrow( + expect.objectContaining>({ + resource: 'retained repo bytes', + observed: 130, + limit: 129 + }) + ) + }) + + it('rejects the first repo config over the count limit', () => { + const budget = new WorktreePollingScanBudget({ maxRepoConfigs: 1 }) + budget.claimRepoConfig('repo-1', 'one') + + expect(() => budget.claimRepoConfig('repo-2', 'two')).toThrow( + expect.objectContaining>({ + resource: 'repo configs', + observed: 2, + limit: 1 + }) + ) + }) + + it('starts a base poller at the exact directory-entry limit', async () => { + const root = await createRoot() + await Promise.all([mkdir(join(root, 'one')), mkdir(join(root, 'two'))]) + const target = makeTarget('base', root) + + const poller = await startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + () => {}, + { + scanLimits: { + maxScannedEntries: 2, + maxRetainedPaths: 3 + } + } + ) + + await poller.unsubscribe() + }) + + it('rejects a base snapshot on the first directory entry over the limit', async () => { + const root = await createRoot() + await Promise.all([ + mkdir(join(root, 'one')), + mkdir(join(root, 'two')), + mkdir(join(root, 'three')) + ]) + const target = makeTarget('base', root) + + await expect( + startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + () => {}, + { + scanLimits: { maxScannedEntries: 2 } + } + ) + ).rejects.toMatchObject({ resource: 'scanned entries', observed: 3, limit: 2 }) + }) + + it('keeps the last complete snapshot while over capacity and resumes afterward', async () => { + const root = await createRoot() + const knownWorktree = join(root, 'known') + await mkdir(knownWorktree) + await writeFile(join(knownWorktree, '.git'), 'gitdir: elsewhere') + const target = makeTarget('base', root) + const events: { type: string; path: string }[] = [] + const poller = await startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + (nextEvents) => events.push(...nextEvents), + { + pollIntervalMs: 100, + scanLimits: { maxScannedEntries: 2 } + } + ) + + try { + await Promise.all([ + mkdir(join(root, 'overflow-a')), + mkdir(join(root, 'overflow-b')), + mkdir(join(root, 'overflow-c')) + ]) + await rm(knownWorktree, { recursive: true }) + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(events).toEqual([]) + + await Promise.all([ + rm(join(root, 'overflow-b'), { recursive: true }), + rm(join(root, 'overflow-c'), { recursive: true }) + ]) + await vi.waitFor( + () => { + expect(events).toContainEqual({ type: 'delete', path: knownWorktree }) + }, + { timeout: 2_000 } + ) + } finally { + await poller.unsubscribe() + } + }) + + it('pauses snapshots for excess repo configs and resumes when they are removed', async () => { + const root = await createRoot() + const knownWorktree = join(root, 'known') + await mkdir(knownWorktree) + await writeFile(join(knownWorktree, '.git'), 'gitdir: elsewhere') + const target = makeTarget('base', root) + const events: { type: string; path: string }[] = [] + const poller = await startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + (nextEvents) => events.push(...nextEvents), + { + pollIntervalMs: 100, + scanLimits: { maxRepoConfigs: 1 } + } + ) + + try { + target.repos.set('repo-2', { + repoId: 'repo-2', + repoName: 'second', + nestWorkspaces: false + }) + await rm(knownWorktree, { recursive: true }) + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(events).toEqual([]) + + target.repos.delete('repo-2') + await vi.waitFor( + () => { + expect(events).toContainEqual({ type: 'delete', path: knownWorktree }) + }, + { timeout: 2_000 } + ) + } finally { + await poller.unsubscribe() + } + }) + + it('starts git-common polling at the exact linked-worktree limit', async () => { + const root = await createRoot() + await Promise.all([ + mkdir(join(root, 'worktrees', 'one'), { recursive: true }), + mkdir(join(root, 'worktrees', 'two'), { recursive: true }) + ]) + const target = makeTarget('git-common', root) + + const poller = await startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + () => {}, + { + platform: 'linux', + scanLimits: { + maxScannedEntries: 2, + maxRetainedPaths: 2 + } + } + ) + + await poller.unsubscribe() + }) + + it('rejects git-common polling on the first linked worktree over the limit', async () => { + const root = await createRoot() + await Promise.all([ + mkdir(join(root, 'worktrees', 'one'), { recursive: true }), + mkdir(join(root, 'worktrees', 'two'), { recursive: true }), + mkdir(join(root, 'worktrees', 'three'), { recursive: true }) + ]) + const target = makeTarget('git-common', root) + + await expect( + startWorktreeBaseDirectoryPoller( + target, + () => target.repos, + () => {}, + { + platform: 'linux', + scanLimits: { maxScannedEntries: 2 } + } + ) + ).rejects.toMatchObject({ resource: 'scanned entries', observed: 3, limit: 2 }) + }) +}) diff --git a/src/main/ipc/worktree-polling-scan-budget.ts b/src/main/ipc/worktree-polling-scan-budget.ts new file mode 100644 index 00000000000..727b587ce41 --- /dev/null +++ b/src/main/ipc/worktree-polling-scan-budget.ts @@ -0,0 +1,128 @@ +export const WORKTREE_POLLING_MAX_SCANNED_ENTRIES = 100_000 +export const WORKTREE_POLLING_MAX_RETAINED_PATHS = 16_384 +export const WORKTREE_POLLING_MAX_RETAINED_PATH_BYTES = 16 * 1024 * 1024 +export const WORKTREE_POLLING_MAX_REPO_CONFIGS = 4_096 +export const WORKTREE_POLLING_MAX_RETAINED_REPO_BYTES = 4 * 1024 * 1024 + +export type WorktreePollingScanLimits = { + maxScannedEntries: number + maxRetainedPaths: number + maxRetainedPathBytes: number + maxRepoConfigs: number + maxRetainedRepoBytes: number +} + +export class WorktreePollingCapacityError extends Error { + constructor( + readonly resource: + | 'scanned entries' + | 'retained paths' + | 'retained path bytes' + | 'repo configs' + | 'retained repo bytes', + readonly observed: number, + readonly limit: number + ) { + super(`Worktree polling exceeded ${limit} ${resource} (observed ${observed})`) + this.name = 'WorktreePollingCapacityError' + } +} + +function resolveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} + +export class WorktreePollingScanBudget { + private scannedEntries = 0 + private retainedPaths = 0 + private retainedPathBytes = 0 + private repoConfigs = 0 + private retainedRepoBytes = 0 + private readonly limits: WorktreePollingScanLimits + + constructor(requested: Partial = {}) { + this.limits = { + maxScannedEntries: resolveLimit( + requested.maxScannedEntries, + WORKTREE_POLLING_MAX_SCANNED_ENTRIES, + 'maxScannedEntries' + ), + maxRetainedPaths: resolveLimit( + requested.maxRetainedPaths, + WORKTREE_POLLING_MAX_RETAINED_PATHS, + 'maxRetainedPaths' + ), + maxRetainedPathBytes: resolveLimit( + requested.maxRetainedPathBytes, + WORKTREE_POLLING_MAX_RETAINED_PATH_BYTES, + 'maxRetainedPathBytes' + ), + maxRepoConfigs: resolveLimit( + requested.maxRepoConfigs, + WORKTREE_POLLING_MAX_REPO_CONFIGS, + 'maxRepoConfigs' + ), + maxRetainedRepoBytes: resolveLimit( + requested.maxRetainedRepoBytes, + WORKTREE_POLLING_MAX_RETAINED_REPO_BYTES, + 'maxRetainedRepoBytes' + ) + } + } + + claimEntry(): void { + this.scannedEntries += 1 + if (this.scannedEntries > this.limits.maxScannedEntries) { + throw new WorktreePollingCapacityError( + 'scanned entries', + this.scannedEntries, + this.limits.maxScannedEntries + ) + } + } + + claimRetainedPath(path: string): void { + this.retainedPaths += 1 + if (this.retainedPaths > this.limits.maxRetainedPaths) { + throw new WorktreePollingCapacityError( + 'retained paths', + this.retainedPaths, + this.limits.maxRetainedPaths + ) + } + this.retainedPathBytes += Buffer.byteLength(path, 'utf8') + if (this.retainedPathBytes > this.limits.maxRetainedPathBytes) { + throw new WorktreePollingCapacityError( + 'retained path bytes', + this.retainedPathBytes, + this.limits.maxRetainedPathBytes + ) + } + } + + claimRepoConfig(repoId: string, repoName: string): void { + this.repoConfigs += 1 + if (this.repoConfigs > this.limits.maxRepoConfigs) { + throw new WorktreePollingCapacityError( + 'repo configs', + this.repoConfigs, + this.limits.maxRepoConfigs + ) + } + this.retainedRepoBytes += + Buffer.byteLength(repoId, 'utf8') + Buffer.byteLength(repoName, 'utf8') + 128 + if (this.retainedRepoBytes > this.limits.maxRetainedRepoBytes) { + throw new WorktreePollingCapacityError( + 'retained repo bytes', + this.retainedRepoBytes, + this.limits.maxRetainedRepoBytes + ) + } + } +} diff --git a/src/main/ipc/worktree-remote.ts b/src/main/ipc/worktree-remote.ts index 014e04a583a..feabd181e4a 100644 --- a/src/main/ipc/worktree-remote.ts +++ b/src/main/ipc/worktree-remote.ts @@ -57,7 +57,9 @@ import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import type { SshGitProvider } from '../providers/ssh-git-provider' import { TUI_AGENT_CONFIG, isTuiAgent } from '../../shared/tui-agent-config' import { isWindowsAbsolutePathLike } from '../../shared/cross-platform-path' +import { MAX_ORCA_YAML_BYTES, MAX_ORCA_YAML_CODE_UNITS } from '../../shared/orca-yaml-file-limit' import { getSshGitUsername } from '../git/git-username' +import { readFilesystemProviderBoundedText } from '../filesystem-provider-bounded-text' import { runWorktreeChangeInvalidators } from './worktree-change-invalidators' import { registerOptionalSshWorktreeCreateRoots, @@ -1186,8 +1188,12 @@ async function readRemoteOrcaYaml( hooksRootPath: string ): Promise> { try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(hooksRootPath, 'orca.yaml')) - return result.isBinary ? null : parseOrcaYaml(result.content) + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(hooksRootPath, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + return result.kind === 'text' ? parseOrcaYaml(result.content) : null } catch { return null } diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 6e126be2091..955700a35c8 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -8,6 +8,8 @@ import type { CreateWorktreeResult, GitWorktreeInfo, Worktree } from '../../shar import * as localWorktreeFilesystem from '../local-worktree-filesystem' const ORIGINAL_PLATFORM = process.platform +const TEST_MAX_HOOK_GITIGNORE_BYTES = 4 * 1024 * 1024 +const TEST_MAX_ISSUE_COMMAND_BYTES = 1024 * 1024 const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) const findExistingWorktreeSymlinkPathsMock = vi.hoisted(() => vi.fn()) @@ -193,6 +195,8 @@ vi.mock('../hooks', () => ({ getEffectiveHooksFromConfig: getEffectiveHooksFromConfigMock, getDefaultTabsLaunch: getDefaultTabsLaunchMock, getSetupRunnerEnvVars: getSetupRunnerEnvVarsMock, + MAX_HOOK_GITIGNORE_BYTES: 4 * 1024 * 1024, + MAX_ISSUE_COMMAND_BYTES: 1024 * 1024, loadHooks: loadHooksMock, parseOrcaYaml: parseOrcaYamlMock, runHook: runHookMock, @@ -3697,6 +3701,12 @@ describe('registerWorktreeHandlers', () => { ]) } const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('orca.yaml')) { + return { size: 32, type: 'file', mtime: 0 } + } + throw Object.assign(new Error('missing'), { code: 'ENOENT' }) + }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n setup: pnpm install\n', isBinary: false @@ -7086,6 +7096,7 @@ describe('registerWorktreeHandlers', () => { }) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 36, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: echo archived\n', isBinary: false @@ -7163,6 +7174,7 @@ describe('registerWorktreeHandlers', () => { }) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 36, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: echo archived\n', isBinary: false @@ -7222,6 +7234,7 @@ describe('registerWorktreeHandlers', () => { }) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 36, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: echo archived\n', isBinary: false @@ -7280,6 +7293,7 @@ describe('registerWorktreeHandlers', () => { }) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 30, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: exit 7\n', isBinary: false @@ -7338,6 +7352,7 @@ describe('registerWorktreeHandlers', () => { execNonInteractive: vi.fn().mockRejectedValue(new Error('relay disconnected')) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 36, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: echo archived\n', isBinary: false @@ -7400,6 +7415,7 @@ describe('registerWorktreeHandlers', () => { }) } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 36, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: echo archived\n', isBinary: false @@ -7560,6 +7576,7 @@ describe('registerWorktreeHandlers', () => { } const sshRepo = { ...localRepo, path: '/remote/repo', connectionId: 'conn-1' } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 35, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n archive: remote-cleanup', isBinary: false @@ -8689,6 +8706,12 @@ describe('registerWorktreeHandlers', () => { worktreeBaseRef: null } const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('/.orca/issue-command')) { + return { size: 14, type: 'file', mtime: 0 } + } + throw new Error('shared read failed') + }), readFile: vi.fn(async (filePath: string) => { if (filePath.endsWith('/.orca/issue-command')) { return { content: 'local command\n', isBinary: false } @@ -8712,6 +8735,46 @@ describe('registerWorktreeHandlers', () => { }) }) + it('admits an exact-size SSH issue command and rejects +1 before reading', async () => { + const repo = { + id: 'repo-ssh', + path: '/remote/repo', + displayName: 'ssh', + badgeColor: '#000', + addedAt: 0, + connectionId: 'conn-1', + worktreeBaseRef: null + } + let issueCommandSize = TEST_MAX_ISSUE_COMMAND_BYTES + const exactContent = 'x'.repeat(TEST_MAX_ISSUE_COMMAND_BYTES) + const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('/.orca/issue-command')) { + return { size: issueCommandSize, type: 'file', mtime: 0 } + } + throw Object.assign(new Error('missing'), { code: 'ENOENT' }) + }), + readFile: vi.fn().mockResolvedValue({ content: exactContent, isBinary: false }) + } + store.getRepo.mockReturnValue(repo) + getSshFilesystemProviderMock.mockReturnValue(fsProvider) + + const exactResult = (await handlers['hooks:readIssueCommand'](null, { + repoId: 'repo-ssh' + })) as { localContent: string | null } + expect(exactResult.localContent).toHaveLength(TEST_MAX_ISSUE_COMMAND_BYTES) + expect(fsProvider.readFile).toHaveBeenCalledOnce() + + issueCommandSize += 1 + fsProvider.readFile.mockClear() + await expect( + handlers['hooks:readIssueCommand'](null, { + repoId: 'repo-ssh' + }) + ).resolves.toMatchObject({ localContent: null, effectiveContent: null, source: 'none' }) + expect(fsProvider.readFile).not.toHaveBeenCalled() + }) + it('writes SSH issue-command overrides without clobbering .gitignore on read failure', async () => { const repo = { id: 'repo-ssh', @@ -8724,6 +8787,7 @@ describe('registerWorktreeHandlers', () => { } const fsProvider = { createDir: vi.fn().mockResolvedValue(undefined), + stat: vi.fn().mockRejectedValue(new Error('ssh read failed')), readFile: vi.fn().mockRejectedValue(new Error('ssh read failed')), writeFile: vi.fn().mockResolvedValue(undefined), deletePath: vi.fn().mockResolvedValue(undefined) @@ -8756,6 +8820,12 @@ describe('registerWorktreeHandlers', () => { connectionId: 'conn-1' } const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('/.orca/issue-command')) { + return { size: 15, type: 'file', mtime: 0 } + } + throw Object.assign(new Error('missing'), { code: 'ENOENT' }) + }), readFile: vi.fn(async (filePath: string) => { if (filePath.endsWith('/.orca/issue-command')) { return { content: 'remote command\n', isBinary: false } @@ -8793,6 +8863,7 @@ describe('registerWorktreeHandlers', () => { const enoent = Object.assign(new Error('missing'), { code: 'ENOENT' }) const fsProvider = { createDir: vi.fn().mockResolvedValue(undefined), + stat: vi.fn().mockRejectedValue(enoent), readFile: vi.fn().mockRejectedValue(enoent), writeFile: vi.fn().mockResolvedValue(undefined), deletePath: vi.fn().mockResolvedValue(undefined) @@ -8813,6 +8884,56 @@ describe('registerWorktreeHandlers', () => { ) }) + it('admits an exact-size SSH .gitignore and rejects +1 before reading', async () => { + const repo = { + id: 'repo-ssh', + path: '/remote/repo', + displayName: 'ssh', + badgeColor: '#000', + addedAt: 0, + connectionId: 'conn-1', + worktreeBaseRef: null + } + let gitignoreSize = TEST_MAX_HOOK_GITIGNORE_BYTES + const exactContent = 'x'.repeat(TEST_MAX_HOOK_GITIGNORE_BYTES) + const fsProvider = { + createDir: vi.fn().mockResolvedValue(undefined), + stat: vi.fn().mockImplementation(async () => ({ + size: gitignoreSize, + type: 'file', + mtime: 0 + })), + readFile: vi.fn().mockResolvedValue({ content: exactContent, isBinary: false }), + writeFile: vi.fn().mockResolvedValue(undefined), + deletePath: vi.fn().mockResolvedValue(undefined) + } + store.getRepo.mockReturnValue(repo) + getSshFilesystemProviderMock.mockReturnValue(fsProvider) + + await handlers['hooks:writeIssueCommand'](null, { + repoId: 'repo-ssh', + content: 'orca issue command' + }) + const gitignoreWrite = fsProvider.writeFile.mock.calls.find(([filePath]) => + filePath.endsWith('/.gitignore') + ) + expect(gitignoreWrite?.[1]).toHaveLength(TEST_MAX_HOOK_GITIGNORE_BYTES + 7) + expect(gitignoreWrite?.[1]).toMatch(/\n\.orca\n$/) + expect(fsProvider.readFile).toHaveBeenCalledOnce() + + gitignoreSize += 1 + fsProvider.readFile.mockClear() + fsProvider.writeFile.mockClear() + await expect( + handlers['hooks:writeIssueCommand'](null, { + repoId: 'repo-ssh', + content: 'orca issue command' + }) + ).rejects.toThrow('Remote .gitignore exceeds the supported size limit') + expect(fsProvider.readFile).not.toHaveBeenCalled() + expect(fsProvider.writeFile).not.toHaveBeenCalled() + }) + it('rejects SSH issue-command writes when the remote filesystem provider is unavailable', async () => { const repo = { id: 'repo-ssh', diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 40cf7efd17d..399ddcfd172 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1,7 +1,7 @@ /* oxlint-disable max-lines */ import type { BrowserWindow } from 'electron' import { ipcMain } from 'electron' -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import { randomUUID } from 'node:crypto' import type { Store } from '../persistence' import { isFolderRepo } from '../../shared/repo-kind' @@ -12,6 +12,7 @@ import { worktreeWorkspaceKey } from '../../shared/workspace-scope' import { inspectSetupScriptImportCandidates } from '../../shared/setup-script-imports' +import { MAX_ORCA_YAML_BYTES, MAX_ORCA_YAML_CODE_UNITS } from '../../shared/orca-yaml-file-limit' import { getProjectHostSetupWorktreeMeta } from '../../shared/project-host-setup-projection' import { projectResolvedWorktreeLineage } from '../../shared/resolved-worktree-lineage' import { deleteWorktreeHistoryDir } from '../terminal-history' @@ -60,6 +61,8 @@ import { getEffectiveHooks, getEffectiveHooksFromConfig, getSetupRunnerEnvVars, + MAX_HOOK_GITIGNORE_BYTES, + MAX_ISSUE_COMMAND_BYTES, loadHooks, parseOrcaYaml, readIssueCommand, @@ -104,6 +107,12 @@ import { resolveAutomationWorkspaceProvenance } from '../automations/workspace-provenance' import { shouldEmitBoundedWarning } from './bounded-warning-dedupe' +import { readFilesystemProviderBoundedText } from '../filesystem-provider-bounded-text' +import { + readSetupScriptImportFile, + SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS +} from '../setup-script-import-file' type CreateWorktreeArgsWithSystemProvenance = CreateWorktreeArgs & { automationProvenance?: AutomationWorkspaceProvenance @@ -349,8 +358,12 @@ async function getArchiveHooksForRemoval(repo: Repo): Promise } try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repo.path, 'orca.yaml')) - const yamlHooks = result.isBinary ? null : parseOrcaYaml(result.content) + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repo.path, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + const yamlHooks = result.kind === 'text' ? parseOrcaYaml(result.content) : null return getEffectiveHooksFromConfig(repo, yamlHooks) } catch { return getEffectiveHooksFromConfig(repo, null) @@ -2107,11 +2120,15 @@ export function registerWorktreeHandlers( return { status: 'error', hasHooks: false, hooks: null, mayNeedUpdate: false } } try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repo.path, 'orca.yaml')) + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repo.path, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) return { status: 'ok', - hasHooks: !result.isBinary, - hooks: result.isBinary ? null : parseOrcaYaml(result.content), + hasHooks: result.kind !== 'binary', + hooks: result.kind === 'text' ? parseOrcaYaml(result.content) : null, mayNeedUpdate: false } } catch (error) { @@ -2169,15 +2186,18 @@ export function registerWorktreeHandlers( return null } try { - const result = await fsProvider.readFile(filePath) - return result.isBinary ? null : result.content + const result = await readFilesystemProviderBoundedText(fsProvider, filePath, { + maxBytes: SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + maxCodeUnits: SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS + }) + return result.kind === 'text' ? result.content : null } catch { return null } } try { - return await readFile(filePath, 'utf-8') + return await readSetupScriptImportFile(filePath) } catch (error) { if (!isENOENT(error)) { console.warn('[hooks] Failed to inspect setup script import candidate:', error) @@ -2247,18 +2267,26 @@ export function registerWorktreeHandlers( let localContent: string | null = null let sharedContent: string | null = null try { - const result = await fsProvider.readFile(issueCommandPath) - localContent = result.isBinary ? null : result.content.trim() || null + const result = await readFilesystemProviderBoundedText(fsProvider, issueCommandPath, { + maxBytes: MAX_ISSUE_COMMAND_BYTES, + maxCodeUnits: MAX_ISSUE_COMMAND_BYTES + }) + localContent = result.kind === 'text' ? result.content.trim() || null : null } catch (error) { if (!isENOENT(error)) { status = 'error' } } try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repo.path, 'orca.yaml')) - sharedContent = result.isBinary - ? null - : parseOrcaYaml(result.content)?.issueCommand?.trim() || null + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repo.path, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + sharedContent = + result.kind === 'text' + ? parseOrcaYaml(result.content)?.issueCommand?.trim() || null + : null } catch (error) { if (!isENOENT(error)) { status = 'error' @@ -2309,8 +2337,14 @@ export function registerWorktreeHandlers( await fsProvider.createDir(joinWorktreeRelativePath(repo.path, '.orca')) const gitignorePath = joinWorktreeRelativePath(repo.path, '.gitignore') try { - const result = await fsProvider.readFile(gitignorePath) - if (!result.isBinary && !/^\.orca\/?$/m.test(result.content)) { + const result = await readFilesystemProviderBoundedText(fsProvider, gitignorePath, { + maxBytes: MAX_HOOK_GITIGNORE_BYTES, + maxCodeUnits: MAX_HOOK_GITIGNORE_BYTES + }) + if (result.kind === 'oversized') { + throw new Error('Remote .gitignore exceeds the supported size limit') + } + if (result.kind === 'text' && !/^\.orca\/?$/m.test(result.content)) { const separator = result.content.endsWith('\n') ? '' : '\n' await fsProvider.writeFile(gitignorePath, `${result.content}${separator}.orca\n`) } diff --git a/src/main/ipc/wsl-watcher-snapshot.test.ts b/src/main/ipc/wsl-watcher-snapshot.test.ts new file mode 100644 index 00000000000..ff80833ce7f --- /dev/null +++ b/src/main/ipc/wsl-watcher-snapshot.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' +import { MAX_WSL_SNAPSHOT_RETAINED_ENTRIES, parseWslSnapshotFrame } from './wsl-watcher-snapshot' + +describe('parseWslSnapshotFrame', () => { + it('rejects a snapshot with more retained paths than the entry budget', () => { + const records = Array.from( + { length: MAX_WSL_SNAPSHOT_RETAINED_ENTRIES + 1 }, + (_, index) => `f\t1\t/repo/file-${index}\0` + ).join('') + + expect(parseWslSnapshotFrame(records, 'Ubuntu')).toBeNull() + }) + + it('does not charge duplicate path updates as distinct retained entries', () => { + const records = Array.from( + { length: MAX_WSL_SNAPSHOT_RETAINED_ENTRIES + 1 }, + (_, index) => `f\t${index}\t/repo/same\0` + ).join('') + + const snapshot = parseWslSnapshotFrame(records, 'Ubuntu') + expect(snapshot).not.toBeNull() + expect(snapshot?.size).toBe(1) + expect(snapshot?.get('\\\\wsl.localhost\\Ubuntu\\repo\\same')?.mtime).toBe( + String(MAX_WSL_SNAPSHOT_RETAINED_ENTRIES) + ) + }) +}) diff --git a/src/main/ipc/wsl-watcher-snapshot.ts b/src/main/ipc/wsl-watcher-snapshot.ts new file mode 100644 index 00000000000..c1192d77de7 --- /dev/null +++ b/src/main/ipc/wsl-watcher-snapshot.ts @@ -0,0 +1,72 @@ +import type { Event as WatcherEvent } from '@parcel/watcher' +import { iterateNulDelimitedFields } from '../../shared/nul-delimited-fields' + +export const MAX_WSL_SNAPSHOT_RETAINED_ENTRIES = 50_000 + +export type WslSnapshotEntry = { + path: string + type: string + mtime: string +} + +export type WslSnapshot = Map + +function toWslUncPath(linuxPath: string, distro: string): string { + return `\\\\wsl.localhost\\${distro}${linuxPath.replace(/\//g, '\\')}` +} + +export function parseWslSnapshotFrame(frame: string, distro: string): WslSnapshot | null { + const snapshot: WslSnapshot = new Map() + for (const rawEntry of iterateNulDelimitedFields(frame)) { + if (!rawEntry) { + continue + } + const firstTab = rawEntry.indexOf('\t') + const secondTab = firstTab === -1 ? -1 : rawEntry.indexOf('\t', firstTab + 1) + if (firstTab <= 0 || secondTab <= firstTab + 1) { + continue + } + const linuxPath = rawEntry.slice(secondTab + 1) + if (!linuxPath.startsWith('/')) { + continue + } + const path = toWslUncPath(linuxPath, distro) + if (!snapshot.has(path) && snapshot.size >= MAX_WSL_SNAPSHOT_RETAINED_ENTRIES) { + return null + } + snapshot.set(path, { + type: rawEntry.slice(0, firstTab), + mtime: rawEntry.slice(firstTab + 1, secondTab), + path + }) + } + return snapshot +} + +export function diffWslSnapshots(prev: WslSnapshot, next: WslSnapshot): WatcherEvent[] { + const events: WatcherEvent[] = [] + + for (const [entryPath, nextEntry] of next) { + const prevEntry = prev.get(entryPath) + if (!prevEntry) { + events.push({ type: 'create', path: entryPath } as WatcherEvent) + continue + } + if (prevEntry.type !== nextEntry.type) { + events.push({ type: 'delete', path: entryPath } as WatcherEvent) + events.push({ type: 'create', path: entryPath } as WatcherEvent) + continue + } + if (prevEntry.mtime !== nextEntry.mtime) { + events.push({ type: 'update', path: entryPath } as WatcherEvent) + } + } + + for (const entryPath of prev.keys()) { + if (!next.has(entryPath)) { + events.push({ type: 'delete', path: entryPath } as WatcherEvent) + } + } + + return events +} diff --git a/src/main/jira/client.test.ts b/src/main/jira/client.test.ts index 0c0ee11511c..71893f71d1d 100644 --- a/src/main/jira/client.test.ts +++ b/src/main/jira/client.test.ts @@ -3,6 +3,10 @@ import { tmpdir } from 'node:os' import type * as Os from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + MAX_INTEGRATION_ACCOUNTS, + MAX_INTEGRATION_CREDENTIAL_BYTES +} from '../integration-account-persistence-limits' const OLD_FETCH = globalThis.fetch const { closeAllConnectionsMock, netFetchMock, resolveProxyMock, setProxyMock } = vi.hoisted( @@ -650,4 +654,73 @@ describe('Jira client credential storage', () => { expect(headers.get('User-Agent')).toBe('Orca') expect(fetchMock).not.toHaveBeenCalled() }) + + it('admits the exact saved-site boundary without changing order', async () => { + const sites = Array.from({ length: MAX_INTEGRATION_ACCOUNTS }, (_, index) => ({ + id: `site-${index}`, + token: `token-${index}` + })) + writeMultiSiteFiles(sites, 'all') + const jira = await loadClientModule() + + const status = jira.getStatus() + expect(status.sites).toHaveLength(MAX_INTEGRATION_ACCOUNTS) + expect(status.sites?.map((site) => site.id)).toEqual(sites.map((site) => site.id)) + }) + + it('preserves an over-limit saved-site file and refuses to overwrite it', async () => { + const sites = Array.from({ length: MAX_INTEGRATION_ACCOUNTS + 1 }, (_, index) => ({ + id: `site-${index}`, + token: `token-${index}` + })) + writeMultiSiteFiles(sites, 'all') + const path = join(tempHome, '.orca', 'jira-sites.json') + const before = readFileSync(path, 'utf8') + const jira = await loadClientModule() + + expect(jira.getStatus()).toMatchObject({ connected: false, sites: [] }) + await expect( + jira.connect({ + siteUrl: 'example.atlassian.net', + email: 'ada@example.com', + apiToken: 'token-alpha' + }) + ).resolves.toMatchObject({ ok: false, error: expect.stringContaining('left unchanged') }) + expect(netFetchMock).not.toHaveBeenCalled() + expect(readFileSync(path, 'utf8')).toBe(before) + }) + + it('admits an exact-size Jira credential and rejects credential byte +1 before fetch', async () => { + const exactToken = 't'.repeat(MAX_INTEGRATION_CREDENTIAL_BYTES) + netFetchMock.mockResolvedValueOnce( + new Response( + JSON.stringify({ + accountId: 'account-alpha', + displayName: 'Ada', + emailAddress: 'ada@example.com' + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ) + ) + const jira = await loadClientModule() + + await expect( + jira.connect({ + siteUrl: 'example.atlassian.net', + email: 'ada@example.com', + apiToken: exactToken + }) + ).resolves.toMatchObject({ ok: true }) + await expect( + jira.connect({ + siteUrl: 'another.atlassian.net', + email: 'ada@example.com', + apiToken: `${exactToken}t` + }) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining(`${MAX_INTEGRATION_CREDENTIAL_BYTES} UTF-8 bytes`) + }) + expect(netFetchMock).toHaveBeenCalledTimes(1) + }) }) diff --git a/src/main/jira/client.ts b/src/main/jira/client.ts index 4c182ee2ad5..fa5e2ad8a47 100644 --- a/src/main/jira/client.ts +++ b/src/main/jira/client.ts @@ -2,17 +2,36 @@ request plumbing share one boundary so encrypted token lifecycle and multi-site selection cannot drift between task operations. */ import { createHash } from 'node:crypto' -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import { net, safeStorage, session } from 'electron' import { CredentialDecryptionError, credentialFileHasContent, + readIntegrationCredentialFileSync, + readIntegrationCredentialFileSyncText, readStoredCredentialToken } from '../integration-credential-file' import { ensureElectronProxyFromEnvironment } from '../network/proxy-settings' import { withSpan } from '../observability/tracer' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' +import { IntegrationApiConcurrencyGate } from '../integration-api-concurrency' +import { + assertIntegrationAccountCount, + assertIntegrationCredentialBytes, + assertIntegrationStringBytes, + IntegrationAccountPersistenceLimitError, + MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES, + MAX_INTEGRATION_ACCOUNT_FILE_BYTES, + MAX_INTEGRATION_ACCOUNT_ID_BYTES, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES, + MAX_INTEGRATION_ACCOUNTS, + MAX_INTEGRATION_ACCOUNT_URL_BYTES, + serializeIntegrationAccountFile, + unreadableIntegrationAccountFileError +} from '../integration-account-persistence-limits' +import { boundedIntegrationErrorMessage } from '../integration-error-message' import type { JiraAuthType, JiraConnectArgs, @@ -30,28 +49,14 @@ import type { const JIRA_API_USER_AGENT = 'Orca' const MAX_CONCURRENT = 4 -let running = 0 -const queue: (() => void)[] = [] +const concurrencyGate = new IntegrationApiConcurrencyGate(MAX_CONCURRENT) export function acquire(): Promise { - if (running < MAX_CONCURRENT) { - running += 1 - return Promise.resolve() - } - return new Promise((resolve) => - queue.push(() => { - running += 1 - resolve() - }) - ) + return concurrencyGate.acquire() } export function release(): void { - running -= 1 - const next = queue.shift() - if (next) { - next() - } + concurrencyGate.release() } type JiraSiteFile = { @@ -77,18 +82,29 @@ export class JiraApiError extends Error { status: number | null constructor(message: string, status: number | null = null) { - super(message) + super(boundedIntegrationErrorMessage(message)) this.status = status } } let cachedSiteFile: JiraSiteFile | null = null let siteFileLoaded = false +let siteFileReadError: Error | null = null const cachedTokens = new Map() // Why: decrypt failures are recorded per site so getStatus can explain // failing reads without re-touching the keychain on every status poll. const credentialErrors = new Map() +function cacheToken(siteId: string, token: string): void { + if (!cachedTokens.has(siteId) && cachedTokens.size >= MAX_INTEGRATION_ACCOUNTS) { + const oldestSiteId = cachedTokens.keys().next().value + if (oldestSiteId !== undefined) { + cachedTokens.delete(oldestSiteId) + } + } + cachedTokens.set(siteId, token) +} + function getOrcaDir(): string { return join(homedir(), '.orca') } @@ -102,6 +118,7 @@ function getTokenDir(): string { } function getTokenPath(siteId: string): string { + assertIntegrationStringBytes('Jira', 'site ID', siteId, MAX_INTEGRATION_ACCOUNT_ID_BYTES) return join(getTokenDir(), `${Buffer.from(siteId).toString('base64url')}.enc`) } @@ -157,19 +174,95 @@ function normalizeSite(input: unknown): JiraSite | null { } } +function assertSiteBounds(site: JiraSite): void { + assertIntegrationStringBytes('Jira', 'site ID', site.id, MAX_INTEGRATION_ACCOUNT_ID_BYTES) + assertIntegrationStringBytes('Jira', 'site URL', site.siteUrl, MAX_INTEGRATION_ACCOUNT_URL_BYTES) + assertIntegrationStringBytes('Jira', 'email', site.email, MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES) + assertIntegrationStringBytes( + 'Jira', + 'display name', + site.displayName, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + assertIntegrationStringBytes( + 'Jira', + 'account ID', + site.accountId, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) +} + +function assertStoredSiteBounds(input: unknown): void { + if (!input || typeof input !== 'object') { + return + } + const record = input as Record + const fields = [ + ['site ID', record.id, MAX_INTEGRATION_ACCOUNT_ID_BYTES], + ['site URL', record.siteUrl, MAX_INTEGRATION_ACCOUNT_URL_BYTES], + ['email', record.email, MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES], + ['display name', record.displayName, MAX_INTEGRATION_ACCOUNT_LABEL_BYTES], + ['account ID', record.accountId, MAX_INTEGRATION_ACCOUNT_LABEL_BYTES] + ] as const + for (const [field, value, maxBytes] of fields) { + if (typeof value === 'string') { + assertIntegrationStringBytes('Jira', field, value, maxBytes) + } + } +} + +function assertSiteFileBounds(file: JiraSiteFile): void { + assertIntegrationAccountCount('Jira', file.sites.length) + for (const site of file.sites) { + assertSiteBounds(site) + } +} + function readSiteFileFromDisk(): JiraSiteFile { const path = getSiteFilePath() if (!existsSync(path)) { + siteFileReadError = null return emptySiteFile() } try { - const parsed = JSON.parse(readFileSync(path, { encoding: 'utf-8' })) as Partial - const sites = Array.isArray(parsed.sites) - ? parsed.sites - .map((site) => normalizeSite(site)) - .filter((site): site is JiraSite => site !== null) - .filter((site) => hasStoredToken(site.id)) - : [] + const parsed = JSON.parse(readIntegrationCredentialFileSyncText(path)) as Partial + if ( + !parsed || + typeof parsed !== 'object' || + Array.isArray(parsed) || + parsed.version !== 1 || + !Array.isArray(parsed.sites) + ) { + throw unreadableIntegrationAccountFileError('Jira') + } + if (typeof parsed.activeSiteId === 'string') { + assertIntegrationStringBytes( + 'Jira', + 'active site ID', + parsed.activeSiteId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + } + if (typeof parsed.selectedSiteId === 'string' && parsed.selectedSiteId !== 'all') { + assertIntegrationStringBytes( + 'Jira', + 'selected site ID', + parsed.selectedSiteId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + } + const sites: JiraSite[] = [] + assertIntegrationAccountCount('Jira', parsed.sites.length) + for (const input of parsed.sites) { + assertStoredSiteBounds(input) + const site = normalizeSite(input) + if (!site) { + throw unreadableIntegrationAccountFileError('Jira') + } + if (hasStoredToken(site.id)) { + sites.push(site) + } + } const activeSiteId = typeof parsed.activeSiteId === 'string' && sites.some((site) => site.id === parsed.activeSiteId) @@ -181,8 +274,10 @@ function readSiteFileFromDisk(): JiraSiteFile { sites.some((site) => site.id === parsed.selectedSiteId)) ? parsed.selectedSiteId : activeSiteId + siteFileReadError = null return { version: 1, activeSiteId, selectedSiteId, sites } } catch { + siteFileReadError = unreadableIntegrationAccountFileError('Jira') return emptySiteFile() } } @@ -196,6 +291,10 @@ function getSiteFile(): JiraSiteFile { } function writeSiteFile(file: JiraSiteFile): void { + if (siteFileReadError) { + throw siteFileReadError + } + assertSiteFileBounds(file) ensureOrcaDir() const sites = file.sites.filter((site) => hasStoredToken(site.id)) const activeSiteId = @@ -209,22 +308,31 @@ function writeSiteFile(file: JiraSiteFile): void { ? file.selectedSiteId : activeSiteId - cachedSiteFile = { + const nextFile: JiraSiteFile = { version: 1, activeSiteId, selectedSiteId, sites } - siteFileLoaded = true - writeFileSync(getSiteFilePath(), JSON.stringify(cachedSiteFile, null, 2), { + const serialized = serializeIntegrationAccountFile(nextFile) + writeFileSync(getSiteFilePath(), serialized, { encoding: 'utf-8', mode: 0o600 }) + cachedSiteFile = nextFile + siteFileLoaded = true } function writeEncryptedToken(path: string, apiToken: string): void { + assertIntegrationCredentialBytes('Jira', apiToken) if (safeStorage.isEncryptionAvailable()) { - writeFileSync(path, safeStorage.encryptString(apiToken), { mode: 0o600 }) + const encrypted = safeStorage.encryptString(apiToken) + if (encrypted.length > MAX_INTEGRATION_ACCOUNT_FILE_BYTES) { + throw new IntegrationAccountPersistenceLimitError( + `Jira encrypted credential exceeds ${MAX_INTEGRATION_ACCOUNT_FILE_BYTES} bytes.` + ) + } + writeFileSync(path, encrypted, { mode: 0o600 }) return } console.warn('[jira] safeStorage encryption unavailable — storing token in plaintext') @@ -241,15 +349,19 @@ function readToken(siteId: string): string | null { return null } try { - const raw = readFileSync(path) + const raw = readIntegrationCredentialFileSync(path) const token = readStoredCredentialToken('Jira', raw) if (token) { - cachedTokens.set(siteId, token) + assertIntegrationCredentialBytes('Jira', token) + cacheToken(siteId, token) } credentialErrors.delete(siteId) return token } catch (error) { - if (error instanceof CredentialDecryptionError) { + if ( + error instanceof CredentialDecryptionError || + error instanceof IntegrationAccountPersistenceLimitError + ) { credentialErrors.set(siteId, error.message) throw error } @@ -261,7 +373,7 @@ function saveToken(siteId: string, apiToken: string): void { ensureOrcaDir() ensureTokenDir() writeEncryptedToken(getTokenPath(siteId), apiToken) - cachedTokens.set(siteId, apiToken) + cacheToken(siteId, apiToken) credentialErrors.delete(siteId) } @@ -344,9 +456,9 @@ function describeErrorCause(error: unknown): string | undefined { } const cause = (error as { cause?: unknown }).cause if (cause instanceof Error) { - return `${cause.name}: ${cause.message}` + return boundedIntegrationErrorMessage(`${cause.name}: ${cause.message}`) } - return cause === undefined ? undefined : String(cause) + return cause === undefined ? undefined : boundedIntegrationErrorMessage(cause) } async function jiraFetch(url: string, init: RequestInit): Promise { @@ -360,7 +472,7 @@ async function jiraFetch(url: string, init: RequestInit): Promise { }).catch((error) => { span.addEvent('jira.proxySetupFailed', { errorName: error instanceof Error ? error.name : typeof error, - errorMessage: error instanceof Error ? error.message : String(error) + errorMessage: boundedIntegrationErrorMessage(error) }) }) try { @@ -372,10 +484,7 @@ async function jiraFetch(url: string, init: RequestInit): Promise { 'jira.transportErrorName', error instanceof Error ? error.name : typeof error ) - span.setAttribute( - 'jira.transportErrorMessage', - error instanceof Error ? error.message : String(error) - ) + span.setAttribute('jira.transportErrorMessage', boundedIntegrationErrorMessage(error)) const cause = describeErrorCause(error) if (cause) { span.setAttribute('jira.transportErrorCause', cause) @@ -410,16 +519,16 @@ async function requestWithCredentials( if (response.status === 204) { return null } - return response.json() + return readFetchResponseJsonWithinLimit(response) } async function readJiraError(response: Response): Promise { try { - const data = (await response.json()) as { + const data = await readFetchResponseJsonWithinLimit<{ errorMessages?: string[] errors?: Record message?: string - } + }>(response) const messages = [ ...(Array.isArray(data.errorMessages) ? data.errorMessages : []), ...Object.values(data.errors ?? {}), @@ -454,7 +563,7 @@ export async function jiraRequest( if (response.status === 204) { return null as T } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } export function getClients(selection?: JiraSiteSelection | null): JiraClientForSite[] { @@ -475,7 +584,11 @@ export function getClients(selection?: JiraSiteSelection | null): JiraClientForS // credentialError for getStatus to surface, so skip this site like a // missing token. A specific-site selection still rethrows so the renderer // can surface the decrypt banner promptly. - if (isAllSelection && error instanceof CredentialDecryptionError) { + if ( + isAllSelection && + (error instanceof CredentialDecryptionError || + error instanceof IntegrationAccountPersistenceLimitError) + ) { return [] } throw error @@ -504,9 +617,25 @@ export function getStatus(): JiraConnectionStatus { export async function connect( args: JiraConnectArgs ): Promise<{ ok: true; viewer: JiraViewer } | { ok: false; error: string }> { + try { + assertIntegrationStringBytes( + 'Jira', + 'site URL', + args.siteUrl, + MAX_INTEGRATION_ACCOUNT_URL_BYTES + ) + assertIntegrationStringBytes('Jira', 'email', args.email, MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES) + assertIntegrationCredentialBytes('Jira', args.apiToken) + } catch (error) { + return { + ok: false, + error: error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Connection failed.' + } + } let siteUrl: string try { siteUrl = normalizeJiraSiteUrl(args.siteUrl) + assertIntegrationStringBytes('Jira', 'site URL', siteUrl, MAX_INTEGRATION_ACCOUNT_URL_BYTES) } catch { return { ok: false, error: 'Enter a valid Jira site URL.' } } @@ -527,6 +656,10 @@ export async function connect( return { ok: false, error: 'Email and API token are required.' } } + getSiteFile() + if (siteFileReadError) { + return { ok: false, error: siteFileReadError.message } + } await acquire() try { const myselfPath = authType === 'server' ? '/rest/api/2/myself' : '/rest/api/3/myself' @@ -554,17 +687,23 @@ export async function connect( accountId: viewer.accountId, authType } - saveToken(id, apiToken) const file = getSiteFile() - writeSiteFile({ + const nextFile: JiraSiteFile = { version: 1, activeSiteId: id, selectedSiteId: id, sites: [site, ...file.sites.filter((entry) => entry.id !== id)] - }) + } + assertSiteFileBounds(nextFile) + serializeIntegrationAccountFile(nextFile) + saveToken(id, apiToken) + writeSiteFile(nextFile) return { ok: true, viewer } } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : 'Connection failed.' } + return { + ok: false, + error: error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Connection failed.' + } } finally { release() } @@ -572,6 +711,9 @@ export async function connect( export function disconnect(siteId?: string): void { const file = getSiteFile() + if (siteFileReadError) { + throw siteFileReadError + } const ids = siteId ? [siteId] : file.sites.map((site) => site.id) for (const id of ids) { deleteToken(id) @@ -604,7 +746,10 @@ export async function testConnection( try { client = getClients(siteId)[0] } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : 'Connection failed.' } + return { + ok: false, + error: error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Connection failed.' + } } if (!client) { return { ok: false, error: 'Not connected to Jira.' } @@ -617,15 +762,21 @@ export async function testConnection( ) return { ok: true, viewer } } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : 'Connection failed.' } + return { + ok: false, + error: error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Connection failed.' + } } finally { release() } } export function clearToken(siteId: string): void { - deleteToken(siteId) const file = getSiteFile() + if (siteFileReadError) { + throw siteFileReadError + } + deleteToken(siteId) writeSiteFile({ ...file, sites: file.sites.filter((site) => site.id !== siteId) }) } diff --git a/src/main/jira/issues.ts b/src/main/jira/issues.ts index 2ce24a87420..9fcc774889f 100644 --- a/src/main/jira/issues.ts +++ b/src/main/jira/issues.ts @@ -21,6 +21,16 @@ import type { JiraTransition, JiraUser } from '../../shared/types' +import { + boundedIntegrationErrorLog, + boundedIntegrationErrorMessage +} from '../integration-error-message' +import { + INTEGRATION_PAGINATION_MAX_ITEMS, + IntegrationPaginationBudget, + INTEGRATION_PAGINATION_MAX_PAGES +} from '../integration-pagination-budget' +import { runBoundedIntegrationFanout } from '../integration-fanout' import { acquire, apiBasePath, @@ -83,16 +93,13 @@ function getErrorStatus(error: unknown): number | null { return typeof status === 'number' && Number.isFinite(status) ? status : null } -function toIssueSearchFailureError(error: unknown): unknown { +function toIssueSearchFailureError(error: unknown): Error { const status = getErrorStatus(error) - if ( - status === null || - !(error instanceof Error) || - error.message.startsWith(`Error ${status}:`) - ) { - return error + const message = boundedIntegrationErrorMessage(error) + if (status === null || message.startsWith(`Error ${status}:`)) { + return new Error(message) } - return new Error(`Error ${status}: ${error.message}`) + return new Error(boundedIntegrationErrorMessage(`Error ${status}: ${message}`)) } function shouldSurfaceSiteFailure( @@ -164,17 +171,28 @@ async function fetchPagedRecords( maxResults = 100 ): Promise { const records: JiraRecord[] = [] + const budget = new IntegrationPaginationBudget() let startAt = 0 - for (let guard = 0; guard < 100; guard += 1) { + for (let guard = 0; guard < INTEGRATION_PAGINATION_MAX_PAGES; guard += 1) { const response = await jiraRequest>( entry, pathForPage(startAt, maxResults) ) const items = getPageItems(response, key) - records.push(...items) + if (!budget.admitPage(items)) { + console.warn('[jira] Paginated result exceeded its retained result budget; truncating.') + break + } + for (const item of items) { + records.push(item) + } if (!shouldFetchNextPage(response, startAt, items, maxResults)) { break } + if (!budget.canRequestPage) { + console.warn('[jira] Paginated result reached its retained result budget; truncating.') + break + } startAt += asFiniteNumber(response.maxResults) ?? maxResults } return records @@ -374,7 +392,7 @@ async function searchIssuesForClient( fields: ISSUE_FIELDS }) }) - return (result.issues ?? []).map((issue) => mapJiraIssue(entry.site, issue)) + return (result.issues ?? []).slice(0, limit).map((issue) => mapJiraIssue(entry.site, issue)) } export async function listIssues( @@ -397,8 +415,9 @@ export async function searchIssues( const safeLimit = clampLimit(limit) const failures: (JiraIssueSearchFailure | undefined)[] = Array.from({ length: entries.length }) const surfaceSiteFailure = shouldSurfaceSiteFailure(siteId, entries.length) - const results = await Promise.all( - entries.map(async (entry, index) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry, index) => { await acquire() try { return await searchIssuesForClient(entry, jql.trim(), safeLimit) @@ -410,13 +429,14 @@ export async function searchIssues( if (surfaceSiteFailure) { throw toIssueSearchFailureError(error) } - console.warn('[jira] searchIssues failed:', error) + console.warn('[jira] searchIssues failed:', boundedIntegrationErrorLog(error)) failures[index] = { error: toIssueSearchFailureError(error), auth: authFailure } return [] as JiraIssue[] } finally { release() } - }) + }, + (issues) => issues ) // 'all' fan-out: only surface an error when every connected site failed, so a // partial success (or a genuinely empty result) is not reported as an error. @@ -426,9 +446,11 @@ export async function searchIssues( if (recordedFailures.length === entries.length) { throw (recordedFailures.find((failure) => !failure.auth) ?? recordedFailures[0]).error } - return entries.length === 1 - ? results.flat().slice(0, safeLimit) - : sortAndLimitIssues(results.flat(), safeLimit) + if (fanout.truncated) { + console.warn('[jira] Cross-site search exceeded its aggregate result budget; truncating.') + } + const results = fanout.results.flat() + return entries.length === 1 ? results.slice(0, safeLimit) : sortAndLimitIssues(results, safeLimit) } export async function getIssue( @@ -453,7 +475,7 @@ export async function getIssue( throw error } } else { - console.warn('[jira] getIssue failed:', error) + console.warn('[jira] getIssue failed:', boundedIntegrationErrorLog(error)) } } finally { release() @@ -631,7 +653,7 @@ export async function getIssueComments( clearToken(entry.site.id) throw error } - console.warn('[jira] getIssueComments failed:', error) + console.warn('[jira] getIssueComments failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -643,8 +665,9 @@ export async function listProjects(siteId?: JiraSiteSelection | null): Promise { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { // Server/DC has no /project/search resource; /project returns the @@ -659,7 +682,11 @@ export async function listProjects(siteId?: JiraSiteSelection | null): Promise mapProject(project, entry.site)) + const retainedProjects = projects.slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) + if (projects.length > retainedProjects.length) { + console.warn('[jira] Projects returned more rows than supported; truncating.') + } + return retainedProjects.map((project) => mapProject(project, entry.site)) } catch (error) { if (isAuthError(error)) { clearToken(entry.site.id) @@ -667,15 +694,19 @@ export async function listProjects(siteId?: JiraSiteSelection | null): Promise projects ) - return results.flat().sort((a, b) => a.name.localeCompare(b.name)) + if (fanout.truncated) { + console.warn('[jira] Cross-site projects exceeded their aggregate result budget; truncating.') + } + return fanout.results.flat().sort((a, b) => a.name.localeCompare(b.name)) } export async function listIssueTypes( @@ -704,7 +735,7 @@ export async function listIssueTypes( clearToken(entry.site.id) throw error } - console.warn('[jira] listIssueTypes failed:', error) + console.warn('[jira] listIssueTypes failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -723,9 +754,10 @@ export async function listCreateFields( await acquire() try { const fields: JiraCreateField[] = [] + const budget = new IntegrationPaginationBudget() let startAt = 0 const maxResults = 100 - for (let guard = 0; guard < 100; guard += 1) { + for (let guard = 0; guard < INTEGRATION_PAGINATION_MAX_PAGES; guard += 1) { const params = new URLSearchParams({ maxResults: String(maxResults), startAt: String(startAt) @@ -737,14 +769,23 @@ export async function listCreateFields( )}/issuetypes/${encodeURIComponent(issueTypeId)}?${params.toString()}` ) const records = getCreateFieldRecords(response) - fields.push( - ...records - .map((record) => mapCreateField(record)) - .filter((field): field is JiraCreateField => field !== null) - ) + if (!budget.admitPage(records)) { + console.warn('[jira] Create fields exceeded their retained result budget; truncating.') + break + } + for (const record of records) { + const field = mapCreateField(record) + if (field) { + fields.push(field) + } + } if (!shouldFetchNextPage(response, startAt, records, maxResults)) { break } + if (!budget.canRequestPage) { + console.warn('[jira] Create fields reached their retained result budget; truncating.') + break + } startAt += asFiniteNumber(response.maxResults) ?? maxResults } return fields @@ -753,7 +794,7 @@ export async function listCreateFields( clearToken(entry.site.id) throw error } - console.warn('[jira] listCreateFields failed:', error) + console.warn('[jira] listCreateFields failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -768,13 +809,16 @@ export async function listPriorities(siteId?: string | null): Promise(entry, `${apiBasePath(entry.site)}/priority`) - return response.map(mapPriority).filter((priority): priority is JiraPriority => !!priority) + return response + .slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) + .map(mapPriority) + .filter((priority): priority is JiraPriority => !!priority) } catch (error) { if (isAuthError(error)) { clearToken(entry.site.id) throw error } - console.warn('[jira] listPriorities failed:', error) + console.warn('[jira] listPriorities failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -802,13 +846,16 @@ export async function listAssignableUsers( entry, `${apiBasePath(entry.site)}/user/assignable/search?${params.toString()}` ) - return response.map(mapUser).filter((user): user is JiraUser => !!user) + return response + .slice(0, 50) + .map(mapUser) + .filter((user): user is JiraUser => !!user) } catch (error) { if (isAuthError(error)) { clearToken(entry.site.id) throw error } - console.warn('[jira] listAssignableUsers failed:', error) + console.warn('[jira] listAssignableUsers failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -829,17 +876,19 @@ export async function listTransitions( entry, `${apiBasePath(entry.site)}/issue/${encodeURIComponent(key)}/transitions` ) - return (response.transitions ?? []).map((transition) => ({ - id: asString(transition.id), - name: asString(transition.name), - to: mapStatus(transition.to) - })) + return (response.transitions ?? []) + .slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) + .map((transition) => ({ + id: asString(transition.id), + name: asString(transition.name), + to: mapStatus(transition.to) + })) } catch (error) { if (isAuthError(error)) { clearToken(entry.site.id) throw error } - console.warn('[jira] listTransitions failed:', error) + console.warn('[jira] listTransitions failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -891,12 +940,18 @@ export async function getProjectStatusOrder( const seenStatusIds = new Set() const statusIdsByColumn: string[][] = [] for (const column of columns) { + if (seenStatusIds.size >= INTEGRATION_PAGINATION_MAX_ITEMS) { + break + } const statuses = asRecord(column).statuses if (!Array.isArray(statuses)) { continue } const columnStatusIds: string[] = [] for (const status of statuses) { + if (seenStatusIds.size >= INTEGRATION_PAGINATION_MAX_ITEMS) { + break + } const statusId = asIdentifier(asRecord(status).id) if (statusId && !seenStatusIds.has(statusId)) { seenStatusIds.add(statusId) @@ -913,7 +968,7 @@ export async function getProjectStatusOrder( clearToken(entry.site.id) throw error } - console.warn('[jira] getProjectStatusOrder failed:', error) + console.warn('[jira] getProjectStatusOrder failed:', boundedIntegrationErrorLog(error)) return { statusIdsByColumn: [] } } finally { release() diff --git a/src/main/keybindings/keybinding-file-bounds.test.ts b/src/main/keybindings/keybinding-file-bounds.test.ts new file mode 100644 index 00000000000..75dc15a4b35 --- /dev/null +++ b/src/main/keybindings/keybinding-file-bounds.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { mkdtempSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + MAX_KEYBINDING_FILE_BYTES, + MAX_KEYBINDING_JSON_STRUCTURAL_TOKENS, + readKeybindingFile, + writeKeybindingOverride +} from './keybinding-file' + +describe('keybinding file bounds', () => { + const roots: string[] = [] + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + }) + + function makePath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-keybinding-bounds-')) + roots.push(root) + return join(root, 'keybindings.json') + } + + it('parses a valid document exactly at the byte limit', () => { + const filePath = makePath() + const prefix = '{"padding":"' + const suffix = '"}' + writeFileSync( + filePath, + `${prefix}${'x'.repeat(MAX_KEYBINDING_FILE_BYTES - prefix.length - suffix.length)}${suffix}` + ) + + const snapshot = readKeybindingFile(filePath, 'linux') + + expect(snapshot.exists).toBe(true) + expect(snapshot.diagnostics.some((entry) => entry.message.startsWith('Could not read'))).toBe( + false + ) + }) + + it('rejects an oversized sparse document and never overwrites it', () => { + const filePath = makePath() + writeFileSync(filePath, '{}') + truncateSync(filePath, MAX_KEYBINDING_FILE_BYTES + 1) + const originalSize = statSync(filePath).size + + expect(readKeybindingFile(filePath, 'linux')).toMatchObject({ + exists: true, + overrides: {}, + diagnostics: [{ severity: 'error' }] + }) + expect(() => + writeKeybindingOverride(filePath, 'linux', 'worktree.quickOpen', ['Ctrl+P']) + ).toThrow() + expect(statSync(filePath).size).toBe(originalSize) + }) + + it('rejects structurally amplified JSON before parsing', () => { + const filePath = makePath() + writeFileSync(filePath, `{"padding":[${'0,'.repeat(MAX_KEYBINDING_JSON_STRUCTURAL_TOKENS)}0]}`) + const parseSpy = vi.spyOn(JSON, 'parse') + + expect(readKeybindingFile(filePath, 'linux')).toMatchObject({ + exists: true, + overrides: {}, + diagnostics: [{ severity: 'error' }] + }) + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('preserves the prior file when pretty serialization exceeds the read ceiling', () => { + const filePath = makePath() + const prefix = '{"padding":"' + const suffix = '"}' + const before = `${prefix}${'x'.repeat( + MAX_KEYBINDING_FILE_BYTES - prefix.length - suffix.length + )}${suffix}` + writeFileSync(filePath, before) + + expect(() => + writeKeybindingOverride(filePath, 'linux', 'terminal.search', ['Ctrl+Shift+F']) + ).toThrow('JSON output exceeds') + expect(readFileSync(filePath, 'utf8')).toBe(before) + }) +}) diff --git a/src/main/keybindings/keybinding-file.ts b/src/main/keybindings/keybinding-file.ts index 4d41a42ba87..32d3f9316bb 100644 --- a/src/main/keybindings/keybinding-file.ts +++ b/src/main/keybindings/keybinding-file.ts @@ -1,5 +1,5 @@ /* eslint-disable max-lines -- Why: parsing, sanitizing, migrating, and writing the keybindings file must stay together so file-format edge cases share one validation path. */ -import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { dirname, join } from 'node:path' import { findKeybindingConflicts, @@ -15,12 +15,18 @@ import { type KeybindingOverrides, type KeybindingPlatform } from '../../shared/keybindings' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' type JsonObject = Record const FILE_VERSION = 1 const PLATFORM_KEYS: readonly KeybindingPlatform[] = ['darwin', 'linux', 'win32'] const ROOT_KEYS = new Set(['$schema', 'version', 'keybindings', 'platforms']) +export const MAX_KEYBINDING_FILE_BYTES = 1024 * 1024 +export const MAX_KEYBINDING_JSON_STRUCTURAL_TOKENS = 256 * 1024 +export const MAX_KEYBINDING_JSON_NESTING_DEPTH = 64 export function getUserKeybindingsPath(homePath: string): string { return join(homePath, '.orca', 'keybindings.json') @@ -51,7 +57,14 @@ function readJsonDocument(path: string): { return { exists: false, document: createEmptyDocument() } } try { - const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown + const serialized = readNodeFileSyncWithinLimit(path, MAX_KEYBINDING_FILE_BYTES).buffer.toString( + 'utf8' + ) + assertJsonTextStructureWithinLimits(serialized, { + structuralTokens: MAX_KEYBINDING_JSON_STRUCTURAL_TOKENS, + nestingDepth: MAX_KEYBINDING_JSON_NESTING_DEPTH + }) + const parsed = JSON.parse(serialized) as unknown if (!isJsonObject(parsed)) { return { exists: true, document: null, error: 'Keybindings file must contain a JSON object.' } } @@ -69,7 +82,8 @@ function writeJsonDocument(path: string, document: JsonObject): void { mkdirSync(dirname(path), { recursive: true }) const tempPath = `${path}.tmp` try { - writeFileSync(tempPath, `${JSON.stringify(document, null, 2)}\n`, 'utf8') + const { serialized } = stringifyJsonWithinByteLimit(document, MAX_KEYBINDING_FILE_BYTES - 1, 2) + writeFileSync(tempPath, `${serialized}\n`, 'utf8') renameSync(tempPath, path) } catch (error) { try { diff --git a/src/main/kimi/hook-service.ts b/src/main/kimi/hook-service.ts index 94f5d11ea3e..94a7ae018b3 100644 --- a/src/main/kimi/hook-service.ts +++ b/src/main/kimi/hook-service.ts @@ -1,17 +1,14 @@ -import { - copyFileSync, - existsSync, - mkdirSync, - readFileSync, - renameSync, - unlinkSync, - writeFileSync -} from 'node:fs' +import { copyFileSync, existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join, posix as pathPosix } from 'node:path' import { randomUUID } from 'node:crypto' import type { SFTPWrapper } from 'ssh2' import type { AgentHookInstallState, AgentHookInstallStatus } from '../../shared/agent-hook-types' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' +import { AGENT_HOOK_CONFIG_MAX_BYTES } from '../agent-hooks/agent-hook-file-limits' import { createManagedCommandMatcher, getSharedManagedScriptPath, @@ -99,7 +96,9 @@ function readConfigToml(configPath: string): string | null { return '' } try { - return readFileSync(configPath, 'utf-8') + return readNodeFileSyncWithinLimit(configPath, AGENT_HOOK_CONFIG_MAX_BYTES).buffer.toString( + 'utf8' + ) } catch { return null } @@ -112,10 +111,17 @@ function writeConfigToml(configPath: string, text: string): void { mkdirSync(dir, { recursive: true }) if (existsSync(configPath)) { try { - if (readFileSync(configPath, 'utf-8') === text) { + const existing = readNodeFileSyncWithinLimit( + configPath, + AGENT_HOOK_CONFIG_MAX_BYTES + ).buffer.toString('utf8') + if (existing === text) { return } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + throw error + } // Fall through to the atomic write path. } } diff --git a/src/main/lib/fetch-response-body.test.ts b/src/main/lib/fetch-response-body.test.ts new file mode 100644 index 00000000000..197c2196ee8 --- /dev/null +++ b/src/main/lib/fetch-response-body.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi } from 'vitest' +import { + FetchResponseBodyTooLargeError, + readFetchResponseBytesWithinLimit, + readFetchResponseJsonWithinLimit, + readFetchResponseTextWithinLimit +} from './fetch-response-body' + +function chunkedResponse(chunks: Uint8Array[], onCancel?: () => void): Response { + return new Response( + new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk) + } + controller.close() + }, + cancel() { + onCancel?.() + } + }) + ) +} + +describe('bounded fetch response bodies', () => { + it('preserves bytes, split UTF-8, and parsed JSON below the limit', async () => { + const encoded = new TextEncoder().encode('{"message":"hello 🐋"}') + const chunks = [encoded.subarray(0, 20), encoded.subarray(20, 22), encoded.subarray(22)] + + await expect(readFetchResponseBytesWithinLimit(chunkedResponse(chunks), 1024)).resolves.toEqual( + encoded + ) + await expect(readFetchResponseTextWithinLimit(chunkedResponse(chunks), 1024)).resolves.toBe( + '{"message":"hello 🐋"}' + ) + await expect( + readFetchResponseJsonWithinLimit<{ message: string }>(chunkedResponse(chunks), 1024) + ).resolves.toEqual({ message: 'hello 🐋' }) + }) + + it('rejects an oversized declared body before reading it', async () => { + let cancelled = false + const response = chunkedResponse([new Uint8Array([1])], () => { + cancelled = true + }) + response.headers.set('content-length', '1025') + + await expect(readFetchResponseBytesWithinLimit(response, 1024)).rejects.toThrow( + FetchResponseBodyTooLargeError + ) + expect(cancelled).toBe(true) + }) + + it('cancels a chunked body as soon as accumulated bytes exceed the limit', async () => { + let cancelled = false + const chunks = [new Uint8Array(700), new Uint8Array(400)] + let index = 0 + const response = new Response( + new ReadableStream({ + pull(controller) { + controller.enqueue(chunks[index++] ?? new Uint8Array()) + }, + cancel() { + cancelled = true + } + }) + ) + + await expect(readFetchResponseBytesWithinLimit(response, 1024)).rejects.toThrow( + FetchResponseBodyTooLargeError + ) + expect(cancelled).toBe(true) + }) + + it('rejects invalid limits and preserves native JSON parse failures', async () => { + await expect(readFetchResponseBytesWithinLimit(new Response('ok'), -1)).rejects.toThrow( + RangeError + ) + await expect(readFetchResponseJsonWithinLimit(new Response(''), 10)).rejects.toThrow( + SyntaxError + ) + }) + + it('rejects structural amplification before JSON.parse', async () => { + const parseSpy = vi.spyOn(JSON, 'parse') + + await expect( + readFetchResponseJsonWithinLimit(new Response('[0,0]'), 1024, { + structuralTokens: 2, + nestingDepth: 4 + }) + ).rejects.toThrow('JSON structure exceeds 2 tokens') + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('preserves injected adapters that expose only high-level response methods', async () => { + const textResponse = { text: async () => 'adapter text' } as Response + const jsonResponse = { json: async () => ({ source: 'adapter' }) } as Response + + await expect(readFetchResponseTextWithinLimit(textResponse)).resolves.toBe('adapter text') + await expect(readFetchResponseJsonWithinLimit(jsonResponse)).resolves.toEqual({ + source: 'adapter' + }) + }) +}) diff --git a/src/main/lib/fetch-response-body.ts b/src/main/lib/fetch-response-body.ts new file mode 100644 index 00000000000..b59b540fc35 --- /dev/null +++ b/src/main/lib/fetch-response-body.ts @@ -0,0 +1 @@ +export * from '../../shared/fetch-response-body' diff --git a/src/main/lib/html-to-pdf-memory-limit.test.ts b/src/main/lib/html-to-pdf-memory-limit.test.ts new file mode 100644 index 00000000000..fc9e92e3804 --- /dev/null +++ b/src/main/lib/html-to-pdf-memory-limit.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { + assertHtmlToPdfInputWithinMemoryLimit, + HTML_TO_PDF_MEMORY_LIMIT_ERROR +} from './html-to-pdf-memory-limit' + +describe('HTML to PDF memory limit', () => { + it('measures UTF-8 bytes and accepts the boundary', () => { + expect(() => assertHtmlToPdfInputWithinMemoryLimit('éé', 4)).not.toThrow() + }) + + it('rejects the next byte', () => { + expect(() => assertHtmlToPdfInputWithinMemoryLimit('ééa', 4)).toThrow( + HTML_TO_PDF_MEMORY_LIMIT_ERROR + ) + }) +}) diff --git a/src/main/lib/html-to-pdf-memory-limit.ts b/src/main/lib/html-to-pdf-memory-limit.ts new file mode 100644 index 00000000000..6b04e8b59e4 --- /dev/null +++ b/src/main/lib/html-to-pdf-memory-limit.ts @@ -0,0 +1 @@ +export * from '../../shared/html-to-pdf-memory-limit' diff --git a/src/main/lib/html-to-pdf.test.ts b/src/main/lib/html-to-pdf.test.ts new file mode 100644 index 00000000000..9a0f197b69e --- /dev/null +++ b/src/main/lib/html-to-pdf.test.ts @@ -0,0 +1,90 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + BrowserWindowMock, + destroyMock, + executeJavaScriptMock, + loadFileMock, + printToPDFMock, + unlinkMock, + writeFileMock, + setDidFinishLoad +} = vi.hoisted(() => { + let didFinishLoad: (() => void) | undefined + const setDidFinishLoad = (listener: (() => void) | undefined): void => { + didFinishLoad = listener + } + const webContentsOnceMock = vi.fn( + (event: string, listener: (...args: unknown[]) => void): void => { + if (event === 'did-finish-load') { + didFinishLoad = () => listener() + } + } + ) + const executeJavaScriptMock = vi.fn(async () => undefined) + const printToPDFMock = vi.fn(async () => Buffer.from('%PDF-export')) + const loadFileMock = vi.fn(async () => { + didFinishLoad?.() + }) + const destroyMock = vi.fn() + const exportWindow = { + webContents: { + once: webContentsOnceMock, + executeJavaScript: executeJavaScriptMock, + printToPDF: printToPDFMock + }, + loadFile: loadFileMock, + isDestroyed: vi.fn(() => false), + destroy: destroyMock + } + return { + BrowserWindowMock: vi.fn(function () { + return exportWindow + }), + destroyMock, + executeJavaScriptMock, + loadFileMock, + printToPDFMock, + unlinkMock: vi.fn(async () => undefined), + writeFileMock: vi.fn(async () => undefined), + setDidFinishLoad + } +}) + +vi.mock('electron', () => ({ + app: { getPath: vi.fn(() => '/tmp') }, + BrowserWindow: BrowserWindowMock +})) +vi.mock('node:fs/promises', () => ({ writeFile: writeFileMock, unlink: unlinkMock })) +vi.mock('node:crypto', () => ({ randomUUID: vi.fn(() => 'test-id') })) + +import { htmlToPdf } from './html-to-pdf' + +describe('htmlToPdf', () => { + beforeEach(() => { + vi.clearAllMocks() + setDidFinishLoad(undefined) + printToPDFMock.mockResolvedValue(Buffer.from('%PDF-export')) + }) + + it('preserves ordinary export rendering and cleanup under the memory admission', async () => { + await expect(htmlToPdf('Hello')).resolves.toEqual( + Buffer.from('%PDF-export') + ) + + expect(writeFileMock).toHaveBeenCalledWith( + '/tmp/orca-export-test-id.html', + 'Hello', + 'utf-8' + ) + expect(loadFileMock).toHaveBeenCalledWith('/tmp/orca-export-test-id.html') + expect(executeJavaScriptMock).toHaveBeenCalledOnce() + expect(printToPDFMock).toHaveBeenCalledWith({ + printBackground: true, + pageSize: 'A4', + margins: { top: 0.75, bottom: 0.75, left: 0.75, right: 0.75 } + }) + expect(destroyMock).toHaveBeenCalledOnce() + expect(unlinkMock).toHaveBeenCalledWith('/tmp/orca-export-test-id.html') + }) +}) diff --git a/src/main/lib/html-to-pdf.ts b/src/main/lib/html-to-pdf.ts index f0cc467e515..a50578fcee5 100644 --- a/src/main/lib/html-to-pdf.ts +++ b/src/main/lib/html-to-pdf.ts @@ -2,6 +2,12 @@ import { app, BrowserWindow } from 'electron' import { writeFile, unlink } from 'node:fs/promises' import path from 'node:path' import { randomUUID } from 'node:crypto' +import { + acquireBrowserPdfAdmission, + BROWSER_PDF_BUSY_ERROR +} from '../browser/browser-pdf-admission' +import { assertCdpPdfWithinMemoryLimit } from '../browser/cdp-print-to-pdf' +import { assertHtmlToPdfInputWithinMemoryLimit } from './html-to-pdf-memory-limit' export class ExportTimeoutError extends Error { constructor(message = 'Export timed out') { @@ -31,40 +37,42 @@ new Promise((resolve) => { ` export async function htmlToPdf(html: string): Promise { + assertHtmlToPdfInputWithinMemoryLimit(html) const tempDir = app.getPath('temp') const tempPath = path.join(tempDir, `orca-export-${randomUUID()}.html`) - await writeFile(tempPath, html, 'utf-8') - - const win = new BrowserWindow({ - show: false, - webPreferences: { - sandbox: true, - contextIsolation: true, - nodeIntegration: false, - // Why: image-wait needs to run a short script inside the export page, and - // the exported renderer DOM may already embed scripts/SVGs (e.g. Mermaid) - // that need JS to paint correctly. The window stays sandboxed and - // isolated so this is safe. - javascript: true - } - }) - + const pdfAdmission = acquireBrowserPdfAdmission() + if (!pdfAdmission) { + throw new Error(BROWSER_PDF_BUSY_ERROR) + } + let win: BrowserWindow | null = null let timer: NodeJS.Timeout | undefined try { + await writeFile(tempPath, html, 'utf-8') + win = new BrowserWindow({ + show: false, + webPreferences: { + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + // Why: image-wait needs script execution to finish remote images and rendered SVGs. + javascript: true + } + }) + const exportWindow = win const loadPromise = new Promise((resolve, reject) => { - win.webContents.once('did-finish-load', () => resolve()) - win.webContents.once('did-fail-load', (_event, errorCode, errorDescription) => { + exportWindow.webContents.once('did-finish-load', () => resolve()) + exportWindow.webContents.once('did-fail-load', (_event, errorCode, errorDescription) => { reject(new Error(`Failed to load export document: ${errorDescription} (${errorCode})`)) }) }) - await win.loadFile(tempPath) + await exportWindow.loadFile(tempPath) await loadPromise const renderAndPrint = (async (): Promise => { - await win.webContents.executeJavaScript(WAIT_FOR_IMAGES_SCRIPT, true) - return win.webContents.printToPDF({ + await exportWindow.webContents.executeJavaScript(WAIT_FOR_IMAGES_SCRIPT, true) + return pdfAdmission.print(exportWindow.webContents, { printBackground: true, pageSize: 'A4', margins: { @@ -80,12 +88,15 @@ export async function htmlToPdf(html: string): Promise { timer = setTimeout(() => reject(new ExportTimeoutError()), EXPORT_TIMEOUT_MS) }) - return await Promise.race([renderAndPrint, timeoutPromise]) + const pdf = await Promise.race([renderAndPrint, timeoutPromise]) + assertCdpPdfWithinMemoryLimit(pdf) + return pdf } finally { + pdfAdmission.releaseIfIdle() if (timer) { clearTimeout(timer) } - if (!win.isDestroyed()) { + if (win && !win.isDestroyed()) { win.destroy() } try { diff --git a/src/main/linear/client.test.ts b/src/main/linear/client.test.ts index 7f518afe6e8..82fb079bdd6 100644 --- a/src/main/linear/client.test.ts +++ b/src/main/linear/client.test.ts @@ -3,6 +3,10 @@ import { tmpdir } from 'node:os' import type * as Os from 'node:os' import { join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + MAX_INTEGRATION_ACCOUNTS, + MAX_INTEGRATION_CREDENTIAL_BYTES +} from '../integration-account-persistence-limits' type ViewerFixture = { displayName: string @@ -389,4 +393,57 @@ describe('Linear client workspace storage', () => { expect(() => linear.getClients('bad')).toThrow('Could not decrypt') }) + + it('admits the exact saved-workspace boundary without changing order', async () => { + const workspaces = Array.from({ length: MAX_INTEGRATION_ACCOUNTS }, (_, index) => ({ + id: `workspace-${index}`, + token: `token-${index}` + })) + writeMultiWorkspaceFiles(workspaces, 'all') + const linear = await loadClientModule() + + const status = linear.getStatus() + expect(status.workspaces).toHaveLength(MAX_INTEGRATION_ACCOUNTS) + expect(status.workspaces?.map((workspace) => workspace.id)).toEqual( + workspaces.map((workspace) => workspace.id) + ) + }) + + it('preserves an over-limit saved-workspace file and refuses to overwrite it', async () => { + const workspaces = Array.from({ length: MAX_INTEGRATION_ACCOUNTS + 1 }, (_, index) => ({ + id: `workspace-${index}`, + token: `token-${index}` + })) + writeMultiWorkspaceFiles(workspaces, 'all') + const path = join(tempHome, '.orca', 'linear-workspaces.json') + const before = readFileSync(path, 'utf8') + const linear = await loadClientModule() + + expect(linear.getStatus()).toMatchObject({ connected: false, workspaces: [] }) + await expect(linear.connect('token-alpha')).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('left unchanged') + }) + expect(linearClientMock).not.toHaveBeenCalled() + expect(readFileSync(path, 'utf8')).toBe(before) + }) + + it('admits an exact-size Linear credential and rejects credential byte +1 before SDK use', async () => { + const exactToken = 't'.repeat(MAX_INTEGRATION_CREDENTIAL_BYTES) + fixtures.set(exactToken, { + displayName: 'Ada', + email: 'ada@example.com', + organizationId: 'org-exact', + organizationName: 'Exact', + organizationUrlKey: 'exact' + }) + const linear = await loadClientModule() + + await expect(linear.connect(exactToken)).resolves.toMatchObject({ ok: true }) + await expect(linear.connect(`${exactToken}t`)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining(`${MAX_INTEGRATION_CREDENTIAL_BYTES} UTF-8 bytes`) + }) + expect(linearClientMock).toHaveBeenCalledTimes(1) + }) }) diff --git a/src/main/linear/client.ts b/src/main/linear/client.ts index 7058d130597..ca71dbb3d0d 100644 --- a/src/main/linear/client.ts +++ b/src/main/linear/client.ts @@ -3,13 +3,15 @@ stay in one consistency boundary. */ import { safeStorage } from 'electron' import type { LinearClient } from '@linear/sdk' -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import { loadLinearSdk } from './linear-sdk' import { CredentialDecryptionError, credentialFileHasContent, + readIntegrationCredentialFileSync, + readIntegrationCredentialFileSyncText, readStoredCredentialToken } from '../integration-credential-file' import type { @@ -18,31 +20,32 @@ import type { LinearWorkspace, LinearWorkspaceSelection } from '../../shared/types' +import { IntegrationApiConcurrencyGate } from '../integration-api-concurrency' +import { + assertIntegrationAccountCount, + assertIntegrationCredentialBytes, + assertIntegrationStringBytes, + IntegrationAccountPersistenceLimitError, + MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES, + MAX_INTEGRATION_ACCOUNT_FILE_BYTES, + MAX_INTEGRATION_ACCOUNT_ID_BYTES, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES, + MAX_INTEGRATION_ACCOUNTS, + serializeIntegrationAccountFile, + unreadableIntegrationAccountFileError +} from '../integration-account-persistence-limits' +import { boundedIntegrationErrorMessage } from '../integration-error-message' // ── Concurrency limiter — max 4 parallel Linear API calls ──────────── const MAX_CONCURRENT = 4 -let running = 0 -const queue: (() => void)[] = [] +const concurrencyGate = new IntegrationApiConcurrencyGate(MAX_CONCURRENT) export function acquire(): Promise { - if (running < MAX_CONCURRENT) { - running++ - return Promise.resolve() - } - return new Promise((resolve) => - queue.push(() => { - running++ - resolve() - }) - ) + return concurrencyGate.acquire() } export function release(): void { - running-- - const next = queue.shift() - if (next) { - next() - } + concurrencyGate.release() } // ── Token + workspace storage ──────────────────────────────────────── @@ -74,6 +77,17 @@ let cachedLegacyViewer: LinearViewer | null = null let legacyViewerLoadedFromDisk = false let cachedWorkspaceFile: LinearWorkspaceFile | null = null let workspaceFileLoadedFromDisk = false +let workspaceFileReadError: Error | null = null + +function cacheToken(workspaceId: string, token: string): void { + if (!cachedTokens.has(workspaceId) && cachedTokens.size >= MAX_INTEGRATION_ACCOUNTS) { + const oldestWorkspaceId = cachedTokens.keys().next().value + if (oldestWorkspaceId !== undefined) { + cachedTokens.delete(oldestWorkspaceId) + } + } + cachedTokens.set(workspaceId, token) +} function getOrcaDir(): string { return join(homedir(), '.orca') @@ -99,6 +113,12 @@ function getWorkspaceTokenPath(workspaceId: string): string { if (workspaceId === LEGACY_WORKSPACE_ID) { return getLegacyTokenPath() } + assertIntegrationStringBytes( + 'Linear', + 'workspace ID', + workspaceId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) return join(getWorkspaceTokenDir(), `${Buffer.from(workspaceId).toString('base64url')}.enc`) } @@ -122,11 +142,47 @@ function readLegacyViewerFromDisk(): LinearViewer | null { return null } try { - const raw = readFileSync(path, { encoding: 'utf-8' }) + const raw = readIntegrationCredentialFileSyncText(path) const parsed = JSON.parse(raw) as Partial if (typeof parsed?.displayName !== 'string' || typeof parsed?.organizationName !== 'string') { return null } + assertIntegrationStringBytes( + 'Linear', + 'legacy display name', + parsed.displayName, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + assertIntegrationStringBytes( + 'Linear', + 'legacy organization name', + parsed.organizationName, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + if (typeof parsed.email === 'string') { + assertIntegrationStringBytes( + 'Linear', + 'legacy email', + parsed.email, + MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES + ) + } + if (typeof parsed.organizationId === 'string') { + assertIntegrationStringBytes( + 'Linear', + 'legacy organization ID', + parsed.organizationId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + } + if (typeof parsed.organizationUrlKey === 'string') { + assertIntegrationStringBytes( + 'Linear', + 'legacy organization URL key', + parsed.organizationUrlKey, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + } return { displayName: parsed.displayName, email: typeof parsed.email === 'string' ? parsed.email : null, @@ -180,6 +236,76 @@ function normalizeWorkspace(input: unknown): LinearWorkspace | null { } } +function assertWorkspaceBounds(workspace: LinearWorkspace): void { + assertIntegrationStringBytes( + 'Linear', + 'workspace ID', + workspace.id, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + assertIntegrationStringBytes( + 'Linear', + 'organization ID', + workspace.organizationId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + assertIntegrationStringBytes( + 'Linear', + 'organization name', + workspace.organizationName, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + if (workspace.organizationUrlKey !== undefined) { + assertIntegrationStringBytes( + 'Linear', + 'organization URL key', + workspace.organizationUrlKey, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + } + assertIntegrationStringBytes( + 'Linear', + 'display name', + workspace.displayName, + MAX_INTEGRATION_ACCOUNT_LABEL_BYTES + ) + if (workspace.email !== null) { + assertIntegrationStringBytes( + 'Linear', + 'email', + workspace.email, + MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES + ) + } +} + +function assertStoredWorkspaceBounds(input: unknown): void { + if (!input || typeof input !== 'object') { + return + } + const record = input as Record + const fields = [ + ['workspace ID', record.id, MAX_INTEGRATION_ACCOUNT_ID_BYTES], + ['organization ID', record.organizationId, MAX_INTEGRATION_ACCOUNT_ID_BYTES], + ['organization name', record.organizationName, MAX_INTEGRATION_ACCOUNT_LABEL_BYTES], + ['organization URL key', record.organizationUrlKey, MAX_INTEGRATION_ACCOUNT_LABEL_BYTES], + ['display name', record.displayName, MAX_INTEGRATION_ACCOUNT_LABEL_BYTES], + ['email', record.email, MAX_INTEGRATION_ACCOUNT_EMAIL_BYTES] + ] as const + for (const [field, value, maxBytes] of fields) { + if (typeof value === 'string') { + assertIntegrationStringBytes('Linear', field, value, maxBytes) + } + } +} + +function assertWorkspaceFileBounds(file: LinearWorkspaceFile): void { + assertIntegrationAccountCount('Linear', file.workspaces.length) + for (const workspace of file.workspaces) { + assertWorkspaceBounds(workspace) + } +} + function emptyWorkspaceFile(): LinearWorkspaceFile { return { version: 1, @@ -192,17 +318,49 @@ function emptyWorkspaceFile(): LinearWorkspaceFile { function readWorkspaceFileFromDisk(): LinearWorkspaceFile { const path = getWorkspaceFilePath() if (!existsSync(path)) { + workspaceFileReadError = null return emptyWorkspaceFile() } try { - const raw = readFileSync(path, { encoding: 'utf-8' }) + const raw = readIntegrationCredentialFileSyncText(path) const parsed = JSON.parse(raw) as Partial - const workspaces = Array.isArray(parsed.workspaces) - ? parsed.workspaces - .map((workspace) => normalizeWorkspace(workspace)) - .filter((workspace): workspace is LinearWorkspace => workspace !== null) - .filter((workspace) => hasStoredToken(workspace.id)) - : [] + if ( + !parsed || + typeof parsed !== 'object' || + Array.isArray(parsed) || + parsed.version !== 1 || + !Array.isArray(parsed.workspaces) + ) { + throw unreadableIntegrationAccountFileError('Linear') + } + if (typeof parsed.activeWorkspaceId === 'string') { + assertIntegrationStringBytes( + 'Linear', + 'active workspace ID', + parsed.activeWorkspaceId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + } + if (typeof parsed.selectedWorkspaceId === 'string' && parsed.selectedWorkspaceId !== 'all') { + assertIntegrationStringBytes( + 'Linear', + 'selected workspace ID', + parsed.selectedWorkspaceId, + MAX_INTEGRATION_ACCOUNT_ID_BYTES + ) + } + const workspaces: LinearWorkspace[] = [] + assertIntegrationAccountCount('Linear', parsed.workspaces.length) + for (const input of parsed.workspaces) { + assertStoredWorkspaceBounds(input) + const workspace = normalizeWorkspace(input) + if (!workspace) { + throw unreadableIntegrationAccountFileError('Linear') + } + if (hasStoredToken(workspace.id)) { + workspaces.push(workspace) + } + } const activeWorkspaceId = typeof parsed.activeWorkspaceId === 'string' && workspaces.some((workspace) => workspace.id === parsed.activeWorkspaceId) @@ -215,6 +373,7 @@ function readWorkspaceFileFromDisk(): LinearWorkspaceFile { ? parsed.selectedWorkspaceId : activeWorkspaceId + workspaceFileReadError = null return { version: 1, activeWorkspaceId, @@ -222,6 +381,7 @@ function readWorkspaceFileFromDisk(): LinearWorkspaceFile { workspaces } } catch { + workspaceFileReadError = unreadableIntegrationAccountFileError('Linear') return emptyWorkspaceFile() } } @@ -235,6 +395,10 @@ function getWorkspaceFile(): LinearWorkspaceFile { } function writeWorkspaceFile(file: LinearWorkspaceFile): void { + if (workspaceFileReadError) { + throw workspaceFileReadError + } + assertWorkspaceFileBounds(file) ensureOrcaDir() const persistedWorkspaces = file.workspaces.filter( (workspace) => workspace.id !== LEGACY_WORKSPACE_ID @@ -255,17 +419,19 @@ function writeWorkspaceFile(file: LinearWorkspaceFile): void { ? file.selectedWorkspaceId : activeWorkspaceId - cachedWorkspaceFile = { + const nextFile: LinearWorkspaceFile = { version: 1, activeWorkspaceId, selectedWorkspaceId, workspaces: persistedWorkspaces } - workspaceFileLoadedFromDisk = true - writeFileSync(getWorkspaceFilePath(), JSON.stringify(cachedWorkspaceFile, null, 2), { + const serialized = serializeIntegrationAccountFile(nextFile) + writeFileSync(getWorkspaceFilePath(), serialized, { encoding: 'utf-8', mode: 0o600 }) + cachedWorkspaceFile = nextFile + workspaceFileLoadedFromDisk = true } function getLegacyWorkspace(): LinearWorkspace | null { @@ -321,8 +487,14 @@ function clearLegacyViewerOnDisk(): void { } function writeEncryptedToken(path: string, apiKey: string): void { + assertIntegrationCredentialBytes('Linear', apiKey) if (safeStorage.isEncryptionAvailable()) { const encrypted = safeStorage.encryptString(apiKey) + if (encrypted.length > MAX_INTEGRATION_ACCOUNT_FILE_BYTES) { + throw new IntegrationAccountPersistenceLimitError( + `Linear encrypted credential exceeds ${MAX_INTEGRATION_ACCOUNT_FILE_BYTES} bytes.` + ) + } writeFileSync(path, encrypted, { mode: 0o600 }) return } @@ -338,7 +510,7 @@ function saveWorkspaceToken(workspaceId: string, apiKey: string): void { } const tokenPath = getWorkspaceTokenPath(workspaceId) writeEncryptedToken(tokenPath, apiKey) - cachedTokens.set(workspaceId, apiKey) + cacheToken(workspaceId, apiKey) credentialErrors.delete(workspaceId) } @@ -352,6 +524,12 @@ export function loadToken(options: { force?: boolean; workspaceId?: string } = { if (!workspaceId) { return null } + if ( + workspaceId !== LEGACY_WORKSPACE_ID && + !getWorkspaceState().workspaces.some((workspace) => workspace.id === workspaceId) + ) { + return null + } const cached = cachedTokens.get(workspaceId) if (cached !== undefined) { return cached @@ -364,15 +542,19 @@ export function loadToken(options: { force?: boolean; workspaceId?: string } = { return null } try { - const raw = readFileSync(tokenPath) + const raw = readIntegrationCredentialFileSync(tokenPath) const token = readStoredCredentialToken('Linear', raw) if (token) { - cachedTokens.set(workspaceId, token) + assertIntegrationCredentialBytes('Linear', token) + cacheToken(workspaceId, token) } credentialErrors.delete(workspaceId) return token } catch (error) { - if (error instanceof CredentialDecryptionError) { + if ( + error instanceof CredentialDecryptionError || + error instanceof IntegrationAccountPersistenceLimitError + ) { credentialErrors.set(workspaceId, error.message) throw error } @@ -401,6 +583,10 @@ function clearTokenFile(workspaceId: string): void { } export function clearToken(workspaceId?: string): void { + getWorkspaceFile() + if (workspaceFileReadError) { + throw workspaceFileReadError + } if (!workspaceId) { const state = getWorkspaceState() for (const workspace of state.workspaces) { @@ -417,6 +603,9 @@ export function clearToken(workspaceId?: string): void { return } + if (!getWorkspaceState().workspaces.some((workspace) => workspace.id === workspaceId)) { + return + } clearTokenFile(workspaceId) if (workspaceId === LEGACY_WORKSPACE_ID) { cachedLegacyViewer = null @@ -453,7 +642,10 @@ function workspaceFromLinearData( } } -function upsertWorkspace(workspace: LinearWorkspace, options: { select?: boolean } = {}): void { +function workspaceFileWithUpsert( + workspace: LinearWorkspace, + options: { select?: boolean } = {} +): LinearWorkspaceFile { const file = getWorkspaceFile() const current = file.workspaces.find((entry) => entry.id === workspace.id) const credentialRevision = (current?.credentialRevision ?? 0) + 1 @@ -467,12 +659,16 @@ function upsertWorkspace(workspace: LinearWorkspace, options: { select?: boolean : file.selectedWorkspaceId && file.selectedWorkspaceId !== LEGACY_WORKSPACE_ID ? file.selectedWorkspaceId : workspace.id - writeWorkspaceFile({ + return { version: 1, activeWorkspaceId: workspace.id, selectedWorkspaceId, workspaces - }) + } +} + +function upsertWorkspace(workspace: LinearWorkspace, options: { select?: boolean } = {}): void { + writeWorkspaceFile(workspaceFileWithUpsert(workspace, options)) } function replaceLegacyWorkspace(workspace: LinearWorkspace, token: string): void { @@ -486,7 +682,8 @@ function replaceLegacyWorkspace(workspace: LinearWorkspace, token: string): void function resolveWorkspaceId(workspaceId?: string | null): string | null { if (workspaceId && workspaceId !== 'all') { - return workspaceId + const state = getWorkspaceState() + return state.workspaces.some((workspace) => workspace.id === workspaceId) ? workspaceId : null } const state = getWorkspaceState() if ( @@ -539,7 +736,11 @@ export function getClients( // per-workspace credentialError for getStatus to surface, so skip this // workspace like a missing token. A specific-workspace selection still // rethrows so the renderer can surface the decrypt banner promptly. - if (isAllSelection && error instanceof CredentialDecryptionError) { + if ( + isAllSelection && + (error instanceof CredentialDecryptionError || + error instanceof IntegrationAccountPersistenceLimitError) + ) { continue } throw error @@ -580,13 +781,22 @@ export async function connect( { ok: true; viewer: LinearViewer; workspace: LinearWorkspace } | { ok: false; error: string } > { try { + assertIntegrationCredentialBytes('Linear', apiKey) + getWorkspaceFile() + if (workspaceFileReadError) { + throw workspaceFileReadError + } const client = new (loadLinearSdk().LinearClient)({ apiKey }) const me = await client.viewer const org = await me.organization const workspace = workspaceFromLinearData(me, org) - saveWorkspaceToken(workspace.id, apiKey) + assertWorkspaceBounds(workspace) const legacyWorkspace = getLegacyWorkspace() + const candidateFile = workspaceFileWithUpsert(workspace, { select: true }) + assertWorkspaceFileBounds(candidateFile) + serializeIntegrationAccountFile(candidateFile) + saveWorkspaceToken(workspace.id, apiKey) if ( legacyWorkspace && legacyWorkspace.organizationName === workspace.organizationName && @@ -600,7 +810,8 @@ export async function connect( upsertWorkspace(workspace, { select: true }) return { ok: true, viewer: workspace, workspace } } catch (error) { - const message = error instanceof Error ? error.message : 'Failed to validate API key' + const message = + error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Failed to validate API key' return { ok: false, error: message } } } @@ -659,6 +870,10 @@ export async function testConnection( ): Promise< { ok: true; viewer: LinearViewer; workspace: LinearWorkspace } | { ok: false; error: string } > { + getWorkspaceFile() + if (workspaceFileReadError) { + return { ok: false, error: workspaceFileReadError.message } + } const resolvedWorkspaceId = resolveWorkspaceId(workspaceId) if (!resolvedWorkspaceId) { return { ok: false, error: 'No API key stored.' } @@ -667,7 +882,7 @@ export async function testConnection( try { token = loadToken({ force: true, workspaceId: resolvedWorkspaceId }) } catch (error) { - const message = error instanceof Error ? error.message : 'Test failed' + const message = error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Test failed' return { ok: false, error: message } } if (!token) { @@ -679,6 +894,10 @@ export async function testConnection( const me = await client.viewer const org = await me.organization const workspace = workspaceFromLinearData(me, org) + assertWorkspaceBounds(workspace) + const candidateFile = workspaceFileWithUpsert(workspace, { select: true }) + assertWorkspaceFileBounds(candidateFile) + serializeIntegrationAccountFile(candidateFile) if (resolvedWorkspaceId === LEGACY_WORKSPACE_ID) { replaceLegacyWorkspace(workspace, token) } else { @@ -690,7 +909,7 @@ export async function testConnection( if (isAuthError(error)) { clearToken(resolvedWorkspaceId) } - const message = error instanceof Error ? error.message : 'Test failed' + const message = error instanceof Error ? boundedIntegrationErrorMessage(error) : 'Test failed' return { ok: false, error: message } } } diff --git a/src/main/linear/issue-context-client.test.ts b/src/main/linear/issue-context-client.test.ts index c284ae8a5d4..abd3a63bf83 100644 --- a/src/main/linear/issue-context-client.test.ts +++ b/src/main/linear/issue-context-client.test.ts @@ -78,7 +78,7 @@ describe('Linear agent issue context client', () => { }) expect(console.warn).toHaveBeenCalledWith( '[linear] agent issue read failed:', - expect.any(Error) + expect.stringContaining('fetch failed') ) }) diff --git a/src/main/linear/issue-context-client.ts b/src/main/linear/issue-context-client.ts index 1f4863d235a..541430d06cf 100644 --- a/src/main/linear/issue-context-client.ts +++ b/src/main/linear/issue-context-client.ts @@ -24,11 +24,11 @@ import { linearError, linearMessage } from './issue-context-errors' +import { getFanoutClientEntries, type WorkspaceReadFailure } from './issue-context-fanout' import { - getFanoutClientEntries, - workspaceFailure, - type WorkspaceReadFailure -} from './issue-context-fanout' + readLinearIssueWorkspaceFanout, + readLinearSearchWorkspaceFanout +} from './issue-context-fanout-reads' import { ambiguousWorkspace, resolveWorkspaceSelector, @@ -59,11 +59,10 @@ export async function searchLinearIssuesForAgents(args: { }) } - const perWorkspace = await readSearchWorkspaces( + const perWorkspace = await readLinearSearchWorkspaceFanout( entries, - args.query, - limit + 1, workspaceId, + (entry) => readSearchWorkspace(entry, args.query, limit + 1, workspaceId), entryFailures ) const merged = perWorkspace.results @@ -77,8 +76,8 @@ export async function searchLinearIssuesForAgents(args: { workspaceId, limit, returned: limited.length, - limitReached: merged.length > limit, - partial: perWorkspace.failures.length > 0, + limitReached: perWorkspace.truncated || merged.length > limit, + partial: perWorkspace.truncated || perWorkspace.failures.length > 0, workspaceErrors: perWorkspace.failures.map(({ workspace, code, message }) => ({ workspace, code, @@ -106,9 +105,21 @@ export async function resolveIssue( }) } - const results = await readIssueWorkspaces(entries, identifier, selection, entryFailures) + const issueFanout = await readLinearIssueWorkspaceFanout( + entries, + selection, + (entry) => readIssueWorkspace(entry, identifier), + entryFailures + ) + const results = issueFanout.results if (results.length === 0) { + if (issueFanout.truncated) { + throw linearError( + 'linear_partial', + 'Linear issue lookup exceeded its aggregate result budget.' + ) + } throw linearError('linear_issue_not_found', `Linear issue ${identifier} was not found.`) } if (results.length > 1) { @@ -203,42 +214,6 @@ async function readIssueWorkspace( return response ? { issue: mapIssue(response), workspace: entry.workspace } : null } -async function readIssueWorkspaces( - entries: LinearClientForWorkspace[], - identifier: string, - selection: string | 'all', - initialFailures: WorkspaceReadFailure[] = [] -): Promise { - if (selection !== 'all') { - const selected = await readIssueWorkspace(entries[0], identifier) - return selected ? [selected] : [] - } - - const settled = await Promise.allSettled( - entries.map((entry) => readIssueWorkspace(entry, identifier)) - ) - const results: ResolvedIssue[] = [] - const failures: LinearAgentAccessError[] = initialFailures.map((failure) => failure.error) - - for (const result of settled) { - if (result.status === 'fulfilled') { - if (result.value) { - results.push(result.value) - } - continue - } - if (result.reason instanceof LinearAgentAccessError) { - failures.push(result.reason) - } - console.warn('[linear] agent issue read failed:', result.reason) - } - - if (results.length === 0 && failures[0]) { - throw failures[0] - } - return results -} - async function readSearchWorkspace( entry: LinearClientForWorkspace, query: string, @@ -252,7 +227,7 @@ async function readSearchWorkspace( SEARCH_QUERY, { term: query, first: limit } ) - return raw.data?.searchIssues?.nodes ?? [] + return (raw.data?.searchIssues?.nodes ?? []).slice(0, limit) }, workspaceId ) @@ -264,41 +239,3 @@ async function readSearchWorkspace( } })) } - -async function readSearchWorkspaces( - entries: LinearClientForWorkspace[], - query: string, - limit: number, - workspaceId?: string | 'all', - initialFailures: WorkspaceReadFailure[] = [] -): Promise<{ results: LinearSearchIssueSummary[][]; failures: WorkspaceReadFailure[] }> { - if (workspaceId && workspaceId !== 'all') { - return { - results: [await readSearchWorkspace(entries[0], query, limit, workspaceId)], - failures: [] - } - } - - const settled = await Promise.allSettled( - entries.map(async (entry) => readSearchWorkspace(entry, query, limit, workspaceId)) - ) - const attemptedWorkspaceCount = entries.length + initialFailures.length - const results: LinearSearchIssueSummary[][] = [] - const failures: WorkspaceReadFailure[] = [...initialFailures] - for (let index = 0; index < settled.length; index += 1) { - const result = settled[index] - if (result.status === 'fulfilled') { - results.push(result.value) - continue - } - if (result.reason instanceof LinearAgentAccessError) { - failures.push(workspaceFailure(entries[index].workspace, result.reason)) - } - console.warn('[linear] agent search failed:', result.reason) - } - - if (results.length === 0 && failures.length === attemptedWorkspaceCount && failures[0]) { - throw failures[0].error - } - return { results, failures } -} diff --git a/src/main/linear/issue-context-errors.ts b/src/main/linear/issue-context-errors.ts index 4e76835eb6a..4f8d3c6b93c 100644 --- a/src/main/linear/issue-context-errors.ts +++ b/src/main/linear/issue-context-errors.ts @@ -1,11 +1,12 @@ import type { LinearErrorCode, LinearIncludeErrorCode } from '../../shared/linear-agent-access' +import { boundedIntegrationErrorMessage } from '../integration-error-message' export class LinearAgentAccessError extends Error { readonly code: LinearErrorCode readonly data?: unknown constructor(code: LinearErrorCode, message: string, data?: unknown) { - super(message) + super(boundedIntegrationErrorMessage(message)) this.name = 'LinearAgentAccessError' this.code = code this.data = data @@ -58,13 +59,12 @@ export function classifyLinearError(error: unknown): LinearErrorCode { } export function linearMessage(error: unknown): string { - const message = error instanceof Error ? error.message : String(error) - return sanitizeLinearErrorMessage(message) + return sanitizeLinearErrorMessage(boundedIntegrationErrorMessage(error)) } export function sanitizeLinearErrorMessage(message: string): string { // Why: provider text is useful in CLI errors, but raw SDK failures can embed secrets or user payloads. - return stripLinearStackTrace(message) + return stripLinearStackTrace(boundedIntegrationErrorMessage(message)) .replace( /(headers?\s*[:=]\s*)\{[^{}]*(?:authorization|token|api[-_]?key)[^{}]*\}/gi, '$1[REDACTED]' diff --git a/src/main/linear/issue-context-fanout-reads.ts b/src/main/linear/issue-context-fanout-reads.ts new file mode 100644 index 00000000000..4b744d97022 --- /dev/null +++ b/src/main/linear/issue-context-fanout-reads.ts @@ -0,0 +1,96 @@ +import { runBoundedIntegrationSettledFanout } from '../integration-fanout' +import { boundedIntegrationErrorLog } from '../integration-error-message' +import type { LinearClientForWorkspace } from './client' +import { LinearAgentAccessError } from './issue-context-errors' +import { workspaceFailure, type WorkspaceReadFailure } from './issue-context-fanout' + +export async function readLinearIssueWorkspaceFanout( + entries: LinearClientForWorkspace[], + selection: string | 'all', + readWorkspace: (entry: LinearClientForWorkspace) => Promise, + initialFailures: WorkspaceReadFailure[] = [] +): Promise<{ results: TResult[]; truncated: boolean }> { + if (selection !== 'all') { + const selected = await readWorkspace(entries[0]) + return { results: selected ? [selected] : [], truncated: false } + } + + const fanout = await runBoundedIntegrationSettledFanout(entries, readWorkspace, (result) => + result ? [result] : [] + ) + const results: TResult[] = [] + const failures: LinearAgentAccessError[] = initialFailures.map((failure) => failure.error) + + for (const result of fanout.results) { + if (result.status === 'fulfilled') { + if (result.value) { + results.push(result.value) + } + continue + } + if (result.reason instanceof LinearAgentAccessError) { + failures.push(result.reason) + } + console.warn('[linear] agent issue read failed:', boundedIntegrationErrorLog(result.reason)) + } + + if (fanout.truncated) { + console.warn('[linear] Agent issue fan-out exceeded its aggregate result budget; truncating.') + } + if (results.length === 0 && failures[0] && !fanout.truncated) { + throw failures[0] + } + return { results, truncated: fanout.truncated } +} + +export async function readLinearSearchWorkspaceFanout( + entries: LinearClientForWorkspace[], + workspaceId: string | 'all' | undefined, + readWorkspace: (entry: LinearClientForWorkspace) => Promise, + initialFailures: WorkspaceReadFailure[] = [] +): Promise<{ + results: TResult[][] + failures: WorkspaceReadFailure[] + truncated: boolean +}> { + if (workspaceId && workspaceId !== 'all') { + return { + results: [await readWorkspace(entries[0])], + failures: [], + truncated: false + } + } + + const fanout = await runBoundedIntegrationSettledFanout( + entries, + readWorkspace, + (results) => results + ) + const attemptedWorkspaceCount = fanout.attemptedCount + initialFailures.length + const results: TResult[][] = [] + const failures: WorkspaceReadFailure[] = [...initialFailures] + for (let index = 0; index < fanout.results.length; index += 1) { + const result = fanout.results[index] + if (result.status === 'fulfilled') { + results.push(result.value) + continue + } + if (result.reason instanceof LinearAgentAccessError) { + failures.push(workspaceFailure(entries[index].workspace, result.reason)) + } + console.warn('[linear] agent search failed:', boundedIntegrationErrorLog(result.reason)) + } + + if (fanout.truncated) { + console.warn('[linear] Agent search fan-out exceeded its aggregate result budget; truncating.') + } + if ( + results.length === 0 && + failures.length === attemptedWorkspaceCount && + failures[0] && + !fanout.truncated + ) { + throw failures[0].error + } + return { results, failures, truncated: fanout.truncated } +} diff --git a/src/main/linear/issue-context-fanout.ts b/src/main/linear/issue-context-fanout.ts index 484eec818f7..17fe07ead15 100644 --- a/src/main/linear/issue-context-fanout.ts +++ b/src/main/linear/issue-context-fanout.ts @@ -1,5 +1,6 @@ import type { LinearErrorCode, LinearWorkspaceCandidate } from '../../shared/linear-agent-access' import type { LinearWorkspace } from '../../shared/types' +import { boundedIntegrationErrorLog } from '../integration-error-message' import { getClients, getStatus, type LinearClientForWorkspace } from './client' import { LinearAgentAccessError, @@ -35,7 +36,10 @@ export function getFanoutClientEntries(): { } catch (error) { const failure = workspaceFailure(workspace, toLinearAccessError(error)) failures.push(failure) - console.warn('[linear] agent workspace credential read failed:', error) + console.warn( + '[linear] agent workspace credential read failed:', + boundedIntegrationErrorLog(error) + ) } } return { entries, failures } diff --git a/src/main/linear/issue-context-includes.ts b/src/main/linear/issue-context-includes.ts index 13a8f4ae234..0a8dd70d75c 100644 --- a/src/main/linear/issue-context-includes.ts +++ b/src/main/linear/issue-context-includes.ts @@ -17,6 +17,8 @@ import { clampLinearIssueDepth } from '../../shared/linear-agent-access' import { extractLinearInlineMedia } from '../../shared/linear-inline-media' +import { boundedIntegrationErrorMessage } from '../integration-error-message' +import { IntegrationPaginationBudget } from '../integration-pagination-budget' import type { ResolvedIssue } from './issue-context-client' import { getRequiredEntry, withLinearRead } from './issue-context-client' import { getPublicFileUrlClient } from './client' @@ -69,7 +71,7 @@ export async function readOptionalIncludes( includeErrors.push({ include, code: includeErrorCode(error), - message: error instanceof Error ? error.message : String(error) + message: boundedIntegrationErrorMessage(error) }) } } @@ -175,6 +177,7 @@ async function readChildren( let returned = 0 let capReached = false let depthReached = false + const budget = new IntegrationPaginationBudget() const readLevel = async (issueId: string, level: number): Promise => { if (level > depth || returned >= LINEAR_CHILDREN_NODE_CAP) { @@ -182,16 +185,20 @@ async function readChildren( return [] } const remaining = LINEAR_CHILDREN_NODE_CAP - returned - const response = await readConnectionPages(remaining, async (page) => { - return await withLinearRead(entry, async () => { - const client = getPublicFileUrlClient(entry) - const raw = await client.client.rawRequest>( - CHILDREN_QUERY, - { id: issueId, ...page } - ) - return raw.data?.issue?.children ?? null - }) - }) + const response = await readConnectionPages( + remaining, + async (page) => { + return await withLinearRead(entry, async () => { + const client = getPublicFileUrlClient(entry) + const raw = await client.client.rawRequest>( + CHILDREN_QUERY, + { id: issueId, ...page } + ) + return raw.data?.issue?.children ?? null + }) + }, + budget + ) const nodes = response.nodes if (response.hasMore || nodes.length > remaining) { capReached = true diff --git a/src/main/linear/issue-context-pagination.test.ts b/src/main/linear/issue-context-pagination.test.ts new file mode 100644 index 00000000000..b10b36c859a --- /dev/null +++ b/src/main/linear/issue-context-pagination.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from 'vitest' +import { IntegrationPaginationBudget } from '../integration-pagination-budget' +import { readConnectionPages } from './issue-context-pagination' + +describe('Linear issue context pagination', () => { + it('stops before retaining a page that exceeds the aggregate byte budget', async () => { + const loadConnection = vi + .fn() + .mockResolvedValueOnce({ + nodes: [{ id: 'first', body: 'small' }], + pageInfo: { hasNextPage: true, endCursor: 'next' } + }) + .mockResolvedValueOnce({ + nodes: [{ id: 'second', body: 'x'.repeat(100) }], + pageInfo: { hasNextPage: false, endCursor: null } + }) + const budget = new IntegrationPaginationBudget({ + maxPages: 10, + maxItems: 10, + maxRetainedBytes: 64 + }) + + await expect(readConnectionPages(10, loadConnection, budget)).resolves.toEqual({ + nodes: [{ id: 'first', body: 'small' }], + hasMore: true + }) + expect(loadConnection).toHaveBeenCalledTimes(2) + }) + + it('reports truncation when a backend over-delivers the requested page', async () => { + await expect( + readConnectionPages(1, async () => ({ + nodes: [{ id: 'first' }, { id: 'omitted' }], + pageInfo: { hasNextPage: false, endCursor: null } + })) + ).resolves.toEqual({ + nodes: [{ id: 'first' }], + hasMore: true + }) + }) +}) diff --git a/src/main/linear/issue-context-pagination.ts b/src/main/linear/issue-context-pagination.ts index d8c5f239ac9..ad0c5639628 100644 --- a/src/main/linear/issue-context-pagination.ts +++ b/src/main/linear/issue-context-pagination.ts @@ -1,4 +1,5 @@ import { LINEAR_ISSUE_API_PAGE_SIZE_MAX } from '../../shared/linear-issue-read-limits' +import { IntegrationPaginationBudget } from '../integration-pagination-budget' export type LinearPageVariables = { first: number; after?: string } @@ -12,7 +13,8 @@ export type LinearConnection = { export async function readConnectionPages( limit: number, - loadConnection: (page: LinearPageVariables) => Promise> + loadConnection: (page: LinearPageVariables) => Promise>, + budget = new IntegrationPaginationBudget() ): Promise<{ nodes: T[]; hasMore: boolean }> { const nodes: T[] = [] let after: string | undefined @@ -24,13 +26,21 @@ export async function readConnectionPages( const first = Math.min(LINEAR_ISSUE_API_PAGE_SIZE_MAX, limit - nodes.length) const connection = await loadConnection(after ? { first, after } : { first }) const pageNodes = connection?.nodes ?? [] - nodes.push(...pageNodes.slice(0, limit - nodes.length)) - hasMore = Boolean(connection?.pageInfo?.hasNextPage) + const retainedPageNodes = pageNodes.slice(0, limit - nodes.length) + hasMore = + pageNodes.length > retainedPageNodes.length || Boolean(connection?.pageInfo?.hasNextPage) + if (!budget.admitPage(retainedPageNodes)) { + return { nodes, hasMore: true } + } + nodes.push(...retainedPageNodes) const nextCursor = connection?.pageInfo?.endCursor ?? undefined if (!hasMore || !nextCursor || nextCursor === after || pageNodes.length === 0) { break } + if (!budget.canRequestPage) { + return { nodes, hasMore: true } + } after = nextCursor } diff --git a/src/main/linear/issue-context-raw.ts b/src/main/linear/issue-context-raw.ts index 6f29e9e6a01..fe61843035d 100644 --- a/src/main/linear/issue-context-raw.ts +++ b/src/main/linear/issue-context-raw.ts @@ -231,7 +231,7 @@ export function mapIssue(issue: RawIssue): LinearIssueSummary { project: issue.project ?? null, cycle: issue.cycle ?? null, assignee: issue.assignee ?? null, - labels: issue.labels?.nodes ?? [], + labels: (issue.labels?.nodes ?? []).slice(0, 50), priority: issue.priority, estimate: issue.estimate, dueDate: issue.dueDate, diff --git a/src/main/linear/issue-context-relations.ts b/src/main/linear/issue-context-relations.ts index fd39787d720..69e942f1441 100644 --- a/src/main/linear/issue-context-relations.ts +++ b/src/main/linear/issue-context-relations.ts @@ -1,5 +1,6 @@ import type { LinearCollectionMeta, LinearIssueRelation } from '../../shared/linear-agent-access' import { LINEAR_RELATIONS_CAP } from '../../shared/linear-agent-access' +import { IntegrationPaginationBudget } from '../integration-pagination-budget' import type { ResolvedIssue } from './issue-context-client' import { getRequiredEntry, withLinearRead } from './issue-context-client' import { readConnectionPages } from './issue-context-pagination' @@ -16,15 +17,20 @@ export async function readIssueRelations( resolved: ResolvedIssue ): Promise<{ items: LinearIssueRelation[]; meta: LinearCollectionMeta }> { const entry = getRequiredEntry(resolved.workspace.id) - const response = await readConnectionPages(LINEAR_RELATIONS_CAP, async (page) => { - return await withLinearRead(entry, async () => { - const raw = await entry.client.client.rawRequest< - RawRelationsResponse, - Record - >(RELATIONS_QUERY, { id: resolved.issue.id, ...page }) - return raw.data?.issue?.relations ?? null - }) - }) + const budget = new IntegrationPaginationBudget() + const response = await readConnectionPages( + LINEAR_RELATIONS_CAP, + async (page) => { + return await withLinearRead(entry, async () => { + const raw = await entry.client.client.rawRequest< + RawRelationsResponse, + Record + >(RELATIONS_QUERY, { id: resolved.issue.id, ...page }) + return raw.data?.issue?.relations ?? null + }) + }, + budget + ) const outbound = response.nodes .slice(0, LINEAR_RELATIONS_CAP) .map((node) => mapRelation(node, 'outbound', node.relatedIssue)) @@ -32,15 +38,19 @@ export async function readIssueRelations( // Why: when outbound relations exactly fill the cap, probe inverse relations so // the response cannot claim completeness while silently omitting inbound ones. const inverseReadLimit = Math.max(1, remaining) - const inverse = await readConnectionPages(inverseReadLimit, async (page) => { - return await withLinearRead(entry, async () => { - const raw = await entry.client.client.rawRequest< - RawRelationsResponse, - Record - >(INVERSE_RELATIONS_QUERY, { id: resolved.issue.id, ...page }) - return raw.data?.issue?.inverseRelations ?? null - }) - }) + const inverse = await readConnectionPages( + inverseReadLimit, + async (page) => { + return await withLinearRead(entry, async () => { + const raw = await entry.client.client.rawRequest< + RawRelationsResponse, + Record + >(INVERSE_RELATIONS_QUERY, { id: resolved.issue.id, ...page }) + return raw.data?.issue?.inverseRelations ?? null + }) + }, + budget + ) const inbound = inverse.nodes .slice(0, remaining) .map((node) => mapRelation(node, 'inbound', node.issue)) diff --git a/src/main/linear/issues-retention-bounds.test.ts b/src/main/linear/issues-retention-bounds.test.ts new file mode 100644 index 00000000000..c736edb6e1b --- /dev/null +++ b/src/main/linear/issues-retention-bounds.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { LinearClientForWorkspace } from './client' +import { MAX_INTEGRATION_ACCOUNTS } from '../integration-account-persistence-limits' +import { LINEAR_SEARCH_MAX_LIMIT } from '../../shared/linear-agent-access' + +const getClients = vi.fn() + +vi.mock('./client', () => ({ + acquire: vi.fn().mockResolvedValue(undefined), + release: vi.fn(), + getClients: (...args: unknown[]) => getClients(...args), + isAuthError: vi.fn().mockReturnValue(false), + clearToken: vi.fn() +})) + +function issue(id: string) { + return { + id, + identifier: id, + title: id, + description: '', + url: `https://linear.app/${id}`, + estimate: null, + priority: 0, + updatedAt: '2026-01-01T00:00:00.000Z', + labelIds: [], + labels: { nodes: [] } + } +} + +function entry(id: string, rawRequest: ReturnType): LinearClientForWorkspace { + return { + workspace: { + id, + organizationId: id, + organizationName: id, + displayName: 'Ada', + email: null + }, + client: { client: { rawRequest } } + } as unknown as LinearClientForWorkspace +} + +describe('Linear issue retention bounds', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('retains only the requested list page when a provider returns extra rows', async () => { + const rawRequest = vi.fn().mockResolvedValue({ + data: { + issues: { + nodes: [issue('LIN-1'), issue('LIN-2')], + pageInfo: { hasNextPage: false } + } + } + }) + getClients.mockReturnValue([entry('workspace-1', rawRequest)]) + const { listIssues } = await import('./issues') + + await expect(listIssues('all', 1, 'workspace-1')).resolves.toMatchObject({ + items: [{ id: 'LIN-1' }], + hasMore: true + }) + expect(rawRequest.mock.calls[0]?.[1]).toMatchObject({ first: 1 }) + }) + + it('clamps direct search callers and maps no more than the requested rows', async () => { + const rawRequest = vi.fn().mockResolvedValue({ + data: { + searchIssues: { + nodes: Array.from({ length: LINEAR_SEARCH_MAX_LIMIT + 1 }, (_, index) => + issue(`LIN-${index + 1}`) + ) + } + } + }) + getClients.mockReturnValue([entry('workspace-1', rawRequest)]) + const { searchIssues } = await import('./issues') + + await expect(searchIssues('bug', Number.MAX_SAFE_INTEGER, 'workspace-1')).resolves.toHaveLength( + LINEAR_SEARCH_MAX_LIMIT + ) + expect(rawRequest.mock.calls[0]?.[1]).toMatchObject({ first: LINEAR_SEARCH_MAX_LIMIT }) + }) + + it('stops all-workspace scheduling at the aggregate account boundary', async () => { + const requests = Array.from({ length: MAX_INTEGRATION_ACCOUNTS + 1 }, (_, index) => + vi.fn().mockResolvedValue({ + data: { + issues: { + nodes: [issue(`LIN-${index + 1}`)], + pageInfo: { hasNextPage: false } + } + } + }) + ) + getClients.mockReturnValue( + requests.map((request, index) => entry(`workspace-${index}`, request)) + ) + const { listIssues } = await import('./issues') + + await expect(listIssues('all', 1, 'all')).resolves.toMatchObject({ + items: [{ id: 'LIN-1' }], + hasMore: true + }) + expect(requests[MAX_INTEGRATION_ACCOUNTS - 1]).toHaveBeenCalledTimes(1) + expect(requests[MAX_INTEGRATION_ACCOUNTS]).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/linear/issues.ts b/src/main/linear/issues.ts index 1c8d7d8ee74..0d41ddbf40b 100644 --- a/src/main/linear/issues.ts +++ b/src/main/linear/issues.ts @@ -11,6 +11,7 @@ import type { } from '../../shared/types' import type { LinearClient } from '@linear/sdk' import { loadLinearSdk } from './linear-sdk' +import { clampLinearSearchLimit } from '../../shared/linear-agent-access' import { LINEAR_ISSUE_API_PAGE_SIZE_MAX, clampLinearIssueListLimit @@ -19,6 +20,15 @@ import { isEmptyLinearIssueAttributeFilter, type LinearIssueAttributeFilter } from '../../shared/linear-issue-attribute-filter' +import { + boundedIntegrationErrorLog, + boundedIntegrationErrorMessage +} from '../integration-error-message' +import { createIntegrationFanoutBudget, runBoundedIntegrationFanout } from '../integration-fanout' +import { + INTEGRATION_PAGINATION_MAX_ITEMS, + IntegrationPaginationBudget +} from '../integration-pagination-budget' import { acquire, release, @@ -397,7 +407,7 @@ function mapRawIssueForWorkspace( entry: LinearClientForWorkspace, issue: LinearIssueNode ): LinearIssue { - const labelNodes = issue.labels?.nodes ?? [] + const labelNodes = (issue.labels?.nodes ?? []).slice(0, LINEAR_ISSUE_API_PAGE_SIZE_MAX) return { id: issue.id, identifier: issue.identifier, @@ -418,7 +428,9 @@ function mapRawIssueForWorkspace( labels: labelNodes.map((label) => label.name), // Why: labelIds drives full-replace updates. Keep Linear's complete id // list even when display label nodes are paginated. - labelIds: issue.labelIds ?? labelNodes.map((label) => label.id), + labelIds: + issue.labelIds?.slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) ?? + labelNodes.map((label) => label.id), assignee: issue.assignee ? { id: issue.assignee.id, @@ -441,6 +453,7 @@ async function readIssueConnectionPages( loadConnection: LinearIssueConnectionLoader ): Promise<{ items: LinearIssue[]; hasMore: boolean }> { const items: LinearIssue[] = [] + const budget = new IntegrationPaginationBudget() let after: string | undefined let hasMore = false @@ -449,12 +462,22 @@ async function readIssueConnectionPages( // cursors instead of asking for the whole expanded limit in one request. const first = Math.min(LINEAR_ISSUE_API_PAGE_SIZE_MAX, limit - items.length) const connection = await loadConnection(after ? { first, after } : { first }) - const nodes = connection?.nodes ?? [] - items.push(...nodes.map((issue) => mapRawIssueForWorkspace(entry, issue))) - hasMore = Boolean(connection?.pageInfo?.hasNextPage) + const pageNodes = connection?.nodes ?? [] + const nodes = pageNodes.slice(0, first) + const pageItems = nodes.map((issue) => mapRawIssueForWorkspace(entry, issue)) + hasMore = pageNodes.length > nodes.length || Boolean(connection?.pageInfo?.hasNextPage) + if (!budget.admitPage(pageItems)) { + console.warn('[linear] Issue list exceeded its retained result budget; truncating.') + return { items, hasMore: true } + } + items.push(...pageItems) const nextCursor = connection?.pageInfo?.endCursor ?? undefined - if (!hasMore || !nextCursor || nextCursor === after || nodes.length === 0) { + if (!hasMore || !nextCursor || nextCursor === after || pageNodes.length === 0) { + break + } + if (!budget.canRequestPage) { + console.warn('[linear] Issue list reached its retained result budget; truncating.') break } after = nextCursor @@ -539,7 +562,7 @@ function shouldThrowAuthError(selection: LinearWorkspaceSelection | null | undef } function linearWriteMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) + return boundedIntegrationErrorMessage(error) } function isDuplicateIdError(error: unknown): boolean { @@ -643,7 +666,7 @@ async function runLinearLookup( } function isLinearLookupMiss(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) // Why: Linear throws for direct entity lookups that miss; write-id probes // need the same null shape as GraphQL nullable data, not a failed write. return message.includes('Entity not found:') && message.includes('Could not find referenced') @@ -703,7 +726,10 @@ function mapRawIssueWriteRecord( ): LinearIssueWriteRecord { return { ...issue, - labels: issue.labels?.nodes ?? [] + ...(issue.labelIds + ? { labelIds: issue.labelIds.slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) } + : {}), + labels: (issue.labels?.nodes ?? []).slice(0, LINEAR_ISSUE_API_PAGE_SIZE_MAX) } } @@ -731,7 +757,7 @@ export async function getIssue( throw error } } else { - console.warn('[linear] getIssue failed:', error) + console.warn('[linear] getIssue failed:', boundedIntegrationErrorLog(error)) } } finally { release() @@ -814,20 +840,22 @@ export async function searchIssues( limit = 20, workspaceId?: LinearWorkspaceSelection | null ): Promise { + const effectiveLimit = clampLinearSearchLimit(limit) const entries = getClients(workspaceId) if (entries.length === 0) { return [] } - const results = await Promise.all( - entries.map(async (entry) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { const result = await entry.client.client.rawRequest< LinearIssueConnectionResponse, LinearRawVariables - >(SEARCH_ISSUES_QUERY, { term: query, first: limit }) - const nodes = result.data?.searchIssues?.nodes ?? [] + >(SEARCH_ISSUES_QUERY, { term: query, first: effectiveLimit }) + const nodes = (result.data?.searchIssues?.nodes ?? []).slice(0, effectiveLimit) return nodes.map((issue) => mapRawIssueForWorkspace(entry, issue)) } catch (error) { if (isAuthError(error)) { @@ -836,21 +864,28 @@ export async function searchIssues( throw error } } else { - console.warn('[linear] searchIssues failed:', error) + console.warn('[linear] searchIssues failed:', boundedIntegrationErrorLog(error)) } return [] } finally { release() } - }) + }, + (issues) => issues ) // Why: searchIssues returns Linear's relevance ranking. Re-sorting by // updatedAt would discard relevance order for single-workspace results, // diverging from Linear's web UI and pre-PR behavior. - if (entries.length === 1) { - return results.flat().slice(0, limit) + if (fanout.truncated) { + console.warn( + '[linear] Cross-workspace search exceeded its aggregate result budget; truncating.' + ) } - return sortAndLimitIssues(results.flat(), limit) + const results = fanout.results.flat() + if (entries.length === 1) { + return results.slice(0, effectiveLimit) + } + return sortAndLimitIssues(results, effectiveLimit) } export type LinearListFilter = 'assigned' | 'created' | 'all' | 'completed' | 'open' @@ -875,7 +910,7 @@ function linearWorkspaceError( entry: LinearClientForWorkspace, error: unknown ): LinearWorkspaceError { - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) const lower = message.toLocaleLowerCase() const type: LinearWorkspaceError['type'] = isAuthError(error) ? 'auth' @@ -917,7 +952,7 @@ async function readListIssuesForWorkspace( throw error } } else { - console.warn('[linear] listIssues failed:', error) + console.warn('[linear] listIssues failed:', boundedIntegrationErrorLog(error)) } return { items: [], hasMore: false, errors: [linearWorkspaceError(entry, error)] } } finally { @@ -931,10 +966,11 @@ async function readIssueConnectionPage( page: LinearIssuePageRequest ): Promise { const connection = await loadConnection(page) - const nodes = connection?.nodes ?? [] + const pageNodes = connection?.nodes ?? [] + const nodes = pageNodes.slice(0, page.first) return { items: nodes.map((issue) => mapRawIssueForWorkspace(entry, issue)), - hasMore: Boolean(connection?.pageInfo?.hasNextPage), + hasMore: pageNodes.length > nodes.length || Boolean(connection?.pageInfo?.hasNextPage), endCursor: connection?.pageInfo?.endCursor ?? undefined } } @@ -942,8 +978,9 @@ async function readIssueConnectionPage( async function readListIssuesPageForState( state: LinearIssueWorkspacePageState, first: number, - workspaceId: LinearWorkspaceSelection | null | undefined -): Promise { + workspaceId: LinearWorkspaceSelection | null | undefined, + budget?: IntegrationPaginationBudget +): Promise { const previousCursor = state.after await acquire() try { @@ -952,25 +989,38 @@ async function readListIssuesPageForState( state.loadConnection, previousCursor ? { first, after: previousCursor } : { first } ) + if (budget && !budget.admitPage(page.items)) { + state.hasMore = true + state.canPage = false + return false + } state.items.push(...page.items) state.hasMore = page.hasMore state.after = page.endCursor state.canPage = Boolean( page.hasMore && page.endCursor && page.endCursor !== previousCursor && page.items.length > 0 ) + return true } catch (error) { state.items = [] state.hasMore = false state.canPage = false - state.error = linearWorkspaceError(state.entry, error) + const workspaceError = linearWorkspaceError(state.entry, error) + state.error = workspaceError if (isAuthError(error)) { clearToken(state.entry.workspace.id) if (shouldThrowAuthError(workspaceId)) { throw error } } else { - console.warn('[linear] listIssues failed:', error) + console.warn('[linear] listIssues failed:', boundedIntegrationErrorLog(error)) } + if (budget && !budget.admitPage([workspaceError])) { + state.error = undefined + state.hasMore = true + return false + } + return true } finally { release() } @@ -1023,17 +1073,40 @@ async function readListIssuesAcrossWorkspaces( canPage: false })) const first = Math.min(LINEAR_ISSUE_API_PAGE_SIZE_MAX, limit) + const aggregateBudget = createIntegrationFanoutBudget() // Why: "all workspaces" is a global sorted list. Pull one bounded page per // workspace first, then spend additional API calls only where unseen issues // can still change the global updatedAt cutoff. - await Promise.all(states.map((state) => readListIssuesPageForState(state, first, workspaceId))) + const initialFanout = await runBoundedIntegrationFanout( + states, + async (state) => { + await readListIssuesPageForState(state, first, workspaceId) + return state + }, + (state) => [...state.items, ...(state.error ? [state.error] : [])], + { budget: aggregateBudget } + ) + const acceptedStates = new Set(initialFanout.results) + for (const state of states) { + if (!acceptedStates.has(state)) { + state.items = [] + state.error = undefined + state.hasMore = true + state.canPage = false + } + } + let aggregateTruncated = initialFanout.truncated for (;;) { const nextState = findWorkspaceToPageForLimit(states, limit) if (!nextState) { break } + if (!aggregateBudget.canRequestPage) { + aggregateTruncated = true + break + } const itemCount = states.reduce((count, state) => count + state.items.length, 0) const pageSize = itemCount < limit @@ -1042,7 +1115,10 @@ async function readListIssuesAcrossWorkspaces( LINEAR_ISSUE_API_PAGE_SIZE_MAX, Math.max(1, countSelectedIssuesOlderThanWorkspaceBoundary(states, nextState, limit)) ) - await readListIssuesPageForState(nextState, pageSize, workspaceId) + if (!(await readListIssuesPageForState(nextState, pageSize, workspaceId, aggregateBudget))) { + aggregateTruncated = true + break + } } const limited = sortLimitAndDescribeIssues( @@ -1051,7 +1127,7 @@ async function readListIssuesAcrossWorkspaces( ) return { items: limited.items, - hasMore: states.some((state) => state.hasMore) || limited.clipped, + hasMore: aggregateTruncated || states.some((state) => state.hasMore) || limited.clipped, errors: states.flatMap((state) => (state.error ? [state.error] : [])) } } @@ -1148,7 +1224,7 @@ export async function createIssue( clearToken(entry.workspace.id) throw error } - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) return { ok: false, error: message } } finally { release() @@ -1284,7 +1360,7 @@ export async function updateIssue( clearToken(entry.workspace.id) throw error } - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) return { ok: false, error: message } } finally { release() @@ -1381,7 +1457,7 @@ export async function addIssueComment( clearToken(entry.workspace.id) throw error } - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) return { ok: false, error: message } } finally { release() @@ -1511,7 +1587,7 @@ export async function getIssueComments( LinearRawVariables >(ISSUE_COMMENTS_QUERY, { id: issueId }) const nodes = result.data?.issue?.comments?.nodes ?? [] - return nodes.map((node) => ({ + return nodes.slice(0, LINEAR_ISSUE_API_PAGE_SIZE_MAX).map((node) => ({ id: node.id, body: node.body ?? '', // Why: rawRequest returns createdAt as an ISO string already; do not @@ -1529,7 +1605,7 @@ export async function getIssueComments( clearToken(entry.workspace.id) throw error } - console.warn('[linear] getIssueComments failed:', error) + console.warn('[linear] getIssueComments failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() diff --git a/src/main/linear/linear-project-request-coalescer.test.ts b/src/main/linear/linear-project-request-coalescer.test.ts new file mode 100644 index 00000000000..22ccb01732a --- /dev/null +++ b/src/main/linear/linear-project-request-coalescer.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it, vi } from 'vitest' +import { + LINEAR_PROJECT_MAX_INFLIGHT_KEYS, + LINEAR_PROJECT_MAX_INFLIGHT_KEY_BYTES, + LinearProjectRequestCoalescer +} from './linear-project-request-coalescer' + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +describe('LinearProjectRequestCoalescer', () => { + it('preserves ordinary same-key coalescing and cleans up after settlement', async () => { + const coalescer = new LinearProjectRequestCoalescer() + const request = deferred() + const load = vi.fn(() => request.promise) + + const first = coalescer.coalesce('projects:alpha', load) + const second = coalescer.coalesce('projects:alpha', load) + expect(first).toBe(second) + expect(load).toHaveBeenCalledTimes(1) + expect(coalescer.trackedRequestCount).toBe(1) + + request.resolve('done') + await expect(first).resolves.toBe('done') + expect(coalescer.trackedRequestCount).toBe(0) + }) + + it('tracks the exact key-count boundary and executes key +1 untracked', async () => { + const coalescer = new LinearProjectRequestCoalescer() + const requests = Array.from({ length: LINEAR_PROJECT_MAX_INFLIGHT_KEYS + 1 }, () => + deferred() + ) + const promises = requests.map((request, index) => + coalescer.coalesce(`key-${index}`, () => request.promise) + ) + + expect(coalescer.trackedRequestCount).toBe(LINEAR_PROJECT_MAX_INFLIGHT_KEYS) + requests.forEach((request, index) => request.resolve(index)) + await expect(Promise.all(promises)).resolves.toHaveLength(requests.length) + expect(coalescer.trackedRequestCount).toBe(0) + }) + + it('tracks an exact-size key and executes key byte +1 untracked', async () => { + const coalescer = new LinearProjectRequestCoalescer() + const exactRequest = deferred() + const exact = coalescer.coalesce( + 'a'.repeat(LINEAR_PROJECT_MAX_INFLIGHT_KEY_BYTES), + () => exactRequest.promise + ) + const over = coalescer.coalesce( + 'a'.repeat(LINEAR_PROJECT_MAX_INFLIGHT_KEY_BYTES + 1), + async () => 'over' + ) + + expect(coalescer.trackedRequestCount).toBe(1) + await expect(over).resolves.toBe('over') + expect(coalescer.trackedRequestCount).toBe(1) + exactRequest.resolve('exact') + await expect(exact).resolves.toBe('exact') + expect(coalescer.trackedRequestCount).toBe(0) + }) + + it('keeps a forced replacement tracked when the older request settles', async () => { + const coalescer = new LinearProjectRequestCoalescer() + const staleRequest = deferred() + const freshRequest = deferred() + + const stale = coalescer.coalesce('same', () => staleRequest.promise) + const fresh = coalescer.coalesce('same', () => freshRequest.promise, true) + staleRequest.resolve('stale') + await expect(stale).resolves.toBe('stale') + expect(coalescer.trackedRequestCount).toBe(1) + + freshRequest.resolve('fresh') + await expect(fresh).resolves.toBe('fresh') + expect(coalescer.trackedRequestCount).toBe(0) + }) +}) diff --git a/src/main/linear/linear-project-request-coalescer.ts b/src/main/linear/linear-project-request-coalescer.ts new file mode 100644 index 00000000000..35a7983e690 --- /dev/null +++ b/src/main/linear/linear-project-request-coalescer.ts @@ -0,0 +1,33 @@ +export const LINEAR_PROJECT_MAX_INFLIGHT_KEYS = 256 +export const LINEAR_PROJECT_MAX_INFLIGHT_KEY_BYTES = 16 * 1024 + +export class LinearProjectRequestCoalescer { + private readonly inFlight = new Map>() + + coalesce(key: string, load: () => Promise, force = false): Promise { + const existing = this.inFlight.get(key) as Promise | undefined + if (existing && !force) { + return existing + } + + const loaded = load() + const mayTrack = + Buffer.byteLength(key, 'utf8') <= LINEAR_PROJECT_MAX_INFLIGHT_KEY_BYTES && + (this.inFlight.has(key) || this.inFlight.size < LINEAR_PROJECT_MAX_INFLIGHT_KEYS) + if (!mayTrack) { + return loaded + } + + const tracked = loaded.finally(() => { + if (this.inFlight.get(key) === tracked) { + this.inFlight.delete(key) + } + }) + this.inFlight.set(key, tracked) + return tracked + } + + get trackedRequestCount(): number { + return this.inFlight.size + } +} diff --git a/src/main/linear/linear-team-pages.test.ts b/src/main/linear/linear-team-pages.test.ts new file mode 100644 index 00000000000..af4329c0780 --- /dev/null +++ b/src/main/linear/linear-team-pages.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from 'vitest' +import { + INTEGRATION_PAGINATION_MAX_ITEMS, + INTEGRATION_PAGINATION_MAX_PAGES, + IntegrationPaginationLimitError +} from '../integration-pagination-budget' +import { fetchAllTeamLabels } from './linear-team-pages' + +function label(index: number) { + return { id: `label-${index}`, name: `Label ${index}`, color: '#888888' } +} + +function cumulativeLabelPage(totalPages: number, advertisesOneMore = false) { + let pageNumber = 1 + const page = { + nodes: [label(1)], + pageInfo: { hasNextPage: totalPages > 1 || advertisesOneMore }, + fetchNext: vi.fn(async () => { + pageNumber += 1 + page.nodes.push(label(pageNumber)) + page.pageInfo.hasNextPage = pageNumber < totalPages || advertisesOneMore + return page + }) + } + return page +} + +describe('bounded Linear team metadata pagination', () => { + it('preserves cumulative node order through the exact page boundary', async () => { + const page = cumulativeLabelPage(INTEGRATION_PAGINATION_MAX_PAGES) + + const labels = await fetchAllTeamLabels({ labels: async () => page }) + expect(labels).toHaveLength(INTEGRATION_PAGINATION_MAX_PAGES) + expect(labels[0]).toMatchObject({ id: 'label-1' }) + expect(labels.at(-1)).toMatchObject({ id: `label-${INTEGRATION_PAGINATION_MAX_PAGES}` }) + expect(page.fetchNext).toHaveBeenCalledTimes(INTEGRATION_PAGINATION_MAX_PAGES - 1) + }) + + it('rejects page +1 before asking the SDK to materialize it', async () => { + const page = cumulativeLabelPage(INTEGRATION_PAGINATION_MAX_PAGES, true) + + await expect(fetchAllTeamLabels({ labels: async () => page })).rejects.toThrow( + IntegrationPaginationLimitError + ) + expect(page.fetchNext).toHaveBeenCalledTimes(INTEGRATION_PAGINATION_MAX_PAGES - 1) + }) + + it('admits the exact item boundary and rejects item +1', async () => { + const exact = { + nodes: Array.from({ length: INTEGRATION_PAGINATION_MAX_ITEMS }, (_, index) => label(index)), + pageInfo: { hasNextPage: false }, + fetchNext: vi.fn() + } + const over = { + ...exact, + nodes: [...exact.nodes, label(INTEGRATION_PAGINATION_MAX_ITEMS)] + } + + await expect(fetchAllTeamLabels({ labels: async () => exact })).resolves.toHaveLength( + INTEGRATION_PAGINATION_MAX_ITEMS + ) + await expect(fetchAllTeamLabels({ labels: async () => over })).rejects.toThrow( + IntegrationPaginationLimitError + ) + }) +}) diff --git a/src/main/linear/linear-team-pages.ts b/src/main/linear/linear-team-pages.ts index 1da7fd6242f..e3c9f318b7e 100644 --- a/src/main/linear/linear-team-pages.ts +++ b/src/main/linear/linear-team-pages.ts @@ -1,5 +1,9 @@ import type { LinearLabel, LinearMember, LinearTeam, LinearWorkflowState } from '../../shared/types' import { buildLinearTeamUrl } from '../../shared/linear-links' +import { + IntegrationPaginationBudget, + IntegrationPaginationLimitError +} from '../integration-pagination-budget' import type { LinearClientForWorkspace } from './client' const TEAM_PAGE_SIZE = 100 @@ -32,13 +36,26 @@ type TeamStateNode = { position: number } +async function fetchBoundedConnection( + page: LinearConnectionPage +): Promise> { + const budget = new IntegrationPaginationBudget() + while (true) { + budget.assertCumulativePage(page.nodes) + if (!page.pageInfo.hasNextPage) { + return page + } + if (!budget.canRequestPage) { + throw new IntegrationPaginationLimitError() + } + await page.fetchNext() + } +} + export async function fetchAllTeamsForWorkspace( entry: LinearClientForWorkspace ): Promise { - let page = await entry.client.teams({ first: TEAM_PAGE_SIZE }) - while (page.pageInfo.hasNextPage) { - await page.fetchNext() - } + const page = await fetchBoundedConnection(await entry.client.teams({ first: TEAM_PAGE_SIZE })) return page.nodes.map((t) => ({ id: t.id, workspaceId: entry.workspace.id, @@ -56,10 +73,7 @@ export async function fetchAllTeamsForWorkspace( export async function fetchAllTeamStates(team: { states: (variables?: { first?: number }) => Promise> }): Promise { - const states = await team.states({ first: TEAM_PAGE_SIZE }) - while (states.pageInfo.hasNextPage) { - await states.fetchNext() - } + const states = await fetchBoundedConnection(await team.states({ first: TEAM_PAGE_SIZE })) return states.nodes .map((s) => ({ id: s.id, @@ -74,20 +88,14 @@ export async function fetchAllTeamStates(team: { export async function fetchAllTeamLabels(team: { labels: (variables?: { first?: number }) => Promise> }): Promise { - const labels = await team.labels({ first: TEAM_PAGE_SIZE }) - while (labels.pageInfo.hasNextPage) { - await labels.fetchNext() - } + const labels = await fetchBoundedConnection(await team.labels({ first: TEAM_PAGE_SIZE })) return labels.nodes.map((l) => ({ id: l.id, name: l.name, color: l.color })) } export async function fetchAllTeamMembers(team: { members: (variables?: { first?: number }) => Promise> }): Promise { - const members = await team.members({ first: TEAM_PAGE_SIZE }) - while (members.pageInfo.hasNextPage) { - await members.fetchNext() - } + const members = await fetchBoundedConnection(await team.members({ first: TEAM_PAGE_SIZE })) return members.nodes.map((m) => ({ id: m.id, displayName: m.displayName, diff --git a/src/main/linear/mappers.test.ts b/src/main/linear/mappers.test.ts index 0fba648efc8..7fd239fef8e 100644 --- a/src/main/linear/mappers.test.ts +++ b/src/main/linear/mappers.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { mapLinearIssue } from './mappers' describe('mapLinearIssue', () => { @@ -47,4 +47,44 @@ describe('mapLinearIssue', () => { project: undefined }) }) + + it('retains only requested lazy-relation rows when the SDK returns extra nodes', async () => { + const labels = vi.fn().mockResolvedValue({ + nodes: Array.from({ length: 51 }, (_, index) => ({ + id: `label-${index}`, + name: `Label ${index}` + })) + }) + const children = vi.fn().mockResolvedValue({ + nodes: Array.from({ length: 26 }, (_, index) => ({ + id: `child-${index}`, + identifier: `LIN-${index}`, + title: `Child ${index}`, + url: `https://linear.app/child-${index}` + })) + }) + const issue = { + id: 'issue-1', + identifier: 'LIN-1', + title: 'Bound relations', + description: null, + url: 'https://linear.app/LIN-1', + estimate: null, + priority: 0, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + state: null, + team: null, + assignee: null, + labels, + children + } + + const mapped = await mapLinearIssue(issue as never, { includeChildren: true }) + + expect(mapped.labels).toHaveLength(50) + expect(mapped.labelIds).toHaveLength(50) + expect(mapped.subIssues).toHaveLength(25) + expect(labels).toHaveBeenCalledWith({ first: 50 }) + expect(children).toHaveBeenCalledWith({ first: 25 }) + }) }) diff --git a/src/main/linear/mappers.ts b/src/main/linear/mappers.ts index 7ea55be7fca..ba35743fba4 100644 --- a/src/main/linear/mappers.ts +++ b/src/main/linear/mappers.ts @@ -1,5 +1,7 @@ import type { Issue, IssueSearchResult } from '@linear/sdk' import type { LinearIssue, LinearIssueChildSummary } from '../../shared/types' +import { LINEAR_ISSUE_API_PAGE_SIZE_MAX } from '../../shared/linear-issue-read-limits' +import { INTEGRATION_PAGINATION_MAX_ITEMS } from '../integration-pagination-budget' type IssueWithChildren = Issue & { children: Issue['children'] @@ -51,21 +53,24 @@ export async function mapLinearIssue( let labelIds: string[] = [] if ('labels' in issue && typeof issue.labels === 'function') { try { - const labelsConnection = await (issue as Issue).labels() - labelNames = labelsConnection.nodes.map((l) => l.name) - labelIds = labelsConnection.nodes.map((l) => l.id) + const labelsConnection = await (issue as Issue).labels({ + first: LINEAR_ISSUE_API_PAGE_SIZE_MAX + }) + const labels = labelsConnection.nodes.slice(0, LINEAR_ISSUE_API_PAGE_SIZE_MAX) + labelNames = labels.map((label) => label.name) + labelIds = labels.map((label) => label.id) } catch { // Swallow — labels are non-critical display data. } } else if ('labelIds' in issue && Array.isArray(issue.labelIds)) { - labelIds = issue.labelIds as string[] + labelIds = issue.labelIds.slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) as string[] } let subIssues: LinearIssueChildSummary[] | undefined if (options.includeChildren && 'children' in issue && typeof issue.children === 'function') { try { const childrenConnection = await (issue as IssueWithChildren).children({ first: 25 }) - subIssues = childrenConnection.nodes.map(mapLinearIssueChild) + subIssues = childrenConnection.nodes.slice(0, 25).map(mapLinearIssueChild) } catch { // Swallow — child issues are secondary display data and creation still works without them. } diff --git a/src/main/linear/mcp-issue-list.ts b/src/main/linear/mcp-issue-list.ts index 3906442879e..33b1c187dd0 100644 --- a/src/main/linear/mcp-issue-list.ts +++ b/src/main/linear/mcp-issue-list.ts @@ -2,6 +2,7 @@ import type { LinearMcpIssueListRequest, LinearMcpIssueListResult } from '../../shared/linear-agent-access' +import { runBoundedIntegrationSettledFanout } from '../integration-fanout' import { getClients, getStatus, type LinearClientForWorkspace } from './client' import { withLinearRead } from './issue-context-client' import { linearError } from './issue-context-errors' @@ -70,7 +71,7 @@ export async function listMcpIssues( nextSteps: ['Connect Linear from Orca settings, then retry the issue list.'] }) } - const { pages, failures } = await readIssueListWorkspaces( + const { pages, failures, truncated } = await readIssueListWorkspaces( entries, request, limit, @@ -78,7 +79,7 @@ export async function listMcpIssues( entryFailures ) const issues = pages.flatMap((page) => page.issues) - let hasMore = pages.some((page) => page.hasMore) + let hasMore = truncated || pages.some((page) => page.hasMore) issues.sort((left, right) => compareIssues(left, right, orderBy)) if (issues.length > limit) { @@ -96,7 +97,7 @@ export async function listMcpIssues( : {}), orderBy, workspaceId: request.workspaceId === 'all' ? 'all' : entries[0].workspace.id, - partial: failures.length > 0, + partial: truncated || failures.length > 0, workspaceErrors: failures.map(({ workspace, code, message }) => ({ workspace, code, @@ -125,31 +126,45 @@ async function readIssueListWorkspaces( limit: number, orderBy: 'createdAt' | 'updatedAt', initialFailures: WorkspaceReadFailure[] -): Promise<{ pages: WorkspaceIssuePage[]; failures: WorkspaceReadFailure[] }> { +): Promise<{ + pages: WorkspaceIssuePage[] + failures: WorkspaceReadFailure[] + truncated: boolean +}> { if (request.workspaceId !== 'all') { return { pages: [await readIssueListWorkspace(entries[0], request, limit, orderBy)], - failures: [] + failures: [], + truncated: false } } - const settled = await Promise.allSettled( - entries.map((entry) => readIssueListWorkspace(entry, request, limit, orderBy)) + const fanout = await runBoundedIntegrationSettledFanout( + entries, + (entry) => readIssueListWorkspace(entry, request, limit, orderBy), + (page) => page.issues ) const pages: WorkspaceIssuePage[] = [] const failures = [...initialFailures] - for (let index = 0; index < settled.length; index += 1) { - const result = settled[index] + for (let index = 0; index < fanout.results.length; index += 1) { + const result = fanout.results[index] if (result.status === 'fulfilled') { pages.push(result.value) continue } failures.push(workspaceFailure(entries[index].workspace, result.reason)) } - if (pages.length === 0 && failures.length === entries.length + initialFailures.length) { + if ( + pages.length === 0 && + failures.length === fanout.attemptedCount + initialFailures.length && + !fanout.truncated + ) { throw failures[0].error } - return { pages, failures } + if (fanout.truncated) { + console.warn('[linear] MCP issue list exceeded its aggregate result budget; truncating.') + } + return { pages, failures, truncated: fanout.truncated } } async function readIssueListWorkspace( @@ -170,12 +185,14 @@ async function readIssueListWorkspace( includeArchived: request.includeArchived ?? false }) const connection = raw.data?.issues + const rawIssues = connection?.nodes ?? [] + const issues = rawIssues.slice(0, limit) return { - issues: (connection?.nodes ?? []).map((issue) => ({ + issues: issues.map((issue) => ({ ...mapIssue(issue), workspace: { id: entry.workspace.id, name: entry.workspace.organizationName } })), - hasMore: connection?.pageInfo?.hasNextPage === true, + hasMore: rawIssues.length > issues.length || connection?.pageInfo?.hasNextPage === true, nextCursor: connection?.pageInfo?.endCursor ?? undefined } }) diff --git a/src/main/linear/projects.ts b/src/main/linear/projects.ts index 5945c3fc95b..d32289ed4fc 100644 --- a/src/main/linear/projects.ts +++ b/src/main/linear/projects.ts @@ -16,6 +16,15 @@ import { LINEAR_ISSUE_API_PAGE_SIZE_MAX, clampLinearIssueListLimit } from '../../shared/linear-issue-read-limits' +import { + boundedIntegrationErrorLog, + boundedIntegrationErrorMessage +} from '../integration-error-message' +import { runBoundedIntegrationFanout } from '../integration-fanout' +import { + INTEGRATION_PAGINATION_MAX_ITEMS, + IntegrationPaginationBudget +} from '../integration-pagination-budget' import { acquire, clearToken, @@ -24,6 +33,7 @@ import { release, type LinearClientForWorkspace } from './client' +import { LinearProjectRequestCoalescer } from './linear-project-request-coalescer' type LinearRawVariables = Record @@ -508,7 +518,7 @@ const CUSTOM_VIEW_PROJECTS_QUERY = ` } ` -const inFlight = new Map>() +const requestCoalescer = new LinearProjectRequestCoalescer() const LINEAR_PROJECT_API_PAGE_SIZE_MAX = 50 function clampLimit(limit = 20): number { @@ -516,17 +526,7 @@ function clampLimit(limit = 20): number { } function coalesce(key: string, load: () => Promise, force = false): Promise { - const existing = inFlight.get(key) as Promise | undefined - if (existing && !force) { - return existing - } - const promise = load().finally(() => { - if (inFlight.get(key) === promise) { - inFlight.delete(key) - } - }) - inFlight.set(key, promise) - return promise + return requestCoalescer.coalesce(key, load, force) } function normalizeConcreteWorkspaceId(workspaceId: unknown): LinearConcreteWorkspaceId { @@ -547,7 +547,7 @@ function workspaceError(entry: LinearClientForWorkspace, error: unknown): Linear } const record = error as { name?: string; message?: string; status?: number; response?: unknown } - const message = record.message || 'Linear request failed.' + const message = boundedIntegrationErrorMessage(record.message || 'Linear request failed.') const status = typeof record.status === 'number' ? record.status @@ -629,14 +629,15 @@ function mapProjectForWorkspace( priorityLabel: project.priorityLabel ?? null, lead: mapUser(project.lead), members: project.members?.nodes + ?.slice(0, 10) ?.map(mapUser) .filter((user): user is LinearProjectMemberSummary => !!user), - teams: project.teams?.nodes?.map((team) => ({ + teams: project.teams?.nodes?.slice(0, 10).map((team) => ({ id: team.id, name: team.name ?? '', key: team.key ?? undefined })), - labels: project.labels?.nodes?.map((label) => ({ + labels: project.labels?.nodes?.slice(0, 20).map((label) => ({ id: label.id, name: label.name ?? '', color: label.color ?? undefined @@ -661,7 +662,7 @@ function mapProjectDetailForWorkspace( ): LinearProjectDetail { return { ...mapProjectForWorkspace(entry, project), - milestones: project.projectMilestones?.nodes?.map((milestone) => ({ + milestones: project.projectMilestones?.nodes?.slice(0, 20).map((milestone) => ({ id: milestone.id, name: milestone.name ?? '', status: milestone.status ?? undefined, @@ -669,6 +670,7 @@ function mapProjectDetailForWorkspace( progress: milestone.progress ?? null })), resources: project.externalLinks?.nodes + ?.slice(0, 20) ?.filter((link) => link.url) .map((link) => ({ id: link.id, @@ -694,7 +696,7 @@ function mapIssueForWorkspace( entry: LinearClientForWorkspace, issue: LinearIssueNode ): LinearIssue { - const labelNodes = issue.labels?.nodes ?? [] + const labelNodes = (issue.labels?.nodes ?? []).slice(0, LINEAR_ISSUE_API_PAGE_SIZE_MAX) return { id: issue.id, identifier: issue.identifier, @@ -712,7 +714,9 @@ function mapIssueForWorkspace( key: issue.team?.key ?? '' }, labels: labelNodes.map((label) => label.name), - labelIds: issue.labelIds ?? labelNodes.map((label) => label.id), + labelIds: + issue.labelIds?.slice(0, INTEGRATION_PAGINATION_MAX_ITEMS) ?? + labelNodes.map((label) => label.id), assignee: mapUser(issue.assignee), estimate: issue.estimate ?? null, priority: issue.priority, @@ -777,6 +781,7 @@ async function readIssueConnectionPages( }) => Promise | null | undefined> ): Promise> { const items: LinearIssue[] = [] + const budget = new IntegrationPaginationBudget() let after: string | undefined let hasMore = false @@ -785,12 +790,22 @@ async function readIssueConnectionPages( // Orca reads must follow cursors to show more than one backend page. const first = Math.min(LINEAR_ISSUE_API_PAGE_SIZE_MAX, limit - items.length) const connection = await loadConnection(after ? { first, after } : { first }) - const nodes = connection?.nodes ?? [] - items.push(...nodes.map((issue) => mapIssueForWorkspace(entry, issue))) - hasMore = Boolean(connection?.pageInfo?.hasNextPage) + const pageNodes = connection?.nodes ?? [] + const nodes = pageNodes.slice(0, first) + const pageItems = nodes.map((issue) => mapIssueForWorkspace(entry, issue)) + hasMore = pageNodes.length > nodes.length || Boolean(connection?.pageInfo?.hasNextPage) + if (!budget.admitPage(pageItems)) { + console.warn('[linear] Project issue list exceeded its retained result budget; truncating.') + return { items, hasMore: true } + } + items.push(...pageItems) const nextCursor = connection?.pageInfo?.endCursor ?? undefined - if (!hasMore || !nextCursor || nextCursor === after || nodes.length === 0) { + if (!hasMore || !nextCursor || nextCursor === after || pageNodes.length === 0) { + break + } + if (!budget.canRequestPage) { + console.warn('[linear] Project issue list reached its retained result budget; truncating.') break } after = nextCursor @@ -813,8 +828,9 @@ async function readCollection( return { items: [] } } - const results = await Promise.all( - entries.map(async (entry) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { return await load(entry) @@ -822,7 +838,7 @@ async function readCollection( if (isAuthError(error)) { clearToken(entry.workspace.id) } else { - console.warn('[linear] project/view read failed:', error) + console.warn('[linear] project/view read failed:', boundedIntegrationErrorLog(error)) } if (shouldFailWholeRequest(workspaceId)) { throw error @@ -831,15 +847,22 @@ async function readCollection( } finally { release() } - }) + }, + (result) => [...result.items, ...(result.errors ?? [])] ) + if (fanout.truncated) { + console.warn( + '[linear] Cross-workspace project metadata exceeded its aggregate result budget; truncating.' + ) + } + const results = fanout.results return { items: results.flatMap((result) => result.items), errors: results.flatMap((result) => result.errors ?? []).length ? results.flatMap((result) => result.errors ?? []) : undefined, - hasMore: results.some((result) => result.hasMore) + hasMore: fanout.truncated || results.some((result) => result.hasMore) } }, force @@ -875,9 +898,11 @@ export async function listProjects( LinearRawVariables >(trimmed ? SEARCH_PROJECTS_QUERY : PROJECTS_QUERY, variables) const connection = trimmed ? result.data?.searchProjects : result.data?.projects + const rawProjects = connection?.nodes ?? [] + const projects = rawProjects.slice(0, first) return { - items: (connection?.nodes ?? []).map((project) => mapProjectForWorkspace(entry, project)), - hasMore: !!connection?.pageInfo?.hasNextPage + items: projects.map((project) => mapProjectForWorkspace(entry, project)), + hasMore: rawProjects.length > projects.length || !!connection?.pageInfo?.hasNextPage } }, force @@ -907,6 +932,7 @@ export async function listProjectsByExactName( await acquire() try { const matches: LinearProjectSummary[] = [] + const budget = new IntegrationPaginationBudget() let after: string | undefined while (true) { const result = await entry.client.client.rawRequest< @@ -918,17 +944,31 @@ export async function listProjectsByExactName( ...(after ? { after } : {}) }) const connection = result.data?.searchProjects - for (const project of connection?.nodes ?? []) { + const rawNodes = connection?.nodes ?? [] + const nodes = rawNodes.slice(0, LINEAR_PROJECT_API_PAGE_SIZE_MAX) + if (!budget.admitPage(nodes)) { + console.warn('[linear] Project search exceeded its retained result budget; truncating.') + break + } + for (const project of nodes) { if (project.name.trim().toLowerCase() === normalized) { matches.push(mapProjectForWorkspace(entry, project)) } } + if (rawNodes.length > nodes.length) { + console.warn('[linear] Project search returned more rows than requested; truncating.') + break + } const nextCursor = connection?.pageInfo?.endCursor ?? undefined + if (connection?.pageInfo?.hasNextPage === true && !budget.canRequestPage) { + console.warn('[linear] Project search reached its retained result budget; truncating.') + break + } if ( connection?.pageInfo?.hasNextPage !== true || !nextCursor || nextCursor === after || - (connection.nodes ?? []).length === 0 + nodes.length === 0 ) { break } @@ -1015,7 +1055,7 @@ export async function createProject( clearToken(entry.workspace.id) throw error } - const message = error instanceof Error ? error.message : String(error) + const message = boundedIntegrationErrorMessage(error) return { ok: false, error: message } } finally { release() @@ -1073,6 +1113,7 @@ export async function listProjectTeams( return [] } const teams: NonNullable = [] + const budget = new IntegrationPaginationBudget() let after: string | undefined await acquire() try { @@ -1090,15 +1131,30 @@ export async function listProjectTeams( throw new Error('Project was not found') } const connection = project.teams - const nodes = connection?.nodes ?? [] - teams.push( - ...nodes.map((team) => ({ + const rawNodes = connection?.nodes ?? [] + const nodes = rawNodes.slice(0, LINEAR_PROJECT_API_PAGE_SIZE_MAX) + if (!budget.admitPage(nodes)) { + console.warn( + '[linear] Project teams exceeded their retained result budget; truncating.' + ) + break + } + for (const team of nodes) { + teams.push({ id: team.id, name: team.name ?? '', key: team.key ?? undefined - })) - ) + }) + } + if (rawNodes.length > nodes.length) { + console.warn('[linear] Project teams returned more rows than requested; truncating.') + break + } const nextCursor = connection?.pageInfo?.endCursor ?? undefined + if (connection?.pageInfo?.hasNextPage === true && !budget.canRequestPage) { + console.warn('[linear] Project teams reached their retained result budget; truncating.') + break + } if ( !connection?.pageInfo?.hasNextPage || !nextCursor || @@ -1141,11 +1197,13 @@ export async function listCustomViews( LinearRawVariables >(CUSTOM_VIEWS_QUERY, { first, filter, orderBy: 'updatedAt' }) const connection = result.data?.customViews + const rawViews = connection?.nodes ?? [] + const views = rawViews.slice(0, first) return { - items: (connection?.nodes ?? []) + items: views .map((view) => mapCustomViewForWorkspace(entry, view)) .filter((view): view is LinearCustomViewSummary => !!view && view.model === model), - hasMore: !!connection?.pageInfo?.hasNextPage + hasMore: rawViews.length > views.length || !!connection?.pageInfo?.hasNextPage } }, force @@ -1251,9 +1309,11 @@ export async function listCustomViewProjects( throw new Error('Custom view does not contain projects') } const connection = view?.projects + const rawProjects = connection?.nodes ?? [] + const projects = rawProjects.slice(0, first) return { - items: (connection?.nodes ?? []).map((project) => mapProjectForWorkspace(entry, project)), - hasMore: !!connection?.pageInfo?.hasNextPage + items: projects.map((project) => mapProjectForWorkspace(entry, project)), + hasMore: rawProjects.length > projects.length || !!connection?.pageInfo?.hasNextPage } }, force diff --git a/src/main/linear/teams.ts b/src/main/linear/teams.ts index e06697e8d46..c587a1a97d7 100644 --- a/src/main/linear/teams.ts +++ b/src/main/linear/teams.ts @@ -6,6 +6,11 @@ import type { LinearWorkspaceError, LinearWorkspaceSelection } from '../../shared/types' +import { + boundedIntegrationErrorLog, + boundedIntegrationErrorMessage +} from '../integration-error-message' +import { runBoundedIntegrationFanout } from '../integration-fanout' import { acquire, release, getClients, isAuthError, clearToken } from './client' import { fetchAllTeamLabels, @@ -22,11 +27,12 @@ export async function listTeams( return [] } - const results = await Promise.all( - entries.map(async (entry) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { - return fetchAllTeamsForWorkspace(entry) + return await fetchAllTeamsForWorkspace(entry) } catch (error) { if (isAuthError(error)) { clearToken(entry.workspace.id) @@ -34,15 +40,21 @@ export async function listTeams( throw error } } else { - console.warn('[linear] listTeams failed:', error) + console.warn('[linear] listTeams failed:', boundedIntegrationErrorLog(error)) } return [] } finally { release() } - }) + }, + (teams) => teams ) - return results.flat().sort((a, b) => a.name.localeCompare(b.name)) + if (fanout.truncated) { + console.warn( + '[linear] Cross-workspace teams exceeded their aggregate result budget; truncating.' + ) + } + return fanout.results.flat().sort((a, b) => a.name.localeCompare(b.name)) } export async function listTeamsOrThrow( @@ -53,8 +65,9 @@ export async function listTeamsOrThrow( return [] } - const results = await Promise.all( - entries.map(async (entry) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { return await fetchAllTeamsForWorkspace(entry) @@ -66,9 +79,15 @@ export async function listTeamsOrThrow( } finally { release() } - }) + }, + (teams) => teams ) - return results.flat().sort((a, b) => a.name.localeCompare(b.name)) + if (fanout.truncated) { + console.warn( + '[linear] Cross-workspace teams exceeded their aggregate result budget; truncating.' + ) + } + return fanout.results.flat().sort((a, b) => a.name.localeCompare(b.name)) } export async function listTeamsForAgent( @@ -79,8 +98,9 @@ export async function listTeamsForAgent( return { teams: [], errors: [] } } - const results = await Promise.all( - entries.map(async (entry) => { + const fanout = await runBoundedIntegrationFanout( + entries, + async (entry) => { await acquire() try { return { teams: await fetchAllTeamsForWorkspace(entry), error: null } @@ -94,17 +114,23 @@ export async function listTeamsForAgent( workspaceId: entry.workspace.id, workspaceName: entry.workspace.organizationName, type: isAuthError(error) ? 'auth' : 'unknown', - message: error instanceof Error ? error.message : String(error) + message: boundedIntegrationErrorMessage(error) } satisfies LinearWorkspaceError } } finally { release() } - }) + }, + (result) => [...result.teams, ...(result.error ? [result.error] : [])] ) + if (fanout.truncated) { + console.warn('[linear] Agent team list exceeded its aggregate result budget; truncating.') + } return { - teams: results.flatMap((result) => result.teams).sort((a, b) => a.name.localeCompare(b.name)), - errors: results.flatMap((result) => (result.error ? [result.error] : [])) + teams: fanout.results + .flatMap((result) => result.teams) + .sort((a, b) => a.name.localeCompare(b.name)), + errors: fanout.results.flatMap((result) => (result.error ? [result.error] : [])) } } @@ -126,7 +152,7 @@ export async function getTeamStates( clearToken(entry.workspace.id) throw error } - console.warn('[linear] getTeamStates failed:', error) + console.warn('[linear] getTeamStates failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -174,7 +200,7 @@ export async function getTeamLabels( clearToken(entry.workspace.id) throw error } - console.warn('[linear] getTeamLabels failed:', error) + console.warn('[linear] getTeamLabels failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() @@ -222,7 +248,7 @@ export async function getTeamMembers( clearToken(entry.workspace.id) throw error } - console.warn('[linear] getTeamMembers failed:', error) + console.warn('[linear] getTeamMembers failed:', boundedIntegrationErrorLog(error)) return [] } finally { release() diff --git a/src/main/local-downloaded-folder-promotion-budget.test.ts b/src/main/local-downloaded-folder-promotion-budget.test.ts new file mode 100644 index 00000000000..b0f11ec1645 --- /dev/null +++ b/src/main/local-downloaded-folder-promotion-budget.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { + LocalDownloadedFolderPromotionBudget, + type LocalDownloadedFolderPromotionLimits +} from './local-downloaded-folder-promotion-budget' + +const limits: LocalDownloadedFolderPromotionLimits = { + maximumEntries: 2, + maximumDepth: 2, + maximumPathBytes: 4, + maximumRetainedPathBytes: 8 +} + +describe('LocalDownloadedFolderPromotionBudget', () => { + it('accepts every exact boundary', () => { + const budget = new LocalDownloadedFolderPromotionBudget(limits) + + budget.recordEntry('12', '34', 2) + budget.recordEntry('56', '78', 2) + }) + + it.each([ + [ + 'entries', + (budget: LocalDownloadedFolderPromotionBudget) => { + budget.recordEntry('', '', 0) + budget.recordEntry('', '', 0) + budget.recordEntry('', '', 0) + } + ], + ['depth', (budget: LocalDownloadedFolderPromotionBudget) => budget.recordEntry('', '', 3)], + ['path', (budget: LocalDownloadedFolderPromotionBudget) => budget.recordEntry('12345', '', 0)] + ] as const)('rejects one unit beyond the %s boundary', (reason, exceed) => { + expect(() => exceed(new LocalDownloadedFolderPromotionBudget(limits))).toThrow( + expect.objectContaining({ reason }) + ) + }) + + it('rejects one unit beyond the retained-path boundary', () => { + const budget = new LocalDownloadedFolderPromotionBudget({ + ...limits, + maximumEntries: 3 + }) + budget.recordEntry('12', '34', 0) + budget.recordEntry('56', '78', 0) + + expect(() => budget.recordEntry('9', '', 0)).toThrow( + expect.objectContaining({ reason: 'paths' }) + ) + }) +}) diff --git a/src/main/local-downloaded-folder-promotion-budget.ts b/src/main/local-downloaded-folder-promotion-budget.ts new file mode 100644 index 00000000000..e7821d31313 --- /dev/null +++ b/src/main/local-downloaded-folder-promotion-budget.ts @@ -0,0 +1,49 @@ +export type LocalDownloadedFolderPromotionLimits = { + maximumEntries: number + maximumDepth: number + maximumPathBytes: number + maximumRetainedPathBytes: number +} + +export const LOCAL_DOWNLOADED_FOLDER_PROMOTION_LIMITS: LocalDownloadedFolderPromotionLimits = { + maximumEntries: 50_000, + maximumDepth: 64, + maximumPathBytes: 64 * 1024, + maximumRetainedPathBytes: 16 * 1024 * 1024 +} + +export class LocalDownloadedFolderPromotionCapacityError extends Error { + constructor(readonly reason: 'entries' | 'depth' | 'path' | 'paths') { + super(`Downloaded folder promotion exceeds the ${reason} limit`) + this.name = 'LocalDownloadedFolderPromotionCapacityError' + } +} + +export class LocalDownloadedFolderPromotionBudget { + private entries = 0 + private retainedPathBytes = 0 + + constructor(private readonly limits = LOCAL_DOWNLOADED_FOLDER_PROMOTION_LIMITS) {} + + recordEntry(sourcePath: string, destinationPath: string, depth: number): void { + if (depth > this.limits.maximumDepth) { + throw new LocalDownloadedFolderPromotionCapacityError('depth') + } + this.entries += 1 + if (this.entries > this.limits.maximumEntries) { + throw new LocalDownloadedFolderPromotionCapacityError('entries') + } + const sourceBytes = Buffer.byteLength(sourcePath, 'utf8') + const destinationBytes = Buffer.byteLength(destinationPath, 'utf8') + if ( + sourceBytes > this.limits.maximumPathBytes || + destinationBytes > this.limits.maximumPathBytes + ) { + throw new LocalDownloadedFolderPromotionCapacityError('path') + } + this.retainedPathBytes += sourceBytes + destinationBytes + if (this.retainedPathBytes > this.limits.maximumRetainedPathBytes) { + throw new LocalDownloadedFolderPromotionCapacityError('paths') + } + } +} diff --git a/src/main/local-downloaded-folder-promotion-traversal.ts b/src/main/local-downloaded-folder-promotion-traversal.ts new file mode 100644 index 00000000000..a7681c225e2 --- /dev/null +++ b/src/main/local-downloaded-folder-promotion-traversal.ts @@ -0,0 +1,51 @@ +import type { Dirent } from 'node:fs' +import { opendir } from 'node:fs/promises' +import { join } from 'node:path' +import { LocalDownloadedFolderPromotionBudget } from './local-downloaded-folder-promotion-budget' + +export async function readPromotionDirectoryEntries( + sourcePath: string, + destinationPath: string, + budget: LocalDownloadedFolderPromotionBudget, + depth: number +): Promise { + const entries: Dirent[] = [] + const directory = await opendir(sourcePath) + try { + for await (const entry of directory) { + budget.recordEntry(join(sourcePath, entry.name), join(destinationPath, entry.name), depth + 1) + entries.push(entry) + } + } finally { + await directory.close().catch(() => undefined) + } + return entries.sort((a, b) => a.name.localeCompare(b.name)) +} + +export async function assertPromotionTreeWithinCapacity( + sourceRoot: string, + destinationRoot: string, + signal?: AbortSignal +): Promise { + const budget = new LocalDownloadedFolderPromotionBudget() + const pending = [{ sourcePath: sourceRoot, destinationPath: destinationRoot, depth: 0 }] + while (pending.length > 0) { + signal?.throwIfAborted() + const current = pending.pop()! + const directory = await opendir(current.sourcePath) + try { + for await (const entry of directory) { + signal?.throwIfAborted() + const sourcePath = join(current.sourcePath, entry.name) + const destinationPath = join(current.destinationPath, entry.name) + const depth = current.depth + 1 + budget.recordEntry(sourcePath, destinationPath, depth) + if (entry.isDirectory()) { + pending.push({ sourcePath, destinationPath, depth }) + } + } + } finally { + await directory.close().catch(() => undefined) + } + } +} diff --git a/src/main/local-downloaded-folder-promotion.test.ts b/src/main/local-downloaded-folder-promotion.test.ts index 57d82edd6e2..47168f19408 100644 --- a/src/main/local-downloaded-folder-promotion.test.ts +++ b/src/main/local-downloaded-folder-promotion.test.ts @@ -19,6 +19,7 @@ import { copyLocalDownloadedFileNoClobber, publishLocalDownloadedFileNoClobber } from './local-downloaded-folder-promotion' +import { LOCAL_DOWNLOADED_FOLDER_PROMOTION_LIMITS } from './local-downloaded-folder-promotion-budget' describe('promoteLocalDownloadedFolder', () => { const roots: string[] = [] @@ -115,6 +116,30 @@ describe('promoteLocalDownloadedFolder', () => { await expect(readdir(destinationPath)).rejects.toMatchObject({ code: 'ENOENT' }) }) + it('rejects excessive depth before claiming the destination', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-folder-promotion-')) + roots.push(root) + const tempPath = join(root, '.transfer.download') + const destinationPath = join(root, 'downloaded') + await mkdir(tempPath) + let nested = tempPath + for ( + let depth = 0; + depth <= LOCAL_DOWNLOADED_FOLDER_PROMOTION_LIMITS.maximumDepth; + depth += 1 + ) { + nested = join(nested, 'd') + await mkdir(nested) + } + + await expect(promoteLocalDownloadedFolder(tempPath, destinationPath)).rejects.toMatchObject({ + reason: 'depth' + }) + + await expect(lstat(destinationPath)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(lstat(nested)).resolves.toMatchObject({ isDirectory: expect.any(Function) }) + }) + it('rolls back unchanged entries after a mid-publication failure', async () => { const { tempPath, destinationPath } = await createPaths() await writeFile(join(tempPath, 'a-first.txt'), 'remote') diff --git a/src/main/local-downloaded-folder-promotion.ts b/src/main/local-downloaded-folder-promotion.ts index f5ba45f7a5c..9c962d57e75 100644 --- a/src/main/local-downloaded-folder-promotion.ts +++ b/src/main/local-downloaded-folder-promotion.ts @@ -1,8 +1,12 @@ -import { fstatSync, lstatSync } from 'node:fs' -import { link, lstat, mkdir, open, readdir, rm, rmdir, unlink } from 'node:fs/promises' -import type { BigIntStats } from 'node:fs' +import { fstatSync, lstatSync, type BigIntStats } from 'node:fs' +import { link, lstat, mkdir, open, rm, rmdir, unlink } from 'node:fs/promises' import type { FileHandle } from 'node:fs/promises' import { join } from 'node:path' +import { LocalDownloadedFolderPromotionBudget } from './local-downloaded-folder-promotion-budget' +import { + assertPromotionTreeWithinCapacity, + readPromotionDirectoryEntries +} from './local-downloaded-folder-promotion-traversal' const LOCAL_COPY_CHUNK_BYTES = 1024 * 1024 @@ -210,6 +214,8 @@ async function publishDirectoryNoClobber( sourcePath: string, destinationPath: string, publishedEntries: PublishedEntry[], + budget: LocalDownloadedFolderPromotionBudget, + depth: number, signal?: AbortSignal ): Promise { await mkdir(destinationPath, { recursive: false }) @@ -217,9 +223,7 @@ async function publishDirectoryNoClobber( // claim so no async gap exists before rollback ownership is registered. const destinationStats = lstatSync(destinationPath, { bigint: true }) publishedEntries.push(publishedEntryFromStats('directory', destinationPath, destinationStats)) - const entries = (await readdir(sourcePath, { withFileTypes: true })).toSorted((a, b) => - a.name.localeCompare(b.name) - ) + const entries = await readPromotionDirectoryEntries(sourcePath, destinationPath, budget, depth) for (const entry of entries) { signal?.throwIfAborted() const sourceEntryPath = join(sourcePath, entry.name) @@ -229,6 +233,8 @@ async function publishDirectoryNoClobber( sourceEntryPath, destinationEntryPath, publishedEntries, + budget, + depth + 1, signal ) } else if (entry.isFile()) { @@ -251,12 +257,21 @@ export async function promoteLocalDownloadedFolder( destinationPath: string, signal?: AbortSignal ): Promise { + signal?.throwIfAborted() + await assertPromotionTreeWithinCapacity(tempPath, destinationPath, signal) signal?.throwIfAborted() const publishedEntries: PublishedEntry[] = [] try { // Why: Node has no portable atomic no-replace directory rename. Claiming // the destination first preserves no-clobber while promotion stays local. - await publishDirectoryNoClobber(tempPath, destinationPath, publishedEntries, signal) + await publishDirectoryNoClobber( + tempPath, + destinationPath, + publishedEntries, + new LocalDownloadedFolderPromotionBudget(), + 0, + signal + ) } catch (error) { await rollbackPublishedEntries(publishedEntries) if (isEEXIST(error)) { diff --git a/src/main/local-worktree-filesystem.test.ts b/src/main/local-worktree-filesystem.test.ts index 496d88b0eae..a8cf9eec6ae 100644 --- a/src/main/local-worktree-filesystem.test.ts +++ b/src/main/local-worktree-filesystem.test.ts @@ -1,9 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { execFileMock, lstatMock, readFileMock, rmMock } = vi.hoisted(() => ({ +const { execFileMock, lstatMock, readNodeFileWithinLimitMock, rmMock } = vi.hoisted(() => ({ execFileMock: vi.fn(), lstatMock: vi.fn(), - readFileMock: vi.fn(), + readNodeFileWithinLimitMock: vi.fn(), rmMock: vi.fn() })) @@ -13,10 +13,13 @@ vi.mock('node:child_process', () => ({ vi.mock('node:fs/promises', () => ({ lstat: lstatMock, - readFile: readFileMock, rm: rmMock })) +vi.mock('../shared/node-bounded-file-reader', () => ({ + readNodeFileWithinLimit: readNodeFileWithinLimitMock +})) + import { getLocalWorktreePathAccess, removeLocalWorktreePath, @@ -49,7 +52,7 @@ describe('local worktree filesystem runtime access', () => { beforeEach(() => { execFileMock.mockReset() lstatMock.mockReset() - readFileMock.mockReset() + readNodeFileWithinLimitMock.mockReset() rmMock.mockReset() completeExecFile() }) @@ -61,7 +64,9 @@ describe('local worktree filesystem runtime access', () => { it('uses host filesystem operations when no WSL distro is selected', async () => { lstatMock.mockResolvedValue({ type: 'file' }) - readFileMock.mockResolvedValue('gitdir: ../.git/worktrees/feature') + readNodeFileWithinLimitMock.mockResolvedValue({ + buffer: Buffer.from('gitdir: ../.git/worktrees/feature') + }) const access = getLocalWorktreePathAccess() await access.statPath('C:\\repo\\.git') @@ -69,7 +74,7 @@ describe('local worktree filesystem runtime access', () => { await removeLocalWorktreePath('C:\\repo\\feature') expect(lstatMock).toHaveBeenCalledWith('C:\\repo\\.git') - expect(readFileMock).toHaveBeenCalledWith('C:\\repo\\.git', 'utf8') + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith('C:\\repo\\.git', 64 * 1024) expect(rmMock).toHaveBeenCalledWith( toHostRemovalPath('C:\\repo\\feature'), expect.objectContaining({ @@ -165,6 +170,8 @@ describe('local worktree filesystem runtime access', () => { expect.objectContaining({ encoding: 'utf8' }), expect.any(Function) ) + const readArgs = execFileMock.mock.calls[1]?.[1] as string[] + expect(readArgs.at(-1)).toContain('head -c 65537 --') const removeArgs = execFileMock.mock.calls[2]?.[1] as string[] expect(removeArgs.at(-1)).toContain('rm -rf --') expect(removeArgs.at(-1)).toContain( @@ -174,6 +181,17 @@ describe('local worktree filesystem runtime access', () => { }) }) + it('rejects oversized WSL Git pointer output', async () => { + await withPlatform('win32', async () => { + completeExecFile('x'.repeat(64 * 1024 + 1)) + const access = getLocalWorktreePathAccess({ wslDistro: 'Ubuntu' }) + + await expect(access.readPath('/home/me/repo/.git')).rejects.toThrow( + 'Worktree Git pointer exceeds the safe read limit' + ) + }) + }) + it('reports missing WSL stat targets with an ENOENT-shaped error', async () => { await withPlatform('win32', async () => { failExecFile(Object.assign(new Error('missing'), { code: 2 })) diff --git a/src/main/local-worktree-filesystem.ts b/src/main/local-worktree-filesystem.ts index aafb5ea93fa..f82929eec2f 100644 --- a/src/main/local-worktree-filesystem.ts +++ b/src/main/local-worktree-filesystem.ts @@ -1,6 +1,6 @@ import { execFile } from 'node:child_process' import type { RmOptions } from 'node:fs' -import { lstat, readFile, rm } from 'node:fs/promises' +import { lstat, rm } from 'node:fs/promises' import { win32 } from 'node:path' import { setTimeout as delay } from 'node:timers/promises' import { @@ -9,7 +9,12 @@ import { quotePosixShell } from '../shared/wsl-login-shell-command' import { toLinuxPath } from './wsl' -import type { ReadPath, StatPath } from './worktree-orphan-gitdir-proof' +import { + MAX_WORKTREE_GIT_POINTER_BYTES, + type ReadPath, + type StatPath +} from './worktree-orphan-gitdir-proof' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' export type LocalWorktreeFilesystemOptions = { wslDistro?: string @@ -37,13 +42,13 @@ function shouldUseWslFilesystem(options: LocalWorktreeFilesystemOptions): boolea function execFileText( file: string, args: string[], - options: { timeout: number } + options: { maxBuffer?: number; timeout: number } ): Promise { return new Promise((resolve, reject) => { execFile( file, args, - { encoding: 'utf8', timeout: options.timeout }, + { encoding: 'utf8', timeout: options.timeout, maxBuffer: options.maxBuffer }, (error, stdout, stderr) => { if (error) { reject(error) @@ -97,7 +102,10 @@ export function getLocalWorktreePathAccess( if (!shouldUseWslFilesystem(options) || !distro) { return { statPath: lstat, - readPath: (path) => readFile(path, 'utf8') + readPath: async (path) => + (await readNodeFileWithinLimit(path, MAX_WORKTREE_GIT_POINTER_BYTES)).buffer.toString( + 'utf8' + ) } } @@ -120,7 +128,26 @@ export function getLocalWorktreePathAccess( }, readPath: async (path) => { const target = quotePosixShell(toLinuxPath(path)) - const { stdout } = await runWslLoginShellCommand(distro, `cat -- ${target}`) + const { stdout } = await execFileText( + 'wsl.exe', + [ + '-d', + distro, + '--', + 'sh', + '-lc', + escapeWslShCommandForWindows( + buildWslLoginShellCommand(`head -c ${MAX_WORKTREE_GIT_POINTER_BYTES + 1} -- ${target}`) + ) + ], + { + timeout: WSL_FILE_OPERATION_TIMEOUT_MS, + maxBuffer: (MAX_WORKTREE_GIT_POINTER_BYTES + 1) * 2 + } + ) + if (Buffer.byteLength(stdout, 'utf8') > MAX_WORKTREE_GIT_POINTER_BYTES) { + throw new Error('Worktree Git pointer exceeds the safe read limit') + } return stdout } } diff --git a/src/main/memory/collector.test.ts b/src/main/memory/collector.test.ts index 63f8026b870..e8992b3acc9 100644 --- a/src/main/memory/collector.test.ts +++ b/src/main/memory/collector.test.ts @@ -653,6 +653,53 @@ describe('collectMemorySnapshot', () => { expectProcessSweepCount(2) }) + it('bounds retained worktree histories and recovers an evicted active history', async () => { + mockPsResponse('') + const { + collectMemorySnapshot, + getMemoryHistoryWorktreeCountForTests, + MEMORY_HISTORY_MAX_WORKTREES + } = await loadCollector() + const registrations = Array.from({ length: MEMORY_HISTORY_MAX_WORKTREES }, (_, index) => ({ + ptyId: `pty-${index}`, + worktreeId: `repo::/worktree-${index}`, + sessionId: `session-${index}`, + paneKey: `pane-${index}`, + pid: null + })) + listRegisteredPtysMock.mockReturnValue(registrations) + + await collectMemorySnapshot(emptyStore) + const atBoundary = await collectMemorySnapshot(emptyStore) + + expect(atBoundary.worktrees.every((worktree) => worktree.history.length === 2)).toBe(true) + expect(getMemoryHistoryWorktreeCountForTests()).toBe(MEMORY_HISTORY_MAX_WORKTREES) + + listRegisteredPtysMock.mockReturnValue([ + ...registrations, + { + ptyId: 'pty-overflow', + worktreeId: 'repo::/worktree-overflow', + sessionId: 'session-overflow', + paneKey: 'pane-overflow', + pid: null + } + ]) + const overflow = await collectMemorySnapshot(emptyStore) + + expect(overflow.worktrees).toHaveLength(MEMORY_HISTORY_MAX_WORKTREES + 1) + expect(overflow.worktrees[0].history).toEqual([0]) + expect(getMemoryHistoryWorktreeCountForTests()).toBe(MEMORY_HISTORY_MAX_WORKTREES) + + listRegisteredPtysMock.mockReturnValue([registrations[0]]) + const reactivated = await collectMemorySnapshot(emptyStore) + const retained = await collectMemorySnapshot(emptyStore) + + expect(reactivated.worktrees[0].history).toEqual([0]) + expect(retained.worktrees[0].history).toEqual([0, 0]) + expect(getMemoryHistoryWorktreeCountForTests()).toBe(MEMORY_HISTORY_MAX_WORKTREES) + }) + it('uses host process RSS for Electron app metrics when available', async () => { mockPsResponse(['10 1 1.5 111', '20 10 2.5 222', '30 10 3.5 333'].join('\n')) appMetricsMock.mockReturnValue([ diff --git a/src/main/memory/collector.ts b/src/main/memory/collector.ts index 171c07f175b..527af81e87b 100644 --- a/src/main/memory/collector.ts +++ b/src/main/memory/collector.ts @@ -126,6 +126,7 @@ function emptySnapshot(): MemorySnapshot { const APP_HISTORY_KEY = '__app__' const HISTORY_CAPACITY = 60 const HISTORY_STALE_MS = 10 * 60 * 1000 +export const MEMORY_HISTORY_MAX_WORKTREES = 1024 type HistoryRing = { samples: number[] @@ -138,18 +139,39 @@ function pushHistorySample(key: string, memoryBytes: number, now: number): void let ring = historyByKey.get(key) if (!ring) { ring = { samples: [], touchedAt: now } - historyByKey.set(key, ring) } ring.samples.push(memoryBytes) if (ring.samples.length > HISTORY_CAPACITY) { ring.samples.shift() } ring.touchedAt = now + // Why: worktree ids can churn faster than the stale TTL in a long-lived + // main process; recency ordering gives the history cache a hard ceiling. + historyByKey.delete(key) + historyByKey.set(key, ring) + trimWorktreeHistory() } -function readHistory(key: string): number[] { +function trimWorktreeHistory(): void { + const appEntryCount = historyByKey.has(APP_HISTORY_KEY) ? 1 : 0 + while (historyByKey.size - appEntryCount > MEMORY_HISTORY_MAX_WORKTREES) { + let oldestWorktreeKey: string | undefined + for (const key of historyByKey.keys()) { + if (key !== APP_HISTORY_KEY) { + oldestWorktreeKey = key + break + } + } + if (oldestWorktreeKey === undefined) { + break + } + historyByKey.delete(oldestWorktreeKey) + } +} + +function readHistory(key: string, currentSample?: number): number[] { const ring = historyByKey.get(key) - return ring ? [...ring.samples] : [] + return ring ? [...ring.samples] : currentSample === undefined ? [] : [currentSample] } function sweepStaleHistory(now: number): void { @@ -160,6 +182,11 @@ function sweepStaleHistory(now: number): void { } } +/** @internal — test-only */ +export function getMemoryHistoryWorktreeCountForTests(): number { + return historyByKey.size - (historyByKey.has(APP_HISTORY_KEY) ? 1 : 0) +} + // ─── Host process enumeration ─────────────────────────────────────── async function enumerateProcesses(): Promise { @@ -433,7 +460,9 @@ async function runSnapshot(store: MemorySnapshotStore): Promise const worktrees: WorktreeMemory[] = bucketList.map((b) => ({ ...b, - history: readHistory(b.worktreeId) + // Why: an over-cap active worktree still gets its current point even when + // an older retained ring had to be evicted during this same snapshot. + history: readHistory(b.worktreeId, b.memory) })) let sessionCpuTotal = 0 diff --git a/src/main/mimo/hook-service.test.ts b/src/main/mimo/hook-service.test.ts index e30e415ca97..f175debdffc 100644 --- a/src/main/mimo/hook-service.test.ts +++ b/src/main/mimo/hook-service.test.ts @@ -1,5 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -75,4 +84,47 @@ describe('MimoCodeHookService buildPtyEnv', () => { readFileSync(join(overlayHome, 'config', 'plugins', 'orca-mimocode-status.js'), 'utf8') ).toContain('/hook/mimo-code') }) + + it.skipIf(process.platform === 'win32')( + 'keeps a symlinked user plugins directory isolated from Orca writes', + () => { + const sourcePlugins = join(mimocodeHome, 'config', 'plugins') + const realPlugins = join(mimocodeHome, 'real-plugins') + rmSync(sourcePlugins, { recursive: true, force: true }) + mkdirSync(realPlugins) + writeFileSync(join(realPlugins, 'user-plugin.js'), 'USER PLUGIN') + symlinkSync(realPlugins, sourcePlugins, 'dir') + + const env = new MimoCodeHookService().buildPtyEnv('pty-1', mimocodeHome) + const overlayPlugins = join(env.MIMOCODE_HOME!, 'config', 'plugins') + + expect(existsSync(join(realPlugins, 'orca-mimocode-status.js'))).toBe(false) + expect(lstatSync(overlayPlugins).isSymbolicLink()).toBe(false) + expect(readFileSync(join(overlayPlugins, 'user-plugin.js'), 'utf8')).toBe('USER PLUGIN') + expect(readFileSync(join(overlayPlugins, 'orca-mimocode-status.js'), 'utf8')).toContain( + '/hook/mimo-code' + ) + } + ) + + it('falls back to the original home when the config plan exceeds capacity', async () => { + const mirroring = await import('../pty/config-overlay-mirroring') + const planSpy = vi.spyOn(mirroring, 'createConfigOverlayPlan').mockImplementation(() => { + throw new mirroring.ConfigOverlayCapacityError('entries', 4_097, 4_096) + }) + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + expect(new MimoCodeHookService().buildPtyEnv('pty-1', mimocodeHome)).toEqual({ + MIMOCODE_HOME: mimocodeHome + }) + expect(warnSpy).toHaveBeenCalledWith( + '[mimocode-hooks] config overlay exceeded its memory limit; using the original MiMo home without Orca status integration' + ) + expect(readFileSync(join(mimocodeHome, 'config', 'mimocode.json'), 'utf8')).toBe( + '{"theme":"dark"}' + ) + } finally { + planSpy.mockRestore() + } + }) }) diff --git a/src/main/mimo/hook-service.ts b/src/main/mimo/hook-service.ts index 3f879d93063..7d5420ba2de 100644 --- a/src/main/mimo/hook-service.ts +++ b/src/main/mimo/hook-service.ts @@ -1,9 +1,14 @@ import { app } from 'electron' import { join } from 'node:path' -import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { getOpenCodeFamilyPluginSource } from '../opencode/hook-service' -import { mirrorEntry, safeRemoveTree } from '../pty/overlay-mirror' +import { + ConfigOverlayCapacityError, + applyConfigOverlayPlan, + createConfigOverlayPlan +} from '../pty/config-overlay-mirroring' +import { safeRemoveTree } from '../pty/overlay-mirror' const ORCA_MIMOCODE_PLUGIN_FILE = 'orca-mimocode-status.js' const MIMOCODE_HOOKS_DIR = 'mimocode-hooks' @@ -24,30 +29,9 @@ function resolveSourceConfigDir(existingHome: string | undefined): string | unde return existsSync(xdg) ? xdg : undefined } -function mirrorConfigDir(sourceConfigDir: string, targetConfigDir: string): void { - mkdirSync(targetConfigDir, { recursive: true }) - for (const entry of readdirSync(sourceConfigDir, { withFileTypes: true })) { - if (entry.name === 'plugins' && entry.isDirectory()) { - const overlayPlugins = join(targetConfigDir, 'plugins') - mkdirSync(overlayPlugins, { recursive: true }) - for (const pluginEntry of readdirSync(join(sourceConfigDir, 'plugins'), { - withFileTypes: true - })) { - if (pluginEntry.name === ORCA_MIMOCODE_PLUGIN_FILE) { - continue - } - mirrorEntry( - join(sourceConfigDir, 'plugins', pluginEntry.name), - join(overlayPlugins, pluginEntry.name) - ) - } - continue - } - mirrorEntry(join(sourceConfigDir, entry.name), join(targetConfigDir, entry.name)) - } -} - export class MimoCodeHookService { + private warnedOverlayCapacity = false + clearPty(_ptyId: string): void {} buildPtyEnv(_ptyId: string, existingMimocodeHome?: string): Record { @@ -61,16 +45,33 @@ export class MimoCodeHookService { const overlayConfig = join(home, 'config') const sourceConfig = resolveSourceConfigDir(existingMimocodeHome) if (sourceConfig) { - safeRemoveTree(overlayConfig) - mirrorConfigDir(sourceConfig, overlayConfig) + const plan = createConfigOverlayPlan(sourceConfig, { + reservedPluginFile: ORCA_MIMOCODE_PLUGIN_FILE + }) + if (!safeRemoveTree(overlayConfig)) { + throw new Error('Unable to clear the MiMo config overlay') + } + mkdirSync(overlayConfig, { recursive: true }) + applyConfigOverlayPlan(plan, overlayConfig) } const pluginsDir = join(home, 'config', 'plugins') mkdirSync(pluginsDir, { recursive: true }) - writeFileSync( - join(pluginsDir, ORCA_MIMOCODE_PLUGIN_FILE), - getOpenCodeFamilyPluginSource('/hook/mimo-code') - ) - } catch { + const pluginPath = join(pluginsDir, ORCA_MIMOCODE_PLUGIN_FILE) + try { + unlinkSync(pluginPath) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + writeFileSync(pluginPath, getOpenCodeFamilyPluginSource('/hook/mimo-code')) + } catch (error) { + if (!this.warnedOverlayCapacity && error instanceof ConfigOverlayCapacityError) { + this.warnedOverlayCapacity = true + console.warn( + '[mimocode-hooks] config overlay exceeded its memory limit; using the original MiMo home without Orca status integration' + ) + } return existingMimocodeHome ? { MIMOCODE_HOME: existingMimocodeHome } : {} } return { MIMOCODE_HOME: home } diff --git a/src/main/minimax/minimax-cookie-store.test.ts b/src/main/minimax/minimax-cookie-store.test.ts index 4362dd2df93..1f2466b2237 100644 --- a/src/main/minimax/minimax-cookie-store.test.ts +++ b/src/main/minimax/minimax-cookie-store.test.ts @@ -22,10 +22,13 @@ const homedirMock = vi.fn(() => '/home/test') vi.mock('node:fs', () => ({ existsSync: existsSyncMock, - readFileSync: readFileSyncMock, rmSync: rmSyncMock })) +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileSync: readFileSyncMock +})) + vi.mock('node:os', () => ({ homedir: homedirMock })) diff --git a/src/main/minimax/minimax-cookie-store.ts b/src/main/minimax/minimax-cookie-store.ts index 3864f892bfb..e9bbcbcb508 100644 --- a/src/main/minimax/minimax-cookie-store.ts +++ b/src/main/minimax/minimax-cookie-store.ts @@ -1,8 +1,9 @@ import { safeStorage } from 'electron' -import { existsSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, rmSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import { hardenExistingSecureFile, writeSecureFile } from '../../shared/secure-file' +import { readIntegrationCredentialFileSync } from '../integration-credential-file' const MINIMAX_COOKIE_FILE = 'minimax-session-cookie.enc' const COOKIE_ENVELOPE_PREFIX = 'orca-minimax-cookie:v1:' @@ -149,7 +150,7 @@ export function readMiniMaxSessionCookie(): string | null { console.warn('[minimax] Failed to harden MiniMax cookie file while reading', error) } try { - const raw = readFileSync(keyPath) + const raw = readIntegrationCredentialFileSync(keyPath) const envelope = decodeCookieEnvelope(raw) cachedMiniMaxCookie = envelope ? readEnvelope(envelope) : readLegacyCookie(raw) return cachedMiniMaxCookie diff --git a/src/main/native-chat/session-file-resolver.ts b/src/main/native-chat/session-file-resolver.ts index 33cf8c23823..bb23a88512e 100644 --- a/src/main/native-chat/session-file-resolver.ts +++ b/src/main/native-chat/session-file-resolver.ts @@ -3,7 +3,7 @@ import { homedir } from 'node:os' import { basename, extname, join } from 'node:path' import type { AgentType } from '../../shared/native-chat-types' import { resolveNativeChatTranscriptAgent } from '../../shared/native-chat-agent-support' -import { walkSessionFiles } from '../ai-vault/session-scanner-discovery' +import { findFirstSessionFile } from '../ai-vault/session-scanner-discovery' import { getOrcaManagedCodexHomePath } from '../codex/codex-home-paths' import { findGrokChatHistoryBySessionId, @@ -102,11 +102,10 @@ async function resolveClaudeSessionFile( projectsDir: string ): Promise { const targetName = `${sessionId}.jsonl` - const files = await walkSessionFiles(projectsDir, 'claude', [], { + return findFirstSessionFile(projectsDir, { extensions: new Set(['.jsonl']), filePredicate: (path) => basename(path) === targetName }) - return files[0] ?? null } async function resolveCodexSessionFile( @@ -120,15 +119,15 @@ async function resolveCodexSessionFile( if (!existsSync(sessionsDir)) { continue } - const files = await walkSessionFiles(sessionsDir, 'codex', [], { + const file = await findFirstSessionFile(sessionsDir, { extensions: new Set(['.jsonl']), filePredicate: (path) => { const name = basename(path, extname(path)) return name === sessionId || name.endsWith(`-${sessionId}`) } }) - if (files[0]) { - return files[0] + if (file) { + return file } } return null diff --git a/src/main/native-chat/transcript-incremental-reader.test.ts b/src/main/native-chat/transcript-incremental-reader.test.ts new file mode 100644 index 00000000000..d97a264176f --- /dev/null +++ b/src/main/native-chat/transcript-incremental-reader.test.ts @@ -0,0 +1,185 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import type { NativeChatMessage } from '../../shared/native-chat-types' +import { + APPEND_BATCH_RETAINED_BYTE_LIMIT, + createIncrementalTranscriptState, + readIncrementalTranscriptMessages +} from './transcript-incremental-reader' +import { + estimateTranscriptMessageRetainedBytes, + MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES +} from './transcript-message-retention' +import { transcriptFallbackId } from './transcript-fallback-id' +import { MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES } from './transcript-tail-reader' + +const tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('readIncrementalTranscriptMessages', () => { + it('keeps the newest shared-retention window for an initial snapshot without batching', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-transcript-incremental-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + const extraMessages = 5 + await writeFile( + filePath, + Array.from( + { length: MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES + extraMessages }, + (_unused, index) => `row-${index}\n` + ).join('') + ) + const state = createIncrementalTranscriptState() + + const messages = await readIncrementalTranscriptMessages( + filePath, + state, + (line): NativeChatMessage => ({ + id: line, + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + }) + ) + + expect(messages).toHaveLength(MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES) + expect(messages[0]?.id).toBe(`row-${extraMessages}`) + expect(messages.at(-1)?.id).toBe( + `row-${MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES + extraMessages - 1}` + ) + }) + + it('flushes append batches by retained bytes before the message-count limit', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-transcript-incremental-batch-bytes-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + const payload = 'x'.repeat(1024 * 1024) + const lines = Array.from({ length: 5 }, (_unused, index) => `row-${index}:${payload}`) + await writeFile(filePath, `${lines.join('\n')}\n`) + const state = createIncrementalTranscriptState() + const batches: NativeChatMessage[][] = [] + const decode = (line: string): NativeChatMessage => ({ + id: line.slice(0, line.indexOf(':')), + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + }) + + const remaining = await readIncrementalTranscriptMessages(filePath, state, decode, (messages) => + batches.push(messages) + ) + const delivered = [...batches.flat(), ...remaining] + + expect(batches.map((batch) => batch.length)).toEqual([3]) + expect(delivered.map((message) => message.id)).toEqual( + Array.from({ length: 5 }, (_unused, index) => `row-${index}`) + ) + for (const batch of [...batches, remaining]) { + const retainedBytes = batch.reduce( + (total, message) => + total + + estimateTranscriptMessageRetainedBytes( + lines.find((line) => line.startsWith(`${message.id}:`))!.length + ), + 0 + ) + expect(retainedBytes).toBeLessThanOrEqual(APPEND_BATCH_RETAINED_BYTE_LIMIT) + } + }) + + it('pauses an aggregate drain at a record boundary and resumes without gaps', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-transcript-incremental-drain-budget-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + await writeFile(filePath, 'row-0\nrow-1\nrow-2\nrow-3\nrow-4\n') + const state = createIncrementalTranscriptState() + const decode = (line: string): NativeChatMessage => ({ + id: line, + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + }) + const drain = (): Promise => + readIncrementalTranscriptMessages(filePath, state, decode, () => {}, undefined, undefined, { + maxDrainRetainedBytes: 600 + }) + + const first = await drain() + const firstOffset = state.offset + const second = await drain() + const third = await drain() + + expect(first.map((message) => message.id)).toEqual(['row-0', 'row-1']) + expect(second.map((message) => message.id)).toEqual(['row-2', 'row-3']) + expect(third.map((message) => message.id)).toEqual(['row-4']) + expect(firstOffset).toBeGreaterThan(0) + expect(state.pendingRecord.byteLength).toBe(0) + }) + + it('advances one bounded record when the drain budget is smaller than that record', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-transcript-incremental-small-budget-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + await writeFile(filePath, 'row-0\nrow-1\n') + const state = createIncrementalTranscriptState() + const decode = (line: string): NativeChatMessage => ({ + id: line, + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + }) + const drain = (): Promise => + readIncrementalTranscriptMessages(filePath, state, decode, () => {}, undefined, undefined, { + maxDrainRetainedBytes: 1 + }) + + const first = await drain() + const firstOffset = state.offset + const second = await drain() + const secondOffset = state.offset + const exhausted = await drain() + + expect(first.map((message) => message.id)).toEqual(['row-0']) + expect(second.map((message) => message.id)).toEqual(['row-1']) + expect(exhausted).toEqual([]) + expect(firstOffset).toBeGreaterThan(0) + expect(secondOffset).toBeGreaterThan(firstOffset) + }) + + it('keeps the exact fallback offset after discarding an oversized record', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-transcript-incremental-oversized-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + const oversized = Buffer.alloc(MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES + 1, 0x78) + const valid = Buffer.from('valid\n') + await writeFile(filePath, Buffer.concat([oversized, Buffer.from('\n'), valid])) + const state = createIncrementalTranscriptState() + + const messages = await readIncrementalTranscriptMessages( + filePath, + state, + (line, fallbackId): NativeChatMessage => ({ + id: fallbackId, + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + }) + ) + + expect(messages.map((message) => message.id)).toEqual([ + transcriptFallbackId(filePath, oversized.byteLength + 1) + ]) + expect(state.offset).toBe(oversized.byteLength + 1 + valid.byteLength) + expect(state.pendingRecord.byteLength).toBe(0) + }) +}) diff --git a/src/main/native-chat/transcript-incremental-reader.ts b/src/main/native-chat/transcript-incremental-reader.ts index 8ad8cc335d1..1c1df1ddef5 100644 --- a/src/main/native-chat/transcript-incremental-reader.ts +++ b/src/main/native-chat/transcript-incremental-reader.ts @@ -1,27 +1,38 @@ import { open, stat } from 'node:fs/promises' import type { NativeChatMessage, NativeChatTurnLifecycle } from '../../shared/native-chat-types' import { transcriptFallbackId } from './transcript-fallback-id' +import { TranscriptRecordBuffer } from './transcript-record-buffer' import { MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES, type NativeChatLineDecoder } from './transcript-tail-reader' +import { + estimateTranscriptMessageRetainedBytes, + TranscriptMessageRetention +} from './transcript-message-retention' -const APPEND_BATCH_MESSAGE_LIMIT = 40 +export const APPEND_BATCH_MESSAGE_LIMIT = 40 +export const APPEND_BATCH_RETAINED_BYTE_LIMIT = 8 * 1024 * 1024 +export const INCREMENTAL_DRAIN_RETAINED_BYTE_LIMIT = 32 * 1024 * 1024 export type IncrementalTranscriptState = { offset: number - pendingChunks: Buffer[] + pendingRecord: TranscriptRecordBuffer pendingStart: number - pendingBytes: number - droppingOversizedRecord: boolean +} + +export function createIncrementalTranscriptState(): IncrementalTranscriptState { + return { + offset: 0, + pendingRecord: new TranscriptRecordBuffer(MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES), + pendingStart: 0 + } } export function resetIncrementalTranscriptState(state: IncrementalTranscriptState): void { state.offset = 0 - state.pendingChunks.length = 0 + state.pendingRecord.clear() state.pendingStart = 0 - state.pendingBytes = 0 - state.droppingOversizedRecord = false } export async function readIncrementalTranscriptMessages( @@ -30,27 +41,41 @@ export async function readIncrementalTranscriptMessages( decode: NativeChatLineDecoder, onBatch?: (messages: NativeChatMessage[]) => void, decodeLifecycle?: (line: string, fallbackId: string) => NativeChatTurnLifecycle | null, - onLifecycle?: (lifecycle: NativeChatTurnLifecycle) => void + onLifecycle?: (lifecycle: NativeChatTurnLifecycle) => void, + options: { maxDrainRetainedBytes?: number } = {} ): Promise { const end = (await stat(filePath)).size if (end <= state.offset) { return [] } const messages: NativeChatMessage[] = [] + let messageBatchBytes = 0 + let drainRetainedBytes = 0 + const requestedDrainLimit = options.maxDrainRetainedBytes + const maxDrainRetainedBytes = + Number.isSafeInteger(requestedDrainLimit) && (requestedDrainLimit ?? 0) > 0 + ? Math.min(INCREMENTAL_DRAIN_RETAINED_BYTE_LIMIT, requestedDrainLimit ?? 0) + : INCREMENTAL_DRAIN_RETAINED_BYTE_LIMIT + const retainedSnapshot = onBatch ? null : new TranscriptMessageRetention() const handle = await open(filePath, 'r') try { const stream = handle.createReadStream({ start: state.offset, end: end - 1, autoClose: false }) let absoluteOffset = state.offset - for await (const rawChunk of stream) { + streamChunks: for await (const rawChunk of stream) { const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) let segmentStart = 0 let newline = chunk.indexOf(0x0a) while (newline >= 0) { retainPart(chunk.subarray(segmentStart, newline)) - if (!state.droppingOversizedRecord) { - decodeLine() + if (!state.pendingRecord.isOversized && !decodeLine()) { + const retryOffset = state.pendingStart + resetPendingLine(retryOffset) + state.offset = retryOffset + break streamChunks } - resetPendingLine(absoluteOffset + newline + 1) + const nextOffset = absoluteOffset + newline + 1 + resetPendingLine(nextOffset) + state.offset = nextOffset segmentStart = newline + 1 newline = chunk.indexOf(0x0a, segmentStart) } @@ -60,51 +85,69 @@ export async function readIncrementalTranscriptMessages( absoluteOffset += chunk.length state.offset = absoluteOffset } - return messages + return retainedSnapshot?.values() ?? messages } finally { await handle.close() } function retainPart(part: Buffer): void { - if (state.droppingOversizedRecord) { - return - } - state.pendingBytes += part.length - if (state.pendingBytes > MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES) { - state.pendingChunks.length = 0 - state.droppingOversizedRecord = true - return - } - state.pendingChunks.push(part) + state.pendingRecord.append(part) } function resetPendingLine(nextStart: number): void { - state.pendingChunks.length = 0 - state.pendingBytes = 0 - state.droppingOversizedRecord = false + state.pendingRecord.clear() state.pendingStart = nextStart } - function decodeLine(): void { - let line = Buffer.concat(state.pendingChunks).toString('utf8') + function decodeLine(): boolean { + let line = state.pendingRecord.toString() if (line.endsWith('\r')) { line = line.slice(0, -1) } if (!line) { - return + return true } const fallbackId = transcriptFallbackId(filePath, state.pendingStart) + const message = decode(line, fallbackId) + const estimatedBytes = message + ? estimateTranscriptMessageRetainedBytes(state.pendingRecord.byteLength) + : 0 + if ( + message && + onBatch && + drainRetainedBytes > 0 && + estimatedBytes > maxDrainRetainedBytes - drainRetainedBytes + ) { + // Why: a budget below one bounded record must still advance instead of + // retrying that record forever. + return false + } const lifecycle = decodeLifecycle?.(line, fallbackId) if (lifecycle) { onLifecycle?.(lifecycle) } - const message = decode(line, fallbackId) if (!message) { - return + return true + } + if (retainedSnapshot) { + retainedSnapshot.add(message, state.pendingRecord.byteLength) + return true + } + drainRetainedBytes += estimatedBytes + if ( + onBatch && + messages.length > 0 && + estimatedBytes > APPEND_BATCH_RETAINED_BYTE_LIMIT - messageBatchBytes + ) { + onBatch(messages.splice(0)) + messageBatchBytes = 0 } messages.push(message) + messageBatchBytes += estimatedBytes if (onBatch && messages.length >= APPEND_BATCH_MESSAGE_LIMIT) { onBatch(messages.splice(0)) + messageBatchBytes = 0 } + return true } } diff --git a/src/main/native-chat/transcript-message-retention.test.ts b/src/main/native-chat/transcript-message-retention.test.ts new file mode 100644 index 00000000000..9dae78eb069 --- /dev/null +++ b/src/main/native-chat/transcript-message-retention.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' +import type { NativeChatMessage } from '../../shared/native-chat-types' +import { + MAX_NATIVE_CHAT_TRANSCRIPT_RETAINED_BYTES, + TranscriptMessageRetention +} from './transcript-message-retention' + +function message(id: string): NativeChatMessage { + return { + id, + role: 'user', + blocks: [{ type: 'text', text: id }], + timestamp: null, + source: 'transcript' + } +} + +describe('TranscriptMessageRetention', () => { + it('keeps the newest messages within both count and byte budgets', () => { + const retention = new TranscriptMessageRetention(2, 600) + + retention.add(message('one'), 10) + retention.add(message('two'), 10) + retention.add(message('three'), 10) + + expect(retention.values().map(({ id }) => id)).toEqual(['two', 'three']) + expect(retention.size).toBe(2) + expect(retention.retainedBytes).toBeLessThanOrEqual(600) + }) + + it('applies the production 64 MiB policy without retaining older oversized history', () => { + const retention = new TranscriptMessageRetention() + const sourceBytes = 2 * 1024 * 1024 + + for (let index = 0; index < 17; index += 1) { + retention.add(message(`message-${index}`), sourceBytes) + } + + expect(retention.values().map(({ id }) => id)).toEqual( + Array.from({ length: 15 }, (_unused, index) => `message-${index + 2}`) + ) + expect(retention.retainedBytes).toBeLessThanOrEqual(MAX_NATIVE_CHAT_TRANSCRIPT_RETAINED_BYTES) + }) +}) diff --git a/src/main/native-chat/transcript-message-retention.ts b/src/main/native-chat/transcript-message-retention.ts new file mode 100644 index 00000000000..aee50bacd55 --- /dev/null +++ b/src/main/native-chat/transcript-message-retention.ts @@ -0,0 +1,59 @@ +import type { NativeChatMessage } from '../../shared/native-chat-types' + +export const MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES = 50_000 +export const MAX_NATIVE_CHAT_TRANSCRIPT_RETAINED_BYTES = 64 * 1024 * 1024 + +type RetainedMessage = { + message: NativeChatMessage + estimatedBytes: number +} + +export function estimateTranscriptMessageRetainedBytes(sourceBytes: number): number { + return sourceBytes * 2 + 256 +} + +export class TranscriptMessageRetention { + private entries: (RetainedMessage | undefined)[] = [] + private head = 0 + private retained = 0 + + constructor( + private readonly maxMessages = MAX_NATIVE_CHAT_TRANSCRIPT_MESSAGES, + private readonly maxBytes = MAX_NATIVE_CHAT_TRANSCRIPT_RETAINED_BYTES + ) {} + + add(message: NativeChatMessage, sourceBytes: number): void { + const estimatedBytes = estimateTranscriptMessageRetainedBytes(sourceBytes) + this.entries.push({ message, estimatedBytes }) + this.retained += estimatedBytes + while (this.size > this.maxMessages || this.retained > this.maxBytes) { + const oldest = this.entries[this.head] + this.entries[this.head] = undefined + this.head += 1 + this.retained -= oldest?.estimatedBytes ?? 0 + } + if (this.head >= 1_024 && this.head * 2 >= this.entries.length) { + this.entries.splice(0, this.head) + this.head = 0 + } + } + + values(): NativeChatMessage[] { + const messages: NativeChatMessage[] = [] + for (let index = this.head; index < this.entries.length; index += 1) { + const entry = this.entries[index] + if (entry) { + messages.push(entry.message) + } + } + return messages + } + + get size(): number { + return this.entries.length - this.head + } + + get retainedBytes(): number { + return this.retained + } +} diff --git a/src/main/native-chat/transcript-read-admission.test.ts b/src/main/native-chat/transcript-read-admission.test.ts new file mode 100644 index 00000000000..fbb395d4fcf --- /dev/null +++ b/src/main/native-chat/transcript-read-admission.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from 'vitest' +import { NativeChatTranscriptReadAdmission } from './transcript-read-admission' + +describe('NativeChatTranscriptReadAdmission', () => { + it('admits only reads that fit both concurrency and aggregate byte limits', async () => { + const admission = new NativeChatTranscriptReadAdmission(10, 2, 4) + const releaseSix = await admission.acquire(6) + const releaseFour = await admission.acquire(4) + let thirdAdmitted = false + const third = admission.acquire(1).then((release) => { + thirdAdmitted = true + return release + }) + + await Promise.resolve() + expect(thirdAdmitted).toBe(false) + expect(admission.activeCount).toBe(2) + expect(admission.retainedBytes).toBe(10) + expect(admission.queuedCount).toBe(1) + + releaseFour() + const releaseThird = await third + expect(thirdAdmitted).toBe(true) + expect(admission.retainedBytes).toBe(7) + + releaseSix() + releaseThird() + expect(admission.activeCount).toBe(0) + expect(admission.retainedBytes).toBe(0) + }) + + it('removes an aborted queued read and immediately reuses the slot', async () => { + const admission = new NativeChatTranscriptReadAdmission(10, 2, 1) + const releaseActive = await admission.acquire(10) + const controller = new AbortController() + const queued = admission.acquire(1, controller.signal) + + expect(admission.queuedCount).toBe(1) + controller.abort() + await expect(queued).rejects.toMatchObject({ name: 'AbortError' }) + expect(admission.queuedCount).toBe(0) + + const replacement = admission.acquire(1) + expect(admission.queuedCount).toBe(1) + releaseActive() + const releaseReplacement = await replacement + releaseReplacement() + }) + + it('bounds queued closures and releases reservations idempotently', async () => { + const admission = new NativeChatTranscriptReadAdmission(1, 1, 2) + const releaseActive = await admission.acquire(1) + const first = admission.acquire(1) + const second = admission.acquire(1) + + expect(() => admission.acquire(1)).toThrow('Too many queued native chat transcript reads') + releaseActive() + releaseActive() + const releaseFirst = await first + releaseFirst() + const releaseSecond = await second + releaseSecond() + + expect(admission.activeCount).toBe(0) + expect(admission.retainedBytes).toBe(0) + }) + + it('detaches abort listeners after admission', async () => { + const admission = new NativeChatTranscriptReadAdmission(1, 1, 1) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + + const release = await admission.acquire(1, controller.signal) + + expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function)) + controller.abort() + expect(admission.activeCount).toBe(1) + release() + }) +}) diff --git a/src/main/native-chat/transcript-read-admission.ts b/src/main/native-chat/transcript-read-admission.ts new file mode 100644 index 00000000000..54a4a37e3e4 --- /dev/null +++ b/src/main/native-chat/transcript-read-admission.ts @@ -0,0 +1,144 @@ +import { INCREMENTAL_DRAIN_RETAINED_BYTE_LIMIT } from './transcript-incremental-reader' +import { MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES } from './transcript-tail-reader' + +export const MAX_NATIVE_CHAT_TRANSCRIPT_READ_CONCURRENCY = 8 +export const MAX_NATIVE_CHAT_TRANSCRIPT_READ_WAITERS = 256 +export const MAX_NATIVE_CHAT_TRANSCRIPT_PROCESS_RETAINED_BYTES = 128 * 1024 * 1024 +export const NATIVE_CHAT_TRANSCRIPT_PAGE_RESERVATION_BYTES = + MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES +export const NATIVE_CHAT_TRANSCRIPT_WATCH_DRAIN_RESERVATION_BYTES = + MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES + INCREMENTAL_DRAIN_RETAINED_BYTE_LIMIT + +type AdmissionWaiter = { + abort: (() => void) | null + bytes: number + reject: (error: Error) => void + resolve: (release: () => void) => void + signal?: AbortSignal +} + +function abortedReadError(): Error { + const error = new Error('Native chat transcript read was canceled') + error.name = 'AbortError' + return error +} + +export class NativeChatTranscriptReadAdmission { + private activeBytes = 0 + private activeReads = 0 + private readonly waiters: AdmissionWaiter[] = [] + + constructor( + private readonly maxBytes = MAX_NATIVE_CHAT_TRANSCRIPT_PROCESS_RETAINED_BYTES, + private readonly maxActive = MAX_NATIVE_CHAT_TRANSCRIPT_READ_CONCURRENCY, + private readonly maxWaiters = MAX_NATIVE_CHAT_TRANSCRIPT_READ_WAITERS + ) { + if ( + !Number.isSafeInteger(maxBytes) || + maxBytes < 0 || + !Number.isSafeInteger(maxActive) || + maxActive < 1 || + !Number.isSafeInteger(maxWaiters) || + maxWaiters < 0 + ) { + throw new RangeError('Invalid native chat transcript read admission limits') + } + } + + acquire(bytes: number, signal?: AbortSignal): Promise<() => void> { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.maxBytes) { + throw new RangeError('Native chat transcript read exceeds the process memory budget') + } + if (signal?.aborted) { + return Promise.reject(abortedReadError()) + } + const canAdmitImmediately = + this.waiters.length === 0 && + this.activeReads < this.maxActive && + bytes <= this.maxBytes - this.activeBytes + if (!canAdmitImmediately && this.waiters.length >= this.maxWaiters) { + throw new Error('Too many queued native chat transcript reads') + } + + return new Promise((resolve, reject) => { + const waiter: AdmissionWaiter = { + abort: null, + bytes, + reject, + resolve, + ...(signal ? { signal } : {}) + } + if (signal) { + waiter.abort = (): void => { + const index = this.waiters.indexOf(waiter) + if (index < 0) { + return + } + this.waiters.splice(index, 1) + signal.removeEventListener('abort', waiter.abort!) + waiter.abort = null + reject(abortedReadError()) + this.admitWaiters() + } + signal.addEventListener('abort', waiter.abort, { once: true }) + } + this.waiters.push(waiter) + this.admitWaiters() + }) + } + + get activeCount(): number { + return this.activeReads + } + + get queuedCount(): number { + return this.waiters.length + } + + get retainedBytes(): number { + return this.activeBytes + } + + private admitWaiters(): void { + while (this.waiters.length > 0) { + const waiter = this.waiters[0] + if (this.activeReads >= this.maxActive || waiter.bytes > this.maxBytes - this.activeBytes) { + return + } + this.waiters.shift() + if (waiter.signal && waiter.abort) { + waiter.signal.removeEventListener('abort', waiter.abort) + waiter.abort = null + } + this.activeReads += 1 + this.activeBytes += waiter.bytes + let released = false + waiter.resolve(() => { + if (released) { + return + } + released = true + this.activeReads -= 1 + this.activeBytes -= waiter.bytes + this.admitWaiters() + }) + } + } +} + +export const nativeChatTranscriptReadAdmission = new NativeChatTranscriptReadAdmission() + +export async function withNativeChatTranscriptWatchDrainAdmission( + signal: AbortSignal, + run: () => Promise +): Promise { + const release = await nativeChatTranscriptReadAdmission.acquire( + NATIVE_CHAT_TRANSCRIPT_WATCH_DRAIN_RESERVATION_BYTES, + signal + ) + try { + return await run() + } finally { + release() + } +} diff --git a/src/main/native-chat/transcript-read-cache.ts b/src/main/native-chat/transcript-read-cache.ts index 8943c223dd2..9a828f0d291 100644 --- a/src/main/native-chat/transcript-read-cache.ts +++ b/src/main/native-chat/transcript-read-cache.ts @@ -13,9 +13,8 @@ import { readNativeChatTranscript, type ReadTranscriptResult } from './transcrip // share one sessionId yet resolve to DIFFERENT files (the same session resumed // into a second worktree, which writes a new transcript file), and a // sessionId-only key let one worktree's cached parse be served to another when -// their file mtimes momentarily coincided (#7326). The cache stores ONE -// canonical, unwindowed parse; windowing and per-surface truncation stay in the -// callers so the same parse is reused across all `limit` values and every client kind. +// their file mtimes momentarily coincided (#7326). The cache stores one +// canonical bounded parse; surface-specific windowing stays in callers. type CachedTranscript = { result: ReadTranscriptResult @@ -34,7 +33,7 @@ const cache = new Map() const MAX_CACHE_ENTRIES = 50 // Why: a heavy Claude/Codex coding session's JSONL is routinely tens of MB (tool // results embed whole file contents, command output, and diffs), and each cached -// entry is the full unwindowed parse. The count cap alone let 50 such entries +// entry is a large bounded parse. The count cap alone let 50 such entries // retain multiple GB in the one process that now serves desktop + every paired // web/mobile client. Bound total cached file bytes too; we always keep the most- // recent entry (see setCached) so an active transcript is never re-parsed on @@ -79,9 +78,7 @@ async function fileStat(filePath: string): Promise<{ mtimeMs: number; bytes: num } /** - * Read the full transcript for an agent + session, returning the cached parse on - * an mtime hit and re-reading (and re-caching) when the file changed. Returns the - * canonical, unwindowed result; callers apply their own windowing/truncation. + * Read the canonical bounded transcript for a session, refreshing on file changes. */ export async function readNativeChatTranscriptCached( agent: AgentType, diff --git a/src/main/native-chat/transcript-reader.test.ts b/src/main/native-chat/transcript-reader.test.ts index ff48548804d..06c1c3ec6d1 100644 --- a/src/main/native-chat/transcript-reader.test.ts +++ b/src/main/native-chat/transcript-reader.test.ts @@ -2,8 +2,10 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' +import type { NativeChatMessage } from '../../shared/native-chat-types' import { readNativeChatTranscript } from './transcript-reader' import { + MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES, nativeChatLineDecoderForAgent, readNativeChatTranscriptTail, readNativeChatTranscriptTailFile @@ -332,4 +334,41 @@ describe('readNativeChatTranscriptTailFile', () => { expect(result.messages).toEqual([]) expect(result.hasMore).toBe(false) }) + + it('preserves the newest messages and a correct pagination boundary at the page byte cap', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-native-chat-tail-page-bytes-')) + tempRoots.push(root) + const filePath = join(root, 'transcript.jsonl') + const payload = 'x'.repeat(1024 * 1024) + const ids = Array.from({ length: 10 }, (_unused, index) => `message-${index}`) + await writeFile(filePath, ids.map((id) => `${id}:${payload}\n`).join('')) + const decode = (line: string): NativeChatMessage => { + const id = line.slice(0, line.indexOf(':')) + return { + id, + role: 'user', + blocks: [{ type: 'text', text: line }], + timestamp: null, + source: 'transcript' + } + } + + const newest = await readNativeChatTranscriptTailFile(filePath, 100, decode, true) + const older = await readNativeChatTranscriptTailFile( + filePath, + 100, + decode, + true, + newest.beforeOffset + ) + + expect(newest.hasMore).toBe(true) + expect(newest.messages.length).toBeGreaterThan(0) + expect(newest.messages.length).toBeLessThan(ids.length) + expect(newest.messages.at(-1)?.id).toBe(ids.at(-1)) + expect([...older.messages, ...newest.messages].map((message) => message.id)).toEqual(ids) + expect(newest.messages.length * (payload.length * 2 + 256)).toBeLessThanOrEqual( + MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES + ) + }) }) diff --git a/src/main/native-chat/transcript-reader.ts b/src/main/native-chat/transcript-reader.ts index 86353604e20..55542e3a1d6 100644 --- a/src/main/native-chat/transcript-reader.ts +++ b/src/main/native-chat/transcript-reader.ts @@ -28,13 +28,7 @@ export type ReadTranscriptOptions = ResolveSessionFileOptions & { filePath?: string } -/** - * Read the ENTIRE Claude/Codex JSONL transcript for an agent + session id into - * the NativeChatMessage model. Unlike the AI-Vault preview scan, this applies - * NO message cap. Unknown record types are skipped rather than throwing, so a - * single malformed/unrecognized line cannot fail the whole read. The per-line - * record-to-message mapping is shared with the live tailer. - */ +/** Reads the newest bounded transcript window; malformed records are skipped. */ export async function readNativeChatTranscript( agent: AgentType, sessionId: string, diff --git a/src/main/native-chat/transcript-record-buffer.test.ts b/src/main/native-chat/transcript-record-buffer.test.ts new file mode 100644 index 00000000000..6e5d4c3b735 --- /dev/null +++ b/src/main/native-chat/transcript-record-buffer.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { TranscriptRecordBuffer } from './transcript-record-buffer' + +describe('TranscriptRecordBuffer', () => { + it('preserves UTF-8 across 100,000 one-byte fragments', () => { + const expected = Buffer.from(`${'a'.repeat(99_996)}😀`) + const record = new TranscriptRecordBuffer(expected.byteLength) + + for (const byte of expected) { + record.append(Uint8Array.of(byte)) + } + + expect(record.byteLength).toBe(expected.byteLength) + expect(record.toString()).toBe(expected.toString()) + expect(record.isOversized).toBe(false) + }) + + it('drops retained storage at the byte cap while tracking the full record length', () => { + const record = new TranscriptRecordBuffer(4) + + record.append(Buffer.from('1234')) + record.append(Buffer.from('56789')) + record.append(Buffer.from('abc')) + + expect(record.byteLength).toBe(12) + expect(record.toString()).toBe('') + expect(record.isOversized).toBe(true) + + record.clear() + record.append(Buffer.from('ok')) + expect(record.toString()).toBe('ok') + }) +}) diff --git a/src/main/native-chat/transcript-record-buffer.ts b/src/main/native-chat/transcript-record-buffer.ts new file mode 100644 index 00000000000..e5f09bf326a --- /dev/null +++ b/src/main/native-chat/transcript-record-buffer.ts @@ -0,0 +1,44 @@ +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' + +export class TranscriptRecordBuffer { + private readonly retained = new GrowingByteBuffer() + private observedBytes = 0 + private oversized = false + + constructor(private readonly maxRetainedBytes: number) { + if (!Number.isSafeInteger(maxRetainedBytes) || maxRetainedBytes < 0) { + throw new RangeError('Transcript record limit must be a non-negative safe integer') + } + } + + append(part: Buffer | Uint8Array): void { + this.observedBytes += part.byteLength + if (this.oversized) { + return + } + if (this.observedBytes > this.maxRetainedBytes) { + this.retained.clear() + this.oversized = true + return + } + this.retained.append(part) + } + + clear(): void { + this.retained.clear() + this.observedBytes = 0 + this.oversized = false + } + + toString(): string { + return this.retained.toString('utf8') + } + + get byteLength(): number { + return this.observedBytes + } + + get isOversized(): boolean { + return this.oversized + } +} diff --git a/src/main/native-chat/transcript-stream-lines.test.ts b/src/main/native-chat/transcript-stream-lines.test.ts index 2bfa6f6f9d2..ddcb2a4edf3 100644 --- a/src/main/native-chat/transcript-stream-lines.test.ts +++ b/src/main/native-chat/transcript-stream-lines.test.ts @@ -1,6 +1,8 @@ import { Readable } from 'node:stream' import { describe, expect, it } from 'vitest' +import { transcriptFallbackId } from './transcript-fallback-id' import { decodeTranscriptStream } from './transcript-stream-lines' +import { MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES } from './transcript-tail-reader' const decode = (line: string, id: string) => ({ id, @@ -46,4 +48,52 @@ describe('decodeTranscriptStream', () => { expect(result.messages).toHaveLength(1) expect(result.consumedBytes).toBe(Buffer.byteLength(complete, 'utf8')) }) + + it('preserves multibyte UTF-8 split across one-byte stream chunks', async () => { + const line = `prefix-😀-suffix` + const encoded = Buffer.from(`${line}\n`) + const result = await decodeTranscriptStream( + Readable.from([...encoded].map((byte) => Uint8Array.of(byte))), + '/chat.jsonl', + 0, + decode, + false + ) + + expect(result.messages[0]?.blocks).toEqual([{ type: 'text', text: line }]) + expect(result.consumedBytes).toBe(encoded.byteLength) + }) + + it('drops an oversized record without retaining it or losing the next line', async () => { + const oversized = 'x'.repeat(MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES + 1) + const valid = '{"valid":true}' + const result = await decodeTranscriptStream( + Readable.from([oversized, `\n${valid}\n`]), + '/chat.jsonl', + 0, + decode, + true + ) + + expect(result.messages).toHaveLength(1) + expect(result.messages[0]?.blocks).toEqual([{ type: 'text', text: valid }]) + expect(result.messages[0]?.id).toBe( + transcriptFallbackId('/chat.jsonl', Buffer.byteLength(`${oversized}\n`, 'utf8')) + ) + expect(result.consumedBytes).toBe(Buffer.byteLength(`${oversized}\n${valid}\n`, 'utf8')) + }) + + it('consumes an oversized trailing record when trailing lines are included', async () => { + const oversized = 'x'.repeat(MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES + 1) + const result = await decodeTranscriptStream( + Readable.from([oversized]), + '/chat.jsonl', + 0, + decode, + true + ) + + expect(result.messages).toEqual([]) + expect(result.consumedBytes).toBe(Buffer.byteLength(oversized, 'utf8')) + }) }) diff --git a/src/main/native-chat/transcript-stream-lines.ts b/src/main/native-chat/transcript-stream-lines.ts index a764941ad30..50339043dc5 100644 --- a/src/main/native-chat/transcript-stream-lines.ts +++ b/src/main/native-chat/transcript-stream-lines.ts @@ -1,6 +1,9 @@ import type { Readable } from 'node:stream' import type { NativeChatMessage } from '../../shared/native-chat-types' import { transcriptFallbackId } from './transcript-fallback-id' +import { TranscriptRecordBuffer } from './transcript-record-buffer' +import { MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES } from './transcript-tail-reader' +import { TranscriptMessageRetention } from './transcript-message-retention' type TranscriptDecoder = (line: string, fallbackId: string) => NativeChatMessage | null @@ -11,28 +14,42 @@ export async function decodeTranscriptStream( decode: TranscriptDecoder, includeTrailingLine: boolean ): Promise<{ messages: NativeChatMessage[]; consumedBytes: number }> { - const messages: NativeChatMessage[] = [] - let pending = '' + const messages = new TranscriptMessageRetention() + const pending = new TranscriptRecordBuffer(MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES) let consumedBytes = 0 for await (const chunk of stream) { - pending += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8') - let newlineIndex = pending.indexOf('\n') + const bytes = + typeof chunk === 'string' + ? Buffer.from(chunk, 'utf8') + : Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(chunk) + let segmentStart = 0 + let newlineIndex = bytes.indexOf(0x0a) while (newlineIndex !== -1) { - const segment = pending.slice(0, newlineIndex + 1) - decodeLine(segment.slice(0, -1), consumedBytes) - consumedBytes += Buffer.byteLength(segment, 'utf8') - pending = pending.slice(newlineIndex + 1) - newlineIndex = pending.indexOf('\n') + pending.append(bytes.subarray(segmentStart, newlineIndex)) + if (!pending.isOversized) { + decodeLine(pending.toString(), consumedBytes) + } + consumedBytes += pending.byteLength + 1 + pending.clear() + segmentStart = newlineIndex + 1 + newlineIndex = bytes.indexOf(0x0a, segmentStart) + } + if (segmentStart < bytes.length) { + pending.append(bytes.subarray(segmentStart)) } } - if (includeTrailingLine && pending.length > 0) { - decodeLine(pending, consumedBytes) - consumedBytes += Buffer.byteLength(pending, 'utf8') + if (includeTrailingLine && pending.byteLength > 0) { + if (!pending.isOversized) { + decodeLine(pending.toString(), consumedBytes) + } + consumedBytes += pending.byteLength } - return { messages, consumedBytes } + return { messages: messages.values(), consumedBytes } function decodeLine(rawLine: string, relativeOffset: number): void { const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine @@ -41,7 +58,7 @@ export async function decodeTranscriptStream( } const message = decode(line, transcriptFallbackId(filePath, start + relativeOffset)) if (message) { - messages.push(message) + messages.add(message, Buffer.byteLength(line, 'utf8')) } } } diff --git a/src/main/native-chat/transcript-tail-reader.ts b/src/main/native-chat/transcript-tail-reader.ts index 2f874e95746..f852b881b16 100644 --- a/src/main/native-chat/transcript-tail-reader.ts +++ b/src/main/native-chat/transcript-tail-reader.ts @@ -16,8 +16,10 @@ import { nativeChatTurnLifecycleDecoderForAgent, type NativeChatTurnLifecycleDecoder } from './transcript-turn-lifecycle' +import { estimateTranscriptMessageRetainedBytes } from './transcript-message-retention' export const MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES = 2 * 1024 * 1024 +export const MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES = 16 * 1024 * 1024 const TAIL_CHUNK_BYTES = 64 * 1024 export type NativeChatLineDecoder = (line: string, fallbackId: string) => NativeChatMessage | null @@ -59,6 +61,8 @@ export async function readNativeChatTranscriptTailFile( let lineBytes = 0 let lineOversized = false let lifecycle: NativeChatTurnLifecycle | undefined + let retainedMessageBytes = 0 + let pageBudgetReached = false try { const consumedTo = includeTrailingLine ? end : await findLastCompleteLineEnd(handle, end) if (consumedTo === 0) { @@ -68,12 +72,16 @@ export async function readNativeChatTranscriptTailFile( const finalByte = Buffer.allocUnsafe(1) await handle.read(finalByte, 0, 1, consumedTo - 1) let cursor = consumedTo - (finalByte[0] === 0x0a ? 1 : 0) - while (cursor > 0 && newestFirst.length <= limit) { + while (cursor > 0 && newestFirst.length <= limit && !pageBudgetReached) { const start = Math.max(0, cursor - TAIL_CHUNK_BYTES) const buffer = Buffer.allocUnsafe(cursor - start) const { bytesRead } = await handle.read(buffer, 0, buffer.length, start) let segmentEnd = bytesRead - for (let index = bytesRead - 1; index >= 0 && newestFirst.length <= limit; index--) { + for ( + let index = bytesRead - 1; + index >= 0 && newestFirst.length <= limit && !pageBudgetReached; + index-- + ) { if (buffer[index] !== 0x0a) { continue } @@ -89,7 +97,7 @@ export async function readNativeChatTranscriptTailFile( } cursor = start } - if (cursor === 0 && lineParts.length > 0 && newestFirst.length <= limit) { + if (cursor === 0 && lineParts.length > 0 && newestFirst.length <= limit && !pageBudgetReached) { decodeLine(0, newestFirst) } const chronological = newestFirst.toReversed() @@ -100,7 +108,7 @@ export async function readNativeChatTranscriptTailFile( messages: selected.map((entry) => entry.message), ...(lifecycle ? { lifecycle } : {}), consumedTo, - hasMore: limit > 0 && chronological.length > limit, + hasMore: limit > 0 && (chronological.length > limit || pageBudgetReached), beforeOffset: selected[0]?.offset ?? end } } finally { @@ -144,6 +152,12 @@ export async function readNativeChatTranscriptTailFile( lifecycle ??= decodeLifecycle?.(line, fallbackId) ?? undefined const message = decode(line, fallbackId) if (message) { + const estimatedBytes = estimateTranscriptMessageRetainedBytes(lineBytes) + if (estimatedBytes > MAX_NATIVE_CHAT_TRANSCRIPT_PAGE_RETAINED_BYTES - retainedMessageBytes) { + pageBudgetReached = true + return + } + retainedMessageBytes += estimatedBytes messages.push({ message, offset: lineOffset }) } } diff --git a/src/main/native-chat/transcript-watch-drain-admission.test.ts b/src/main/native-chat/transcript-watch-drain-admission.test.ts new file mode 100644 index 00000000000..87a0a19d753 --- /dev/null +++ b/src/main/native-chat/transcript-watch-drain-admission.test.ts @@ -0,0 +1,76 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as TranscriptTailReader from './transcript-tail-reader' + +const { tailRead } = vi.hoisted(() => ({ tailRead: vi.fn() })) + +vi.mock('./transcript-tail-reader', async () => { + const actual = await vi.importActual('./transcript-tail-reader') + return { ...actual, readNativeChatTranscriptTailFile: tailRead } +}) + +import { nativeChatTranscriptReadAdmission } from './transcript-read-admission' +import { getActiveNativeChatWatcherCount, subscribeNativeChatTranscript } from './transcript-watch' +import type { NativeChatTranscriptSubscription } from './transcript-watch-contract' + +let root: string | null = null +const subscriptions: NativeChatTranscriptSubscription[] = [] + +afterEach(async () => { + for (const subscription of subscriptions.splice(0)) { + subscription.unsubscribe() + } + await vi.waitFor(() => expect(nativeChatTranscriptReadAdmission.activeCount).toBe(0)) + if (root) { + await rm(root, { recursive: true, force: true }) + root = null + } +}) + +describe('native chat transcript watcher drain admission', () => { + it('removes a queued drain immediately when its watcher unsubscribes', async () => { + root = await mkdtemp(join(tmpdir(), 'orca-native-chat-drain-admission-')) + const filePath = join(root, 'transcript.jsonl') + await writeFile(filePath, '{}\n') + let finishReads!: () => void + const readGate = new Promise<{ + messages: never[] + consumedTo: number + hasMore: boolean + beforeOffset: number + }>((resolve) => { + finishReads = () => resolve({ messages: [], consumedTo: 3, hasMore: false, beforeOffset: 0 }) + }) + tailRead.mockReturnValue(readGate) + const activeBefore = getActiveNativeChatWatcherCount() + + for (let index = 0; index < 3; index++) { + subscriptions.push( + await subscribeNativeChatTranscript({ + agent: 'claude', + sessionId: `session-${index}`, + filePath, + initialLimit: 40, + onInitialSnapshot: () => {}, + onAppend: () => {}, + debounceMs: 0, + reconciliationIntervalMs: 10_000 + }) + ) + } + await vi.waitFor(() => { + expect(nativeChatTranscriptReadAdmission.activeCount).toBe(2) + expect(nativeChatTranscriptReadAdmission.queuedCount).toBe(1) + }) + + subscriptions.pop()?.unsubscribe() + + await vi.waitFor(() => expect(nativeChatTranscriptReadAdmission.queuedCount).toBe(0)) + expect(getActiveNativeChatWatcherCount()).toBe(activeBefore + 2) + + finishReads() + await vi.waitFor(() => expect(nativeChatTranscriptReadAdmission.activeCount).toBe(0)) + }) +}) diff --git a/src/main/native-chat/transcript-watch-engine.ts b/src/main/native-chat/transcript-watch-engine.ts index ff1df71ee0f..59825e6a727 100644 --- a/src/main/native-chat/transcript-watch-engine.ts +++ b/src/main/native-chat/transcript-watch-engine.ts @@ -6,11 +6,12 @@ import { type TranscriptFileVersion } from './transcript-file-version' import { + createIncrementalTranscriptState, readIncrementalTranscriptMessages, - resetIncrementalTranscriptState, - type IncrementalTranscriptState + resetIncrementalTranscriptState } from './transcript-incremental-reader' import { createTranscriptNativeWatcher } from './transcript-native-watcher' +import { withNativeChatTranscriptWatchDrainAdmission } from './transcript-read-admission' import { readNativeChatTranscriptTailFile } from './transcript-tail-reader' import { nativeChatTurnLifecycleDecoderForAgent } from './transcript-turn-lifecycle' import type { @@ -61,13 +62,7 @@ export async function installTranscriptWatcher( const { onAppend, onInitialSnapshot, onReplace, initialLimit } = args const decodeLifecycle = nativeChatTurnLifecycleDecoderForAgent(args.agent) - const state: IncrementalTranscriptState = { - offset: 0, - pendingChunks: [], - pendingStart: 0, - pendingBytes: 0, - droppingOversizedRecord: false - } + const state = createIncrementalTranscriptState() let watchedVersion: TranscriptFileVersion | null = null let watchedBoundary = '' let initialDrain = true @@ -78,6 +73,7 @@ export async function installTranscriptWatcher( let reading = false let pendingReadRequested = false let rotationRetryCount = 0 + const drainController = new AbortController() function scheduleRotationRetry(): void { if (closed) { @@ -134,7 +130,7 @@ export async function installTranscriptWatcher( scheduleRotationRetry() } - async function drainOnce(): Promise { + async function drainOnceAdmitted(): Promise { const current = await readTranscriptFileVersion(filePath) const currentBoundary = await boundaryFingerprint(filePath, state.offset) if (closed) { @@ -239,6 +235,14 @@ export async function installTranscriptWatcher( await finishSuccessfulDrain(current) } + async function drainOnce(): Promise { + await withNativeChatTranscriptWatchDrainAdmission(drainController.signal, async () => { + if (!closed) { + await drainOnceAdmitted() + } + }) + } + async function drain(): Promise { if (closed) { return @@ -317,6 +321,7 @@ export async function installTranscriptWatcher( return } closed = true + drainController.abort() scheduler.dispose() nativeWatcher.dispose() activeWatcherCount-- diff --git a/src/main/network/macos-system-resolver-health.test.ts b/src/main/network/macos-system-resolver-health.test.ts index 804847e6bbf..62f706a980d 100644 --- a/src/main/network/macos-system-resolver-health.test.ts +++ b/src/main/network/macos-system-resolver-health.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { spawn } from 'node:child_process' import { classifyMacSystemResolverHealth, + MAC_RESOLVER_OUTPUT_MAX_BYTES, readCurrentProcessMacSystemResolverHealth } from './macos-system-resolver-health' import type { ChildProcessWithoutNullStreams } from 'node:child_process' @@ -110,6 +111,44 @@ resolver #1 expect(child.kill).toHaveBeenCalledTimes(1) }) + it('kills scutil and fails open before decoding oversized output', async () => { + mockPlatform('darwin') + const child = createMockScutilProcess() + vi.mocked(spawn).mockReturnValue(child) + + const healthPromise = readCurrentProcessMacSystemResolverHealth() + child.stdout.emit('data', 'x'.repeat(MAC_RESOLVER_OUTPUT_MAX_BYTES + 1)) + + await expect(healthPromise).resolves.toBe('unknown') + expect(child.kill).toHaveBeenCalledTimes(1) + expect(child.stdout.listenerCount('data')).toBe(0) + expect(child.stderr.listenerCount('data')).toBe(0) + }) + + it('accepts the exact combined output cap across many tiny chunks', async () => { + mockPlatform('darwin') + const child = createMockScutilProcess() + vi.mocked(spawn).mockReturnValue(child) + + const healthPromise = readCurrentProcessMacSystemResolverHealth() + const resolverOutput = Buffer.from( + 'DNS configuration\nresolver #1\n nameserver[0] : 1.1.1.1\n' + ) + child.stdout.emit('data', resolverOutput) + const tinyChunk = Buffer.alloc(16, 0x20) + let remainingBytes = MAC_RESOLVER_OUTPUT_MAX_BYTES - resolverOutput.byteLength + while (remainingBytes > 0) { + const chunk = + remainingBytes >= tinyChunk.byteLength ? tinyChunk : tinyChunk.subarray(0, remainingBytes) + child.stderr.emit('data', chunk) + remainingBytes -= chunk.byteLength + } + child.emit('close', 0) + + await expect(healthPromise).resolves.toBe('healthy') + expect(child.kill).not.toHaveBeenCalled() + }) + it('removes scutil listeners when the timeout settles before child close', async () => { vi.useFakeTimers() mockPlatform('darwin') diff --git a/src/main/network/macos-system-resolver-health.ts b/src/main/network/macos-system-resolver-health.ts index 9a3dd6f1eb6..43699c6494b 100644 --- a/src/main/network/macos-system-resolver-health.ts +++ b/src/main/network/macos-system-resolver-health.ts @@ -1,7 +1,9 @@ import { spawn } from 'node:child_process' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import type { SystemResolverHealth } from '../daemon/types' const MAC_RESOLVER_CHECK_TIMEOUT_MS = 1_500 +export const MAC_RESOLVER_OUTPUT_MAX_BYTES = 1024 * 1024 const MAC_NO_DNS_CONFIGURATION_RE = /\bNo DNS configuration available\b/i const MAC_DNS_CONFIGURATION_RE = /^DNS configuration\b/m const MAC_NAMESERVER_RE = /nameserver\[\d+\]\s*:/m @@ -24,18 +26,33 @@ export async function readCurrentProcessMacSystemResolverHealth( } return new Promise((resolve) => { - let stdout = '' - let stderr = '' + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() + let outputBytes = 0 + let outputLimitExceeded = false let settled = false let timer: ReturnType | null = null const child = spawn('/usr/sbin/scutil', ['--dns'], { stdio: ['ignore', 'pipe', 'pipe'] }) - const onStdoutData = (chunk: string): void => { - stdout += chunk + const retainOutput = (target: GrowingByteBuffer, chunk: Buffer | string): void => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, 'utf8') + if (bytes.byteLength > MAC_RESOLVER_OUTPUT_MAX_BYTES - outputBytes) { + outputLimitExceeded = true + stdout.clear() + stderr.clear() + child.kill() + finish() + return + } + target.append(bytes) + outputBytes += bytes.byteLength } - const onStderrData = (chunk: string): void => { - stderr += chunk + const onStdoutData = (chunk: Buffer | string): void => { + retainOutput(stdout, chunk) + } + const onStderrData = (chunk: Buffer | string): void => { + retainOutput(stderr, chunk) } const onAbort = (): void => { child.kill('SIGKILL') @@ -55,7 +72,11 @@ export async function readCurrentProcessMacSystemResolverHealth( child.off('error', finish) child.off('close', finish) signal?.removeEventListener('abort', onAbort) - resolve(classifyMacSystemResolverHealth(`${stdout}\n${stderr}`)) + resolve( + outputLimitExceeded + ? 'unknown' + : classifyMacSystemResolverHealth(`${stdout.takeString()}\n${stderr.takeString()}`) + ) } timer = setTimeout(() => { child.kill() @@ -63,8 +84,6 @@ export async function readCurrentProcessMacSystemResolverHealth( // cap the RPC even if scutil is slow to exit after SIGTERM. finish() }, MAC_RESOLVER_CHECK_TIMEOUT_MS) - child.stdout.setEncoding('utf8') - child.stderr.setEncoding('utf8') child.stdout.on('data', onStdoutData) child.stderr.on('data', onStderrData) child.on('error', finish) diff --git a/src/main/observability/bundle.test.ts b/src/main/observability/bundle.test.ts index 347e1025893..0ba1c9ef342 100644 --- a/src/main/observability/bundle.test.ts +++ b/src/main/observability/bundle.test.ts @@ -1,7 +1,7 @@ // Bundle collection + upload tests. Upload helpers live outside bundle.ts, but // this suite keeps the diagnostic bundle contract in one place. -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { createServer, type RequestListener, type Server } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -344,6 +344,23 @@ describe('bundle — collection', () => { ).not.toThrow() }) + it('skips a sparse source file above the bounded read limit', () => { + writeFileSync(traceFile, '') + truncateSync(traceFile, 50 * 1024 * 1024 + 1) + + const bundle = collectBundle({ + traceFilePath: traceFile, + maxFiles: 10, + appVersion: '1', + platform: 'darwin', + arch: 'arm64', + osRelease: '24', + orcaChannel: 'dev' + }) + + expect(bundle.spanCount).toBe(0) + }) + it('skips valid JSON lines that are not span objects without throwing', () => { writeFileSync( traceFile, diff --git a/src/main/observability/bundle.ts b/src/main/observability/bundle.ts index 750f41fead9..12328eea5f4 100644 --- a/src/main/observability/bundle.ts +++ b/src/main/observability/bundle.ts @@ -7,12 +7,13 @@ // upload body-size cap, token-handling discipline). import { randomBytes } from 'node:crypto' -import { readFileSync, statSync } from 'node:fs' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import { MAX_BUNDLE_BYTES } from './diagnostic-bundle-limits' import { listRotatedFiles } from './local-file-sink' import { redactValue } from './redactor' const DEFAULT_LOOKBACK_MINUTES = 30 +const MAX_BUNDLE_SOURCE_FILE_BYTES = 50 * 1024 * 1024 export type CollectBundleOptions = { readonly traceFilePath: string @@ -103,12 +104,7 @@ export function collectBundle(opts: CollectBundleOptions): CollectedBundle { outer: for (const file of files) { let text: string try { - // stat first: the sink caps at 10 MB/file, so a tampered oversize file could panic-allocate on read. - const size = statSync(file).size - if (size > 50 * 1024 * 1024) { - continue - } - text = readFileSync(file, 'utf8') + text = readNodeFileSyncWithinLimit(file, MAX_BUNDLE_SOURCE_FILE_BYTES).buffer.toString('utf8') } catch { continue } diff --git a/src/main/observability/diagnostic-upload-http.ts b/src/main/observability/diagnostic-upload-http.ts index 17e6b186085..bb5823d28b4 100644 --- a/src/main/observability/diagnostic-upload-http.ts +++ b/src/main/observability/diagnostic-upload-http.ts @@ -1,6 +1,7 @@ import { request as httpRequest, type ClientRequest, type IncomingMessage } from 'node:http' import { request as httpsRequest } from 'node:https' import { URL } from 'node:url' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' export const MAX_RESPONSE_BYTES = 1024 * 1024 @@ -40,7 +41,7 @@ function postRaw( let settled = false let req: ClientRequest | null = null let res: IncomingMessage | null = null - const chunks: Buffer[] = [] + const responseBody = new GrowingByteBuffer() let responseBytes = 0 function cleanupListeners(): void { req?.off('error', onRequestError) @@ -54,6 +55,7 @@ function postRaw( return } settled = true + responseBody.clear() cleanupListeners() resolve(value) } @@ -65,6 +67,7 @@ function postRaw( return } settled = true + responseBody.clear() if (options.destroyRequest) { req?.destroy() } @@ -85,11 +88,11 @@ function postRaw( }) return } - chunks.push(chunk) + responseBody.append(chunk) } function onResponseEnd(): void { const status = res?.statusCode ?? 0 - const text = Buffer.concat(chunks).toString('utf8') + const text = responseBody.takeString() if (status >= 200 && status < 300) { try { resolveOnce(text.length > 0 ? JSON.parse(text) : {}) diff --git a/src/main/observability/local-file-sink.test.ts b/src/main/observability/local-file-sink.test.ts index 4420c043288..2f9d2bd5f28 100644 --- a/src/main/observability/local-file-sink.test.ts +++ b/src/main/observability/local-file-sink.test.ts @@ -12,7 +12,12 @@ import { import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { createLocalFileSink, getRotatedFamilySize, listRotatedFiles } from './local-file-sink' +import { + createLocalFileSink, + DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD, + getRotatedFamilySize, + listRotatedFiles +} from './local-file-sink' let dir: string @@ -67,6 +72,50 @@ describe('local-file-sink — basic write', () => { sink.close() }) + it('flushes at the default byte threshold before the count threshold', () => { + const file = join(dir, 'test.ndjson') + const emptyLineBytes = Buffer.byteLength(`${JSON.stringify({ payload: '' })}\n`, 'utf8') + const record = { + payload: 'x'.repeat(DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD - emptyLineBytes) + } + const sink = createLocalFileSink({ + filePath: file, + batchWindowMs: 100_000, + flushBufferThreshold: 10_000 + }) + + sink.push(record) + + expect(statSync(file).size).toBe(DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD) + sink.close() + }) + + it('flushes a prior byte batch without reordering the next record', () => { + const file = join(dir, 'test.ndjson') + const first = { i: 1, payload: '😀'.repeat(8) } + const second = { i: 2, payload: '😀'.repeat(8) } + const firstLineBytes = Buffer.byteLength(`${JSON.stringify(first)}\n`, 'utf8') + const secondLineBytes = Buffer.byteLength(`${JSON.stringify(second)}\n`, 'utf8') + const sink = createLocalFileSink({ + filePath: file, + batchWindowMs: 100_000, + flushBufferThreshold: 100, + flushBufferByteThreshold: firstLineBytes + secondLineBytes - 1 + }) + + sink.push(first) + expect(statSync(file).size).toBe(0) + sink.push(second) + expect(statSync(file).size).toBe(firstLineBytes) + sink.close() + + const records = readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { i: number }) + expect(records.map((record) => record.i)).toEqual([1, 2]) + }) + it('creates trace directories and files with private POSIX permissions', () => { if (process.platform === 'win32') { return @@ -169,6 +218,31 @@ describe('local-file-sink — rotation', () => { expect(getRotatedFamilySize(file, 3)).toBeLessThanOrEqual(100) }) + it('stops serializing an oversized record before visiting the whole value', () => { + const file = join(dir, 'test.ndjson') + const sink = createLocalFileSink({ + filePath: file, + maxBytes: 100, + batchWindowMs: 100_000, + flushBufferThreshold: 1 + }) + let visits = 0 + const payload = Array.from({ length: 10_000 }, () => ({ + toJSON: () => { + visits += 1 + return 'value' + } + })) + + sink.push({ payload }) + sink.push({ ok: true }) + sink.close() + + expect(visits).toBeLessThan(payload.length) + const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean) + expect(lines.map((line) => JSON.parse(line))).toEqual([{ ok: true }]) + }) + it('splits an oversized buffered batch instead of dropping valid records', () => { const file = join(dir, 'test.ndjson') const sink = createLocalFileSink({ diff --git a/src/main/observability/local-file-sink.ts b/src/main/observability/local-file-sink.ts index b2a74bbbb40..853b016ada3 100644 --- a/src/main/observability/local-file-sink.ts +++ b/src/main/observability/local-file-sink.ts @@ -18,8 +18,10 @@ import { writeSync } from 'node:fs' import { dirname } from 'node:path' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' const DEFAULT_FLUSH_BUFFER_THRESHOLD = 32 +export const DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD = 1024 * 1024 export const DEFAULT_MAX_BYTES = 10 * 1024 * 1024 // 10 MB export const DEFAULT_MAX_FILES = 10 export const DEFAULT_BATCH_WINDOW_MS = 200 @@ -32,6 +34,7 @@ export type LocalFileSinkOptions = { readonly maxFiles?: number readonly batchWindowMs?: number readonly flushBufferThreshold?: number + readonly flushBufferByteThreshold?: number } export type LocalFileSink = { @@ -66,6 +69,14 @@ export function createLocalFileSink(opts: LocalFileSinkOptions): LocalFileSink { const maxFiles = opts.maxFiles ?? DEFAULT_MAX_FILES const batchWindowMs = opts.batchWindowMs ?? DEFAULT_BATCH_WINDOW_MS const flushThreshold = opts.flushBufferThreshold ?? DEFAULT_FLUSH_BUFFER_THRESHOLD + const requestedFlushByteThreshold = + opts.flushBufferByteThreshold ?? DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD + const flushByteThreshold = Number.isFinite(requestedFlushByteThreshold) + ? Math.max( + 1, + Math.min(DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD, Math.trunc(requestedFlushByteThreshold)) + ) + : DEFAULT_FLUSH_BUFFER_BYTE_THRESHOLD // Traces hold paths and crash context; lock to current-user regardless of umask. const traceDirectory = dirname(filePath) @@ -78,6 +89,7 @@ export function createLocalFileSink(opts: LocalFileSinkOptions): LocalFileSink { let currentBytes: number = safeFstatSize(fd) let buffer: string[] = [] + let bufferBytes = 0 let timer: NodeJS.Timeout | null = null let closed = false @@ -135,6 +147,7 @@ export function createLocalFileSink(opts: LocalFileSinkOptions): LocalFileSink { } const lines = buffer buffer = [] + bufferBytes = 0 let pendingChunk: string[] = [] let pendingChunkBytes = 0 @@ -210,14 +223,21 @@ export function createLocalFileSink(opts: LocalFileSinkOptions): LocalFileSink { return } let line: string + let lineBytes: number try { - line = `${JSON.stringify(record)}\n` + const serialized = stringifyJsonWithinByteLimit(record, Math.max(0, maxBytes - 1)) + line = `${serialized.serialized}\n` + lineBytes = serialized.byteLength + 1 } catch { // Redactor handles cycles upstream; a throw here means pre-redact data slipped in — drop rather than crash (best-effort). return } + if (bufferBytes > 0 && bufferBytes + lineBytes > flushByteThreshold) { + flushBuffer() + } buffer.push(line) - if (buffer.length >= flushThreshold) { + bufferBytes += lineBytes + if (buffer.length >= flushThreshold || bufferBytes >= flushByteThreshold) { flushBuffer() } else { ensureTimer() diff --git a/src/main/opencode-usage/scanner.ts b/src/main/opencode-usage/scanner.ts index 57a95df3a62..67d91e67f77 100644 --- a/src/main/opencode-usage/scanner.ts +++ b/src/main/opencode-usage/scanner.ts @@ -1,13 +1,15 @@ /* eslint-disable max-lines -- Why: OpenCode usage analytics need to normalize multiple local DB schema generations, attribute worktrees, and build persisted projections in one auditable pipeline. */ import { existsSync } from 'node:fs' -import { readdir, realpath, stat } from 'node:fs/promises' +import { realpath, stat } from 'node:fs/promises' import { homedir } from 'node:os' import { basename, isAbsolute, join, posix, win32 } from 'node:path' import type { Repo } from '../../shared/types' import { areWorktreePathsEqual } from '../ipc/worktree-logic' import Database from '../sqlite/sync-database' -import { columnExists, tableExists } from './schema-helpers' +import { listOpenCodeDatabaseFiles } from '../opencode/opencode-database-files' import { canonicalizeUsageWorktreePaths } from '../usage-worktree-canonicalizer' +import { getUsageHistoryRetainedBytes, UsageHistoryScanBudget } from '../usage-history-scan-budget' +import { iterateOpenCodeUsageRows, type OpenCodeUsageRow } from './sqlite-usage-row-stream' import type { OpenCodeUsageAttributedEvent, OpenCodeUsageDailyAggregate, @@ -27,34 +29,6 @@ export type OpenCodeUsageWorktreeRef = { displayName: string } -type OpenCodeUsageRow = { - id: string - session_id: string - time_created: number - time_updated: number | null - data: string - directory: string | null - title: string | null - worktree: string | null - session_model: string | null -} - -type OpenCodeSessionUsageRow = { - id: string - session_id: string - time_created: number - time_updated: number | null - directory: string | null - title: string | null - worktree: string | null - session_model: string | null - cost: number - tokens_input: number - tokens_output: number - tokens_reasoning: number - tokens_cache_read: number -} - const YIELD_EVERY_DATABASES = 2 function ensureNumber(value: unknown): number { @@ -111,11 +85,7 @@ export async function listOpenCodeDatabases(): Promise { } try { - const entries = await readdir(getOpenCodeDataDirectory(), { withFileTypes: true }) - return entries - .filter((entry) => entry.isFile() && /^opencode(?:-[A-Za-z0-9_.-]+)?\.db$/.test(entry.name)) - .map((entry) => join(getOpenCodeDataDirectory(), entry.name)) - .sort() + return (await listOpenCodeDatabaseFiles(getOpenCodeDataDirectory())).paths } catch { return [] } @@ -148,140 +118,6 @@ async function yieldToEventLoop(): Promise { await new Promise((resolve) => setTimeout(resolve, 0)) } -function getProjectJoin(db: Database.Database): string { - return tableExists(db, 'project') && columnExists(db, 'session', 'project_id') - ? 'LEFT JOIN project p ON p.id = s.project_id' - : 'LEFT JOIN (SELECT NULL AS id, NULL AS worktree) p ON 1 = 0' -} - -function getSessionModelSelect(db: Database.Database): string { - return columnExists(db, 'session', 'model') ? 's.model AS session_model' : 'NULL AS session_model' -} - -function getAssistantSessionMessageCount(db: Database.Database): number { - if (!tableExists(db, 'session_message')) { - return 0 - } - const assistantPredicate = columnExists(db, 'session_message', 'type') - ? "type = 'assistant' AND json_extract(data, '$.tokens.input') IS NOT NULL" - : "json_extract(data, '$.tokens.input') IS NOT NULL" - const row = db - .prepare(`SELECT COUNT(*) AS count FROM session_message WHERE ${assistantPredicate}`) - .get() as { count?: number } | undefined - return row?.count ?? 0 -} - -function canReadSessionUsageRows(db: Database.Database): boolean { - if (!tableExists(db, 'session')) { - return false - } - return ['cost', 'tokens_input', 'tokens_output', 'tokens_reasoning', 'tokens_cache_read'].every( - (columnName) => columnExists(db, 'session', columnName) - ) -} - -function getSessionUsageRowCount(db: Database.Database): number { - if (!canReadSessionUsageRows(db)) { - return 0 - } - const row = db - .prepare( - `SELECT COUNT(*) AS count - FROM session - WHERE tokens_input + tokens_output + tokens_reasoning + tokens_cache_read > 0` - ) - .get() as { count?: number } | undefined - return row?.count ?? 0 -} - -function selectSessionUsageRows(db: Database.Database): OpenCodeUsageRow[] { - const projectJoin = getProjectJoin(db) - const sessionModelSelect = getSessionModelSelect(db) - const rows = db - .prepare( - `SELECT s.id, s.id AS session_id, s.time_created, s.time_updated, - s.directory, s.title, p.worktree, ${sessionModelSelect}, - s.cost, s.tokens_input, s.tokens_output, s.tokens_reasoning, s.tokens_cache_read - FROM session s - ${projectJoin} - WHERE s.tokens_input + s.tokens_output + s.tokens_reasoning + s.tokens_cache_read > 0 - ORDER BY s.time_created, s.id` - ) - .all() as OpenCodeSessionUsageRow[] - - return rows.map((row) => ({ - id: row.id, - session_id: row.session_id, - time_created: row.time_created, - time_updated: row.time_updated, - directory: row.directory, - title: row.title, - worktree: row.worktree, - session_model: row.session_model, - data: JSON.stringify({ - cost: row.cost, - tokens: { - input: row.tokens_input, - output: row.tokens_output, - reasoning: row.tokens_reasoning, - total: row.tokens_input + row.tokens_output + row.tokens_reasoning, - cache: { - read: row.tokens_cache_read, - write: 0 - } - } - }) - })) -} - -function selectUsageRows(db: Database.Database): OpenCodeUsageRow[] { - if (!tableExists(db, 'session')) { - return [] - } - - // Why: newer OpenCode DBs maintain session-level token/cost totals. Reading - // one aggregate row per session is faster than parsing every message blob. - if (getSessionUsageRowCount(db) > 0) { - return selectSessionUsageRows(db) - } - - const projectJoin = getProjectJoin(db) - const sessionModelSelect = getSessionModelSelect(db) - - if (getAssistantSessionMessageCount(db) > 0) { - const assistantPredicate = columnExists(db, 'session_message', 'type') - ? "sm.type = 'assistant'" - : "json_extract(sm.data, '$.tokens.input') IS NOT NULL" - return db - .prepare( - `SELECT sm.id, sm.session_id, sm.time_created, sm.time_updated, sm.data, - s.directory, s.title, p.worktree, ${sessionModelSelect} - FROM session_message sm - JOIN session s ON s.id = sm.session_id - ${projectJoin} - WHERE ${assistantPredicate} - ORDER BY sm.time_created, sm.id` - ) - .all() as OpenCodeUsageRow[] - } - - if (!tableExists(db, 'message')) { - return [] - } - - return db - .prepare( - `SELECT m.id, m.session_id, m.time_created, m.time_updated, m.data, - s.directory, s.title, p.worktree, ${sessionModelSelect} - FROM message m - JOIN session s ON s.id = m.session_id - ${projectJoin} - WHERE json_extract(m.data, '$.role') = 'assistant' - ORDER BY m.time_created, m.id` - ) - .all() as OpenCodeUsageRow[] -} - function parseJsonObject(value: unknown): Record | null { if (typeof value === 'object' && value !== null && !Array.isArray(value)) { return value as Record @@ -841,11 +677,71 @@ function mergeDailyAggregates( } } +function claimOpenCodeUsageProjection( + budget: UsageHistoryScanBudget, + sessions: readonly OpenCodeUsageSession[], + dailyAggregates: readonly OpenCodeUsageDailyAggregate[] +): void { + for (const session of sessions) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + session.sessionId, + session.firstTimestamp, + session.lastTimestamp, + session.primaryModel, + session.primaryProjectLabel, + session.primaryWorktreeId, + session.primaryRepoId + ]) + ) + for (const location of session.locationBreakdown) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + location.locationKey, + location.projectLabel, + location.repoId, + location.worktreeId + ]) + ) + } + for (const model of session.modelBreakdown) { + budget.claimProjection(getUsageHistoryRetainedBytes([model.modelKey, model.modelLabel])) + } + for (const locationModel of session.locationModelBreakdown) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + locationModel.locationKey, + locationModel.modelKey, + locationModel.modelLabel, + locationModel.repoId, + locationModel.worktreeId + ]) + ) + } + } + for (const daily of dailyAggregates) { + budget.claimProjection( + getUsageHistoryRetainedBytes([ + daily.day, + daily.model, + daily.projectKey, + daily.projectLabel, + daily.repoId, + daily.worktreeId + ]) + ) + } +} + export async function parseOpenCodeUsageDatabase( dbPath: string, worktrees: (OpenCodeUsageWorktreeRef & { canonicalPath: string })[], - options: { claimSession?: (sessionId: string) => boolean } = {} + options: { + claimSession?: (sessionId: string) => boolean + budget?: UsageHistoryScanBudget + } = {} ): Promise { + const budget = options.budget ?? new UsageHistoryScanBudget() const processedDatabase = await getProcessedDatabaseInfo(dbPath) const db = new Database(dbPath, { readonly: true, fileMustExist: true }) try { @@ -853,7 +749,7 @@ export async function parseOpenCodeUsageDatabase( const events: OpenCodeUsageAttributedEvent[] = [] const claimedBySessionId = new Map() let hasDeferredClaims = false - for (const row of selectUsageRows(db)) { + for (const row of iterateOpenCodeUsageRows(db)) { const parsed = parseOpenCodeUsageRow(row) if (!parsed) { continue @@ -862,6 +758,7 @@ export async function parseOpenCodeUsageDatabase( // each session must be counted from exactly one database (#8006). let owned = claimedBySessionId.get(parsed.sessionId) if (owned === undefined) { + budget.claimOwnershipKey(parsed.sessionId) owned = options.claimSession ? options.claimSession(parsed.sessionId) : true claimedBySessionId.set(parsed.sessionId, owned) } @@ -871,12 +768,27 @@ export async function parseOpenCodeUsageDatabase( } const attributed = await attributeOpenCodeUsageEvent(parsed, worktrees) if (attributed) { + budget.claimRecord( + getUsageHistoryRetainedBytes([ + attributed.sessionId, + attributed.timestamp, + attributed.cwd, + attributed.model, + attributed.day, + attributed.projectKey, + attributed.projectLabel, + attributed.repoId, + attributed.worktreeId + ]) + ) events.push(attributed) } } + const aggregates = aggregateOpenCodeUsage(events) + claimOpenCodeUsageProjection(budget, aggregates.sessions, aggregates.dailyAggregates) return { ...processedDatabase, - ...aggregateOpenCodeUsage(events), + ...aggregates, ownedSessionIds: [...claimedBySessionId.entries()] .filter(([, owned]) => owned) .map(([sessionId]) => sessionId), @@ -889,12 +801,14 @@ export async function parseOpenCodeUsageDatabase( export async function scanOpenCodeUsageDatabases( worktrees: OpenCodeUsageWorktreeRef[], - previousProcessedDatabases: OpenCodeUsagePersistedDatabase[] + previousProcessedDatabases: OpenCodeUsagePersistedDatabase[], + options: { budget?: UsageHistoryScanBudget } = {} ): Promise<{ processedDatabases: OpenCodeUsagePersistedDatabase[] sessions: OpenCodeUsageSession[] dailyAggregates: OpenCodeUsageDailyAggregate[] }> { + const budget = options.budget ?? new UsageHistoryScanBudget() const dbPaths = await listOpenCodeDatabases() const previousByPath = new Map( previousProcessedDatabases.map((database) => [database.path, database]) @@ -965,7 +879,12 @@ export async function scanOpenCodeUsageDatabases( const sessionOwnerById = new Map() for (const dbPath of [...reusedByPath.keys()].sort(compareOpenCodeClaimPriority)) { const previous = reusedByPath.get(dbPath) + for (const session of previous?.sessions ?? []) { + budget.claimRecords(session.eventCount) + } + claimOpenCodeUsageProjection(budget, previous?.sessions ?? [], previous?.dailyAggregates ?? []) for (const sessionId of previous?.ownedSessionIds ?? []) { + budget.claimOwnershipKey(sessionId) if (!sessionOwnerById.has(sessionId)) { sessionOwnerById.set(sessionId, dbPath) } @@ -976,6 +895,7 @@ export async function scanOpenCodeUsageDatabases( const orderedPathsToParse = [...pathsToParse].sort(compareOpenCodeClaimPriority) for (const [index, dbPath] of orderedPathsToParse.entries()) { const processed = await parseOpenCodeUsageDatabase(dbPath, worktreesWithCanonicalPaths, { + budget, claimSession: (sessionId) => { const owner = sessionOwnerById.get(sessionId) if (owner !== undefined && owner !== dbPath) { diff --git a/src/main/opencode-usage/sqlite-usage-row-stream.test.ts b/src/main/opencode-usage/sqlite-usage-row-stream.test.ts new file mode 100644 index 00000000000..9244d6728fa --- /dev/null +++ b/src/main/opencode-usage/sqlite-usage-row-stream.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../sqlite/sync-database' +import { UsageHistoryScanBudget, UsageHistoryScanCapacityError } from '../usage-history-scan-budget' +import { parseOpenCodeUsageDatabase } from './scanner' +import { + iterateOpenCodeUsageRows, + OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES +} from './sqlite-usage-row-stream' + +const tempDirs: string[] = [] + +function createDatabase(): { db: Database.Database; path: string } { + const dir = mkdtempSync(join(tmpdir(), 'orca-opencode-retention-')) + tempDirs.push(dir) + const path = join(dir, 'opencode.db') + const db = new Database(path) + db.exec(` + CREATE TABLE session ( + id TEXT PRIMARY KEY, + directory TEXT, + title TEXT, + cost REAL, + tokens_input INTEGER, + tokens_output INTEGER, + tokens_reasoning INTEGER, + tokens_cache_read INTEGER, + time_created INTEGER, + time_updated INTEGER + ); + `) + return { db, path } +} + +function insertSession(db: Database.Database, id: string, title = '', inputTokens = 1): void { + db.prepare( + `INSERT INTO session ( + id, directory, title, cost, + tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, + time_created, time_updated + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run(id, null, title, 0, inputTokens, 0, 0, 0, 1_777_777_700_000, 1_777_777_800_000) +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +describe('OpenCode SQLite usage row retention', () => { + it('preserves an exact-boundary row', () => { + const { db } = createDatabase() + const title = 't'.repeat(OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES - 2) + insertSession(db, 's', title) + + const rows = [...iterateOpenCodeUsageRows(db)] + + expect(rows).toHaveLength(1) + expect(rows[0]?.id).toBe('s') + expect(rows[0]?.title).toBe(title) + db.close() + }) + + it('rejects a row one byte above the retained-text limit', () => { + const { db } = createDatabase() + insertSession(db, 's', 't'.repeat(OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES - 1)) + + expect(() => [...iterateOpenCodeUsageRows(db)]).toThrowError( + new UsageHistoryScanCapacityError('retainedBytes', OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES) + ) + db.close() + }) + + it('rejects oversized legacy JSON before asking SQLite to parse it', () => { + const { db } = createDatabase() + insertSession(db, 's', '', 0) + db.exec(` + CREATE TABLE message ( + id TEXT PRIMARY KEY, + session_id TEXT, + time_created INTEGER, + time_updated INTEGER, + data TEXT + ); + `) + const data = `{"role":"assistant","padding":"${'x'.repeat( + OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES + )}"}` + db.prepare( + 'INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)' + ).run('m', 's', 1, 1, data) + + expect(() => [...iterateOpenCodeUsageRows(db)]).toThrowError( + new UsageHistoryScanCapacityError('retainedBytes', OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES) + ) + db.close() + }) + + it('shares the history record budget across streamed rows', async () => { + const { db, path } = createDatabase() + insertSession(db, 'session-1') + insertSession(db, 'session-2') + db.close() + const budget = new UsageHistoryScanBudget({ + records: 1, + ownershipKeys: 4, + retainedBytes: 16 * 1024 * 1024 + }) + + await expect(parseOpenCodeUsageDatabase(path, [], { budget })).rejects.toMatchObject({ + resource: 'records', + limit: 1 + }) + }) +}) diff --git a/src/main/opencode-usage/sqlite-usage-row-stream.ts b/src/main/opencode-usage/sqlite-usage-row-stream.ts new file mode 100644 index 00000000000..9c7cbca46fd --- /dev/null +++ b/src/main/opencode-usage/sqlite-usage-row-stream.ts @@ -0,0 +1,238 @@ +import type Database from '../sqlite/sync-database' +import { UsageHistoryScanCapacityError } from '../usage-history-scan-budget' +import { columnExists, tableExists } from './schema-helpers' + +export const OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES = 4 * 1024 * 1024 + +export type OpenCodeUsageRow = { + id: string + session_id: string + time_created: number + time_updated: number | null + data: string + directory: string | null + title: string | null + worktree: string | null + session_model: string | null +} + +type OpenCodeSessionUsageRow = Omit & { + cost: number + tokens_input: number + tokens_output: number + tokens_reasoning: number + tokens_cache_read: number +} + +function retainedTextBytesSql(expressions: readonly string[]): string { + return expressions + .map((expression) => `length(CAST(COALESCE(${expression}, '') AS BLOB))`) + .join(' + ') +} + +function assertNoOversizedRows( + db: Database.Database, + fromSql: string, + whereSql: string, + rowBytesSql: string +): void { + const oversized = db + .prepare( + `SELECT 1 + ${fromSql} + WHERE ${whereSql} + AND (${rowBytesSql}) > ${OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES} + LIMIT 1` + ) + .get() + if (oversized) { + throw new UsageHistoryScanCapacityError('retainedBytes', OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES) + } +} + +function assertNoOversizedJsonCandidates( + db: Database.Database, + table: 'message' | 'session_message', + candidateWhereSql: string +): void { + const oversized = db + .prepare( + `SELECT 1 + FROM ${table} + WHERE ${candidateWhereSql} + AND length(CAST(COALESCE(data, '') AS BLOB)) > ${OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES} + LIMIT 1` + ) + .get() + if (oversized) { + throw new UsageHistoryScanCapacityError('retainedBytes', OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES) + } +} + +function boundedJsonPredicate(dataExpression: string, predicate: string): string { + return `CASE + WHEN length(CAST(COALESCE(${dataExpression}, '') AS BLOB)) <= ${OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES} + THEN (${predicate}) + ELSE 0 + END` +} + +function getProjectJoin(db: Database.Database): string { + return tableExists(db, 'project') && columnExists(db, 'session', 'project_id') + ? 'LEFT JOIN project p ON p.id = s.project_id' + : 'LEFT JOIN (SELECT NULL AS id, NULL AS worktree) p ON 1 = 0' +} + +function getSessionModelSelect(db: Database.Database): string { + return columnExists(db, 'session', 'model') ? 's.model AS session_model' : 'NULL AS session_model' +} + +function getAssistantSessionMessageCount(db: Database.Database): number { + if (!tableExists(db, 'session_message')) { + return 0 + } + const hasType = columnExists(db, 'session_message', 'type') + assertNoOversizedJsonCandidates(db, 'session_message', hasType ? "type = 'assistant'" : '1 = 1') + const jsonPredicate = boundedJsonPredicate( + 'data', + "json_extract(data, '$.tokens.input') IS NOT NULL" + ) + const assistantPredicate = hasType ? `type = 'assistant' AND ${jsonPredicate}` : jsonPredicate + const row = db + .prepare(`SELECT COUNT(*) AS count FROM session_message WHERE ${assistantPredicate}`) + .get() as { count?: number } | undefined + return row?.count ?? 0 +} + +function canReadSessionUsageRows(db: Database.Database): boolean { + if (!tableExists(db, 'session')) { + return false + } + return ['cost', 'tokens_input', 'tokens_output', 'tokens_reasoning', 'tokens_cache_read'].every( + (columnName) => columnExists(db, 'session', columnName) + ) +} + +function getSessionUsageRowCount(db: Database.Database): number { + if (!canReadSessionUsageRows(db)) { + return 0 + } + const row = db + .prepare( + `SELECT COUNT(*) AS count + FROM session + WHERE tokens_input + tokens_output + tokens_reasoning + tokens_cache_read > 0` + ) + .get() as { count?: number } | undefined + return row?.count ?? 0 +} + +function* iterateSessionUsageRows(db: Database.Database): Iterable { + const projectJoin = getProjectJoin(db) + const sessionModelSelect = getSessionModelSelect(db) + const fromSql = `FROM session s ${projectJoin}` + const whereSql = 's.tokens_input + s.tokens_output + s.tokens_reasoning + s.tokens_cache_read > 0' + const rowBytesSql = retainedTextBytesSql([ + 's.id', + 's.id', + 's.directory', + 's.title', + 'p.worktree', + columnExists(db, 'session', 'model') ? 's.model' : "''" + ]) + assertNoOversizedRows(db, fromSql, whereSql, rowBytesSql) + + const rows = db + .prepare( + `SELECT s.id, s.id AS session_id, s.time_created, s.time_updated, + s.directory, s.title, p.worktree, ${sessionModelSelect}, + s.cost, s.tokens_input, s.tokens_output, s.tokens_reasoning, s.tokens_cache_read + ${fromSql} + WHERE ${whereSql} + AND (${rowBytesSql}) <= ${OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES} + ORDER BY s.time_created, s.id` + ) + .iterate() as Iterable + + for (const row of rows) { + yield { + id: row.id, + session_id: row.session_id, + time_created: row.time_created, + time_updated: row.time_updated, + directory: row.directory, + title: row.title, + worktree: row.worktree, + session_model: row.session_model, + data: JSON.stringify({ + cost: row.cost, + tokens: { + input: row.tokens_input, + output: row.tokens_output, + reasoning: row.tokens_reasoning, + total: row.tokens_input + row.tokens_output + row.tokens_reasoning, + cache: { read: row.tokens_cache_read, write: 0 } + } + }) + } + } +} + +function iterateMessageUsageRows( + db: Database.Database, + table: 'message' | 'session_message', + assistantPredicate: string +): Iterable { + const alias = table === 'message' ? 'm' : 'sm' + const projectJoin = getProjectJoin(db) + const sessionModelSelect = getSessionModelSelect(db) + const fromSql = `FROM ${table} ${alias} JOIN session s ON s.id = ${alias}.session_id ${projectJoin}` + const rowBytesSql = retainedTextBytesSql([ + `${alias}.id`, + `${alias}.session_id`, + `${alias}.data`, + 's.directory', + 's.title', + 'p.worktree', + columnExists(db, 'session', 'model') ? 's.model' : "''" + ]) + assertNoOversizedRows(db, fromSql, assistantPredicate, rowBytesSql) + + return db + .prepare( + `SELECT ${alias}.id, ${alias}.session_id, ${alias}.time_created, + ${alias}.time_updated, ${alias}.data, + s.directory, s.title, p.worktree, ${sessionModelSelect} + ${fromSql} + WHERE ${assistantPredicate} + AND (${rowBytesSql}) <= ${OPENCODE_USAGE_SQLITE_ROW_MAX_BYTES} + ORDER BY ${alias}.time_created, ${alias}.id` + ) + .iterate() as Iterable +} + +export function iterateOpenCodeUsageRows(db: Database.Database): Iterable { + if (!tableExists(db, 'session')) { + return [] + } + if (getSessionUsageRowCount(db) > 0) { + return iterateSessionUsageRows(db) + } + + if (getAssistantSessionMessageCount(db) > 0) { + const assistantPredicate = columnExists(db, 'session_message', 'type') + ? "sm.type = 'assistant'" + : boundedJsonPredicate('sm.data', "json_extract(sm.data, '$.tokens.input') IS NOT NULL") + return iterateMessageUsageRows(db, 'session_message', assistantPredicate) + } + + if (!tableExists(db, 'message')) { + return [] + } + assertNoOversizedJsonCandidates(db, 'message', '1 = 1') + return iterateMessageUsageRows( + db, + 'message', + boundedJsonPredicate('m.data', "json_extract(m.data, '$.role') = 'assistant'") + ) +} diff --git a/src/main/opencode-usage/store.ts b/src/main/opencode-usage/store.ts index b926f32651c..e2966272a21 100644 --- a/src/main/opencode-usage/store.ts +++ b/src/main/opencode-usage/store.ts @@ -1,7 +1,6 @@ /* eslint-disable max-lines -- Why: this store owns OpenCode analytics persistence, scan policy, and renderer query semantics. Keeping range/scope queries next to scan persistence prevents UI totals from drifting from the SQLite projection. */ import { app } from 'electron' -import { dirname, join } from 'node:path' -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' import type { OpenCodeUsageBreakdownKind, OpenCodeUsageBreakdownRow, @@ -14,6 +13,10 @@ import type { OpenCodeUsageSummary } from '../../shared/opencode-usage-types' import type { Store } from '../persistence' +import { + readUsageProjectionStateFile, + writeUsageProjectionStateFileWithRecovery +} from '../usage-projection-state-file' import { loadKnownUsageWorktreesByRepo, type UsageWorktreeRef } from '../usage-worktree-metadata' import type { OpenCodeUsageDailyAggregate, OpenCodeUsagePersistedState } from './types' import { createWorktreeRefs, scanOpenCodeUsageDatabases } from './scanner' @@ -161,10 +164,11 @@ export class OpenCodeUsageStore { private load(): OpenCodeUsagePersistedState { try { const usageFile = getOpenCodeUsageFile() - if (!existsSync(usageFile)) { + const raw = readUsageProjectionStateFile(usageFile) + if (raw === null) { return getDefaultState() } - const parsed = JSON.parse(readFileSync(usageFile, 'utf-8')) as OpenCodeUsagePersistedState + const parsed = JSON.parse(raw) as OpenCodeUsagePersistedState return normalizePersistedState({ ...getDefaultState(), ...parsed, @@ -181,13 +185,12 @@ export class OpenCodeUsageStore { private writeToDisk(): void { const usageFile = getOpenCodeUsageFile() - const dir = dirname(usageFile) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - const tmpFile = `${usageFile}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp` - writeFileSync(tmpFile, JSON.stringify(this.state, null, 2), 'utf-8') - renameSync(tmpFile, usageFile) + this.state = writeUsageProjectionStateFileWithRecovery(usageFile, this.state, (error) => { + const reset = getDefaultState() + reset.scanState.enabled = this.state.scanState.enabled + reset.scanState.lastScanError = error.message + return reset + }) } async setEnabled(enabled: boolean): Promise { diff --git a/src/main/opencode/config-overlay-manifest.test.ts b/src/main/opencode/config-overlay-manifest.test.ts new file mode 100644 index 00000000000..0818a98cf94 --- /dev/null +++ b/src/main/opencode/config-overlay-manifest.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest' +import { + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES, + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES, + ConfigOverlayCapacityError +} from '../pty/config-overlay-mirroring' +import { _configOverlayManifestInternals } from './config-overlay-manifest' + +const { parseOverlayManifest } = _configOverlayManifestInternals + +describe('OpenCode overlay manifest bounds', () => { + it('accepts the exact entry limit and rejects one more retained entry', () => { + const exact = parseOverlayManifest( + JSON.stringify({ + topLevelEntries: Array.from({ length: CONFIG_OVERLAY_MAX_SOURCE_ENTRIES }, () => 'same'), + pluginEntries: [] + }) + ) + expect(exact.topLevelEntries).toHaveLength(CONFIG_OVERLAY_MAX_SOURCE_ENTRIES) + + expect(() => + parseOverlayManifest( + JSON.stringify({ + topLevelEntries: Array.from( + { length: CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + 1 }, + () => 'same' + ), + pluginEntries: [] + }) + ) + ).toThrowError( + new ConfigOverlayCapacityError( + 'entries', + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + 1, + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + ) + ) + }) + + it('accepts the exact encoded-name budget and rejects the next name', () => { + const name = 'a'.repeat(4_094) + const names = Array.from({ length: CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES / 4_096 }, () => name) + expect( + parseOverlayManifest(JSON.stringify({ topLevelEntries: names })).topLevelEntries + ).toHaveLength(names.length) + + expect(() => + parseOverlayManifest(JSON.stringify({ topLevelEntries: [...names, 'a'] })) + ).toThrowError( + new ConfigOverlayCapacityError( + 'retained-name-bytes', + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES + 3, + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES + ) + ) + }) + + it('never authorizes cleanup of reserved overlay-owned paths', () => { + expect( + parseOverlayManifest( + JSON.stringify({ + topLevelEntries: ['plugins', '.orca-opencode-overlay-manifest.json', 'auth.json'], + pluginEntries: ['orca-opencode-status.js', 'user-plugin.js'] + }) + ) + ).toEqual({ + topLevelEntries: ['auth.json'], + pluginEntries: ['user-plugin.js'] + }) + }) +}) diff --git a/src/main/opencode/config-overlay-manifest.ts b/src/main/opencode/config-overlay-manifest.ts new file mode 100644 index 00000000000..6d9f2ffd9e2 --- /dev/null +++ b/src/main/opencode/config-overlay-manifest.ts @@ -0,0 +1,164 @@ +import { unlinkSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' +import { + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES, + ConfigOverlayCapacityError, + ConfigOverlayEntryBudget, + applyConfigOverlayPlan, + createConfigOverlayPlan, + type AppliedConfigOverlayEntries +} from '../pty/config-overlay-mirroring' +import { safeRemoveTree } from '../pty/overlay-mirror' + +export const ORCA_OPENCODE_PLUGIN_FILE = 'orca-opencode-status.js' +export const OPENCODE_OVERLAY_MANIFEST_FILE = '.orca-opencode-overlay-manifest.json' +export const OPENCODE_OVERLAY_MANIFEST_MAX_BYTES = 2 * 1_024 * 1_024 + +type OpenCodeOverlayManifest = { + topLevelEntries: string[] + pluginEntries: string[] +} + +const RESERVED_TOP_LEVEL_ENTRIES = new Set([OPENCODE_OVERLAY_MANIFEST_FILE]) + +function emptyManifest(): OpenCodeOverlayManifest { + return { topLevelEntries: [], pluginEntries: [] } +} + +function parseManifestEntryNames( + value: unknown, + kind: 'top-level' | 'plugin', + budget: ConfigOverlayEntryBudget +): string[] { + if (!Array.isArray(value)) { + return [] + } + + const names: string[] = [] + for (const candidate of value) { + if (typeof candidate !== 'string') { + continue + } + budget.reserve(candidate) + if ( + (kind === 'top-level' && + (candidate === 'plugins' || candidate === OPENCODE_OVERLAY_MANIFEST_FILE)) || + (kind === 'plugin' && candidate === ORCA_OPENCODE_PLUGIN_FILE) + ) { + continue + } + names.push(candidate) + } + return names +} + +function parseOverlayManifest(contents: string): OpenCodeOverlayManifest { + const parsed = JSON.parse(contents) as Partial + const topLevelCount = Array.isArray(parsed.topLevelEntries) ? parsed.topLevelEntries.length : 0 + const pluginCount = Array.isArray(parsed.pluginEntries) ? parsed.pluginEntries.length : 0 + if (topLevelCount + pluginCount > CONFIG_OVERLAY_MAX_SOURCE_ENTRIES) { + throw new ConfigOverlayCapacityError( + 'entries', + topLevelCount + pluginCount, + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + ) + } + + const budget = new ConfigOverlayEntryBudget() + return { + topLevelEntries: parseManifestEntryNames(parsed.topLevelEntries, 'top-level', budget), + pluginEntries: parseManifestEntryNames(parsed.pluginEntries, 'plugin', budget) + } +} + +function readOverlayManifest(overlayDir: string): OpenCodeOverlayManifest { + try { + const contents = readNodeFileSyncWithinLimit( + join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE), + OPENCODE_OVERLAY_MANIFEST_MAX_BYTES + ).buffer.toString('utf8') + return parseOverlayManifest(contents) + } catch (error) { + if (error instanceof ConfigOverlayCapacityError || error instanceof NodeFileReadTooLargeError) { + throw error + } + return emptyManifest() + } +} + +function removePathBeforeWrite(path: string): void { + try { + unlinkSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } +} + +function writeOverlayManifest(overlayDir: string, manifest: OpenCodeOverlayManifest): void { + const manifestPath = join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE) + const { serialized } = stringifyJsonWithinByteLimit( + manifest, + OPENCODE_OVERLAY_MANIFEST_MAX_BYTES - 1 + ) + removePathBeforeWrite(manifestPath) + writeFileSync(manifestPath, `${serialized}\n`) +} + +function clearManifestEntries(overlayDir: string, manifest: OpenCodeOverlayManifest): void { + for (const entryName of manifest.topLevelEntries) { + if (!safeRemoveTree(join(overlayDir, entryName))) { + throw new Error('Unable to clear an OpenCode config overlay entry') + } + } + + const overlayPluginsDir = join(overlayDir, 'plugins') + for (const entryName of manifest.pluginEntries) { + if (!safeRemoveTree(join(overlayPluginsDir, entryName))) { + throw new Error('Unable to clear an OpenCode plugin overlay entry') + } + } +} + +function toManifest(applied: AppliedConfigOverlayEntries): OpenCodeOverlayManifest { + return { + topLevelEntries: applied.topLevelEntryNames, + pluginEntries: applied.pluginEntryNames + } +} + +export function mirrorOpenCodeConfigWithManifest(sourceDir: string, overlayDir: string): void { + const plan = createConfigOverlayPlan(sourceDir, { + reservedPluginFile: ORCA_OPENCODE_PLUGIN_FILE, + reservedTopLevelEntryNames: RESERVED_TOP_LEVEL_ENTRIES + }) + const previousManifest = readOverlayManifest(overlayDir) + clearManifestEntries(overlayDir, previousManifest) + + const applied: AppliedConfigOverlayEntries = { + topLevelEntryNames: [], + pluginEntryNames: [] + } + try { + applyConfigOverlayPlan(plan, overlayDir, applied) + writeOverlayManifest(overlayDir, toManifest(applied)) + } catch (error) { + try { + clearManifestEntries(overlayDir, toManifest(applied)) + } catch { + // Preserve the original mirror/write failure; a later spawn can retry cleanup. + } + throw error + } +} + +export const _configOverlayManifestInternals = { + parseOverlayManifest, + writeOverlayManifest +} diff --git a/src/main/opencode/hook-service.test.ts b/src/main/opencode/hook-service.test.ts index f3956519ee7..df9fe7b80e7 100644 --- a/src/main/opencode/hook-service.test.ts +++ b/src/main/opencode/hook-service.test.ts @@ -8,10 +8,12 @@ import { readdirSync, rmSync, symlinkSync, + truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { GENERATED_NODE_MANAGED_FILE_MAX_BYTES } from '../generated-node-bounded-file-reader' const { getPathMock } = vi.hoisted(() => ({ getPathMock: vi.fn<(name: string) => string>() @@ -24,6 +26,10 @@ vi.mock('electron', () => ({ })) import { OpenCodeHookService, _internals } from './hook-service' +import { + OPENCODE_OVERLAY_MANIFEST_FILE, + OPENCODE_OVERLAY_MANIFEST_MAX_BYTES +} from './config-overlay-manifest' const { isUsableId, toSafeDirName } = _internals @@ -65,6 +71,11 @@ describe('OpenCode hook plugin source', () => { expect(source).toContain('const coords = resolveHookCoords();') expect(source).toContain('`http://127.0.0.1:${coords.port}/hook/opencode`') expect(source).toContain('"X-Orca-Agent-Hook-Token": coords.token') + expect(source).toContain( + `function readOrcaManagedFileWithinLimit(fs, path, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES})` + ) + expect(source).toContain('readOrcaManagedFileWithinLimit(fs, path)') + expect(source).not.toContain('fs.readFileSync') }) it('caches the parsed endpoint file on mtime+size+inode to skip re-reads per post', () => { @@ -355,6 +366,81 @@ describe('OpenCodeHookService overlay mode (user OPENCODE_CONFIG_DIR set)', () = expectUserConfigIntact() }) + it('does not mirror or overwrite a user file named like the internal manifest', () => { + const userManifest = join(userConfigDir, OPENCODE_OVERLAY_MANIFEST_FILE) + writeFileSync(userManifest, 'USER MANIFEST SENTINEL') + + const env = new OpenCodeHookService().buildPtyEnv(ptyId, userConfigDir) + const overlayManifest = join(env.OPENCODE_CONFIG_DIR!, OPENCODE_OVERLAY_MANIFEST_FILE) + + expect(readFileSync(userManifest, 'utf8')).toBe('USER MANIFEST SENTINEL') + expect(lstatSync(overlayManifest).isSymbolicLink()).toBe(false) + expect(JSON.parse(readFileSync(overlayManifest, 'utf8'))).toMatchObject({ + topLevelEntries: expect.arrayContaining(['auth.json', 'opencode.json']) + }) + }) + + it('falls back before mutation when a retained manifest exceeds its byte limit', () => { + const service = new OpenCodeHookService() + const first = service.buildPtyEnv(ptyId, userConfigDir) + const overlayDir = first.OPENCODE_CONFIG_DIR! + const overlayManifest = join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE) + truncateSync(overlayManifest, OPENCODE_OVERLAY_MANIFEST_MAX_BYTES + 1) + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + expect(service.buildPtyEnv(ptyId, userConfigDir)).toEqual({ + OPENCODE_CONFIG_DIR: userConfigDir + }) + expect(readFileSync(join(overlayDir, 'auth.json'), 'utf8')).toBe('user-auth-token') + expect(warnSpy).toHaveBeenCalledWith( + '[opencode-hooks] config overlay exceeded its memory limit; using the original OPENCODE_CONFIG_DIR without Orca status integration' + ) + expectUserConfigIntact() + }) + + it('ignores forged manifest traversal entries during cleanup', () => { + const overlayDir = join( + userDataDir, + 'opencode-config-overlays', + toSafeDirName(`source:${userConfigDir}`) + ) + mkdirSync(overlayDir, { recursive: true }) + const outsideMarker = join(userDataDir, 'outside-manifest-marker') + writeFileSync(outsideMarker, 'keep') + writeFileSync( + join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE), + JSON.stringify({ topLevelEntries: ['../outside-manifest-marker'] }) + ) + + const env = new OpenCodeHookService().buildPtyEnv(ptyId, userConfigDir) + + expect(env.OPENCODE_CONFIG_DIR).toBe(overlayDir) + expect(readFileSync(outsideMarker, 'utf8')).toBe('keep') + }) + + it.skipIf(process.platform === 'win32')( + 'replaces a stale manifest symlink without writing through to the user file', + () => { + const userManifest = join(userConfigDir, OPENCODE_OVERLAY_MANIFEST_FILE) + writeFileSync(userManifest, 'USER MANIFEST SENTINEL') + const overlayDir = join( + userDataDir, + 'opencode-config-overlays', + toSafeDirName(`source:${userConfigDir}`) + ) + mkdirSync(overlayDir, { recursive: true }) + symlinkSync(userManifest, join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE), 'file') + + const env = new OpenCodeHookService().buildPtyEnv(ptyId, userConfigDir) + + expect(env.OPENCODE_CONFIG_DIR).toBe(overlayDir) + expect(readFileSync(userManifest, 'utf8')).toBe('USER MANIFEST SENTINEL') + expect(lstatSync(join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE)).isSymbolicLink()).toBe( + false + ) + } + ) + it.skipIf(process.platform === 'win32')( 'does not write through a symlinked plugins/ directory into the user filesystem', () => { diff --git a/src/main/opencode/hook-service.ts b/src/main/opencode/hook-service.ts index c2baa488dc9..cc45e229618 100644 --- a/src/main/opencode/hook-service.ts +++ b/src/main/opencode/hook-service.ts @@ -1,29 +1,19 @@ /* eslint-disable max-lines -- Why: holds an inline JS plugin source emitted as one file; splitting across TS modules would scatter tightly coupled string-template logic. */ import { app } from 'electron' import { join } from 'node:path' -import { - existsSync, - mkdirSync, - readFileSync, - readdirSync, - realpathSync, - statSync, - unlinkSync, - writeFileSync -} from 'node:fs' +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { createHash } from 'node:crypto' -import { mirrorEntry, safeRemoveTree } from '../pty/overlay-mirror' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { getGeneratedNodeBoundedFileReaderSourceLines } from '../generated-node-bounded-file-reader' +import { ConfigOverlayCapacityError } from '../pty/config-overlay-mirroring' +import { + ORCA_OPENCODE_PLUGIN_FILE, + mirrorOpenCodeConfigWithManifest +} from './config-overlay-manifest' -const ORCA_OPENCODE_PLUGIN_FILE = 'orca-opencode-status.js' const OPENCODE_LEGACY_HOOKS_DIR = 'opencode-hooks' const OPENCODE_OVERLAY_DIR = 'opencode-config-overlays' const OPENCODE_SHARED_CONFIG_DIR = 'shared' -const OPENCODE_OVERLAY_MANIFEST_FILE = '.orca-opencode-overlay-manifest.json' - -type OpenCodeOverlayManifest = { - topLevelEntries: string[] - pluginEntries: string[] -} // Why: bounds-check only — the id is a daemon sessionId with path separators, hashed downstream to a filesystem-safe name (an old regex rejecting "/"/":" broke every such id, #1148); 1024 just caps pathological hash input. function isUsableId(id: string): boolean { @@ -48,12 +38,13 @@ export function getOpenCodeFamilyPluginSource(hookPathname: string): string { "// OpenCode's Node process (not Orca's) and has no access to server.ts's", '// equivalent warnedVersions / warnedEnvs Sets.', 'let warnedBadEndpoint = false;', + ...getGeneratedNodeBoundedFileReaderSourceLines(), '', '// Why: message.part.updated can fire many times per second during a', '// streaming assistant reply, and each post() calls resolveHookCoords()', '// which reads the endpoint file. The file only changes on Orca restart', '// (rare), so a stat+mtime check is substantially cheaper than a full', - '// readFileSync+parse on every streamed part. On stat error we fall', + '// bounded read+parse on every streamed part. On stat error we fall', '// through to parse so the fail-open behavior is preserved.', 'let cachedEndpointKey = "";', 'let cachedEndpointValues = null;', @@ -74,7 +65,7 @@ export function getOpenCodeFamilyPluginSource(hookPathname: string): string { ' if (cacheKey === cachedEndpointKey && cachedEndpointValues) {', ' return cachedEndpointValues;', ' }', - ' const contents = fs.readFileSync(path, "utf8");', + ' const contents = readOrcaManagedFileWithinLimit(fs, path);', ' const out = {};', ' for (const line of contents.split(/\\r?\\n/)) {', ' // Why: Windows endpoint.cmd uses `set KEY=VALUE`; Unix endpoint.env', @@ -91,7 +82,7 @@ export function getOpenCodeFamilyPluginSource(hookPathname: string): string { ' return out;', ' } catch (ioErr) {', ' // Why: any stat or read failure (file yanked mid-read, permission', - ' // race, unlink between stat and readFileSync) must invalidate the', + ' // race, unlink between stat and bounded read) must invalidate the', ' // cache so a transient failure does not lock in a stale parse for', ' // the remaining process lifetime; rethrow to the outer catch.', ' cachedEndpointKey = "";', @@ -377,6 +368,8 @@ export function getOpenCodeFamilyPluginSource(hookPathname: string): string { // Why: installs the plugin into OPENCODE_CONFIG_DIR so it POSTs to the shared agent-hooks server, unifying OpenCode status with Claude/Codex/Gemini (the old loopback-IPC path never reached agentStatusByPaneKey). export class OpenCodeHookService { + private warnedOverlayCapacity = false + clearPty(_ptyId: string): void { // Why: no-op — config dirs are app/source-scoped now, and recursive delete on the main-process hot path could freeze on Windows. } @@ -407,7 +400,16 @@ export class OpenCodeHookService { mkdirSync(overlayDir, { recursive: true }) this.mirrorUserConfig(existingConfigDir, overlayDir) this.writePluginIntoOverlay(overlayDir) - } catch { + } catch (error) { + if ( + !this.warnedOverlayCapacity && + (error instanceof ConfigOverlayCapacityError || error instanceof NodeFileReadTooLargeError) + ) { + this.warnedOverlayCapacity = true + console.warn( + '[opencode-hooks] config overlay exceeded its memory limit; using the original OPENCODE_CONFIG_DIR without Orca status integration' + ) + } // Why: best-effort — symlink creation needs Windows developer mode (else EPERM) and userData may be read-only; preserve the user's config over dropping their auth/models/keymap. return { OPENCODE_CONFIG_DIR: existingConfigDir } } @@ -427,90 +429,8 @@ export class OpenCodeHookService { return join(app.getPath('userData'), OPENCODE_LEGACY_HOOKS_DIR, OPENCODE_SHARED_CONFIG_DIR) } - private readOverlayManifest(overlayDir: string): OpenCodeOverlayManifest { - try { - const parsed = JSON.parse( - readFileSync(join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE), 'utf8') - ) as Partial - return { - topLevelEntries: Array.isArray(parsed.topLevelEntries) ? parsed.topLevelEntries : [], - pluginEntries: Array.isArray(parsed.pluginEntries) ? parsed.pluginEntries : [] - } - } catch { - return { topLevelEntries: [], pluginEntries: [] } - } - } - - private writeOverlayManifest(overlayDir: string, manifest: OpenCodeOverlayManifest): void { - writeFileSync( - join(overlayDir, OPENCODE_OVERLAY_MANIFEST_FILE), - `${JSON.stringify(manifest, null, 2)}\n` - ) - } - - private clearManifestEntries(overlayDir: string, manifest: OpenCodeOverlayManifest): void { - for (const entryName of manifest.topLevelEntries) { - safeRemoveTree(join(overlayDir, entryName)) - } - - const overlayPluginsDir = join(overlayDir, 'plugins') - for (const entryName of manifest.pluginEntries) { - if (entryName === ORCA_OPENCODE_PLUGIN_FILE) { - continue - } - safeRemoveTree(join(overlayPluginsDir, entryName)) - } - } - - // Why: mirror user config entries as symlinks so edits propagate live; only plugins/ becomes a real overlay dir so Orca can drop a sibling plugin file. private mirrorUserConfig(sourceDir: string, overlayDir: string): void { - const previousManifest = this.readOverlayManifest(overlayDir) - // Why: overlays persist across terminals; remove only Orca-mirrored paths so stale user config clears but OpenCode runtime dirs (node_modules) survive. - this.clearManifestEntries(overlayDir, previousManifest) - - const nextManifest: OpenCodeOverlayManifest = { topLevelEntries: [], pluginEntries: [] } - - for (const entry of readdirSync(sourceDir, { withFileTypes: true })) { - const sourcePath = join(sourceDir, entry.name) - - if (entry.name === 'plugins') { - // Why: check isSymbolicLink before isDirectory — a Windows junction reports both, and the symlink branch must win. - const isSymlink = entry.isSymbolicLink() - let isLinkPointingToDir = false - if (isSymlink) { - try { - isLinkPointingToDir = statSync(sourcePath).isDirectory() - } catch { - // Why: broken/inaccessible symlink — mirror the dangling link verbatim instead of resolving through it. - isLinkPointingToDir = false - } - } - - if ((!isSymlink && entry.isDirectory()) || isLinkPointingToDir) { - // Why: resolve a symlinked plugins/ to its real target so /plugins stays a real dir and writePluginIntoOverlay can't write through the user's link. - const resolvedSource = isLinkPointingToDir ? realpathSync(sourcePath) : sourcePath - const overlayPluginsDir = join(overlayDir, 'plugins') - mkdirSync(overlayPluginsDir, { recursive: true }) - for (const pluginEntry of readdirSync(resolvedSource, { withFileTypes: true })) { - // Why: skip a user plugin sharing Orca's filename; mirroring it would let writePluginIntoOverlay clobber the user's file. - if (pluginEntry.name === ORCA_OPENCODE_PLUGIN_FILE) { - continue - } - mirrorEntry( - join(resolvedSource, pluginEntry.name), - join(overlayPluginsDir, pluginEntry.name) - ) - nextManifest.pluginEntries.push(pluginEntry.name) - } - continue - } - } - - mirrorEntry(sourcePath, join(overlayDir, entry.name)) - nextManifest.topLevelEntries.push(entry.name) - } - - this.writeOverlayManifest(overlayDir, nextManifest) + mirrorOpenCodeConfigWithManifest(sourceDir, overlayDir) } // Why: pre-write unlink guards against POSIX writeFileSync writing through a mirrored symlink and clobbering a same-named user plugin. @@ -520,8 +440,10 @@ export class OpenCodeHookService { const pluginPath = join(pluginsDir, ORCA_OPENCODE_PLUGIN_FILE) try { unlinkSync(pluginPath) - } catch { - // File may not exist on a fresh overlay; a real failure surfaces on writeFileSync below. + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } } writeFileSync(pluginPath, getOpenCodePluginSource()) } @@ -531,7 +453,7 @@ export class OpenCodeHookService { const pluginsDir = join(configDir, 'plugins') try { mkdirSync(pluginsDir, { recursive: true }) - writeFileSync(join(pluginsDir, ORCA_OPENCODE_PLUGIN_FILE), getOpenCodePluginSource()) + this.writePluginIntoOverlay(configDir) } catch { // Why: userData can be locked on Windows (EPERM/EBUSY); plugin is non-critical, so spawn without it. return null diff --git a/src/main/opencode/opencode-database-files.test.ts b/src/main/opencode/opencode-database-files.test.ts new file mode 100644 index 00000000000..f27452e2e89 --- /dev/null +++ b/src/main/opencode/opencode-database-files.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { collectOpenCodeDatabaseFiles } from './opencode-database-files' + +function entry(name: string, file = true): { name: string; isFile(): boolean } { + return { name, isFile: () => file } +} + +describe('OpenCode database file discovery', () => { + it('preserves and sorts every matching file within the limit', async () => { + const result = await collectOpenCodeDatabaseFiles( + '/data', + (async function* () { + yield entry('opencode-z.db') + yield entry('notes.txt') + yield entry('opencode.db') + yield entry('opencode-directory.db', false) + })(), + 2 + ) + + expect(result).toEqual({ + paths: ['/data/opencode-z.db', '/data/opencode.db'].sort(), + truncated: false + }) + }) + + it('stops retaining names immediately after the database count limit', async () => { + let enumerated = 0 + const result = await collectOpenCodeDatabaseFiles( + '/data', + (async function* () { + for (const name of ['opencode-a.db', 'opencode-b.db', 'opencode-c.db', 'opencode-d.db']) { + enumerated += 1 + yield entry(name) + } + })(), + 2 + ) + + expect(result).toEqual({ + paths: ['/data/opencode-a.db', '/data/opencode-b.db'], + truncated: true + }) + expect(enumerated).toBe(3) + }) +}) diff --git a/src/main/opencode/opencode-database-files.ts b/src/main/opencode/opencode-database-files.ts new file mode 100644 index 00000000000..95e3abb9d20 --- /dev/null +++ b/src/main/opencode/opencode-database-files.ts @@ -0,0 +1,40 @@ +import { opendir } from 'node:fs/promises' +import { join } from 'node:path' + +export const OPENCODE_DATABASE_FILE_LIMIT = 256 + +type OpenCodeDatabaseDirectoryEntry = { + name: string + isFile(): boolean +} + +export type OpenCodeDatabaseFiles = { + paths: string[] + truncated: boolean +} + +export async function listOpenCodeDatabaseFiles(dataDir: string): Promise { + try { + return collectOpenCodeDatabaseFiles(dataDir, await opendir(dataDir)) + } catch { + return { paths: [], truncated: false } + } +} + +export async function collectOpenCodeDatabaseFiles( + dataDir: string, + directory: AsyncIterable, + maxFiles = OPENCODE_DATABASE_FILE_LIMIT +): Promise { + const paths: string[] = [] + for await (const entry of directory) { + if (!entry.isFile() || !/^opencode(?:-[A-Za-z0-9_.-]+)?\.db$/.test(entry.name)) { + continue + } + if (paths.length >= maxFiles) { + return { paths: paths.sort(), truncated: true } + } + paths.push(join(dataDir, entry.name)) + } + return { paths: paths.sort(), truncated: false } +} diff --git a/src/main/orca-profiles/profile-cloud-client.ts b/src/main/orca-profiles/profile-cloud-client.ts index e7657bbdb87..e6351f22a12 100644 --- a/src/main/orca-profiles/profile-cloud-client.ts +++ b/src/main/orca-profiles/profile-cloud-client.ts @@ -7,6 +7,7 @@ import type { OrcaCloudAuthConfig } from './profile-cloud-auth-config' import type { OrcaCloudSession } from './profile-cloud-session-store' import type { OrcaCloudSessionExchangeResponse } from './profile-cloud-session-exchange' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' type ExchangeCodeArgs = { code: string @@ -177,7 +178,7 @@ async function postJson(url: string, body: unknown, accessToken?: string): Pr await cancelUnreadResponseBody(response) throw new OrcaCloudRequestError(response.status) } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } export async function exchangeOrcaCloudAuthCode( diff --git a/src/main/orca-profiles/profile-cloud-dev-org-members.test.ts b/src/main/orca-profiles/profile-cloud-dev-org-members.test.ts new file mode 100644 index 00000000000..357e7316fc0 --- /dev/null +++ b/src/main/orca-profiles/profile-cloud-dev-org-members.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import { + _getDevOrcaCloudOrgRosterCountForTests, + _resetDevOrcaCloudOrgRostersForTests, + DEV_ORG_PENDING_INVITE_MAX_ENTRIES, + DEV_ORG_ROSTER_MAX_ENTRIES, + inviteDevOrcaCloudOrgMember, + listDevOrcaCloudOrgMembers +} from './profile-cloud-dev-org-members' + +beforeEach(() => { + _resetDevOrcaCloudOrgRostersForTests() +}) + +describe('dev organization roster retention', () => { + it('bounds retained organizations while keeping recently used rosters', () => { + for (let index = 0; index <= DEV_ORG_ROSTER_MAX_ENTRIES; index += 1) { + listDevOrcaCloudOrgMembers(`org-${index}`) + } + expect(_getDevOrcaCloudOrgRosterCountForTests()).toBe(DEV_ORG_ROSTER_MAX_ENTRIES) + }) + + it('bounds pending invites within one retained roster', () => { + const orgId = 'invite-heavy-org' + for (let index = 1; index < DEV_ORG_PENDING_INVITE_MAX_ENTRIES; index += 1) { + expect( + inviteDevOrcaCloudOrgMember({ + orgId, + email: `person-${index}@example.com`, + role: 'member' + }) + ).toEqual({ status: 'ok' }) + } + expect( + inviteDevOrcaCloudOrgMember({ + orgId, + email: 'overflow@example.com', + role: 'member' + }) + ).toEqual({ status: 'failed', error: 'The dev organization invite roster is full.' }) + }) + + it('does not retain oversized invitation text', () => { + expect( + inviteDevOrcaCloudOrgMember({ + orgId: 'org', + email: `${'x'.repeat(321)}@example.com`, + role: 'member' + }) + ).toEqual({ status: 'failed', error: 'The dev organization invite roster is full.' }) + expect(listDevOrcaCloudOrgMembers('org').pendingInvites).toHaveLength(1) + }) +}) diff --git a/src/main/orca-profiles/profile-cloud-dev-org-members.ts b/src/main/orca-profiles/profile-cloud-dev-org-members.ts index 4f61e03df6a..7e69bc008a0 100644 --- a/src/main/orca-profiles/profile-cloud-dev-org-members.ts +++ b/src/main/orca-profiles/profile-cloud-dev-org-members.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import type { OrcaOrgMember, OrcaOrgMembersRoster, @@ -8,6 +9,7 @@ import type { OrcaProfileOrgMemberRemoveArgs, OrcaProfileOrgInviteRevokeArgs } from '../../shared/orca-profiles' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' // Why: dev-auth mode has no server, so the whole teammate UI is exercised // against this in-memory per-org roster. It mirrors the shape the real client @@ -19,10 +21,17 @@ type DevOrgRoster = { } const devRostersByOrg = new Map() +export const DEV_ORG_ROSTER_MAX_ENTRIES = 64 +export const DEV_ORG_PENDING_INVITE_MAX_ENTRIES = 256 +const DEV_ORG_ENV_FIELD_MAX_BYTES = 4 * 1024 +const DEV_ORG_INVITE_EMAIL_MAX_BYTES = 320 function cleanEnvString(value: string | undefined, fallback: string): string { const trimmed = value?.trim() - return trimmed || fallback + return trimmed && + !measureUtf8ByteLength(trimmed, { stopAfterBytes: DEV_ORG_ENV_FIELD_MAX_BYTES }).exceededLimit + ? trimmed + : fallback } function devSelf(): OrcaOrgMember { @@ -52,12 +61,22 @@ function seedDevRoster(): DevOrgRoster { } function getDevRoster(orgId: string): DevOrgRoster { - const existing = devRostersByOrg.get(orgId) + const cacheKey = createHash('sha256').update(orgId).digest('base64url') + const existing = devRostersByOrg.get(cacheKey) if (existing) { + devRostersByOrg.delete(cacheKey) + devRostersByOrg.set(cacheKey, existing) return existing } const seeded = seedDevRoster() - devRostersByOrg.set(orgId, seeded) + devRostersByOrg.set(cacheKey, seeded) + while (devRostersByOrg.size > DEV_ORG_ROSTER_MAX_ENTRIES) { + const oldest = devRostersByOrg.keys().next().value + if (oldest === undefined) { + break + } + devRostersByOrg.delete(oldest) + } return seeded } @@ -82,6 +101,14 @@ export function inviteDevOrcaCloudOrgMember( if (roster.pendingInvites.some((invite) => invite.email.toLowerCase() === email)) { return { status: 'conflict', reason: 'already_invited' } } + if ( + roster.pendingInvites.length >= DEV_ORG_PENDING_INVITE_MAX_ENTRIES || + measureUtf8ByteLength(args.email, { + stopAfterBytes: DEV_ORG_INVITE_EMAIL_MAX_BYTES + }).exceededLimit + ) { + return { status: 'failed', error: 'The dev organization invite roster is full.' } + } roster.pendingInvites.push({ email: args.email, role: args.role, createdAt: Date.now() }) return { status: 'ok' } } @@ -128,3 +155,11 @@ export function removeDevOrcaCloudOrgMember( roster.members.splice(index, 1) return { status: 'ok' } } + +export function _resetDevOrcaCloudOrgRostersForTests(): void { + devRostersByOrg.clear() +} + +export function _getDevOrcaCloudOrgRosterCountForTests(): number { + return devRostersByOrg.size +} diff --git a/src/main/orca-profiles/profile-cloud-org-members-client.ts b/src/main/orca-profiles/profile-cloud-org-members-client.ts index cd1b106e489..b3c0e8e3743 100644 --- a/src/main/orca-profiles/profile-cloud-org-members-client.ts +++ b/src/main/orca-profiles/profile-cloud-org-members-client.ts @@ -7,6 +7,7 @@ import type { import type { OrcaCloudAuthConfig } from './profile-cloud-auth-config' import type { OrcaCloudSession } from './profile-cloud-session-store' import { OrcaCloudRequestError } from './profile-cloud-client' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' const CLOUD_REQUEST_TIMEOUT_MS = 30_000 const ORG_ROLES: readonly OrcaOrgRole[] = ['owner', 'admin', 'member'] @@ -106,7 +107,7 @@ function orgMembersUrl(config: OrcaCloudAuthConfig, orgId: string, path: string) async function extractErrorCode(response: Response): Promise { try { - const body = (await response.json()) as unknown + const body = await readFetchResponseJsonWithinLimit(response) if ( body && typeof body === 'object' && @@ -145,7 +146,7 @@ async function requestOrgMembers( if (!response.ok) { throw new OrcaCloudRequestError(response.status, await extractErrorCode(response)) } - return parse((await response.json()) as unknown) + return parse(await readFetchResponseJsonWithinLimit(response)) } export async function listOrcaCloudOrgMembers( diff --git a/src/main/orca-profiles/profile-cloud-session-mutation.test.ts b/src/main/orca-profiles/profile-cloud-session-mutation.test.ts index a8196a7f1a3..1a4b309566b 100644 --- a/src/main/orca-profiles/profile-cloud-session-mutation.test.ts +++ b/src/main/orca-profiles/profile-cloud-session-mutation.test.ts @@ -1,10 +1,13 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { captureCloudSessionMutation, isCloudSessionMutationCurrent, + MAX_CLOUD_SESSION_IDENTITY_KEY_BYTES, + MAX_CLOUD_SESSION_MUTATION_STATE_FILE_BYTES, + MAX_CLOUD_SESSION_TOMBSTONES, recordCloudSessionIdentityMutation, recordSuccessfulCloudSessionLogin, tombstoneCloudSession, @@ -59,4 +62,58 @@ describe('cloud session mutation fence', () => { true ) }) + + it('rejects an oversized sparse mutation fence before parsing it', () => { + const profileDirectory = join(userDataPath, 'profiles', identity.localProfileId) + mkdirSync(profileDirectory, { recursive: true }) + const path = join(profileDirectory, 'account-session-mutation.json') + writeFileSync(path, '{"version":1}') + truncateSync(path, MAX_CLOUD_SESSION_MUTATION_STATE_FILE_BYTES + 1) + + expect(() => captureCloudSessionMutation(identity, userDataPath)).toThrow( + 'invalid_cloud_session_mutation_state' + ) + }) + + it('bounds tombstones while the advancing epoch keeps old snapshots invalid', () => { + const firstIdentity = { ...identity, cloudUserId: 'user-0' } + const firstSnapshot = captureCloudSessionMutation(firstIdentity, userDataPath) + + for (let index = 0; index <= MAX_CLOUD_SESSION_TOMBSTONES; index += 1) { + tombstoneCloudSession({ ...identity, cloudUserId: `user-${index}` }, userDataPath) + } + + const path = join( + userDataPath, + 'profiles', + identity.localProfileId, + 'account-session-mutation.json' + ) + const state = JSON.parse(readFileSync(path, 'utf8')) as { + tombstonedIdentityKeys: string[] + } + expect(state.tombstonedIdentityKeys).toHaveLength(MAX_CLOUD_SESSION_TOMBSTONES) + expect( + isCloudSessionMutationCurrent(identity.localProfileId, userDataPath, firstSnapshot) + ).toBe(false) + }) + + it('rejects an oversized identity before changing the last readable fence', () => { + captureCloudSessionMutation(identity, userDataPath) + const path = join( + userDataPath, + 'profiles', + identity.localProfileId, + 'account-session-mutation.json' + ) + const before = readFileSync(path, 'utf8') + + expect(() => + recordCloudSessionIdentityMutation( + { ...identity, cloudUserId: 'x'.repeat(MAX_CLOUD_SESSION_IDENTITY_KEY_BYTES) }, + userDataPath + ) + ).toThrow('cloud_session_identity_too_large') + expect(readFileSync(path, 'utf8')).toBe(before) + }) }) diff --git a/src/main/orca-profiles/profile-cloud-session-mutation.ts b/src/main/orca-profiles/profile-cloud-session-mutation.ts index d360cb72ab9..b9c89698aa1 100644 --- a/src/main/orca-profiles/profile-cloud-session-mutation.ts +++ b/src/main/orca-profiles/profile-cloud-session-mutation.ts @@ -1,10 +1,15 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' -import { writeSecureJsonFile } from '../../shared/secure-file' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { JsonStringifyByteLimitError } from '../../shared/node-bounded-json-stringify' +import { writeSecureJsonFileWithinLimit } from '../../shared/bounded-secure-json-file' import type { OrcaProfileCloudSummary } from '../../shared/orca-profiles' import { getOrcaProfileDirectory } from './profile-storage-paths' const MUTATION_STATE_VERSION = 1 +export const MAX_CLOUD_SESSION_MUTATION_STATE_FILE_BYTES = 1024 * 1024 +export const MAX_CLOUD_SESSION_IDENTITY_KEY_BYTES = 16 * 1024 +export const MAX_CLOUD_SESSION_TOMBSTONES = 512 export type CloudSessionIdentity = { localProfileId: string @@ -26,7 +31,11 @@ type CloudSessionMutationState = { } function identityKey(identity: CloudSessionIdentity): string { - return `${identity.localProfileId}\0${identity.cloudUserId}\0${identity.cloudProfileId}\0${identity.organizationId}` + const key = `${identity.localProfileId}\0${identity.cloudUserId}\0${identity.cloudProfileId}\0${identity.organizationId}` + if (Buffer.byteLength(key, 'utf8') > MAX_CLOUD_SESSION_IDENTITY_KEY_BYTES) { + throw new Error('cloud_session_identity_too_large') + } + return key } function statePath(profileId: string, userDataPath: string): string { @@ -54,7 +63,12 @@ function readState(profileId: string, userDataPath: string): CloudSessionMutatio return null } try { - const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + const parsed: unknown = JSON.parse( + readNodeFileSyncWithinLimit( + path, + MAX_CLOUD_SESSION_MUTATION_STATE_FILE_BYTES + ).buffer.toString('utf8') + ) if (!isState(parsed)) { throw new Error('invalid_cloud_session_mutation_state') } @@ -69,7 +83,22 @@ function saveState( userDataPath: string, state: CloudSessionMutationState ): void { - writeSecureJsonFile(statePath(profileId, userDataPath), state) + const tombstonedIdentityKeys = state.tombstonedIdentityKeys.slice(-MAX_CLOUD_SESSION_TOMBSTONES) + while (true) { + try { + writeSecureJsonFileWithinLimit( + statePath(profileId, userDataPath), + { ...state, tombstonedIdentityKeys }, + MAX_CLOUD_SESSION_MUTATION_STATE_FILE_BYTES + ) + return + } catch (error) { + if (!(error instanceof JsonStringifyByteLimitError) || tombstonedIdentityKeys.length === 0) { + throw error + } + tombstonedIdentityKeys.shift() + } + } } export function cloudSessionIdentity( diff --git a/src/main/orca-profiles/profile-cloud-session-store.test.ts b/src/main/orca-profiles/profile-cloud-session-store.test.ts index ca32f0529f1..47c88827d76 100644 --- a/src/main/orca-profiles/profile-cloud-session-store.test.ts +++ b/src/main/orca-profiles/profile-cloud-session-store.test.ts @@ -1,5 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { OrcaCloudSession } from './profile-cloud-session-store' @@ -169,4 +177,102 @@ describe('Orca cloud session store', () => { error: 'Unsafe session format.' }) }) + + it('rejects an oversized sparse encrypted-session wrapper before decrypting', async () => { + writePlaintextSessionFile('profile-1', makeSession()) + const store = await loadSessionStore() + truncateSync( + store.getOrcaCloudSessionPath('profile-1', userDataPath), + store.MAX_ORCA_CLOUD_SESSION_FILE_BYTES + 1 + ) + + expect(store.readOrcaCloudSession('profile-1', userDataPath)).toEqual({ + status: 'decrypt-failed', + persistence: 'none', + error: 'Could not decrypt saved Orca account session.' + }) + expect(safeStorageMock.decryptString).not.toHaveBeenCalled() + }) + + it('rejects an oversized session without replacing the prior durable or cached session', async () => { + const store = await loadSessionStore() + const session = makeSession() + store.saveOrcaCloudSession('profile-1', userDataPath, session) + const path = store.getOrcaCloudSessionPath('profile-1', userDataPath) + const durableBefore = readFileSync(path, 'utf-8') + + expect(() => + store.saveOrcaCloudSession('profile-1', userDataPath, { + ...session, + accessToken: 'x'.repeat(store.MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES) + }) + ).toThrow(`JSON output exceeds ${store.MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES} bytes`) + expect(readFileSync(path, 'utf-8')).toBe(durableBefore) + expect(store.readOrcaCloudSession('profile-1', userDataPath)).toEqual({ + status: 'found', + session, + persistence: 'encrypted' + }) + }) + + it('removes stale durable credentials when encryption expansion exceeds the file limit', async () => { + const store = await loadSessionStore() + const prior = makeSession() + store.saveOrcaCloudSession('profile-1', userDataPath, prior) + const session = { + ...prior, + accessToken: 'x'.repeat(Math.floor(store.MAX_ORCA_CLOUD_SESSION_FILE_BYTES * 0.8)) + } + + expect(store.saveOrcaCloudSession('profile-1', userDataPath, session)).toBe('memory-only') + expect(existsSync(store.getOrcaCloudSessionPath('profile-1', userDataPath))).toBe(false) + expect(store.readOrcaCloudSession('profile-1', userDataPath)).toEqual({ + status: 'found', + session, + persistence: 'memory-only' + }) + }) + + it('evicts the least-recently-used memory-only session at the entry limit', async () => { + safeStorageMock.isEncryptionAvailable.mockReturnValue(false) + const store = await loadSessionStore() + const session = makeSession() + + for (let index = 0; index <= store.MAX_ORCA_CLOUD_MEMORY_SESSIONS; index += 1) { + store.saveOrcaCloudSession(`profile-${index}`, userDataPath, session) + } + + expect(store.readOrcaCloudSession('profile-0', userDataPath)).toEqual({ + status: 'missing', + persistence: 'none' + }) + expect( + store.readOrcaCloudSession(`profile-${store.MAX_ORCA_CLOUD_MEMORY_SESSIONS}`, userDataPath) + ).toMatchObject({ status: 'found', session }) + }) + + it('evicts memory-only sessions before exceeding the aggregate byte limit', async () => { + safeStorageMock.isEncryptionAvailable.mockReturnValue(false) + const store = await loadSessionStore() + const session = { + ...makeSession(), + accessToken: 'x'.repeat(store.MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES - 1024) + } + const writesNeeded = + Math.floor( + store.MAX_ORCA_CLOUD_MEMORY_SESSION_BYTES / store.MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES + ) + 2 + + for (let index = 0; index < writesNeeded; index += 1) { + store.saveOrcaCloudSession(`profile-${index}`, userDataPath, session) + } + + expect(store.readOrcaCloudSession('profile-0', userDataPath)).toMatchObject({ + status: 'missing' + }) + expect(store.readOrcaCloudSession(`profile-${writesNeeded - 1}`, userDataPath)).toMatchObject({ + status: 'found', + session + }) + }) }) diff --git a/src/main/orca-profiles/profile-cloud-session-store.ts b/src/main/orca-profiles/profile-cloud-session-store.ts index 62d779772c1..c8803c6390c 100644 --- a/src/main/orca-profiles/profile-cloud-session-store.ts +++ b/src/main/orca-profiles/profile-cloud-session-store.ts @@ -1,7 +1,12 @@ -import { existsSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, rmSync } from 'node:fs' import { join } from 'node:path' import { safeStorage } from 'electron' -import { writeSecureJsonFile } from '../../shared/secure-file' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../../shared/node-bounded-json-stringify' +import { writeSecureJsonFileWithinLimit } from '../../shared/bounded-secure-json-file' import type { OrcaCloudCapabilities, OrcaCloudOrgSummary, @@ -47,14 +52,54 @@ type PersistedPlaintextSession = { type CachedOrcaCloudSession = { session: OrcaCloudSession persistence: Exclude + bytes: number } const memorySessions = new Map() +export const MAX_ORCA_CLOUD_SESSION_FILE_BYTES = 1024 * 1024 +export const MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES = MAX_ORCA_CLOUD_SESSION_FILE_BYTES +export const MAX_ORCA_CLOUD_MEMORY_SESSIONS = 128 +export const MAX_ORCA_CLOUD_MEMORY_SESSION_BYTES = 16 * 1024 * 1024 +let memorySessionBytes = 0 function sessionCacheKey(profileId: string, userDataPath: string): string { return `${userDataPath}\0${profileId}` } +function rememberMemorySession( + cacheKey: string, + session: OrcaCloudSession, + persistence: Exclude, + bytes: number +): void { + const previous = memorySessions.get(cacheKey) + if (previous) { + memorySessionBytes -= previous.bytes + memorySessions.delete(cacheKey) + } + while ( + memorySessions.size >= MAX_ORCA_CLOUD_MEMORY_SESSIONS || + memorySessionBytes + bytes > MAX_ORCA_CLOUD_MEMORY_SESSION_BYTES + ) { + const oldestKey = memorySessions.keys().next().value as string | undefined + if (oldestKey === undefined) { + break + } + const oldest = memorySessions.get(oldestKey) + memorySessions.delete(oldestKey) + memorySessionBytes -= oldest?.bytes ?? 0 + } + memorySessions.set(cacheKey, { session, persistence, bytes }) + memorySessionBytes += bytes +} + +function serializedSessionWithinLimit( + session: OrcaCloudSession, + maxBytes = MAX_ORCA_CLOUD_SESSION_PAYLOAD_BYTES +): { serialized: string; byteLength: number } { + return stringifyJsonWithinByteLimit(session, maxBytes) +} + function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } @@ -106,15 +151,29 @@ export function saveOrcaCloudSession( session: OrcaCloudSession ): OrcaCloudSessionPersistence { const cacheKey = sessionCacheKey(profileId, userDataPath) + const payload = serializedSessionWithinLimit(session) if (safeStorage.isEncryptionAvailable()) { const encrypted: PersistedEncryptedSession = { version: 1, format: 'electron-safe-storage-v1', savedAt: Date.now(), - ciphertext: safeStorage.encryptString(JSON.stringify(session)).toString('base64') + ciphertext: safeStorage.encryptString(payload.serialized).toString('base64') } - writeSecureJsonFile(getOrcaCloudSessionPath(profileId, userDataPath), encrypted) - memorySessions.set(cacheKey, { session, persistence: 'encrypted' }) + try { + writeSecureJsonFileWithinLimit( + getOrcaCloudSessionPath(profileId, userDataPath), + encrypted, + MAX_ORCA_CLOUD_SESSION_FILE_BYTES + ) + } catch (error) { + if (!(error instanceof JsonStringifyByteLimitError)) { + throw error + } + rmSync(getOrcaCloudSessionPath(profileId, userDataPath), { force: true }) + rememberMemorySession(cacheKey, session, 'memory-only', payload.byteLength) + return 'memory-only' + } + rememberMemorySession(cacheKey, session, 'encrypted', payload.byteLength) return 'encrypted' } @@ -125,14 +184,27 @@ export function saveOrcaCloudSession( savedAt: Date.now(), session } - writeSecureJsonFile(getOrcaCloudSessionPath(profileId, userDataPath), plaintext) - memorySessions.set(cacheKey, { session, persistence: 'dev-plaintext' }) + try { + writeSecureJsonFileWithinLimit( + getOrcaCloudSessionPath(profileId, userDataPath), + plaintext, + MAX_ORCA_CLOUD_SESSION_FILE_BYTES + ) + } catch (error) { + if (!(error instanceof JsonStringifyByteLimitError)) { + throw error + } + rmSync(getOrcaCloudSessionPath(profileId, userDataPath), { force: true }) + rememberMemorySession(cacheKey, session, 'memory-only', payload.byteLength) + return 'memory-only' + } + rememberMemorySession(cacheKey, session, 'dev-plaintext', payload.byteLength) return 'dev-plaintext' } // Why: Orca account refresh tokens must not silently fall back to plaintext // in production. Memory-only keeps cloud features usable until restart. - memorySessions.set(cacheKey, { session, persistence: 'memory-only' }) + rememberMemorySession(cacheKey, session, 'memory-only', payload.byteLength) return 'memory-only' } @@ -172,6 +244,8 @@ export function readOrcaCloudSession( const cacheKey = sessionCacheKey(profileId, userDataPath) const memorySession = memorySessions.get(cacheKey) if (memorySession) { + memorySessions.delete(cacheKey) + memorySessions.set(cacheKey, memorySession) return { status: 'found', session: memorySession.session, @@ -185,9 +259,9 @@ export function readOrcaCloudSession( } try { - const parsed = JSON.parse(readFileSync(path, 'utf-8')) as - | PersistedEncryptedSession - | PersistedPlaintextSession + const parsed = JSON.parse( + readNodeFileSyncWithinLimit(path, MAX_ORCA_CLOUD_SESSION_FILE_BYTES).buffer.toString('utf8') + ) as PersistedEncryptedSession | PersistedPlaintextSession if (parsed.version !== 1) { return { status: 'decrypt-failed', persistence: 'none', error: 'Unsupported session format.' } } @@ -204,14 +278,19 @@ export function readOrcaCloudSession( if (!isOrcaCloudSession(session)) { return { status: 'decrypt-failed', persistence: 'none', error: 'Invalid saved session.' } } - memorySessions.set(cacheKey, { session, persistence: 'encrypted' }) + const payload = serializedSessionWithinLimit(session, MAX_ORCA_CLOUD_SESSION_FILE_BYTES) + rememberMemorySession(cacheKey, session, 'encrypted', payload.byteLength) return { status: 'found', session, persistence: 'encrypted' } } if (parsed.format === 'dev-plaintext-v1' && allowsPlaintextOrcaCloudSession()) { if (!isOrcaCloudSession(parsed.session)) { return { status: 'decrypt-failed', persistence: 'none', error: 'Invalid saved session.' } } - memorySessions.set(cacheKey, { session: parsed.session, persistence: 'dev-plaintext' }) + const payload = serializedSessionWithinLimit( + parsed.session, + MAX_ORCA_CLOUD_SESSION_FILE_BYTES + ) + rememberMemorySession(cacheKey, parsed.session, 'dev-plaintext', payload.byteLength) return { status: 'found', session: parsed.session, persistence: 'dev-plaintext' } } return { status: 'decrypt-failed', persistence: 'none', error: 'Unsafe session format.' } @@ -225,6 +304,11 @@ export function readOrcaCloudSession( } export function clearOrcaCloudSession(profileId: string, userDataPath: string): void { - memorySessions.delete(sessionCacheKey(profileId, userDataPath)) + const cacheKey = sessionCacheKey(profileId, userDataPath) + const cached = memorySessions.get(cacheKey) + if (cached) { + memorySessionBytes -= cached.bytes + memorySessions.delete(cacheKey) + } rmSync(getOrcaCloudSessionPath(profileId, userDataPath), { force: true }) } diff --git a/src/main/orca-profiles/profile-index-store.test.ts b/src/main/orca-profiles/profile-index-store.test.ts index a2d3446a99c..cf9aa5f8b59 100644 --- a/src/main/orca-profiles/profile-index-store.test.ts +++ b/src/main/orca-profiles/profile-index-store.test.ts @@ -1,5 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' import { @@ -170,6 +178,40 @@ describe('profile index store', () => { expect(recovered.profiles.length).toBeGreaterThanOrEqual(2) }) + it('recovers an oversized sparse profile index from the backup copy', async () => { + const store = await loadProfileIndexStore() + store.ensureActiveOrcaProfile() + const created = store.createLocalOrcaProfile({ name: 'Work' }) + store.setActiveOrcaProfile(created.profile.id) + const indexPath = store.getOrcaProfileIndexPath() + truncateSync(indexPath, store.MAX_ORCA_PROFILE_INDEX_FILE_BYTES + 1) + + expect(store.getOrcaProfileListState().profiles.map((profile) => profile.id)).toContain( + created.profile.id + ) + }) + + it('rejects an oversized profile index without changing the primary or backup', async () => { + const store = await loadProfileIndexStore() + store.ensureActiveOrcaProfile() + store.createLocalOrcaProfile({ name: 'Work' }) + const indexPath = store.getOrcaProfileIndexPath() + const primaryBefore = readFileSync(indexPath, 'utf-8') + const backupBefore = readFileSync(`${indexPath}.bak`, 'utf-8') + const oversized = store.readProfileIndex(indexPath)! + oversized.profiles[0] = { + ...oversized.profiles[0], + name: 'x'.repeat(store.MAX_ORCA_PROFILE_INDEX_FILE_BYTES) + } + + expect(() => store.writeProfileIndex(indexPath, oversized)).toThrow( + `JSON output exceeds ${store.MAX_ORCA_PROFILE_INDEX_FILE_BYTES} bytes` + ) + expect(readFileSync(indexPath, 'utf-8')).toBe(primaryBefore) + expect(readFileSync(`${indexPath}.bak`, 'utf-8')).toBe(backupBefore) + expect(existsSync(`${indexPath}.tmp`)).toBe(false) + }) + it('rejects profile ids that are not safe path segments', async () => { const store = await loadProfileIndexStore() const indexPath = store.getOrcaProfileIndexPath() diff --git a/src/main/orca-profiles/profile-index-store.ts b/src/main/orca-profiles/profile-index-store.ts index 903e7a85ff8..366db309fc9 100644 --- a/src/main/orca-profiles/profile-index-store.ts +++ b/src/main/orca-profiles/profile-index-store.ts @@ -1,13 +1,8 @@ -import { - copyFileSync, - existsSync, - mkdirSync, - readFileSync, - renameSync, - writeFileSync -} from 'node:fs' +import { copyFileSync, existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs' import { randomUUID } from 'node:crypto' import { dirname } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' import type { GlobalSettings } from '../../shared/types' import { createDefaultLocalOrcaProfile, @@ -49,6 +44,8 @@ export type ActiveOrcaProfileState = { profileDirectory: string } +export const MAX_ORCA_PROFILE_INDEX_FILE_BYTES = 1024 * 1024 + function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } @@ -105,7 +102,13 @@ function sanitizeProfileName(value: unknown): string { function readProfileIndexFile(indexPath: string): OrcaProfileIndex | null { try { - return normalizeProfileIndex(JSON.parse(readFileSync(indexPath, 'utf-8'))) + return normalizeProfileIndex( + JSON.parse( + readNodeFileSyncWithinLimit(indexPath, MAX_ORCA_PROFILE_INDEX_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) + ) } catch { return null } @@ -118,6 +121,10 @@ export function readProfileIndex(indexPath: string): OrcaProfileIndex | null { } export function writeProfileIndex(indexPath: string, index: OrcaProfileIndex): void { + const serialized = stringifyJsonWithinByteLimit( + index, + MAX_ORCA_PROFILE_INDEX_FILE_BYTES + ).serialized mkdirSync(dirname(indexPath), { recursive: true }) // Why: only a still-parseable current index may refresh the backup; // copying a corrupt file over the backup would destroy the recovery copy. @@ -129,7 +136,7 @@ export function writeProfileIndex(indexPath: string, index: OrcaProfileIndex): v } } const tmpPath = `${indexPath}.tmp` - writeFileSync(tmpPath, JSON.stringify(index, null, 2), 'utf-8') + writeFileSync(tmpPath, serialized, 'utf-8') renameSync(tmpPath, indexPath) } diff --git a/src/main/orca-profiles/profile-project-state-file-bounds.test.ts b/src/main/orca-profiles/profile-project-state-file-bounds.test.ts new file mode 100644 index 00000000000..ffe06889259 --- /dev/null +++ b/src/main/orca-profiles/profile-project-state-file-bounds.test.ts @@ -0,0 +1,71 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getDefaultPersistedState } from '../../shared/constants' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { JsonStringifyByteLimitError } from '../../shared/node-bounded-json-stringify' +import { getOrcaProfileDataFile } from './profile-index-store' +import { readProfileState, writeProfileState } from './profile-project-state-file' + +vi.mock('electron', () => ({ + app: { + getPath: () => '' + } +})) + +describe('profile project state file bounds', () => { + let userDataPath = '' + const profileId = 'work' + + beforeEach(() => { + userDataPath = mkdtempSync(join(tmpdir(), 'orca-profile-state-bounds-')) + }) + + afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) + }) + + it('round-trips ordinary state with compact JSON semantics unchanged', () => { + const state = getDefaultPersistedState('/Users/tester') + state.settings.theme = 'dark' + const native = JSON.stringify(state) + + writeProfileState(profileId, userDataPath, state, Buffer.byteLength(native)) + + expect(readFileSync(getOrcaProfileDataFile(profileId, userDataPath), 'utf8')).toBe(native) + expect(readProfileState(profileId, userDataPath).settings.theme).toBe('dark') + }) + + it('rejects a profile read one byte over its limit before parsing', () => { + const dataFile = getOrcaProfileDataFile(profileId, userDataPath) + mkdirSync(dirname(dataFile), { recursive: true }) + writeFileSync(dataFile, '') + truncateSync(dataFile, 1025) + + expect(() => readProfileState(profileId, userDataPath, 1024)).toThrow(NodeFileReadTooLargeError) + }) + + it('leaves the prior profile and no temp file when output exceeds its limit', () => { + const state = getDefaultPersistedState('/Users/tester') + writeProfileState(profileId, userDataPath, state) + const dataFile = getOrcaProfileDataFile(profileId, userDataPath) + const prior = readFileSync(dataFile) + state.settings.theme = 'light' + const nextBytes = Buffer.byteLength(JSON.stringify(state)) + + expect(() => writeProfileState(profileId, userDataPath, state, nextBytes - 1)).toThrow( + JsonStringifyByteLimitError + ) + expect(readFileSync(dataFile)).toEqual(prior) + expect(readdirSync(dirname(dataFile)).filter((name) => name.endsWith('.tmp'))).toEqual([]) + }) +}) diff --git a/src/main/orca-profiles/profile-project-state-file.ts b/src/main/orca-profiles/profile-project-state-file.ts index 4169fea6ffc..b26bd123822 100644 --- a/src/main/orca-profiles/profile-project-state-file.ts +++ b/src/main/orca-profiles/profile-project-state-file.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto' -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname } from 'node:path' import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../../shared/constants' @@ -13,6 +13,10 @@ import type { SparsePreset, WorkspaceSessionState } from '../../shared/types' +import { + readPersistedStateJsonFileSync, + stringifyPersistedStateWithinLimit +} from '../../shared/persisted-state-file-bounds' import { getOrcaProfileDataFile } from './profile-index-store' export type TransferProfileState = PersistedState @@ -29,13 +33,20 @@ function recordOrEmpty(value: unknown): Record { return isRecord(value) ? (value as Record) : {} } -export function readProfileState(profileId: string, userDataPath: string): TransferProfileState { +export function readProfileState( + profileId: string, + userDataPath: string, + maxBytes?: number +): TransferProfileState { const defaults = getDefaultPersistedState(homedir()) const dataFile = getOrcaProfileDataFile(profileId, userDataPath) if (!existsSync(dataFile)) { return structuredClone(defaults) } - const parsed = JSON.parse(readFileSync(dataFile, 'utf-8')) as Partial + const { value: parsed } = readPersistedStateJsonFileSync>( + dataFile, + maxBytes + ) return rebuildRepoBackedProjectState({ ...defaults, ...parsed, @@ -84,13 +95,20 @@ export function readProfileState(profileId: string, userDataPath: string): Trans export function writeProfileState( profileId: string, userDataPath: string, - state: TransferProfileState + state: TransferProfileState, + maxBytes?: number ): void { const dataFile = getOrcaProfileDataFile(profileId, userDataPath) mkdirSync(dirname(dataFile), { recursive: true }) const tmpPath = `${dataFile}.${process.pid}.${randomUUID()}.tmp` - writeFileSync(tmpPath, JSON.stringify(state, null, 2), 'utf-8') - renameSync(tmpPath, dataFile) + const { serialized } = stringifyPersistedStateWithinLimit(state, maxBytes) + try { + writeFileSync(tmpPath, serialized, 'utf-8') + renameSync(tmpPath, dataFile) + } catch (error) { + rmSync(tmpPath, { force: true }) + throw error + } } function isRepoBackedProjectHostSetup( diff --git a/src/main/persistence-state-bounds.test.ts b/src/main/persistence-state-bounds.test.ts new file mode 100644 index 00000000000..98944036fcd --- /dev/null +++ b/src/main/persistence-state-bounds.test.ts @@ -0,0 +1,157 @@ +import { mkdtempSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + ORCA_PERSISTED_STATE_MAX_BYTES, + ORCA_PERSISTED_STATE_SECRET_MAX_BYTES +} from '../shared/persisted-state-file-bounds' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { + getPath: () => testState.dir + }, + safeStorage: { + isEncryptionAvailable: () => true, + encryptString: (plaintext: string) => Buffer.from(`encrypted:${plaintext}`, 'utf8'), + decryptString: (ciphertext: Buffer) => ciphertext.toString('utf8').replace(/^encrypted:/, '') + } +})) + +vi.mock('./ssh/ssh-config-parser', () => ({ + loadUserSshConfig: vi.fn(), + sshConfigHostsToTargets: vi.fn() +})) + +vi.mock('./telemetry/client', () => ({ + track: vi.fn() +})) + +vi.mock('./telemetry/cohort-classifier', () => ({ + getCohortAtEmit: vi.fn().mockReturnValue({ nth_repo_added: 0 }) +})) + +describe('Store persisted-state bounds', () => { + beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-persistence-bounds-')) + }) + + afterEach(() => { + vi.restoreAllMocks() + rmSync(testState.dir, { recursive: true, force: true }) + }) + + function dataFile(): string { + return join(testState.dir, 'orca-data.json') + } + + function backupFile(index: number): string { + return `${dataFile()}.bak.${index}` + } + + function writeMinimalState(path: string, theme: 'dark' | 'light'): string { + const serialized = JSON.stringify({ + schemaVersion: 1, + repos: [], + worktreeMeta: {}, + settings: { theme }, + ui: {}, + githubCache: { pr: {}, issue: {} }, + workspaceSession: {} + }) + writeFileSync(path, serialized) + return serialized + } + + async function createStore() { + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() + } + + it('recovers atomically from an in-limit backup when the primary is oversized', async () => { + writeFileSync(dataFile(), '') + truncateSync(dataFile(), ORCA_PERSISTED_STATE_MAX_BYTES + 1) + const backup = writeMinimalState(backupFile(0), 'dark') + + const store = await createStore() + + expect(store.getSettings().theme).toBe('dark') + expect(readFileSync(dataFile(), 'utf8')).toBe(backup) + store.updateSettings({ theme: 'light' }) + store.flushOrThrow() + expect(JSON.parse(readFileSync(dataFile(), 'utf8')).settings.theme).toBe('light') + }) + + it('skips an oversized backup slot and recovers from the next bounded slot', async () => { + writeFileSync(dataFile(), '{{corrupt') + writeFileSync(backupFile(0), '') + truncateSync(backupFile(0), ORCA_PERSISTED_STATE_MAX_BYTES + 1) + writeMinimalState(backupFile(1), 'light') + + const store = await createStore() + + expect(store.getSettings().theme).toBe('light') + }) + + it('uses defaults but freezes writes when no bounded recovery source exists', async () => { + vi.useFakeTimers() + try { + writeFileSync(dataFile(), '') + truncateSync(dataFile(), ORCA_PERSISTED_STATE_MAX_BYTES + 1) + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + + const store = await createStore() + store.updateSettings({ theme: 'dark' }) + await vi.advanceTimersByTimeAsync(2000) + await store.waitForPendingWrite() + store.flushOrThrow() + + expect(statSync(dataFile()).size).toBe(ORCA_PERSISTED_STATE_MAX_BYTES + 1) + expect(errorSpy).toHaveBeenCalledWith( + '[persistence] State exceeds the safe load limit; using defaults with state writes frozen' + ) + } finally { + vi.useRealTimers() + } + }) + + it('keeps the last valid file when a secret makes a sync flush exceed its bound', async () => { + const store = await createStore() + store.flushOrThrow() + const validState = readFileSync(dataFile()) + store.updateSettings({ + opencodeSessionCookie: 'x'.repeat(ORCA_PERSISTED_STATE_SECRET_MAX_BYTES + 1) + }) + + expect(() => store.flushOrThrow()).toThrow('Persisted state secret exceeds') + expect(readFileSync(dataFile())).toEqual(validState) + }) + + it('keeps the last valid file when an asynchronous bounded write is rejected', async () => { + vi.useFakeTimers() + try { + const store = await createStore() + store.flushOrThrow() + const validState = readFileSync(dataFile()) + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + store.updateSettings({ + opencodeSessionCookie: 'x'.repeat(ORCA_PERSISTED_STATE_SECRET_MAX_BYTES + 1) + }) + + await vi.advanceTimersByTimeAsync(2000) + await store.waitForPendingWrite() + + expect(readFileSync(dataFile())).toEqual(validState) + expect(errorSpy).toHaveBeenCalledWith( + '[persistence] Failed to write state:', + expect.objectContaining({ name: 'PersistedStateSecretCapacityError' }) + ) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/persistence.test.ts b/src/main/persistence.test.ts index 8e0532a6753..2a897ee18f6 100644 --- a/src/main/persistence.test.ts +++ b/src/main/persistence.test.ts @@ -9,7 +9,8 @@ import { existsSync, realpathSync, statSync, - symlinkSync + symlinkSync, + truncateSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' @@ -38,6 +39,7 @@ import { import { folderWorkspaceKey, worktreeWorkspaceKey } from '../shared/workspace-scope' import { toRuntimeExecutionHostId, toSshExecutionHostId } from '../shared/execution-host' import { SshConnectionStore } from './ssh/ssh-connection-store' +import { ORCA_PERSISTED_STATE_MAX_BYTES } from '../shared/persisted-state-file-bounds' import { setSourceControlActionDefault } from '../shared/source-control-ai-actions' import { LEGACY_DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../shared/ssh-types' import { closeTerminalTabInWorkspaceSession } from '../shared/workspace-session-terminal-tab-close' @@ -5491,6 +5493,16 @@ describe('Store', () => { expect(restarted.getGitHubCache().pr['o/r#7']).toEqual({ fetchedAt: 7 }) }) + it('ignores an oversized sparse GitHub cache sidecar', async () => { + const cacheFile = join(testState.dir, 'orca-github-cache.json') + writeFileSync(cacheFile, '') + truncateSync(cacheFile, ORCA_PERSISTED_STATE_MAX_BYTES + 1) + + const store = await createStore() + + expect(store.getGitHubCache()).toEqual({ pr: {}, issue: {} }) + }) + it('keeps GitHub cache sidecars scoped to explicit profile data files', async () => { const profileADir = join(testState.dir, 'profiles', 'a') const profileBDir = join(testState.dir, 'profiles', 'b') diff --git a/src/main/persistence.ts b/src/main/persistence.ts index 0b070b79083..a480a9a82e6 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -1,7 +1,6 @@ /* eslint-disable max-lines -- Why: persistence keeps schema defaults, migration, and load/save/flush in one file so the storage contract reviews as a unit. */ import { app, safeStorage } from 'electron' import { - readFileSync, writeFileSync, mkdirSync, existsSync, @@ -245,6 +244,19 @@ import { import { track } from './telemetry/client' import { getCohortAtEmit } from './telemetry/cohort-classifier' import { isStartupDiagnosticsEnabled, logStartupDiagnostic } from './startup/startup-diagnostics' +import { + assertPersistedStateSecretWithinLimit, + encodePersistedStateJsonStringContent, + isPersistedStateFileCapacityError, + parsePersistedStateJsonBuffer, + readPersistedStateJsonFileSync, + readPersistedStateFileBytesSync, + replacePersistedStateJsonWithinLimit, + replacedPersistedStateJsonByteLength, + restorePersistedStateBackupSync, + stringifyPersistedStateWithinLimit, + updatePersistedStateHashWithJsonRange +} from '../shared/persisted-state-file-bounds' function encrypt(plaintext: string): string { if (!plaintext || !safeStorage.isEncryptionAvailable()) { @@ -406,7 +418,7 @@ function gcStaleWorktreeMeta(state: PersistedState): number { function readGithubCacheSnapshot(dataFile: string): PersistedState['githubCache'] | null { try { - const parsed = JSON.parse(readFileSync(getGithubCacheFile(dataFile), 'utf-8')) as unknown + const { value: parsed } = readPersistedStateJsonFileSync(getGithubCacheFile(dataFile)) const isPlainRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value) if ( @@ -2568,7 +2580,6 @@ export class Store { private lastWrittenStateHash: string | null = null private firstPendingSaveAt: number | null = null private githubCacheDirty = false - private gitUsernameCache = new Map() private loadNeedsSave = false private settingsChangeListeners = new Set< ( @@ -2757,24 +2768,26 @@ export class Store { } } - private restoreFromBackup(dataFile: string): boolean { + private restoreFromBackup(dataFile: string): { + rejectedOversizedBackup: boolean + restored: boolean + } { + let rejectedOversizedBackup = false for (let i = 0; i < BACKUP_COUNT; i++) { const path = backupPath(dataFile, i) if (!existsSync(path)) { continue } try { - const raw = readFileSync(path, 'utf-8') - JSON.parse(raw) - mkdirSync(dirname(dataFile), { recursive: true }) - writeFileSync(dataFile, raw, 'utf-8') + restorePersistedStateBackupSync(path, dataFile) console.warn(`[persistence] Recovered state from backup slot ${i}: ${path}`) - return true + return { rejectedOversizedBackup, restored: true } } catch (err) { + rejectedOversizedBackup ||= isPersistedStateFileCapacityError(err) console.error(`[persistence] Backup slot ${i} unusable, trying next:`, err) } } - return false + return { rejectedOversizedBackup, restored: false } } private load(allowBackupRecovery = true): PersistedState { @@ -2786,16 +2799,17 @@ export class Store { }) let result: PersistedState | null = null + let rejectedOversizedState = false try { if (fileExistedOnLoad) { const readStartedAt = performance.now() - const raw = readFileSync(dataFile, 'utf-8') + const { buffer } = readPersistedStateFileBytesSync(dataFile) logPersistenceStartupMilestone('persistence-read-done', { - bytes: Buffer.byteLength(raw), + bytes: buffer.byteLength, durationMs: Math.round(performance.now() - readStartedAt) }) logPersistenceStartupMilestone('persistence-json-parse-start') - const parsed = JSON.parse(raw) as PersistedState + const parsed = parsePersistedStateJsonBuffer(buffer) logPersistenceStartupMilestone('persistence-json-parse-done') // Why: secrets are stored encrypted via safeStorage; decrypt at the load boundary so the app sees plaintext. @@ -3339,6 +3353,7 @@ export class Store { } } } catch (err) { + rejectedOversizedState = isPersistedStateFileCapacityError(err) console.error('[persistence] Failed to load primary state, trying backups:', err) } @@ -3352,14 +3367,23 @@ export class Store { } } if (fileExistedOnLoad || hasBackup) { - if (this.restoreFromBackup(dataFile)) { + const recovery = this.restoreFromBackup(dataFile) + if (recovery.restored) { return this.load(false) } + rejectedOversizedState ||= recovery.rejectedOversizedBackup console.error('[persistence] No usable state file or backup found, using defaults') } } if (result === null) { + if (rejectedOversizedState) { + // Why: keep recoverable oversized bytes intact instead of replacing them with fallback defaults. + this.writesFrozen = true + console.error( + '[persistence] State exceeds the safe load limit; using defaults with state writes frozen' + ) + } result = getDefaultPersistedState(homedir()) } @@ -3548,49 +3572,79 @@ export class Store { // on deterministic-IV platforms (macOS/legacy-Linux OSCrypt). A per-slot // random UUID can't occur anywhere else in the serialized state (the user // sets their data before it is minted), so it appears exactly once. - const secretSubs: { sentinel: string; blob: string; plaintext: string }[] = [] - const encryptToSentinel = (plaintext: string): string => { - const blob = encrypt(plaintext) - // Deterministic already (empty secret / safeStorage unavailable / encrypt - // failure): blob === plaintext, so no normalization — and no sentinel, - // which also avoids substituting an empty or plaintext-shaped slot. - if (blob === plaintext) { - return blob - } + const secretSubs: { sentinel: string; plaintext: string }[] = [] + const replaceSecretWithSentinel = (plaintext: string): string => { + assertPersistedStateSecretWithinLimit(plaintext) const sentinel = `orca-secret-slot-${randomUUID()}` - secretSubs.push({ sentinel, blob, plaintext }) + secretSubs.push({ sentinel, plaintext }) return sentinel } - // Why: clone before encrypting secrets so in-memory this.state stays plaintext. + // Why: clone with sentinels so in-memory this.state stays plaintext. const stateToSave = { ...this.getDurableState(), settings: { ...this.state.settings, - opencodeSessionCookie: encryptToSentinel(this.state.settings.opencodeSessionCookie), - httpProxyUrl: encryptToSentinel(this.state.settings.httpProxyUrl ?? '') + opencodeSessionCookie: replaceSecretWithSentinel(this.state.settings.opencodeSessionCookie), + httpProxyUrl: replaceSecretWithSentinel(this.state.settings.httpProxyUrl ?? '') }, ui: { ...this.state.ui, browserKagiSessionLink: this.state.ui.browserKagiSessionLink - ? encryptToSentinel(this.state.ui.browserKagiSessionLink) + ? replaceSecretWithSentinel(this.state.ui.browserKagiSessionLink) : null } } // Why compact: ~20% fewer bytes and less serialize time; all readers JSON.parse so formatting is irrelevant. // One full-state stringify; secret slots currently hold sentinels. - const serialized = JSON.stringify(stateToSave) - // Substitute each unique sentinel exactly once: ciphertext for the on-disk - // payload, plaintext for the guard hash. Function-form replacement keeps - // `$` in blob/plaintext inert; both sides read the sentinel as JSON-escaped - // in `serialized`, so each replace is byte-for-byte position-exact. - let payload = serialized - let hashInput = serialized - for (const { sentinel, blob, plaintext } of secretSubs) { - const escapedSentinel = JSON.stringify(sentinel).slice(1, -1) - payload = payload.replace(escapedSentinel, () => blob) - hashInput = hashInput.replace(escapedSentinel, () => JSON.stringify(plaintext).slice(1, -1)) + const { serialized, byteLength: serializedBytes } = + stringifyPersistedStateWithinLimit(stateToSave) + const preparedSecretSubs: { + escapedPlaintext: string + escapedSentinel: string + index: number + plaintext: string + }[] = [] + let normalizedBytes = serializedBytes + for (const { sentinel, plaintext } of secretSubs) { + const escapedSentinel = encodePersistedStateJsonStringContent(sentinel) + const escapedPlaintext = encodePersistedStateJsonStringContent(plaintext) + const index = serialized.indexOf(escapedSentinel) + if (index === -1 || serialized.includes(escapedSentinel, index + escapedSentinel.length)) { + throw new Error('Persisted state secret sentinel is missing or ambiguous') + } + normalizedBytes = replacedPersistedStateJsonByteLength({ + currentBytes: normalizedBytes, + search: escapedSentinel, + replacement: escapedPlaintext + }) + preparedSecretSubs.push({ escapedPlaintext, escapedSentinel, index, plaintext }) } - const stateHash = createHash('sha1').update(hashInput).digest('hex') + + const normalizedHash = createHash('sha1') + let normalizedOffset = 0 + for (const { escapedPlaintext, escapedSentinel, index } of preparedSecretSubs.sort( + (left, right) => left.index - right.index + )) { + updatePersistedStateHashWithJsonRange(normalizedHash, serialized, normalizedOffset, index) + updatePersistedStateHashWithJsonRange(normalizedHash, escapedPlaintext) + normalizedOffset = index + escapedSentinel.length + } + updatePersistedStateHashWithJsonRange(normalizedHash, serialized, normalizedOffset) + + let payload = serialized + let payloadBytes = serializedBytes + for (const { escapedSentinel, plaintext } of preparedSecretSubs) { + const escapedBlob = encodePersistedStateJsonStringContent(encrypt(plaintext)) + const nextPayload = replacePersistedStateJsonWithinLimit({ + serialized: payload, + currentBytes: payloadBytes, + search: escapedSentinel, + replacement: escapedBlob + }) + payload = nextPayload.serialized + payloadBytes = nextPayload.byteLength + } + const stateHash = normalizedHash.digest('hex') return { payload, stateHash } } @@ -3833,8 +3887,7 @@ export class Store { if (!repo) { return false } - const previous = this.gitUsernameCache.get(repo.path) ?? repo.gitUsername ?? '' - this.gitUsernameCache.set(repo.path, username) + const previous = repo.gitUsername ?? '' if (previous === username) { return false } @@ -4486,9 +4539,7 @@ export class Store { const projectHostSetupMethod = sanitizeRepoProjectHostSetupMethod(rawProjectHostSetupMethod) const forkSyncMode = sanitizeForkSyncMode(rawForkSyncMode) // Why: never spawn git/gh username resolution in hydration — a stuck probe froze Windows startup for minutes (issue #7225); read only cache/persisted value. - const gitUsername = isFolderRepo(repo) - ? '' - : (this.gitUsernameCache.get(repo.path) ?? repo.gitUsername ?? '') + const gitUsername = isFolderRepo(repo) ? '' : (repo.gitUsername ?? '') return { ...repoWithoutIcon, @@ -6506,7 +6557,8 @@ export class Store { const cacheFile = getGithubCacheFile(this.dataFile) const tmpFile = `${cacheFile}.${process.pid}.tmp` try { - writeFileSync(tmpFile, JSON.stringify(this.state.githubCache), 'utf-8') + const { serialized } = stringifyPersistedStateWithinLimit(this.state.githubCache) + writeFileSync(tmpFile, serialized, 'utf-8') renameSync(tmpFile, cacheFile) this.githubCacheDirty = false } catch (err) { diff --git a/src/main/pi/agent-status-extension-source.test.ts b/src/main/pi/agent-status-extension-source.test.ts index fd65d81a6cb..49af5f54fad 100644 --- a/src/main/pi/agent-status-extension-source.test.ts +++ b/src/main/pi/agent-status-extension-source.test.ts @@ -3,6 +3,7 @@ import { runInNewContext } from 'node:vm' import ts from 'typescript-api' import { describe, expect, it, vi } from 'vitest' +import { GENERATED_NODE_MANAGED_FILE_MAX_BYTES } from '../generated-node-bounded-file-reader' import { getPiAgentStatusExtensionSource } from './agent-status-extension-source' type HookContext = { @@ -28,11 +29,16 @@ type Harness = { spawnMock: ReturnType spawnedChildren: FakeCurlChild[] fsMock: { + closeSync: ReturnType existsSync: ReturnType + openSync: ReturnType + readSync: ReturnType readFileSync: ReturnType + statSync: ReturnType } handlers: Record processEnv: Record + warnMock: ReturnType callHook: (name: string, event?: unknown, context?: HookContext) => Promise // Re-invoke the extension factory in the same process (as Pi does on an // in-process extension reload), swapping in the freshly registered handlers. @@ -62,6 +68,7 @@ function createHarness(args: { argv?: string[] existsSync?: (path: string) => boolean readFileSync?: (path: string, encoding: string) => string + statSync?: (path: string) => { ino: number; mtimeMs: number; size: number } fetchImpl?: (...params: Parameters) => Promise }): Harness { const fetchMock = vi.fn( @@ -84,14 +91,50 @@ function createHarness(args: { return child }) + const readFileSyncMock = vi.fn( + args.readFileSync ?? + ((path: string) => { + throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }) + }) + ) + const openFiles = new Map() + let nextDescriptor = 10 const fsMock = { existsSync: vi.fn(args.existsSync ?? (() => false)), - readFileSync: vi.fn( - args.readFileSync ?? - ((path: string) => { - throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }) - }) - ) + readFileSync: readFileSyncMock, + statSync: vi.fn(args.statSync ?? (() => ({ ino: 1, mtimeMs: 1, size: 0 }))), + openSync: vi.fn((path: string) => { + const descriptor = nextDescriptor++ + openFiles.set(descriptor, { + bytes: Buffer.from(String(readFileSyncMock(path, 'utf8')), 'utf8'), + offset: 0 + }) + return descriptor + }), + readSync: vi.fn( + ( + descriptor: number, + target: Buffer, + offset: number, + length: number, + position: number | null + ) => { + const opened = openFiles.get(descriptor) + if (!opened) { + throw new Error(`bad descriptor: ${descriptor}`) + } + const start = position ?? opened.offset + const bytesRead = Math.min(length, Math.max(0, opened.bytes.byteLength - start)) + opened.bytes.copy(target, offset, start, start + bytesRead) + if (position === null) { + opened.offset += bytesRead + } + return bytesRead + } + ), + closeSync: vi.fn((descriptor: number) => { + openFiles.delete(descriptor) + }) } const module = { @@ -117,6 +160,7 @@ function createHarness(args: { argv: args.argv ?? ['node', '/usr/bin/orca'] } + const warnMock = vi.fn() const context = { module, exports: module.exports, @@ -124,7 +168,7 @@ function createHarness(args: { process: processMock, fetch: fetchMock, console: { - warn: vi.fn(), + warn: warnMock, error: vi.fn(), log: vi.fn() }, @@ -168,6 +212,7 @@ function createHarness(args: { fsMock, handlers, processEnv: processMock.env, + warnMock, callHook: async (name, event, hookContext) => { await handlers[name]?.(event, hookContext) }, @@ -181,6 +226,72 @@ function createHarness(args: { } describe('getPiAgentStatusExtensionSource', () => { + it('routes every whole-file runtime probe through the emitted bounded reader', () => { + const source = getPiAgentStatusExtensionSource('pi') + + expect(source).toContain( + `function readOrcaManagedFileWithinLimit(fs: any, path: string, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES})` + ) + expect(source).not.toContain('readFileSync') + }) + + it('uses an ordinary endpoint file before the stale process environment', async () => { + const endpointPath = '/tmp/orca-endpoint.env' + const endpoint = + 'ORCA_AGENT_HOOK_PORT=9876\nORCA_AGENT_HOOK_TOKEN=fresh-token\nORCA_AGENT_HOOK_ENV=fresh-env\n' + const harness = createHarness({ + kind: 'pi', + env: { ORCA_AGENT_HOOK_ENDPOINT: endpointPath }, + readFileSync: (path) => { + if (path === endpointPath) { + return endpoint + } + throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }) + }, + statSync: () => ({ ino: 1, mtimeMs: 1, size: Buffer.byteLength(endpoint) }) + }) + + await harness.callHook('agent_start') + + expect(harness.fetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:9876/hook/pi', + expect.objectContaining({ + headers: expect.objectContaining({ + 'X-Orca-Agent-Hook-Token': 'fresh-token' + }) + }) + ) + }) + + it('fails open to process env when the endpoint file exceeds the generated byte cap', async () => { + const endpointPath = '/tmp/orca-endpoint.env' + const oversized = 'x'.repeat(GENERATED_NODE_MANAGED_FILE_MAX_BYTES + 1) + const harness = createHarness({ + kind: 'pi', + env: { ORCA_AGENT_HOOK_ENDPOINT: endpointPath }, + readFileSync: (path) => { + if (path === endpointPath) { + return oversized + } + throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }) + }, + statSync: () => ({ ino: 1, mtimeMs: 1, size: oversized.length }) + }) + + await harness.callHook('agent_start') + + expect(harness.fetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:4321/hook/pi', + expect.objectContaining({ + headers: expect.objectContaining({ + 'X-Orca-Agent-Hook-Token': 'token-1' + }) + }) + ) + expect(harness.warnMock).toHaveBeenCalledOnce() + expect(harness.fsMock.closeSync).toHaveBeenCalledOnce() + }) + it('includes the session id and file path in Pi status posts after session_start', async () => { const harness = createHarness({ kind: 'pi', diff --git a/src/main/pi/agent-status-extension-source.ts b/src/main/pi/agent-status-extension-source.ts index 881640183de..a4f111bb710 100644 --- a/src/main/pi/agent-status-extension-source.ts +++ b/src/main/pi/agent-status-extension-source.ts @@ -11,6 +11,7 @@ // keep the source body in plain JS without TS types and avoid pulling pi or // any Orca dep into the pi runtime. import type { PiAgentKind } from '../../shared/pi-agent-kind' +import { getGeneratedNodeBoundedFileReaderSourceLines } from '../generated-node-bounded-file-reader' import { getPiAgentStatusHandlerSourceLines } from './agent-status-handler-source' export const ORCA_PI_AGENT_STATUS_EXTENSION_FILE = 'orca-agent-status.ts' @@ -65,6 +66,7 @@ export function getPiAgentStatusExtensionSource(kind: PiAgentKind = 'pi'): strin '// critical path, and the latest-only pending slot prevents a stalled', '// Orca receiver from building an unbounded queue of obsolete snapshots.', 'const HOOK_POST_TIMEOUT_MS = 1000', + ...getGeneratedNodeBoundedFileReaderSourceLines({ typed: true }), 'let activePost = false', 'let pendingPost: { hookEventName: string; extra: Record } | null = null', ...sessionMetadataSourceLines, @@ -86,7 +88,7 @@ export function getPiAgentStatusExtensionSource(kind: PiAgentKind = 'pi'): strin ' if (cacheKey === cachedEndpointKey && cachedEndpointValues) {', ' return cachedEndpointValues', ' }', - " const contents: string = fs.readFileSync(path, 'utf8')", + ' const contents = readOrcaManagedFileWithinLimit(fs, path)', ' const out: Record = {}', ' for (const line of contents.split(/\\r?\\n/)) {', ' // Why: parse `KEY=VALUE` (POSIX endpoint.env) and `set KEY=VALUE`', @@ -232,7 +234,7 @@ export function getPiAgentStatusExtensionSource(kind: PiAgentKind = 'pi'): strin " const fs = require('fs')", " for (const path of ['/proc/sys/kernel/osrelease', '/proc/version']) {", ' try {', - " const contents = String(fs.readFileSync(path, 'utf8'))", + ' const contents = readOrcaManagedFileWithinLimit(fs, path)', ' if (/microsoft|wsl/i.test(contents)) return true', ' } catch {', ' // Why: probe the next runtime hint when a proc file is absent or unreadable.', diff --git a/src/main/pi/legacy-omp-overlay-migration-bounds.test.ts b/src/main/pi/legacy-omp-overlay-migration-bounds.test.ts new file mode 100644 index 00000000000..b709ae437d3 --- /dev/null +++ b/src/main/pi/legacy-omp-overlay-migration-bounds.test.ts @@ -0,0 +1,122 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + LEGACY_OMP_OVERLAY_MIGRATION_MAX_DEPTH, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_ENTRIES, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_PATH_BYTES, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_RETAINED_PATH_BYTES, + migrateLegacyOmpOverlayState +} from './legacy-omp-overlay-migration' + +const roots: string[] = [] +const MARKER = '.orca-omp-overlay-migration-complete' + +function tempPair(): { overlay: string; source: string } { + const root = mkdtempSync(join(tmpdir(), 'orca-legacy-overlay-bounds-')) + roots.push(root) + const overlay = join(root, 'overlay') + const source = join(root, 'source') + mkdirSync(overlay) + return { overlay, source } +} + +afterEach(() => { + vi.restoreAllMocks() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('legacy OMP overlay migration bounds', () => { + it('publishes finite production traversal and retention limits', () => { + expect(LEGACY_OMP_OVERLAY_MIGRATION_MAX_ENTRIES).toBe(100_000) + expect(LEGACY_OMP_OVERLAY_MIGRATION_MAX_DEPTH).toBe(256) + expect(LEGACY_OMP_OVERLAY_MIGRATION_MAX_PATH_BYTES).toBe(64 * 1_024) + expect(LEGACY_OMP_OVERLAY_MIGRATION_MAX_RETAINED_PATH_BYTES).toBe(16 * 1_024 * 1_024) + }) + + it('marks a migration at the exact entry limit', () => { + const { overlay, source } = tempPair() + writeFileSync(join(overlay, 'one'), '1') + writeFileSync(join(overlay, 'two'), '2') + + migrateLegacyOmpOverlayState(source, overlay, { maxEntries: 2 }) + + expect(readFileSync(join(source, 'one'), 'utf8')).toBe('1') + expect(readFileSync(join(source, 'two'), 'utf8')).toBe('2') + expect(existsSync(join(overlay, MARKER))).toBe(true) + }) + + it('withholds the marker when the next streamed entry exceeds the limit', () => { + const { overlay, source } = tempPair() + writeFileSync(join(overlay, 'one'), '1') + writeFileSync(join(overlay, 'two'), '2') + writeFileSync(join(overlay, 'three'), '3') + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + migrateLegacyOmpOverlayState(source, overlay, { maxEntries: 2 }) + + expect(existsSync(join(overlay, MARKER))).toBe(false) + expect(existsSync(join(source, 'one'))).toBe(false) + expect(existsSync(join(source, 'two'))).toBe(false) + expect(existsSync(join(source, 'three'))).toBe(false) + expect(warnSpy).toHaveBeenCalledWith( + '[pi-titlebar-extension] failed to migrate legacy OMP overlay state:', + expect.objectContaining({ name: 'LegacyOverlayMigrationCapacityError' }) + ) + }) + + it('accepts the exact path and retained-path byte boundaries', () => { + const exact = tempPair() + const exactOverlayPath = join(exact.overlay, 'state') + const exactTargetPath = join(exact.source, 'state') + writeFileSync(exactOverlayPath, 'state') + const exactPathBytes = Math.max( + Buffer.byteLength(exactOverlayPath, 'utf8'), + Buffer.byteLength(exactTargetPath, 'utf8') + ) + const exactRetainedBytes = Buffer.byteLength(exactTargetPath, 'utf8') + + migrateLegacyOmpOverlayState(exact.source, exact.overlay, { + maxPathBytes: exactPathBytes, + maxRetainedPathBytes: exactRetainedBytes + }) + + expect(readFileSync(exactTargetPath, 'utf8')).toBe('state') + expect(existsSync(join(exact.overlay, MARKER))).toBe(true) + + const rejected = tempPair() + const rejectedOverlayPath = join(rejected.overlay, 'state') + const rejectedTargetPath = join(rejected.source, 'state') + writeFileSync(rejectedOverlayPath, 'state') + const retainedLimit = Buffer.byteLength(rejectedTargetPath, 'utf8') - 1 + + migrateLegacyOmpOverlayState(rejected.source, rejected.overlay, { + maxRetainedPathBytes: retainedLimit + }) + + expect(existsSync(rejectedTargetPath)).toBe(false) + expect(existsSync(join(rejected.overlay, MARKER))).toBe(false) + }) + + it('accepts the exact path depth and rejects the next level', () => { + const accepted = tempPair() + mkdirSync(join(accepted.overlay, 'one', 'two'), { recursive: true }) + writeFileSync(join(accepted.overlay, 'one', 'two', 'leaf'), 'leaf') + + migrateLegacyOmpOverlayState(accepted.source, accepted.overlay, { maxDepth: 3 }) + + expect(readFileSync(join(accepted.source, 'one', 'two', 'leaf'), 'utf8')).toBe('leaf') + expect(existsSync(join(accepted.overlay, MARKER))).toBe(true) + + const rejected = tempPair() + mkdirSync(join(rejected.overlay, 'one', 'two'), { recursive: true }) + writeFileSync(join(rejected.overlay, 'one', 'two', 'leaf'), 'leaf') + + migrateLegacyOmpOverlayState(rejected.source, rejected.overlay, { maxDepth: 2 }) + + expect(existsSync(join(rejected.overlay, MARKER))).toBe(false) + }) +}) diff --git a/src/main/pi/legacy-omp-overlay-migration-budget.ts b/src/main/pi/legacy-omp-overlay-migration-budget.ts new file mode 100644 index 00000000000..fe4d1aaf440 --- /dev/null +++ b/src/main/pi/legacy-omp-overlay-migration-budget.ts @@ -0,0 +1,98 @@ +export const LEGACY_OMP_OVERLAY_MIGRATION_MAX_ENTRIES = 100_000 +export const LEGACY_OMP_OVERLAY_MIGRATION_MAX_DEPTH = 256 +export const LEGACY_OMP_OVERLAY_MIGRATION_MAX_PATH_BYTES = 64 * 1_024 +export const LEGACY_OMP_OVERLAY_MIGRATION_MAX_RETAINED_PATH_BYTES = 16 * 1_024 * 1_024 + +export type LegacyOverlayMigrationLimits = { + maxDepth: number + maxEntries: number + maxPathBytes: number + maxRetainedPathBytes: number +} + +const DEFAULT_LIMITS: LegacyOverlayMigrationLimits = { + maxDepth: LEGACY_OMP_OVERLAY_MIGRATION_MAX_DEPTH, + maxEntries: LEGACY_OMP_OVERLAY_MIGRATION_MAX_ENTRIES, + maxPathBytes: LEGACY_OMP_OVERLAY_MIGRATION_MAX_PATH_BYTES, + maxRetainedPathBytes: LEGACY_OMP_OVERLAY_MIGRATION_MAX_RETAINED_PATH_BYTES +} + +export class LegacyOverlayMigrationCapacityError extends Error { + constructor(kind: string, observed: number, limit: number) { + super(`Legacy OMP overlay ${kind} exceeded its ${limit} limit (observed ${observed})`) + this.name = 'LegacyOverlayMigrationCapacityError' + } +} + +export class LegacyOverlayMigrationBudget { + private entries = 0 + private retainedPathBytes = 0 + readonly limits: LegacyOverlayMigrationLimits + + constructor(requested?: Partial) { + this.limits = { + maxDepth: resolveLimit(requested?.maxDepth, DEFAULT_LIMITS.maxDepth, 'maxDepth'), + maxEntries: resolveLimit(requested?.maxEntries, DEFAULT_LIMITS.maxEntries, 'maxEntries'), + maxPathBytes: resolveLimit( + requested?.maxPathBytes, + DEFAULT_LIMITS.maxPathBytes, + 'maxPathBytes' + ), + maxRetainedPathBytes: resolveLimit( + requested?.maxRetainedPathBytes, + DEFAULT_LIMITS.maxRetainedPathBytes, + 'maxRetainedPathBytes' + ) + } + } + + visit(depth: number, ...paths: string[]): void { + const nextEntries = this.entries + 1 + if (nextEntries > this.limits.maxEntries) { + throw new LegacyOverlayMigrationCapacityError('entries', nextEntries, this.limits.maxEntries) + } + if (depth > this.limits.maxDepth) { + throw new LegacyOverlayMigrationCapacityError('depth', depth, this.limits.maxDepth) + } + for (const path of paths) { + this.measurePath(path) + } + this.entries = nextEntries + } + + retainPaths(...paths: string[]): number { + const retainedBytes = paths.reduce((total, path) => total + this.measurePath(path), 0) + const nextRetainedBytes = this.retainedPathBytes + retainedBytes + if (nextRetainedBytes > this.limits.maxRetainedPathBytes) { + throw new LegacyOverlayMigrationCapacityError( + 'retained-path-bytes', + nextRetainedBytes, + this.limits.maxRetainedPathBytes + ) + } + this.retainedPathBytes = nextRetainedBytes + return retainedBytes + } + + releasePaths(retainedBytes: number): void { + this.retainedPathBytes = Math.max(0, this.retainedPathBytes - retainedBytes) + } + + private measurePath(path: string): number { + const bytes = Buffer.byteLength(path, 'utf8') + if (bytes > this.limits.maxPathBytes) { + throw new LegacyOverlayMigrationCapacityError('path-bytes', bytes, this.limits.maxPathBytes) + } + return bytes + } +} + +function resolveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} diff --git a/src/main/pi/legacy-omp-overlay-migration.ts b/src/main/pi/legacy-omp-overlay-migration.ts index 88baf78ea9f..2bee66014ca 100644 --- a/src/main/pi/legacy-omp-overlay-migration.ts +++ b/src/main/pi/legacy-omp-overlay-migration.ts @@ -1,7 +1,11 @@ -import { cpSync, lstatSync, mkdirSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs' -import type { Dirent, Stats } from 'node:fs' +import { cpSync, lstatSync, mkdirSync, opendirSync, unlinkSync, writeFileSync } from 'node:fs' +import type { Stats } from 'node:fs' import { dirname, join } from 'node:path' import { ORCA_PI_AGENT_STATUS_EXTENSION_FILE } from './agent-status-extension-source' +import { + LegacyOverlayMigrationBudget, + type LegacyOverlayMigrationLimits +} from './legacy-omp-overlay-migration-budget' import { ORCA_PI_PREFILL_EXTENSION_FILE } from './prefill-extension-source' import { ORCA_PI_EXTENSION_FILE } from './titlebar-extension-source' import { isSafeDescendCandidate } from '../pty/overlay-mirror' @@ -19,8 +23,27 @@ const MANAGED_EXTENSION_FILES = new Set([ type DeferredSidecar = { baseTargetPath: string - entry: Dirent - nextSegments: string[] + overlayPath: string + retainedBytes: number + targetPath: string +} + +export { + LEGACY_OMP_OVERLAY_MIGRATION_MAX_DEPTH, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_ENTRIES, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_PATH_BYTES, + LEGACY_OMP_OVERLAY_MIGRATION_MAX_RETAINED_PATH_BYTES +} from './legacy-omp-overlay-migration-budget' +export type { LegacyOverlayMigrationLimits } from './legacy-omp-overlay-migration-budget' + +function closeDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ERR_DIR_CLOSED') { + throw error + } + } } function getPathStats(path: string): Stats | undefined { @@ -64,11 +87,13 @@ function shouldSkipLegacyOverlayEntry(pathSegments: string[]): boolean { function copyLegacyFile( overlayPath: string, targetPath: string, - copiedFilePaths: Set + copiedFilePaths: Set, + budget: LegacyOverlayMigrationBudget ): boolean { if (getPathStats(targetPath)) { return true } + const retainedBytes = budget.retainPaths(targetPath) try { mkdirSync(dirname(targetPath), { recursive: true }) cpSync(overlayPath, targetPath, { @@ -79,15 +104,21 @@ function copyLegacyFile( copiedFilePaths.add(targetPath) return true } catch { + budget.releasePaths(retainedBytes) return false } } -function removeCopiedFiles(paths: string[], copiedFilePaths: Set): void { +function removeCopiedFiles( + paths: string[], + copiedFilePaths: Set, + budget: LegacyOverlayMigrationBudget +): void { for (const path of paths) { if (!copiedFilePaths.delete(path)) { continue } + budget.releasePaths(Buffer.byteLength(path, 'utf8')) try { unlinkSync(path) } catch { @@ -97,46 +128,50 @@ function removeCopiedFiles(paths: string[], copiedFilePaths: Set): void } function copyDeferredSidecars( - overlayDir: string, - sourceAgentDir: string, + budget: LegacyOverlayMigrationBudget, copiedFilePaths: Set, deferredSidecars: DeferredSidecar[] ): boolean { let completed = true const copiedSidecarsByBase = new Map() - for (const { baseTargetPath, entry, nextSegments } of deferredSidecars) { - if (!copiedFilePaths.has(baseTargetPath)) { - continue - } + for (const sidecar of deferredSidecars) { + const { baseTargetPath, overlayPath, targetPath } = sidecar + try { + if (!copiedFilePaths.has(baseTargetPath)) { + continue + } - const overlayPath = join(overlayDir, entry.name) - const targetPath = join(sourceAgentDir, ...nextSegments) - const stats = getPathStats(overlayPath) - if (!stats) { - removeCopiedFiles( - [baseTargetPath, ...(copiedSidecarsByBase.get(baseTargetPath) ?? [])], - copiedFilePaths - ) - completed = false - continue - } - if (stats.isSymbolicLink() || !stats.isFile()) { - continue - } - const wasCopied = copiedFilePaths.has(targetPath) - if (!copyLegacyFile(overlayPath, targetPath, copiedFilePaths)) { - removeCopiedFiles( - [baseTargetPath, ...(copiedSidecarsByBase.get(baseTargetPath) ?? [])], - copiedFilePaths - ) - completed = false - continue - } - if (!wasCopied && copiedFilePaths.has(targetPath)) { - copiedSidecarsByBase.set(baseTargetPath, [ - ...(copiedSidecarsByBase.get(baseTargetPath) ?? []), - targetPath - ]) + const stats = getPathStats(overlayPath) + if (!stats) { + removeCopiedFiles( + [baseTargetPath, ...(copiedSidecarsByBase.get(baseTargetPath) ?? [])], + copiedFilePaths, + budget + ) + completed = false + continue + } + if (stats.isSymbolicLink() || !stats.isFile()) { + continue + } + const wasCopied = copiedFilePaths.has(targetPath) + if (!copyLegacyFile(overlayPath, targetPath, copiedFilePaths, budget)) { + removeCopiedFiles( + [baseTargetPath, ...(copiedSidecarsByBase.get(baseTargetPath) ?? [])], + copiedFilePaths, + budget + ) + completed = false + continue + } + if (!wasCopied && copiedFilePaths.has(targetPath)) { + copiedSidecarsByBase.set(baseTargetPath, [ + ...(copiedSidecarsByBase.get(baseTargetPath) ?? []), + targetPath + ]) + } + } finally { + budget.releasePaths(sidecar.retainedBytes) } } return completed @@ -145,78 +180,93 @@ function copyDeferredSidecars( function copyMissingLegacyOmpOverlayEntries( overlayDir: string, sourceAgentDir: string, - pathSegments: string[] = [], - copiedFilePaths: Set = new Set() + pathSegments: string[], + copiedFilePaths: Set, + budget: LegacyOverlayMigrationBudget ): boolean { let completed = true - let entries: Dirent[] + let directory try { - entries = readdirSync(overlayDir, { withFileTypes: true }) + directory = opendirSync(overlayDir, { bufferSize: 32 }) } catch { return false } const deferredSidecars: DeferredSidecar[] = [] - for (const entry of entries) { - const nextSegments = [...pathSegments, entry.name] - if (shouldSkipLegacyOverlayEntry(nextSegments)) { - continue - } - - const sidecarBaseName = getSqliteSidecarBaseName(entry.name) - if (sidecarBaseName) { - deferredSidecars.push({ - baseTargetPath: join(sourceAgentDir, ...pathSegments, sidecarBaseName), - entry, - nextSegments - }) - continue - } - - const overlayPath = join(overlayDir, entry.name) - const targetPath = join(sourceAgentDir, ...nextSegments) - const stats = getPathStats(overlayPath) - if (!stats) { - completed = false - continue - } - if (stats.isSymbolicLink()) { - continue - } - if (stats.isDirectory()) { - const targetStats = getPathStats(targetPath) - if (targetStats && !isSafeDescendCandidate(targetStats)) { + try { + while (true) { + const entry = directory.readSync() + if (entry === null) { + break + } + const nextSegments = [...pathSegments, entry.name] + const overlayPath = join(overlayDir, entry.name) + const targetPath = join(sourceAgentDir, ...nextSegments) + budget.visit(nextSegments.length, overlayPath, targetPath) + if (shouldSkipLegacyOverlayEntry(nextSegments)) { continue } - if (!targetStats) { - try { - mkdirSync(targetPath, { recursive: true }) - } catch { - completed = false + + const sidecarBaseName = getSqliteSidecarBaseName(entry.name) + if (sidecarBaseName) { + const baseTargetPath = join(sourceAgentDir, ...pathSegments, sidecarBaseName) + deferredSidecars.push({ + baseTargetPath, + overlayPath, + retainedBytes: budget.retainPaths(baseTargetPath, overlayPath, targetPath), + targetPath + }) + continue + } + + const stats = getPathStats(overlayPath) + if (!stats) { + completed = false + continue + } + if (stats.isSymbolicLink()) { + continue + } + if (stats.isDirectory()) { + const targetStats = getPathStats(targetPath) + if (targetStats && !isSafeDescendCandidate(targetStats)) { continue } + if (!targetStats) { + try { + mkdirSync(targetPath, { recursive: true }) + } catch { + completed = false + continue + } + } + completed = + copyMissingLegacyOmpOverlayEntries( + overlayPath, + sourceAgentDir, + nextSegments, + copiedFilePaths, + budget + ) && completed + continue } - completed = - copyMissingLegacyOmpOverlayEntries( - overlayPath, - sourceAgentDir, - nextSegments, - copiedFilePaths - ) && completed - continue + if (!stats.isFile()) { + continue + } + completed = copyLegacyFile(overlayPath, targetPath, copiedFilePaths, budget) && completed } - if (!stats.isFile()) { - continue - } - completed = copyLegacyFile(overlayPath, targetPath, copiedFilePaths) && completed + } finally { + closeDirectory(directory) } - return ( - copyDeferredSidecars(overlayDir, sourceAgentDir, copiedFilePaths, deferredSidecars) && completed - ) + return copyDeferredSidecars(budget, copiedFilePaths, deferredSidecars) && completed } -export function migrateLegacyOmpOverlayState(sourceAgentDir: string, overlayDir: string): void { +export function migrateLegacyOmpOverlayState( + sourceAgentDir: string, + overlayDir: string, + limits?: Partial +): void { // Why: temporary rescue shim for OMP builds from the legacy overlay window. // Remove after 2026-08-07 once affected users have had a full upgrade window. const overlayStats = getPathStats(overlayDir) @@ -227,16 +277,23 @@ export function migrateLegacyOmpOverlayState(sourceAgentDir: string, overlayDir: if (getPathStats(markerPath)) { return } + const budget = new LegacyOverlayMigrationBudget(limits) + const copiedFilePaths = new Set() try { mkdirSync(sourceAgentDir, { recursive: true }) // Why: some pre-managed-extension builds pointed OMP at this overlay, so // first-login auth/session files can exist only there after an update. - if (copyMissingLegacyOmpOverlayEntries(overlayDir, sourceAgentDir)) { + if ( + copyMissingLegacyOmpOverlayEntries(overlayDir, sourceAgentDir, [], copiedFilePaths, budget) + ) { // Why: legacy source overlays can be large and are intentionally kept // for recovery; mark clean migrations so future OMP spawns stay cheap. writeFileSync(markerPath, 'complete\n') } } catch (error) { + // Why: a capacity stop must not strand a copied SQLite base without a + // deferred sidecar; remove only files this attempt created, then retry later. + removeCopiedFiles([...copiedFilePaths], copiedFilePaths, budget) console.warn('[pi-titlebar-extension] failed to migrate legacy OMP overlay state:', error) } } diff --git a/src/main/pi/managed-extension-ownership.test.ts b/src/main/pi/managed-extension-ownership.test.ts new file mode 100644 index 00000000000..1f14ca77a65 --- /dev/null +++ b/src/main/pi/managed-extension-ownership.test.ts @@ -0,0 +1,52 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + ORCA_MANAGED_PI_EXTENSION_MARKER, + PI_MANAGED_EXTENSION_OWNERSHIP_MAX_BYTES, + isManagedPiExtensionFile +} from './managed-extension-ownership' + +const roots: string[] = [] + +function tempFile(contents: string): string { + const root = mkdtempSync(join(tmpdir(), 'orca-managed-extension-ownership-')) + roots.push(root) + const path = join(root, 'orca-agent-status.ts') + writeFileSync(path, contents) + return path +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('managed Pi extension ownership', () => { + it('accepts a marked file at the exact byte limit', () => { + const prefix = `// ${ORCA_MANAGED_PI_EXTENSION_MARKER}\n` + const path = tempFile( + prefix + 'x'.repeat(PI_MANAGED_EXTENSION_OWNERSHIP_MAX_BYTES - prefix.length) + ) + + expect(isManagedPiExtensionFile(path)).toBe(true) + }) + + it('treats a marked file one byte over the limit as user-owned', () => { + const prefix = `// ${ORCA_MANAGED_PI_EXTENSION_MARKER}\n` + const path = tempFile( + prefix + 'x'.repeat(PI_MANAGED_EXTENSION_OWNERSHIP_MAX_BYTES - prefix.length + 1) + ) + + expect(isManagedPiExtensionFile(path)).toBe(false) + }) + + it('treats missing and unmarked files as user-owned', () => { + const path = tempFile('user extension') + + expect(isManagedPiExtensionFile(path)).toBe(false) + expect(isManagedPiExtensionFile(`${path}.missing`)).toBe(false) + }) +}) diff --git a/src/main/pi/managed-extension-ownership.ts b/src/main/pi/managed-extension-ownership.ts new file mode 100644 index 00000000000..27a6b93f95d --- /dev/null +++ b/src/main/pi/managed-extension-ownership.ts @@ -0,0 +1,21 @@ +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const ORCA_MANAGED_PI_EXTENSION_MARKER = '@orca-managed-pi-extension' +export const PI_MANAGED_EXTENSION_OWNERSHIP_MAX_BYTES = 1_024 * 1_024 + +export function withOrcaManagedPiExtensionMarker(source: string): string { + return source.includes(ORCA_MANAGED_PI_EXTENSION_MARKER) + ? source + : `// ${ORCA_MANAGED_PI_EXTENSION_MARKER}\n${source}` +} + +export function isManagedPiExtensionFile(path: string): boolean { + try { + return readNodeFileSyncWithinLimit(path, PI_MANAGED_EXTENSION_OWNERSHIP_MAX_BYTES) + .buffer.toString('utf8') + .includes(ORCA_MANAGED_PI_EXTENSION_MARKER) + } catch { + // Unreadable or oversized files cannot safely be claimed as Orca-owned. + return false + } +} diff --git a/src/main/pi/titlebar-extension-service.ts b/src/main/pi/titlebar-extension-service.ts index 37d030b5e31..4fafe42d024 100644 --- a/src/main/pi/titlebar-extension-service.ts +++ b/src/main/pi/titlebar-extension-service.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import { app } from 'electron' @@ -18,6 +18,10 @@ import { safeRemoveOverlay } from '../pty/overlay-mirror' import { migrateLegacyOmpOverlayState } from './legacy-omp-overlay-migration' +import { + isManagedPiExtensionFile, + withOrcaManagedPiExtensionMarker +} from './managed-extension-ownership' import type { PiAgentKind } from '../../shared/pi-agent-kind' // Why: the Pi test suite imports `isSafeDescendCandidate` from this module's @@ -27,7 +31,6 @@ import type { PiAgentKind } from '../../shared/pi-agent-kind' export const isSafeDescendCandidate = sharedIsSafeDescendCandidate const PI_AGENT_SUBDIR = 'agent' -const ORCA_MANAGED_EXTENSION_MARKER = '@orca-managed-pi-extension' const OMP_MANAGED_STATUS_EXTENSION_DIR = 'omp-managed-status-extension' type ManagedExtensionWriteResult = 'written' | 'skipped-user-owned' | 'failed' @@ -64,12 +67,6 @@ function toSafeOverlayDirName(ptyId: string): string { return createHash('sha256').update(ptyId).digest('hex').slice(0, 32) } -function withOrcaManagedExtensionMarker(source: string): string { - return source.includes(ORCA_MANAGED_EXTENSION_MARKER) - ? source - : `// ${ORCA_MANAGED_EXTENSION_MARKER}\n${source}` -} - export class PiTitlebarExtensionService { private getOverlayRoot(kind: PiAgentKind): string { return join(app.getPath('userData'), OVERLAY_ROOT_DIR_NAME[kind]) @@ -98,16 +95,8 @@ export class PiTitlebarExtensionService { safeRemoveOverlay(overlayDir, this.getOverlayRoot(kind)) } - private canOverwriteManagedExtension(path: string): boolean { - try { - return readFileSync(path, 'utf8').includes(ORCA_MANAGED_EXTENSION_MARKER) - } catch { - return true - } - } - private writeManagedExtension(path: string, source: string): ManagedExtensionWriteResult { - if (existsSync(path) && !this.canOverwriteManagedExtension(path)) { + if (existsSync(path) && !isManagedPiExtensionFile(path)) { return 'skipped-user-owned' } @@ -144,14 +133,14 @@ export class PiTitlebarExtensionService { this.writeManagedExtension( join(extensionsDir, ORCA_PI_EXTENSION_FILE), - withOrcaManagedExtensionMarker(getPiTitlebarExtensionSource()) + withOrcaManagedPiExtensionMarker(getPiTitlebarExtensionSource()) ) this.writeManagedExtension( join(extensionsDir, ORCA_PI_PREFILL_EXTENSION_FILE), - withOrcaManagedExtensionMarker(getPiPrefillExtensionSource(kind)) + withOrcaManagedPiExtensionMarker(getPiPrefillExtensionSource(kind)) ) const statusExtensionPath = join(extensionsDir, ORCA_PI_AGENT_STATUS_EXTENSION_FILE) - const statusSource = withOrcaManagedExtensionMarker(getPiAgentStatusExtensionSource(kind)) + const statusSource = withOrcaManagedPiExtensionMarker(getPiAgentStatusExtensionSource(kind)) const statusResult = this.writeManagedExtension(statusExtensionPath, statusSource) return { diff --git a/src/main/ports/local-workspace-port-scanner.test.ts b/src/main/ports/local-workspace-port-scanner.test.ts index a6eeb21e5ee..5b79e499644 100644 --- a/src/main/ports/local-workspace-port-scanner.test.ts +++ b/src/main/ports/local-workspace-port-scanner.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import path from 'node:path' +import { LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES } from '../../shared/linux-proc-port-scan-limits' import { attributePortToWorkspace, isContainerProcess, @@ -110,6 +111,18 @@ describe('local workspace port scanner parsing', () => { expect(ports).toEqual([{ host: '127.0.0.1', port: 3000, inode: 12345 }]) expect(usedWhitespaceFieldSplit).toBe(false) }) + + it('caps retained Linux proc listeners before process attribution', () => { + const rows = Array.from( + { length: LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES + 5 }, + (_, index) => + ` ${index}: 0100007F:${(index + 1).toString(16).padStart(4, '0')} 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 ${index + 1}` + ) + + expect(parseProcNetTcp(['header', ...rows].join('\n'))).toHaveLength( + LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES + ) + }) }) describe('attributePortToWorkspace', () => { diff --git a/src/main/ports/local-workspace-port-scanner.ts b/src/main/ports/local-workspace-port-scanner.ts index 1d8f4dbf891..2e89b47b51e 100644 --- a/src/main/ports/local-workspace-port-scanner.ts +++ b/src/main/ports/local-workspace-port-scanner.ts @@ -1,7 +1,7 @@ /* eslint-disable max-lines -- Why: the platform-specific scan paths share parsing, attribution, and normalization rules that must stay in lockstep. */ import { execFile } from 'node:child_process' -import { readFile, readdir, readlink } from 'node:fs/promises' +import { readlink } from 'node:fs/promises' import path from 'node:path' import type { WorkspacePort, @@ -10,6 +10,14 @@ import type { WorkspacePortScanResult } from '../../shared/workspace-ports' import { getProcessOutputFields } from '../../shared/process-output-field-scanner' +import { mapLinuxSocketInodesToPids } from '../../shared/linux-proc-socket-owner-scanner' +import { + createLinuxProcTextReadBudget, + LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES, + readLinuxProcNetworkTable, + readLinuxProcTextWithinBudget, + type LinuxProcTextReadBudget +} from '../../shared/linux-proc-port-scan-limits' import { advertisedUrlWatcher, type AdvertisedUrlWatcher } from './advertised-url-watcher' import { WorkspacePortScanTimeoutBackoff } from './workspace-port-scan-timeout-backoff' @@ -190,6 +198,9 @@ export function parseProcNetTcp(content: string): { host: string; port: number; continue } results.push({ ...parsed, inode }) + if (results.length >= LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES) { + break + } } return results } @@ -230,15 +241,18 @@ async function scanLinuxProcPorts(): Promise { readProcNet('/proc/net/tcp'), readProcNet('/proc/net/tcp6') ]) - const sockets = [...tcp4, ...tcp6] - const inodeToPid = await mapLinuxInodesToPids(new Set(sockets.map((socket) => socket.inode))) + const sockets = [...tcp4, ...tcp6].slice(0, LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES) + const inodeToPid = await mapLinuxSocketInodesToPids( + new Set(sockets.map((socket) => socket.inode)) + ) const metadata = new Map() + const metadataBudget = createLinuxProcTextReadBudget() const rawPorts: RawListeningPort[] = [] for (const socket of sockets) { const pid = inodeToPid.get(socket.inode) if (pid != null && !metadata.has(pid)) { - metadata.set(pid, await loadLinuxProcessMetadata(pid)) + metadata.set(pid, await loadLinuxProcessMetadata(pid, metadataBudget)) } rawPorts.push({ host: socket.host, @@ -254,59 +268,17 @@ async function scanLinuxProcPorts(): Promise { async function readProcNet( filePath: string ): Promise<{ host: string; port: number; inode: number }[]> { - try { - return parseProcNetTcp(await readFile(filePath, 'utf-8')) - } catch { - return [] - } + const content = await readLinuxProcNetworkTable(filePath) + return content === null ? [] : parseProcNetTcp(content) } -async function mapLinuxInodesToPids(inodes: Set): Promise> { - const result = new Map() - if (inodes.size === 0) { - return result - } - let pids: string[] - try { - pids = (await readdir('/proc')).filter((entry) => /^\d+$/.test(entry)) - } catch { - return result - } - - for (const pidText of pids) { - let fds: string[] - try { - fds = await readdir(`/proc/${pidText}/fd`) - } catch { - continue - } - const pid = Number.parseInt(pidText, 10) - for (const fd of fds) { - let link: string - try { - link = await readlink(`/proc/${pidText}/fd/${fd}`) - } catch { - continue - } - const match = link.match(/^socket:\[(\d+)\]$/) - if (!match) { - continue - } - const inode = Number.parseInt(match[1], 10) - if (inodes.has(inode)) { - result.set(inode, pid) - } - } - } - return result -} - -async function loadLinuxProcessMetadata(pid: number): Promise { - const [comm, cmdline, cwd] = await Promise.all([ - readTextIfAvailable(`/proc/${pid}/comm`), - readTextIfAvailable(`/proc/${pid}/cmdline`), - readlink(`/proc/${pid}/cwd`).catch(() => undefined) - ]) +async function loadLinuxProcessMetadata( + pid: number, + budget: LinuxProcTextReadBudget +): Promise { + const comm = await readLinuxProcTextWithinBudget(`/proc/${pid}/comm`, budget) + const cmdline = await readLinuxProcTextWithinBudget(`/proc/${pid}/cmdline`, budget) + const cwd = await readlink(`/proc/${pid}/cwd`).catch(() => undefined) return { processName: comm?.trim() || undefined, commandLine: cmdline?.split('\u0000').join(' ').trim() || undefined, @@ -440,14 +412,6 @@ function isCommandTimeoutError(error: unknown): boolean { return error instanceof CommandTimeoutError } -async function readTextIfAvailable(filePath: string): Promise { - try { - return await readFile(filePath, 'utf-8') - } catch { - return undefined - } -} - function enrichPort( port: RawListeningPort, worktrees: readonly NormalizedWorkspacePortProbe[], diff --git a/src/main/project-groups/nested-repo-discovery.test.ts b/src/main/project-groups/nested-repo-discovery.test.ts index f38e29ea53b..fc99b18add6 100644 --- a/src/main/project-groups/nested-repo-discovery.test.ts +++ b/src/main/project-groups/nested-repo-discovery.test.ts @@ -1,8 +1,8 @@ -import { mkdtemp, mkdir, writeFile, rm, symlink } from 'node:fs/promises' +import { mkdtemp, mkdir, writeFile, rm, symlink, truncate } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { afterEach, describe, expect, it, vi } from 'vitest' -import { scanNestedRepos } from './nested-repo-discovery' +import { NESTED_REPO_GITIGNORE_MAX_BYTES, scanNestedRepos } from './nested-repo-discovery' let tempDirs: string[] = [] @@ -302,6 +302,34 @@ describe('scanNestedRepos', () => { expect(result.repos.map((repo) => repo.path)).toEqual(['/workspace/active/repo']) }) + it('ignores an oversized gitignore payload instead of retaining its rules', async () => { + const directories = new Map([ + ['/workspace', ['.gitignore', 'repo']], + ['/workspace/repo', []] + ]) + const files = new Map([['/workspace/.gitignore', `repo\n${'x'.repeat(1024 * 1024)}`]]) + const gitRepos = new Set(['/workspace/repo']) + + const result = await scanNestedRepos({ + path: '/workspace', + filesystem: posixTestFilesystem({ directories, gitRepos, files }) + }) + + expect(result.repos.map((repo) => repo.path)).toEqual(['/workspace/repo']) + }) + + it('skips a sparse oversized local gitignore before reading its payload', async () => { + const root = await tempRoot() + const ignorePath = join(root, '.gitignore') + await writeFile(ignorePath, '') + await truncate(ignorePath, NESTED_REPO_GITIGNORE_MAX_BYTES + 1) + await makeGitRepo(join(root, 'repo')) + + const result = await scanNestedRepos({ path: root }) + + expect(result.repos.map((repo) => repo.displayName)).toEqual(['repo']) + }) + it('keeps root-anchored gitignore rules scoped to their base directory', async () => { const directories = new Map([ ['/workspace', ['.gitignore', 'active', 'ignored']], @@ -354,6 +382,49 @@ describe('scanNestedRepos', () => { expect(selectedPathChecks).toEqual(['/workspace']) }) + it('accepts the exact aggregate entry capacity without truncation', async () => { + const directories = new Map([['/workspace', ['api', 'web']]]) + const gitRepos = new Set(['/workspace/api', '/workspace/web']) + + const result = await scanNestedRepos({ + path: '/workspace', + filesystem: posixTestFilesystem({ directories, gitRepos }), + limits: { maxEntries: 2 } + }) + + expect(result.repos.map((repo) => repo.path)).toEqual(['/workspace/api', '/workspace/web']) + expect(result.truncated).toBe(false) + }) + + it('returns bounded partial results and closes the iterator on entry overflow', async () => { + let iteratorClosed = false + const filesystem = { + ...posixTestFilesystem({ + directories: new Map(), + gitRepos: new Set(['/workspace/api', '/workspace/web', '/workspace/worker']) + }), + readDirectory: async function* () { + try { + yield { name: 'api', isDirectory: true } + yield { name: 'web', isDirectory: true } + yield { name: 'worker', isDirectory: true } + } finally { + iteratorClosed = true + } + } + } + + const result = await scanNestedRepos({ + path: '/workspace', + filesystem, + limits: { maxEntries: 2 } + }) + + expect(result.repos.map((repo) => repo.path)).toEqual(['/workspace/api', '/workspace/web']) + expect(result.truncated).toBe(true) + expect(iteratorClosed).toBe(true) + }) + it('skips heavy directories and respects result caps', async () => { const root = await tempRoot() await mkdir(join(root, 'node_modules', 'ignored'), { recursive: true }) diff --git a/src/main/project-groups/nested-repo-discovery.ts b/src/main/project-groups/nested-repo-discovery.ts index 30453748767..4505b62b7ca 100644 --- a/src/main/project-groups/nested-repo-discovery.ts +++ b/src/main/project-groups/nested-repo-discovery.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines -- Why: scanner traversal, ignore matching, and filesystem abstraction stay together so local, SSH, and runtime scans cannot drift. */ -import { readFile, readdir, stat } from 'node:fs/promises' +import { opendir, stat } from 'node:fs/promises' import { basename, join } from 'node:path' import type { NestedRepoCandidate, @@ -8,6 +8,8 @@ import type { NestedRepoScanResult } from '../../shared/types' import { isGitRepo } from '../git/repo' +import { NestedRepoScanBudget, type NestedRepoScanLimits } from './nested-repo-scan-budget' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' type NestedRepoDirectoryEntry = { name: string @@ -16,7 +18,12 @@ type NestedRepoDirectoryEntry = { } type NestedRepoScanFilesystem = { - readDirectory: (dirPath: string) => Promise + readDirectory: ( + dirPath: string + ) => + | AsyncIterable + | Iterable + | Promise | Iterable> readTextFile?: (filePath: string) => Promise joinPath: (parentPath: string, childName: string) => string basename: (path: string) => string @@ -46,6 +53,7 @@ type NormalizedNestedRepoScanOptions = { const DEFAULT_MAX_DEPTH = 3 const DEFAULT_MAX_REPOS = 100 +export const NESTED_REPO_GITIGNORE_MAX_BYTES = 1024 * 1024 const SKIPPED_DIRS = new Set([ 'node_modules', @@ -121,24 +129,35 @@ function pathSegmentsMatch(patternSegments: string[], candidateSegments: string[ return matchFrom(0, 0) } -function parseGitignoreRules(content: string, baseSegments: string[]): IgnoreRule[] { - return content - .split(/\r?\n/) - .map((rawLine) => rawLine.trim()) - .filter((line) => line.length > 0 && !line.startsWith('#')) - .map((line) => { - const negate = line.startsWith('!') - const unprefixed = negate ? line.slice(1) : line - const anchored = unprefixed.startsWith('/') - const pattern = unprefixed.replace(/^\/+/, '').replace(/\/+$/, '') - return { - pattern, - negate, - basenameOnly: !anchored && !pattern.includes('/'), - baseSegments - } +function parseGitignoreRules( + content: string, + baseSegments: string[], + budget: NestedRepoScanBudget +): IgnoreRule[] { + const rules: IgnoreRule[] = [] + for (const match of content.matchAll(/[^\r\n]+/g)) { + const line = match[0].trim() + if (!line || line.startsWith('#')) { + continue + } + const negate = line.startsWith('!') + const unprefixed = negate ? line.slice(1) : line + const anchored = unprefixed.startsWith('/') + const pattern = unprefixed.replace(/^\/+/, '').replace(/\/+$/, '') + if (!pattern) { + continue + } + if (!budget.tryRetainIgnoreRule(pattern)) { + break + } + rules.push({ + pattern, + negate, + basenameOnly: !anchored && !pattern.includes('/'), + baseSegments }) - .filter((rule) => rule.pattern.length > 0) + } + return rules } function isIgnoredByRules(name: string, segments: string[], rules: IgnoreRule[]): boolean { @@ -164,6 +183,7 @@ async function readGitignoreRules(args: { entries: NestedRepoDirectoryEntry[] filesystem: NestedRepoScanFilesystem baseSegments: string[] + budget: NestedRepoScanBudget }): Promise { if (!args.filesystem.readTextFile || !args.entries.some((entry) => entry.name === '.gitignore')) { return [] @@ -172,7 +192,10 @@ async function readGitignoreRules(args: { const content = await args.filesystem.readTextFile( args.filesystem.joinPath(args.folderPath, '.gitignore') ) - return parseGitignoreRules(content, args.baseSegments) + if (Buffer.byteLength(content, 'utf8') > NESTED_REPO_GITIGNORE_MAX_BYTES) { + return [] + } + return parseGitignoreRules(content, args.baseSegments, args.budget) } catch { return [] } @@ -195,15 +218,15 @@ async function hasGitMarker(dirPath: string): Promise { return head?.isFile() === true && objects?.isDirectory() === true && refs?.isDirectory() === true } -async function readLocalDirectory(dirPath: string): Promise { - // Why: Dirent data avoids one stat per child and keeps symlinked directories - // from expanding the scan outside the selected folder. - const entries = await readdir(dirPath, { withFileTypes: true }) - return entries.map((entry) => ({ - name: entry.name, - isDirectory: entry.isDirectory(), - isSymlink: entry.isSymbolicLink() - })) +async function* readLocalDirectory(dirPath: string): AsyncGenerator { + const directory = await opendir(dirPath) + for await (const entry of directory) { + yield { + name: entry.name, + isDirectory: entry.isDirectory(), + isSymlink: entry.isSymbolicLink() + } + } } export async function scanNestedRepos(args: { @@ -212,6 +235,7 @@ export async function scanNestedRepos(args: { filesystem?: NestedRepoScanFilesystem signal?: AbortSignal onProgress?: (scan: NestedRepoScanResult) => void + limits?: Partial }): Promise { const startedAt = Date.now() const options = normalizeScanOptions(args.options) @@ -219,9 +243,13 @@ export async function scanNestedRepos(args: { let truncated = false let timedOut = false let stopped = false + const scanBudget = new NestedRepoScanBudget(args.limits) const filesystem = args.filesystem ?? { readDirectory: readLocalDirectory, - readTextFile: (path: string) => readFile(path, 'utf8'), + readTextFile: async (path: string) => + (await readNodeFileWithinLimit(path, NESTED_REPO_GITIGNORE_MAX_BYTES)).buffer.toString( + 'utf8' + ), joinPath: join, basename, hasGitMarker, @@ -281,7 +309,12 @@ export async function scanNestedRepos(args: { let entries: NestedRepoDirectoryEntry[] try { - entries = await filesystem.readDirectory(currentFolder.path) + entries = await collectNestedRepoDirectoryEntries( + await filesystem.readDirectory(currentFolder.path), + currentFolder.path, + filesystem, + scanBudget + ) } catch { continue } @@ -294,7 +327,8 @@ export async function scanNestedRepos(args: { folderPath: currentFolder.path, entries, filesystem, - baseSegments: currentFolder.segments + baseSegments: currentFolder.segments, + budget: scanBudget })) ] @@ -347,7 +381,28 @@ export async function scanNestedRepos(args: { }) } } + if (scanBudget.capacityReached) { + truncated = true + break + } } return buildResult('non_git_folder') } + +async function collectNestedRepoDirectoryEntries( + source: AsyncIterable | Iterable, + directoryPath: string, + filesystem: NestedRepoScanFilesystem, + budget: NestedRepoScanBudget +): Promise { + const entries: NestedRepoDirectoryEntry[] = [] + for await (const entry of source) { + const entryPath = filesystem.joinPath(directoryPath, entry.name) + if (!budget.tryVisitEntry(entryPath)) { + break + } + entries.push(entry) + } + return entries +} diff --git a/src/main/project-groups/nested-repo-scan-budget.test.ts b/src/main/project-groups/nested-repo-scan-budget.test.ts new file mode 100644 index 00000000000..d53d8392ebc --- /dev/null +++ b/src/main/project-groups/nested-repo-scan-budget.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { NestedRepoScanBudget } from './nested-repo-scan-budget' + +describe('NestedRepoScanBudget', () => { + it('accepts the exact path-memory capacity and rejects the next entry', () => { + const budget = new NestedRepoScanBudget({ maxPathBytes: 258 }) + + expect(budget.tryVisitEntry('a')).toBe(true) + expect(budget.tryVisitEntry('b')).toBe(false) + expect(budget.capacityReached).toBe(true) + }) + + it('accepts the exact ignore-rule capacity and rejects the next rule', () => { + const budget = new NestedRepoScanBudget({ maxIgnoreBytes: 130 }) + + expect(budget.tryRetainIgnoreRule('a')).toBe(true) + expect(budget.tryRetainIgnoreRule('b')).toBe(false) + expect(budget.capacityReached).toBe(true) + }) +}) diff --git a/src/main/project-groups/nested-repo-scan-budget.ts b/src/main/project-groups/nested-repo-scan-budget.ts new file mode 100644 index 00000000000..aeb944be8a3 --- /dev/null +++ b/src/main/project-groups/nested-repo-scan-budget.ts @@ -0,0 +1,61 @@ +export const NESTED_REPO_SCAN_MAX_ENTRIES = 100_000 +export const NESTED_REPO_SCAN_MAX_PATH_BYTES = 32 * 1024 * 1024 +export const NESTED_REPO_SCAN_MAX_IGNORE_RULES = 100_000 +export const NESTED_REPO_SCAN_MAX_IGNORE_BYTES = 8 * 1024 * 1024 + +const NESTED_REPO_ENTRY_OVERHEAD_BYTES = 256 +const NESTED_REPO_IGNORE_RULE_OVERHEAD_BYTES = 128 + +export type NestedRepoScanLimits = { + maxEntries: number + maxIgnoreBytes: number + maxIgnoreRules: number + maxPathBytes: number +} + +export class NestedRepoScanBudget { + private entries = 0 + private ignoreBytes = 0 + private ignoreRules = 0 + private pathBytes = 0 + readonly limits: NestedRepoScanLimits + capacityReached = false + + constructor(requested?: Partial) { + this.limits = { + maxEntries: clampLimit(requested?.maxEntries, NESTED_REPO_SCAN_MAX_ENTRIES), + maxIgnoreBytes: clampLimit(requested?.maxIgnoreBytes, NESTED_REPO_SCAN_MAX_IGNORE_BYTES), + maxIgnoreRules: clampLimit(requested?.maxIgnoreRules, NESTED_REPO_SCAN_MAX_IGNORE_RULES), + maxPathBytes: clampLimit(requested?.maxPathBytes, NESTED_REPO_SCAN_MAX_PATH_BYTES) + } + } + + tryVisitEntry(path: string): boolean { + const nextPathBytes = this.pathBytes + path.length * 2 + NESTED_REPO_ENTRY_OVERHEAD_BYTES + if (this.entries >= this.limits.maxEntries || nextPathBytes > this.limits.maxPathBytes) { + this.capacityReached = true + return false + } + this.entries += 1 + this.pathBytes = nextPathBytes + return true + } + + tryRetainIgnoreRule(pattern: string): boolean { + const nextBytes = this.ignoreBytes + pattern.length * 2 + NESTED_REPO_IGNORE_RULE_OVERHEAD_BYTES + if (this.ignoreRules >= this.limits.maxIgnoreRules || nextBytes > this.limits.maxIgnoreBytes) { + this.capacityReached = true + return false + } + this.ignoreRules += 1 + this.ignoreBytes = nextBytes + return true + } +} + +function clampLimit(value: number | undefined, maximum: number): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + return maximum + } + return Math.min(value, maximum) +} diff --git a/src/main/providers/filesystem-path-listing-provider.ts b/src/main/providers/filesystem-path-listing-provider.ts new file mode 100644 index 00000000000..951fe981166 --- /dev/null +++ b/src/main/providers/filesystem-path-listing-provider.ts @@ -0,0 +1,7 @@ +export type FilesystemPathListingProvider = { + listFiles( + rootPath: string, + options?: { excludePaths?: string[]; signal?: AbortSignal; maxResults?: number } + ): Promise + listMarkdownDocuments?(rootPath: string): Promise +} diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index c2dcfcff523..0ef13db58d1 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -231,7 +231,6 @@ describe('LocalPtyProvider', () => { expect(second).toEqual({ id: 'serve-session-1', pid: 12345, - wslDistro: null, isReattach: true }) expect(mockProc.resize).toHaveBeenCalledWith(120, 40) @@ -1788,6 +1787,28 @@ describe('LocalPtyProvider', () => { expect(newEntries[0]).toHaveProperty('title', 'zsh') expect(newEntries[0]).toHaveProperty('cwd', '/tmp/owned-cwd') expect(newEntries[0]).toHaveProperty('worktreeId', 'repo::/tmp/owned-cwd') + expect(newEntries[0]).not.toHaveProperty('wslDistro') + expect(newEntries[1]).not.toHaveProperty('wslDistro') + }) + + it('reports native and WSL ownership explicitly on Windows', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const native = await provider.spawn({ + cols: 80, + rows: 24, + cwd: 'C:\\repo', + shellOverride: 'powershell.exe' + }) + const wsl = await provider.spawn({ + cols: 80, + rows: 24, + cwd: '\\\\wsl.localhost\\Ubuntu\\home\\jin\\repo' + }) + + const processes = await provider.listProcesses() + + expect(processes.find((process) => process.id === native.id)?.wslDistro).toBeNull() + expect(processes.find((process) => process.id === wsl.id)?.wslDistro).toBe('Ubuntu') }) }) diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index 226b5167e8d..1eef45e63db 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -847,7 +847,11 @@ export class LocalPtyProvider implements IPtyProvider { const proc = spawnResult.process const spawnedShellIsWsl = process.platform === 'win32' && pathWin32.basename(shellPath).toLowerCase() === 'wsl.exe' - const spawnedWslDistro = spawnedShellIsWsl ? (launchWslDistro ?? undefined) : null + const spawnedWslDistro = spawnedShellIsWsl + ? (launchWslDistro ?? undefined) + : process.platform === 'win32' + ? null + : undefined createPtyPhysicalExit(id) ptyProcesses.set(id, proc) ptyInitialCwd.set(id, cwd) @@ -1323,7 +1327,8 @@ export class LocalPtyProvider implements IPtyProvider { cwd: ptyInitialCwd.get(id) ?? '', title: proc.process || ptyShellName.get(id) || 'shell', ...(ptyWorktreeId.get(id) ? { worktreeId: ptyWorktreeId.get(id) } : {}), - ...(ptyTerminalHandle.get(id) ? { terminalHandle: ptyTerminalHandle.get(id) } : {}) + ...(ptyTerminalHandle.get(id) ? { terminalHandle: ptyTerminalHandle.get(id) } : {}), + ...(ptyWslDistroById.has(id) ? { wslDistro: ptyWslDistroById.get(id) ?? null } : {}) })) } diff --git a/src/main/providers/provider-dispatch.test.ts b/src/main/providers/provider-dispatch.test.ts index be45aa04ff3..a6d1b20d4d6 100644 --- a/src/main/providers/provider-dispatch.test.ts +++ b/src/main/providers/provider-dispatch.test.ts @@ -56,6 +56,7 @@ vi.mock('../pi/titlebar-extension-service', () => ({ import { deletePtyOwnership, + MAX_REGISTERED_SSH_PTY_PROVIDERS, registerPtyHandlers, registerSshPtyProvider, setPtyOwnership, @@ -178,6 +179,28 @@ describe('PTY provider dispatch', () => { ).rejects.toThrow('No PTY provider for connection "conn-456"') }) + it('fails closed when the process-wide SSH provider registry is saturated', () => { + const ids = Array.from( + { length: MAX_REGISTERED_SSH_PTY_PROVIDERS }, + (_, index) => `capacity-${index}` + ) + try { + for (const id of ids) { + registerSshPtyProvider(id, createMockProvider(id)) + } + + expect(() => + registerSshPtyProvider('capacity-overflow', createMockProvider('overflow')) + ).toThrow('ssh_pty_provider_capacity') + expect(() => registerSshPtyProvider(ids[0], createMockProvider('replacement'))).not.toThrow() + } finally { + for (const id of ids) { + unregisterSshPtyProvider(id) + } + unregisterSshPtyProvider('capacity-overflow') + } + }) + it('keeps same relay PTY ids distinct across SSH targets', () => { setup() const providerA = createMockProvider('ssh:conn-a@@pty-1') diff --git a/src/main/providers/pty-process-info.ts b/src/main/providers/pty-process-info.ts new file mode 100644 index 00000000000..a34746a6267 --- /dev/null +++ b/src/main/providers/pty-process-info.ts @@ -0,0 +1,16 @@ +import type { AgentSessionOwnerBinding } from '../../shared/agent-session-host-authority' +import type { PtyIncarnationId } from '../../shared/pty-incarnation' + +export type PtyProcessInfo = { + id: string + incarnationId?: PtyIncarnationId + cwd: string + title: string + /** Owning worktree when the provider can report it authoritatively. */ + worktreeId?: string + /** Trusted ORCA_TERMINAL_HANDLE exported into this PTY, when known. */ + terminalHandle?: string + /** Exact WSL owner reported by the PTY provider; null means native Windows. */ + wslDistro?: string | null + agentSessionOwners?: AgentSessionOwnerBinding[] +} diff --git a/src/main/providers/pty-process-inspection.test.ts b/src/main/providers/pty-process-inspection.test.ts new file mode 100644 index 00000000000..60ce53bc568 --- /dev/null +++ b/src/main/providers/pty-process-inspection.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from 'vitest' +import type { IPtyProvider } from './types' +import { inspectPtyProviderProcess } from './pty-process-inspection' + +describe('PTY provider process inspection', () => { + it('rejects a missing provider PTY instead of returning idle evidence', async () => { + const provider = { + hasPty: vi.fn(() => false), + getForegroundProcess: vi.fn().mockResolvedValue(null), + hasChildProcesses: vi.fn().mockResolvedValue(false) + } as unknown as IPtyProvider + + await expect(inspectPtyProviderProcess(provider, 'pty-missing')).rejects.toThrow( + 'terminal_gone' + ) + expect(provider.getForegroundProcess).not.toHaveBeenCalled() + }) + + it('preserves a completion-sensitive provider failure', async () => { + const failure = new Error('daemon unavailable') + const inspectProcess = vi.fn().mockRejectedValue(failure) + const provider = { inspectProcess } as unknown as IPtyProvider + + await expect(inspectPtyProviderProcess(provider, 'pty-1')).rejects.toBe(failure) + expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('pty-1') + }) + + it('falls back to the existing provider process APIs', async () => { + const getForegroundProcess = vi.fn().mockResolvedValue('codex') + const hasChildProcesses = vi.fn().mockResolvedValue(true) + const provider = { + getForegroundProcess, + hasChildProcesses + } as Pick as IPtyProvider + + await expect(inspectPtyProviderProcess(provider, 'pty-1')).resolves.toEqual({ + foregroundProcess: 'codex', + hasChildProcesses: true + }) + }) +}) diff --git a/src/main/providers/pty-process-inspection.ts b/src/main/providers/pty-process-inspection.ts new file mode 100644 index 00000000000..68e174ff886 --- /dev/null +++ b/src/main/providers/pty-process-inspection.ts @@ -0,0 +1,26 @@ +import type { IPtyProvider } from './types' + +export type PtyProcessInspection = { + foregroundProcess: string | null + hasChildProcesses: boolean +} + +type CompletionSensitivePtyProvider = IPtyProvider & { + inspectProcess?: (id: string) => Promise +} + +export async function inspectPtyProviderProcess( + provider: IPtyProvider, + ptyId: string +): Promise { + if (provider.hasPty?.(ptyId) === false) { + throw new Error('terminal_gone') + } + const inspectProcess = (provider as CompletionSensitivePtyProvider).inspectProcess + if (inspectProcess) { + return inspectProcess.call(provider, ptyId) + } + const foregroundProcess = await provider.getForegroundProcess(ptyId) + const hasChildProcesses = await provider.hasChildProcesses(ptyId) + return { foregroundProcess, hasChildProcesses } +} diff --git a/src/main/providers/pty-process-list-admission.test.ts b/src/main/providers/pty-process-list-admission.test.ts new file mode 100644 index 00000000000..19daccbd2a5 --- /dev/null +++ b/src/main/providers/pty-process-list-admission.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from 'vitest' +import { + MAX_AGGREGATED_PTY_PROCESS_LIST_BYTES, + MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES, + MAX_AGGREGATED_PTY_PROCESS_LIST_OWNERS, + PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE, + PtyProcessListAdmission, + visitPtyProcessListingsInBatches +} from './pty-process-list-admission' + +describe('PtyProcessListAdmission', () => { + it('strips unknown provider payloads from admitted process metadata', () => { + const admission = new PtyProcessListAdmission() + + expect( + admission.admit({ + id: 'pty-1', + cwd: '/repo', + title: 'shell', + unknownPayload: 'x'.repeat(1024 * 1024) + } as never) + ).toEqual({ id: 'pty-1', cwd: '/repo', title: 'shell' }) + }) + + it('rejects aggregate entry and byte amplification', () => { + const entryAdmission = new PtyProcessListAdmission() + for (let index = 0; index < MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES; index += 1) { + entryAdmission.admit({ id: `pty-${index}`, cwd: '', title: 'shell' }) + } + expect(() => entryAdmission.admit({ id: 'one-more', cwd: '', title: 'shell' })).toThrow( + 'pty_process_list_capacity' + ) + + const byteAdmission = new PtyProcessListAdmission() + expect(() => + byteAdmission.admit({ + id: 'pty-large', + cwd: 'x'.repeat(MAX_AGGREGATED_PTY_PROCESS_LIST_BYTES), + title: 'shell' + }) + ).toThrow('pty_process_list_capacity') + + expect(() => + new PtyProcessListAdmission().admit({ + id: 'pty-owner-flood', + cwd: '', + title: 'shell', + agentSessionOwners: Array.from( + { length: MAX_AGGREGATED_PTY_PROCESS_LIST_OWNERS + 1 }, + () => ({}) + ) + } as never) + ).toThrow('pty_process_list_capacity') + }) +}) + +describe('visitPtyProcessListingsInBatches', () => { + it('never starts more than the bounded provider batch concurrently', async () => { + let active = 0 + let peak = 0 + const finishes: (() => void)[] = [] + const load = vi.fn( + async (source: number) => + await new Promise<{ id: string; cwd: string; title: string }[]>((resolve) => { + active += 1 + peak = Math.max(peak, active) + finishes.push(() => { + active -= 1 + resolve([{ id: `pty-${source}`, cwd: '', title: 'shell' }]) + }) + }) + ) + const visiting = visitPtyProcessListingsInBatches( + Array.from({ length: PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE + 1 }, (_, index) => index), + load, + () => {} + ) + + await vi.waitFor(() => expect(finishes).toHaveLength(PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE)) + finishes.splice(0).forEach((finish) => finish()) + await vi.waitFor(() => expect(finishes).toHaveLength(1)) + finishes.splice(0).forEach((finish) => finish()) + await visiting + + expect(peak).toBe(PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE) + }) +}) diff --git a/src/main/providers/pty-process-list-admission.ts b/src/main/providers/pty-process-list-admission.ts new file mode 100644 index 00000000000..f65a86aacea --- /dev/null +++ b/src/main/providers/pty-process-list-admission.ts @@ -0,0 +1,161 @@ +import { isAgentSessionOwnerBinding } from '../../shared/agent-session-host-authority' +import { MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES } from '../../shared/claimed-agent-pty-owner' +import { cloneAgentSessionOwnerBinding } from '../../shared/claimed-agent-pty-owner-snapshot' +import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import type { PtyProcessInfo } from './types' + +export const MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES = 4096 +export const MAX_AGGREGATED_PTY_PROCESS_LIST_BYTES = 32 * 1024 * 1024 +export const MAX_AGGREGATED_PTY_PROCESS_LIST_OWNERS = MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES +export const PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE = 4 + +function retainedStringBytes(value: unknown): number | null { + return typeof value === 'string' ? Buffer.byteLength(value, 'utf8') : null +} + +function retainedOptionalStringBytes(value: unknown): number | null { + return value === undefined ? 0 : retainedStringBytes(value) +} + +function retainedOwnerBytes(owner: unknown, ptyId: string): number | null { + if (!isAgentSessionOwnerBinding(owner) || owner.phase !== 'live' || owner.ptyId !== ptyId) { + return null + } + return [ + owner.claim.keyId, + owner.claim.identityDigest, + owner.claim.worktreeScopeDigest, + owner.claim.agent, + owner.generation, + owner.ptyId, + owner.surface.worktreeId, + owner.surface.tabId, + owner.surface.leafId, + owner.surface.terminalHandle + ].reduce((total, value) => total + Buffer.byteLength(value, 'utf8'), 0) +} + +export class PtyProcessListAdmission { + private entries = 0 + private retainedBytes = 0 + private owners = 0 + + constructor(private readonly capacityError = 'pty_process_list_capacity') {} + + admit(value: PtyProcessInfo): PtyProcessInfo { + if (typeof value !== 'object' || value === null) { + throw new Error('invalid_pty_process_list') + } + const idBytes = retainedStringBytes(value.id) + const cwdBytes = retainedStringBytes(value.cwd) + const titleBytes = retainedStringBytes(value.title) + const worktreeIdBytes = retainedOptionalStringBytes(value.worktreeId) + const terminalHandleBytes = retainedOptionalStringBytes(value.terminalHandle) + const wslDistroBytes = + value.wslDistro === null ? 0 : retainedOptionalStringBytes(value.wslDistro) + if ( + idBytes === null || + cwdBytes === null || + titleBytes === null || + worktreeIdBytes === null || + terminalHandleBytes === null || + wslDistroBytes === null || + (value.incarnationId !== undefined && !isPtyIncarnationId(value.incarnationId)) || + (value.agentSessionOwners !== undefined && !Array.isArray(value.agentSessionOwners)) + ) { + throw new Error('invalid_pty_process_list') + } + if ( + (value.agentSessionOwners?.length ?? 0) > + MAX_AGGREGATED_PTY_PROCESS_LIST_OWNERS - this.owners + ) { + throw new Error(this.capacityError) + } + + let ownerBytes = 0 + const normalizedOwners = value.agentSessionOwners?.map((owner) => { + const bytes = retainedOwnerBytes(owner, value.id) + if (bytes === null) { + throw new Error('agent_session_ownership_unknown') + } + ownerBytes += bytes + return cloneAgentSessionOwnerBinding(owner) + }) + const nextEntries = this.entries + 1 + const nextOwners = this.owners + (normalizedOwners?.length ?? 0) + const nextBytes = + this.retainedBytes + + idBytes + + cwdBytes + + titleBytes + + worktreeIdBytes + + terminalHandleBytes + + wslDistroBytes + + ownerBytes + if ( + nextEntries > MAX_AGGREGATED_PTY_PROCESS_LIST_ENTRIES || + nextOwners > MAX_AGGREGATED_PTY_PROCESS_LIST_OWNERS || + nextBytes > MAX_AGGREGATED_PTY_PROCESS_LIST_BYTES + ) { + throw new Error(this.capacityError) + } + this.entries = nextEntries + this.owners = nextOwners + this.retainedBytes = nextBytes + + return { + id: value.id, + cwd: value.cwd, + title: value.title, + ...(value.incarnationId !== undefined ? { incarnationId: value.incarnationId } : {}), + ...(value.worktreeId !== undefined ? { worktreeId: value.worktreeId } : {}), + ...(value.terminalHandle !== undefined ? { terminalHandle: value.terminalHandle } : {}), + ...(value.wslDistro !== undefined ? { wslDistro: value.wslDistro } : {}), + ...(normalizedOwners !== undefined ? { agentSessionOwners: normalizedOwners } : {}) + } + } +} + +export async function visitPtyProcessListingsInBatches( + sources: Iterable, + load: (source: T) => Promise, + visit: (source: T, processes: readonly PtyProcessInfo[]) => void +): Promise { + let batch: T[] = [] + for (const source of sources) { + batch.push(source) + if (batch.length < PTY_PROCESS_LIST_PROVIDER_BATCH_SIZE) { + continue + } + const listings = await Promise.all( + batch.map(async (entry) => ({ entry, processes: await load(entry) })) + ) + for (const listing of listings) { + visit(listing.entry, listing.processes) + } + batch = [] + } + if (batch.length === 0) { + return + } + const listings = await Promise.all( + batch.map(async (entry) => ({ entry, processes: await load(entry) })) + ) + for (const listing of listings) { + visit(listing.entry, listing.processes) + } +} + +export async function collectPtyProcessListings( + sources: Iterable, + load: (source: T) => Promise +): Promise { + const admission = new PtyProcessListAdmission() + const processes: PtyProcessInfo[] = [] + await visitPtyProcessListingsInBatches(sources, load, (_source, listing) => { + for (const process of listing) { + processes.push(admission.admit(process)) + } + }) + return processes +} diff --git a/src/main/providers/pty-provider-events.ts b/src/main/providers/pty-provider-events.ts index 6de435cb23f..0048b85f9dd 100644 --- a/src/main/providers/pty-provider-events.ts +++ b/src/main/providers/pty-provider-events.ts @@ -1,11 +1,18 @@ import type { TerminalGitHubPRLink } from '../../shared/terminal-github-pr-link-detector' +export type PtyDataUpstreamCredit = { + charCount: number + acknowledge(charCount: number): void +} + export type PtyDataEvent = { id: string data: string sequenceChars?: number transformed?: boolean seq?: number + /** Main-process-only credit captured from the exact provider generation that emitted this data. */ + upstreamCredit?: PtyDataUpstreamCredit } /** Notification-bearing fact a thinning transport detected while it held diff --git a/src/main/providers/ssh-agent-session-process-list.ts b/src/main/providers/ssh-agent-session-process-list.ts index cb1447aff71..f5287302ca5 100644 --- a/src/main/providers/ssh-agent-session-process-list.ts +++ b/src/main/providers/ssh-agent-session-process-list.ts @@ -1,25 +1,156 @@ import { isAgentSessionOwnerBinding } from '../../shared/agent-session-host-authority' +import { MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES } from '../../shared/claimed-agent-pty-owner' import { isPtyIncarnationId } from '../../shared/pty-incarnation' import type { PtyProcessInfo } from './types' +import { MAX_SSH_PTY_LIVE_ROSTER_ENTRIES } from './ssh-pty-live-roster' +import { admittedSshRelayPtyIdBytes } from './ssh-pty-wire-admission' + +export const MAX_SSH_PTY_PROCESS_LIST_ENTRIES = MAX_SSH_PTY_LIVE_ROSTER_ENTRIES +export const MAX_SSH_PTY_PROCESS_OWNERS_PER_ENTRY = 256 +export const MAX_SSH_PTY_PROCESS_LIST_OWNERS = MAX_CLAIMED_AGENT_PTY_OWNER_ENTRIES +export const MAX_SSH_PTY_PROCESS_LIST_BYTES = 8 * 1024 * 1024 +export const MAX_SSH_PTY_PROCESS_CWD_BYTES = 128 * 1024 +export const MAX_SSH_PTY_PROCESS_TITLE_BYTES = 16 * 1024 +const MAX_SSH_PTY_PROCESS_WORKTREE_ID_BYTES = 128 * 1024 +const MAX_SSH_PTY_PROCESS_TERMINAL_HANDLE_BYTES = 1024 +const MAX_SSH_PTY_PROCESS_WSL_DISTRO_BYTES = 4 * 1024 + +function stringBytes(value: unknown, maxBytes: number): number | null { + if (typeof value !== 'string') { + return null + } + const bytes = Buffer.byteLength(value, 'utf8') + return bytes <= maxBytes ? bytes : null +} + +function optionalStringBytes(value: unknown, maxBytes: number): number | null { + return value === undefined ? 0 : stringBytes(value, maxBytes) +} + +function ownerBytes(owner: unknown): number | null { + if (!isAgentSessionOwnerBinding(owner)) { + return null + } + const strings = [ + owner.claim.keyId, + owner.claim.identityDigest, + owner.claim.worktreeScopeDigest, + owner.claim.agent, + owner.generation, + owner.ptyId, + owner.surface.worktreeId, + owner.surface.tabId, + owner.surface.leafId, + owner.surface.terminalHandle + ] + return strings.reduce((total, value) => total + Buffer.byteLength(value, 'utf8'), 0) +} + +function assertProcessList( + sessions: unknown +): asserts sessions is (PtyProcessInfo & Record)[] { + if (!Array.isArray(sessions) || sessions.length > MAX_SSH_PTY_PROCESS_LIST_ENTRIES) { + throw new Error('invalid_ssh_pty_process_list') + } + let aggregateBytes = 0 + let aggregateOwners = 0 + for (const session of sessions) { + if (typeof session !== 'object' || session === null) { + throw new Error('invalid_ssh_pty_process_list') + } + const idBytes = admittedSshRelayPtyIdBytes(session.id) + const cwdBytes = stringBytes(session.cwd, MAX_SSH_PTY_PROCESS_CWD_BYTES) + const titleBytes = stringBytes(session.title, MAX_SSH_PTY_PROCESS_TITLE_BYTES) + const worktreeBytes = optionalStringBytes( + session.worktreeId, + MAX_SSH_PTY_PROCESS_WORKTREE_ID_BYTES + ) + const terminalHandleBytes = optionalStringBytes( + session.terminalHandle, + MAX_SSH_PTY_PROCESS_TERMINAL_HANDLE_BYTES + ) + const wslDistroBytes = + session.wslDistro === null + ? 0 + : optionalStringBytes(session.wslDistro, MAX_SSH_PTY_PROCESS_WSL_DISTRO_BYTES) + if ( + idBytes === null || + cwdBytes === null || + titleBytes === null || + worktreeBytes === null || + terminalHandleBytes === null || + wslDistroBytes === null || + (session.incarnationId !== undefined && !isPtyIncarnationId(session.incarnationId)) + ) { + throw new Error('invalid_ssh_pty_process_list') + } + aggregateBytes += + idBytes + cwdBytes + titleBytes + worktreeBytes + terminalHandleBytes + wslDistroBytes + if ( + session.agentSessionOwners !== undefined && + (!Array.isArray(session.agentSessionOwners) || + session.agentSessionOwners.length > MAX_SSH_PTY_PROCESS_OWNERS_PER_ENTRY) + ) { + throw new Error('invalid_ssh_pty_process_list') + } + for (const owner of session.agentSessionOwners ?? []) { + const bytes = ownerBytes(owner) + if (bytes === null || owner.ptyId !== session.id) { + throw new Error('agent_session_ownership_unknown') + } + aggregateBytes += bytes + } + aggregateOwners += session.agentSessionOwners?.length ?? 0 + if ( + aggregateBytes > MAX_SSH_PTY_PROCESS_LIST_BYTES || + aggregateOwners > MAX_SSH_PTY_PROCESS_LIST_OWNERS + ) { + throw new Error('invalid_ssh_pty_process_list') + } + } +} export function mapSshPtyProcessList( - sessions: PtyProcessInfo[], + sessions: unknown, toAppPtyId: (id: string) => string ): PtyProcessInfo[] { + assertProcessList(sessions) return sessions.map((session) => { if (session.agentSessionOwners?.length && !isPtyIncarnationId(session.incarnationId)) { throw new Error('agent_session_ownership_unknown') } return { - ...session, id: toAppPtyId(session.id), - ...(session.agentSessionOwners + cwd: session.cwd, + title: session.title, + ...(session.incarnationId ? { incarnationId: session.incarnationId } : {}), + ...(session.worktreeId !== undefined ? { worktreeId: session.worktreeId } : {}), + ...(session.terminalHandle !== undefined ? { terminalHandle: session.terminalHandle } : {}), + ...(session.wslDistro !== undefined ? { wslDistro: session.wslDistro } : {}), + ...(session.agentSessionOwners !== undefined ? { agentSessionOwners: session.agentSessionOwners.map((owner) => { if (!isAgentSessionOwnerBinding(owner) || owner.ptyId !== session.id) { throw new Error('agent_session_ownership_unknown') } - return { ...owner, ptyId: toAppPtyId(owner.ptyId) } + return { + claim: { + digestVersion: owner.claim.digestVersion, + keyId: owner.claim.keyId, + identityDigest: owner.claim.identityDigest, + worktreeScopeDigest: owner.claim.worktreeScopeDigest, + agent: owner.claim.agent + }, + generation: owner.generation, + phase: owner.phase, + ptyId: toAppPtyId(owner.ptyId), + surface: { + worktreeId: owner.surface.worktreeId, + tabId: owner.surface.tabId, + leafId: owner.surface.leafId, + terminalHandle: owner.surface.terminalHandle + } + } }) } : {}) diff --git a/src/main/providers/ssh-filesystem-directory-reader.ts b/src/main/providers/ssh-filesystem-directory-reader.ts new file mode 100644 index 00000000000..5b25a08610f --- /dev/null +++ b/src/main/providers/ssh-filesystem-directory-reader.ts @@ -0,0 +1,59 @@ +import type { DirEntry } from '../../shared/types' +import { + assertFilesystemDirectoryWithinLimit, + resolveFilesystemDirectoryListingLimits +} from '../../shared/filesystem-directory-listing-limit' +import { + assertMobileFileDirectoryWithinLimit, + MOBILE_FILE_DIRECTORY_MAX_ENTRIES, + MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES +} from '../../shared/mobile-file-directory-limit' +import { isMethodNotFoundError } from '../ssh/ssh-filesystem-stream-reader' +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' + +export const SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE = + 'Safe remote directory browsing requires an updated relay. Reconnect the SSH target before retrying.' + +export class SshFilesystemDirectoryReader { + private boundedMethodUnavailable = false + + constructor(private readonly mux: SshChannelMultiplexer) {} + + async readDir( + dirPath: string, + options?: { maxEntries?: number; maxRetainedBytes?: number } + ): Promise { + if (this.boundedMethodUnavailable) { + throw new Error(SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE) + } + const mobileLimits = + options?.maxEntries === MOBILE_FILE_DIRECTORY_MAX_ENTRIES && + options.maxRetainedBytes === MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES + const limits = mobileLimits + ? { + maxEntries: MOBILE_FILE_DIRECTORY_MAX_ENTRIES, + maxRetainedBytes: MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES + } + : resolveFilesystemDirectoryListingLimits(options) + let entries: DirEntry[] + try { + entries = (await this.mux.request('fs.readDirBounded', { + dirPath, + ...limits + })) as DirEntry[] + } catch (error) { + if (!isMethodNotFoundError(error)) { + throw error + } + // Why: fallback would restore remote-side unbounded enumeration. + this.boundedMethodUnavailable = true + throw new Error(SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE) + } + if (mobileLimits) { + assertMobileFileDirectoryWithinLimit(entries) + } else { + assertFilesystemDirectoryWithinLimit(entries, limits) + } + return entries + } +} diff --git a/src/main/providers/ssh-filesystem-download.test.ts b/src/main/providers/ssh-filesystem-download.test.ts index d65735cb806..23af1490e7f 100644 --- a/src/main/providers/ssh-filesystem-download.test.ts +++ b/src/main/providers/ssh-filesystem-download.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { downloadFolderViaSftp } from './ssh-filesystem-download' +import { SSH_DIRECTORY_TRANSFER_LIMITS } from '../ssh/ssh-directory-transfer-budget' type SftpEntryKind = 'directory' | 'file' | 'symlink' | 'fifo' @@ -29,6 +30,31 @@ function sftpEntry(filename: string, kind: SftpEntryKind) { return { filename, longname: filename, attrs: sftpStats(kind) } } +function withDirectoryHandles(sftp: Record): unknown { + if (typeof sftp.opendir === 'function' && typeof sftp.close === 'function') { + return sftp + } + const readDirectory = sftp.readdir as ( + path: string, + callback: (error?: Error, value?: unknown) => void + ) => void + const completed = new Set() + return Object.assign(sftp, { + opendir: (path: string, callback: (error: undefined, handle: Buffer) => void) => + callback(undefined, Buffer.from(path)), + readdir: (handle: Buffer, callback: (error?: Error, value?: unknown) => void) => { + const path = handle.toString('utf8') + if (completed.has(path)) { + callback(undefined, false) + return + } + completed.add(path) + readDirectory(path, callback) + }, + close: (_handle: Buffer, callback: (error?: Error) => void) => callback() + }) +} + describe('downloadFolderViaSftp', () => { const roots: string[] = [] @@ -64,7 +90,11 @@ describe('downloadFolderViaSftp', () => { } await expect( - downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination) + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination + ) ).rejects.toThrow("Remote entries map to the same local name 'a.txt'") expect(sftp.fastGet).toHaveBeenCalledTimes(1) }) @@ -83,7 +113,11 @@ describe('downloadFolderViaSftp', () => { } await expect( - downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination) + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination + ) ).rejects.toThrow("Cannot download unsupported remote entry 'build.pipe'") expect(sftp.fastGet).not.toHaveBeenCalled() }) @@ -103,7 +137,11 @@ describe('downloadFolderViaSftp', () => { } await expect( - downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination) + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination + ) ).rejects.toThrow("Cannot download symbolic link 'creds'") // The link target could be /etc/passwd; rejecting from directory-entry // metadata means it is never followed with stat or opened by fastGet. @@ -125,7 +163,11 @@ describe('downloadFolderViaSftp', () => { } await expect( - downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination) + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination + ) ).rejects.toThrow("Remote entries map to the same local name 'download'") expect(sftp.fastGet).not.toHaveBeenCalled() }) @@ -146,9 +188,12 @@ describe('downloadFolderViaSftp', () => { end: vi.fn() } - await downloadFolderViaSftp(async () => sftp as never, sourcePath, destination, { - windowsRemotePaths: false - }) + await downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + sourcePath, + destination, + { windowsRemotePaths: false } + ) expect(sftp.fastGet).toHaveBeenCalledWith( '/remote/parent\\literal/..\\secret.txt', @@ -171,9 +216,12 @@ describe('downloadFolderViaSftp', () => { } await expect( - downloadFolderViaSftp(async () => sftp as never, 'C:/remote/src', destination, { - windowsRemotePaths: true - }) + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + 'C:/remote/src', + destination, + { windowsRemotePaths: true } + ) ).rejects.toThrow("Invalid remote directory entry '..\\secret.txt'") expect(sftp.fastGet).not.toHaveBeenCalled() }) @@ -195,9 +243,12 @@ describe('downloadFolderViaSftp', () => { } const controller = new AbortController() - const result = downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination, { - signal: controller.signal - }) + const result = downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination, + { signal: controller.signal } + ) await vi.waitFor(() => expect(sftp.fastGet).toHaveBeenCalledTimes(1)) controller.abort(new Error('renderer closed')) @@ -222,22 +273,26 @@ describe('downloadFolderViaSftp', () => { it('cancels a pending SFTP directory read', async () => { const destination = await createDestination() let readDirCallback: ((error?: Error) => void) | undefined + const readdir = vi.fn((_path: string, callback: (error?: Error) => void) => { + readDirCallback = callback + }) const sftp = { stat: vi.fn((_path: string, callback: (err: Error | undefined, value: unknown) => void) => callback(undefined, sftpStats('directory')) ), - readdir: vi.fn((_path: string, callback: (error?: Error) => void) => { - readDirCallback = callback - }), + readdir, fastGet: vi.fn(), end: vi.fn() } const controller = new AbortController() - const result = downloadFolderViaSftp(async () => sftp as never, '/remote/src', destination, { - signal: controller.signal - }) - await vi.waitFor(() => expect(sftp.readdir).toHaveBeenCalledTimes(1)) + const result = downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination, + { signal: controller.signal } + ) + await vi.waitFor(() => expect(readdir).toHaveBeenCalledTimes(1)) controller.abort(new Error('renderer closed')) readDirCallback?.(new Error('channel closed')) @@ -245,4 +300,44 @@ describe('downloadFolderViaSftp', () => { expect(sftp.fastGet).not.toHaveBeenCalled() expect(sftp.end).toHaveBeenCalledTimes(1) }) + + it('streams directory chunks and stops at the retained entry budget', async () => { + const destination = await createDestination() + const handle = Buffer.from('handle') + const chunks = [ + [sftpEntry('one.txt', 'file'), sftpEntry('two.txt', 'file')], + [sftpEntry('three.txt', 'file')], + false + ] + const sftp = { + stat: vi.fn((_path: string, callback: (err: Error | undefined, value: unknown) => void) => + callback(undefined, sftpStats('directory')) + ), + opendir: vi.fn((_path: string, callback: (err: Error | undefined, value: Buffer) => void) => + callback(undefined, handle) + ), + readdir: vi.fn( + (_handle: Buffer, callback: (err: Error | undefined, value: unknown) => void) => + callback(undefined, chunks.shift()) + ), + close: vi.fn((_handle: Buffer, callback: (err?: Error) => void) => callback()), + fastGet: vi.fn(), + end: vi.fn() + } + + await expect( + downloadFolderViaSftp( + async () => withDirectoryHandles(sftp) as never, + '/remote/src', + destination, + { + limits: { ...SSH_DIRECTORY_TRANSFER_LIMITS, maximumEntries: 2 } + } + ) + ).rejects.toThrow('SSH directory transfer exceeds the entries limit') + + expect(sftp.readdir).toHaveBeenCalledTimes(2) + expect(sftp.close).toHaveBeenCalledWith(handle, expect.any(Function)) + expect(sftp.fastGet).not.toHaveBeenCalled() + }) }) diff --git a/src/main/providers/ssh-filesystem-download.ts b/src/main/providers/ssh-filesystem-download.ts index f73ef4c655f..700216ae036 100644 --- a/src/main/providers/ssh-filesystem-download.ts +++ b/src/main/providers/ssh-filesystem-download.ts @@ -7,12 +7,25 @@ import { normalizeRuntimePathSeparators } from '../../shared/cross-platform-path' import { sanitizeLocalDownloadFilename } from '../local-download-filename' -import { fastGetViaSftp, readDirViaSftp, statViaSftp } from './ssh-filesystem-provider-sftp' +import { + fastGetViaSftp, + readDirectoryEntriesViaSftp, + statViaSftp +} from './ssh-filesystem-provider-sftp' +import { + SSH_DIRECTORY_TRANSFER_LIMITS, + SshDirectoryTransferBudget, + type SshDirectoryTransferLimits +} from '../ssh/ssh-directory-transfer-budget' export type SftpFactory = (options?: { signal?: AbortSignal }) => Promise /** When known, windowsRemotePaths drives remote path joining; omit uses path-shape heuristics. */ -export type FolderDownloadOptions = { signal?: AbortSignal; windowsRemotePaths?: boolean } +export type FolderDownloadOptions = { + signal?: AbortSignal + windowsRemotePaths?: boolean + limits?: SshDirectoryTransferLimits +} const DOWNLOAD_UNAVAILABLE_MESSAGE = 'Remote folder download is unavailable. Reconnect the SSH target and retry.' @@ -82,13 +95,11 @@ async function downloadDirectoryTree( sftp: SFTPWrapper, sourceDir: string, destinationDir: string, + budget: SshDirectoryTransferBudget, + depth: number, signal?: AbortSignal, windowsRemotePaths?: boolean ): Promise { - signal?.throwIfAborted() - const entries = (await readDirViaSftp(sftp, sourceDir, { signal })).filter( - (entry) => entry.filename !== '.' && entry.filename !== '..' - ) signal?.throwIfAborted() const usedLocalNames = new Set() const plannedEntries: { @@ -96,7 +107,13 @@ async function downloadDirectoryTree( kind: 'directory' | 'file' localName: string }[] = [] - for (const entry of entries) { + for await (const entry of readDirectoryEntriesViaSftp(sftp, sourceDir, { signal })) { + signal?.throwIfAborted() + if (entry.filename === '.' || entry.filename === '..') { + continue + } + const remotePath = joinSftpChildPath(sourceDir, entry.filename, windowsRemotePaths) + budget.recordPath(remotePath, depth + 1) const localName = sanitizeLocalDownloadFilename(entry.filename) if (usedLocalNames.has(localName)) { throw new Error(`Remote entries map to the same local name '${localName}'`) @@ -115,7 +132,15 @@ async function downloadDirectoryTree( const remotePath = joinSftpChildPath(sourceDir, entry.filename, windowsRemotePaths) const localPath = join(destinationDir, localName) if (kind === 'directory') { - await downloadDirectoryTree(sftp, remotePath, localPath, signal, windowsRemotePaths) + await downloadDirectoryTree( + sftp, + remotePath, + localPath, + budget, + depth + 1, + signal, + windowsRemotePaths + ) continue } // Why: filesystem semantics belong to the selected volume, not the host OS; @@ -170,10 +195,14 @@ export async function downloadFolderViaSftp( if (!rootStats.isDirectory()) { throw new Error('Cannot download a file as a folder') } + const budget = new SshDirectoryTransferBudget(options?.limits ?? SSH_DIRECTORY_TRANSFER_LIMITS) + budget.recordPath(sourcePath, 0, { countEntry: false }) await downloadDirectoryTree( sftp, sourcePath, destinationPath, + budget, + 0, signal, options?.windowsRemotePaths ) diff --git a/src/main/providers/ssh-filesystem-provider-directory-limits.test.ts b/src/main/providers/ssh-filesystem-provider-directory-limits.test.ts new file mode 100644 index 00000000000..36f39f59d6a --- /dev/null +++ b/src/main/providers/ssh-filesystem-provider-directory-limits.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from 'vitest' +import { + FILESYSTEM_DIRECTORY_MAX_ENTRIES, + FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES +} from '../../shared/filesystem-directory-listing-limit' +import { JsonRpcErrorCode } from '../ssh/relay-protocol' +import { SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE } from './ssh-filesystem-directory-reader' +import { SshFilesystemProvider } from './ssh-filesystem-provider' + +function createProvider(request: ReturnType): SshFilesystemProvider { + return new SshFilesystemProvider('conn-1', { + request, + onNotification: vi.fn(() => () => {}) + } as never) +} + +describe('SshFilesystemProvider directory limits', () => { + it('routes normal listings through the bounded method without changing results', async () => { + const entries = [ + { name: 'src', isDirectory: true, isSymlink: false }, + { name: 'README.md', isDirectory: false, isSymlink: false } + ] + const request = vi.fn().mockResolvedValue(entries) + const provider = createProvider(request) + + await expect(provider.readDir('/home/user/project')).resolves.toEqual(entries) + expect(request).toHaveBeenCalledWith('fs.readDirBounded', { + dirPath: '/home/user/project', + maxEntries: FILESYSTEM_DIRECTORY_MAX_ENTRIES, + maxRetainedBytes: FILESYSTEM_DIRECTORY_MAX_RETAINED_BYTES + }) + }) + + it('forwards stricter mobile directory retention limits', async () => { + const request = vi.fn().mockResolvedValue([]) + const provider = createProvider(request) + + await provider.readDir('/home/user/project', { + maxEntries: 10_000, + maxRetainedBytes: 4 * 1024 * 1024 + }) + + expect(request).toHaveBeenCalledWith('fs.readDirBounded', { + dirPath: '/home/user/project', + maxEntries: 10_000, + maxRetainedBytes: 4 * 1024 * 1024 + }) + }) + + it('validates bounded relay results before returning them', async () => { + const request = vi.fn().mockResolvedValue([ + { name: 'one', isDirectory: false, isSymlink: false }, + { name: 'two', isDirectory: false, isSymlink: false }, + { name: 'three', isDirectory: false, isSymlink: false } + ]) + const provider = createProvider(request) + + await expect( + provider.readDir('/home/user/project', { maxEntries: 2, maxRetainedBytes: 1024 }) + ).rejects.toThrow('This folder is too large to list safely') + }) + + it('requires reconnect instead of falling back to an unbounded old relay', async () => { + const request = vi + .fn() + .mockRejectedValue( + Object.assign(new Error('Method not found'), { code: JsonRpcErrorCode.MethodNotFound }) + ) + const provider = createProvider(request) + + await expect(provider.readDir('/home/user/project')).rejects.toThrow( + SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE + ) + await expect(provider.readDir('/home/user/project')).rejects.toThrow( + SSH_BOUNDED_READ_DIR_UNAVAILABLE_MESSAGE + ) + expect(request).toHaveBeenCalledTimes(1) + expect(request).not.toHaveBeenCalledWith('fs.readDir', expect.anything()) + }) +}) diff --git a/src/main/providers/ssh-filesystem-provider-download-folder.test.ts b/src/main/providers/ssh-filesystem-provider-download-folder.test.ts index c875301883d..e5b2979f0f5 100644 --- a/src/main/providers/ssh-filesystem-provider-download-folder.test.ts +++ b/src/main/providers/ssh-filesystem-provider-download-folder.test.ts @@ -29,6 +29,29 @@ function sftpEntry(filename: string, kind: SftpEntryKind) { return { filename, longname: filename, attrs: sftpStats(kind) } } +function createSftpDirectoryHandleMethods( + readEntries: (remotePath: string) => ReturnType[] +) { + const completedPaths = new Set() + return { + opendir: vi.fn((remotePath: string, callback: (error: undefined, handle: Buffer) => void) => + callback(undefined, Buffer.from(remotePath)) + ), + readdir: vi.fn( + (handle: Buffer, callback: (error?: Error & { code?: number }, value?: unknown) => void) => { + const remotePath = handle.toString('utf8') + if (completedPaths.has(remotePath)) { + callback(Object.assign(new Error('EOF'), { code: 1 })) + return + } + completedPaths.add(remotePath) + callback(undefined, readEntries(remotePath)) + } + ), + close: vi.fn((_handle: Buffer, callback: (error?: Error) => void) => callback()) + } +} + function createMockMux() { return { request: vi.fn(), @@ -69,14 +92,10 @@ describe('SshFilesystemProvider downloadFolder', () => { (remotePath: string, callback: (err: Error | undefined, value: unknown) => void) => callback(undefined, sftpStats(remotePath === '/remote/src' ? 'directory' : 'file')) ), - readdir: vi.fn( - (remotePath: string, callback: (err: Error | undefined, value: unknown) => void) => - callback( - undefined, - remotePath === '/remote/src' - ? [sftpEntry('index.ts', 'file'), sftpEntry('lib', 'directory')] - : [sftpEntry('a.ts', 'file')] - ) + ...createSftpDirectoryHandleMethods((remotePath) => + remotePath === '/remote/src' + ? [sftpEntry('index.ts', 'file'), sftpEntry('lib', 'directory')] + : [sftpEntry('a.ts', 'file')] ), fastGet: vi.fn((_source: string, _destination: string, callback: (err?: Error) => void) => callback() @@ -95,6 +114,7 @@ describe('SshFilesystemProvider downloadFolder', () => { ['/remote/src/lib/a.ts', join(destination, 'lib', 'a.ts')] ]) await expect(stat(join(destination, 'lib'))).resolves.toMatchObject({}) + expect(sftp.close).toHaveBeenCalledTimes(2) expect(sftp.end).toHaveBeenCalledTimes(1) expect(mux.request).not.toHaveBeenCalled() }) @@ -107,9 +127,7 @@ describe('SshFilesystemProvider downloadFolder', () => { (remotePath: string, callback: (err: Error | undefined, value: unknown) => void) => callback(undefined, sftpStats(remotePath === '/remote/src' ? 'directory' : 'directory')) ), - readdir: vi.fn((_path: string, callback: (err: Error | undefined, value: unknown) => void) => - callback(undefined, [sftpEntry('linked-dir', 'symlink')]) - ), + ...createSftpDirectoryHandleMethods(() => [sftpEntry('linked-dir', 'symlink')]), fastGet: vi.fn(), end: vi.fn() } @@ -120,6 +138,7 @@ describe('SshFilesystemProvider downloadFolder', () => { ) expect(sftp.fastGet).not.toHaveBeenCalled() + expect(sftp.close).toHaveBeenCalledTimes(1) expect(sftp.end).toHaveBeenCalledTimes(1) }) @@ -130,9 +149,10 @@ describe('SshFilesystemProvider downloadFolder', () => { stat: vi.fn((_path: string, callback: (err: Error | undefined, value: unknown) => void) => callback(undefined, sftpStats('directory')) ), - readdir: vi.fn((_path: string, callback: (err: Error | undefined, value: unknown) => void) => - callback(undefined, [sftpEntry('a:b.txt', 'file'), sftpEntry('a?b.txt', 'file')]) - ), + ...createSftpDirectoryHandleMethods(() => [ + sftpEntry('a:b.txt', 'file'), + sftpEntry('a?b.txt', 'file') + ]), fastGet: vi.fn(), end: vi.fn() } @@ -143,6 +163,7 @@ describe('SshFilesystemProvider downloadFolder', () => { ) expect(sftp.fastGet).not.toHaveBeenCalled() + expect(sftp.close).toHaveBeenCalledTimes(1) expect(sftp.end).toHaveBeenCalledTimes(1) }) diff --git a/src/main/providers/ssh-filesystem-provider-sftp.ts b/src/main/providers/ssh-filesystem-provider-sftp.ts index a3b1b840be3..af1af6c2ac2 100644 --- a/src/main/providers/ssh-filesystem-provider-sftp.ts +++ b/src/main/providers/ssh-filesystem-provider-sftp.ts @@ -95,15 +95,38 @@ export function fastGetViaSftp( ) } -export function readDirViaSftp( +export async function* readDirectoryEntriesViaSftp( sftp: SFTPWrapper, dirPath: string, options?: { signal?: AbortSignal } -): Promise { - return waitForSftpCallback( - (callback) => sftp.readdir(dirPath, callback), +): AsyncGenerator { + const handle = await waitForSftpCallback( + (callback) => sftp.opendir(dirPath, callback), options ) + try { + for (;;) { + const entries = await waitForSftpCallback( + (callback) => + sftp.readdir(handle, (error, value) => { + if ((error as { code?: number } | undefined)?.code === 1) { + callback(null, false) + return + } + callback(error, value && value.length > 0 ? value : false) + }), + options + ) + if (entries === false) { + break + } + yield* entries + } + } finally { + await waitForSftpCallback((callback) => sftp.close(handle, callback), options).catch( + () => undefined + ) + } } export function statViaSftp( diff --git a/src/main/providers/ssh-filesystem-provider-stream.test.ts b/src/main/providers/ssh-filesystem-provider-stream.test.ts index b66b5633503..2563fbeaf7b 100644 --- a/src/main/providers/ssh-filesystem-provider-stream.test.ts +++ b/src/main/providers/ssh-filesystem-provider-stream.test.ts @@ -105,6 +105,54 @@ describe('SshFilesystemProvider readFile streaming', () => { expect(result).toEqual(legacyResult) }) + it('propagates validated raster dimensions from stream metadata', async () => { + const content = Buffer.from('png-header') + mux.request.mockImplementation(async () => { + setImmediate(() => { + mux._emitMethod('fs.streamChunk', { + streamId: 3, + seq: 0, + data: content.toString('base64') + }) + mux._emitMethod('fs.streamEnd', { streamId: 3 }) + }) + return { + streamId: 3, + totalSize: content.length, + isBinary: true, + isImage: true, + mimeType: 'image/png', + imageDimensions: { width: 640, height: 480 }, + resultEncoding: 'base64' + } + }) + + await expect(provider.readFile('/home/image.png')).resolves.toEqual({ + content: content.toString('base64'), + isBinary: true, + isImage: true, + mimeType: 'image/png', + imageDimensions: { width: 640, height: 480 } + }) + }) + + it('rejects unsafe raster dimensions in stream metadata', async () => { + mux.request.mockResolvedValue({ + streamId: 4, + totalSize: 1, + isBinary: true, + isImage: true, + mimeType: 'image/png', + imageDimensions: { width: 32_769, height: 1 }, + resultEncoding: 'base64' + }) + + await expect(provider.readFile('/home/bomb.png')).rejects.toThrow( + 'Malformed file stream metadata' + ) + expect(mux.notify).toHaveBeenCalledWith('fs.cancelStream', { streamId: 4 }) + }) + it('rejects when chunk arrives out of order', async () => { const totalSize = 256 * 1024 * 2 mux.request.mockImplementation(async () => { @@ -178,7 +226,7 @@ describe('SshFilesystemProvider readFile streaming', () => { resultEncoding: 'base64' } }) - await expect(provider.readFile('/home/x.bin')).rejects.toThrow(/count mismatch/i) + await expect(provider.readFile('/home/x.bin')).rejects.toThrow(/incomplete/i) }) it('rejects a short final chunk instead of zero-filling the buffer', async () => { diff --git a/src/main/providers/ssh-filesystem-provider.test.ts b/src/main/providers/ssh-filesystem-provider.test.ts index a7061616c4a..e8409249c0c 100644 --- a/src/main/providers/ssh-filesystem-provider.test.ts +++ b/src/main/providers/ssh-filesystem-provider.test.ts @@ -63,20 +63,6 @@ describe('SshFilesystemProvider', () => { expect(provider.getConnectionId()).toBe('conn-1') }) - describe('readDir', () => { - it('sends fs.readDir request', async () => { - const entries = [ - { name: 'src', isDirectory: true, isSymlink: false }, - { name: 'README.md', isDirectory: false, isSymlink: false } - ] - mux.request.mockResolvedValue(entries) - - const result = await provider.readDir('/home/user/project') - expect(mux.request).toHaveBeenCalledWith('fs.readDir', { dirPath: '/home/user/project' }) - expect(result).toEqual(entries) - }) - }) - describe('readFile', () => { it('short-circuits on empty:true metadata without subscribing to chunks', async () => { mux.request.mockResolvedValue({ totalSize: 0, isBinary: false, empty: true }) diff --git a/src/main/providers/ssh-filesystem-provider.ts b/src/main/providers/ssh-filesystem-provider.ts index 74f31b8e6bf..231e695069b 100644 --- a/src/main/providers/ssh-filesystem-provider.ts +++ b/src/main/providers/ssh-filesystem-provider.ts @@ -25,6 +25,8 @@ import type { DirEntry, FsChangeEvent, SearchOptions, SearchResult } from '../.. import { routeSshFilesystemWatchNotification } from './ssh-filesystem-watch-notifications' import type { WorkspaceSpaceDirectoryScanResult } from '../../shared/workspace-space-types' import { isWindowsRemoteHost, type RemoteHostPlatform } from '../ssh/ssh-remote-platform' +import { SshFilesystemDirectoryReader } from './ssh-filesystem-directory-reader' +import { requestSshMarkdownDocumentPaths } from './ssh-markdown-document-listing' const WORKSPACE_SPACE_SCAN_TIMEOUT_MS = 130_000 export class SshFilesystemProvider implements IFilesystemProvider { @@ -35,6 +37,7 @@ export class SshFilesystemProvider implements IFilesystemProvider { private tempDirPromise: Promise | null = null private disposed = false private loggedStreamFallback = false + private readonly directoryReader: SshFilesystemDirectoryReader readonly downloadFolder?: IFilesystemProvider['downloadFolder'] constructor( @@ -46,6 +49,7 @@ export class SshFilesystemProvider implements IFilesystemProvider { ) { this.connectionId = connectionId this.mux = mux + this.directoryReader = new SshFilesystemDirectoryReader(mux) if (createSftp) { // Why: system SSH has raw single-file transfer but no ssh2 SFTP channel; @@ -83,8 +87,11 @@ export class SshFilesystemProvider implements IFilesystemProvider { return this.connectionId } - async readDir(dirPath: string): Promise { - return (await this.mux.request('fs.readDir', { dirPath })) as DirEntry[] + async readDir( + dirPath: string, + options?: { maxEntries?: number; maxRetainedBytes?: number } + ): Promise { + return this.directoryReader.readDir(dirPath, options) } async readFile(filePath: string): Promise { @@ -319,6 +326,8 @@ export class SshFilesystemProvider implements IFilesystemProvider { })) as string[] } + listMarkdownDocuments = (rootPath: string) => requestSshMarkdownDocumentPaths(this.mux, rootPath) + async watch( rootPath: string, callback: (events: FsChangeEvent[]) => void, diff --git a/src/main/providers/ssh-git-dispatch.test.ts b/src/main/providers/ssh-git-dispatch.test.ts index a975f3db475..8ca20a5e189 100644 --- a/src/main/providers/ssh-git-dispatch.test.ts +++ b/src/main/providers/ssh-git-dispatch.test.ts @@ -12,7 +12,7 @@ describe('SSH Git provider registry', () => { unregisterSshGitProvider(connectionId) }) - it('keeps provider generations monotonic across unregister and re-register', () => { + it('uses a new generation after unregister without retaining the disconnected id', () => { const before = getSshGitProviderGeneration(connectionId) registerSshGitProvider(connectionId, {} as never) const registered = getSshGitProviderGeneration(connectionId) @@ -21,8 +21,18 @@ describe('SSH Git provider registry', () => { registerSshGitProvider(connectionId, {} as never) const reRegistered = getSshGitProviderGeneration(connectionId) - expect(registered).toBe(before + 1) - expect(unregistered).toBe(registered + 1) - expect(reRegistered).toBe(unregistered + 1) + expect(registered).toBeGreaterThan(before) + expect(unregistered).toBe(0) + expect(reRegistered).toBeGreaterThan(registered) + }) + + it('releases generations for thousands of unique disconnected ids', () => { + for (let index = 0; index < 10_000; index += 1) { + const id = `transient-${index}` + registerSshGitProvider(id, {} as never) + expect(getSshGitProviderGeneration(id)).toBeGreaterThan(0) + unregisterSshGitProvider(id) + expect(getSshGitProviderGeneration(id)).toBe(0) + } }) }) diff --git a/src/main/providers/ssh-git-dispatch.ts b/src/main/providers/ssh-git-dispatch.ts index c0acc757ada..ea968a0413e 100644 --- a/src/main/providers/ssh-git-dispatch.ts +++ b/src/main/providers/ssh-git-dispatch.ts @@ -2,19 +2,20 @@ import type { SshGitProvider } from './ssh-git-provider' const sshProviders = new Map() const sshProviderGenerations = new Map() +let nextSshProviderGeneration = 1 export const SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE = 'Remote connection dropped. Click Reconnect on the SSH target before retrying.' export function registerSshGitProvider(connectionId: string, provider: SshGitProvider): void { sshProviders.set(connectionId, provider) - sshProviderGenerations.set(connectionId, (sshProviderGenerations.get(connectionId) ?? 0) + 1) + sshProviderGenerations.set(connectionId, nextSshProviderGeneration) + nextSshProviderGeneration += 1 } export function unregisterSshGitProvider(connectionId: string): void { - if (sshProviders.delete(connectionId)) { - sshProviderGenerations.set(connectionId, (sshProviderGenerations.get(connectionId) ?? 0) + 1) - } + sshProviders.delete(connectionId) + sshProviderGenerations.delete(connectionId) } export function getSshGitProviderGeneration(connectionId: string): number { diff --git a/src/main/providers/ssh-markdown-document-listing.test.ts b/src/main/providers/ssh-markdown-document-listing.test.ts new file mode 100644 index 00000000000..71272b09c5a --- /dev/null +++ b/src/main/providers/ssh-markdown-document-listing.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest' +import { MarkdownDocumentListingCapacityError } from '../../shared/markdown-document-listing-limits' +import { JsonRpcErrorCode, RelayErrorCode } from '../ssh/relay-protocol' +import { requestSshMarkdownDocumentPaths } from './ssh-markdown-document-listing' + +describe('SSH Markdown document listing', () => { + it('requests producer-filtered Markdown paths', async () => { + const request = vi.fn().mockResolvedValue(['README.md', 'docs/guide.mdx']) + + await expect( + requestSshMarkdownDocumentPaths({ request } as never, '/home/user/project') + ).resolves.toEqual(['README.md', 'docs/guide.mdx']) + expect(request).toHaveBeenCalledWith('fs.listMarkdownDocuments', { + rootPath: '/home/user/project' + }) + }) + + it('surfaces typed producer capacity failures', async () => { + const request = vi.fn().mockRejectedValue( + Object.assign(new Error('capacity'), { + code: RelayErrorCode.MarkdownDocumentListingCapacity + }) + ) + + await expect( + requestSshMarkdownDocumentPaths({ request } as never, '/home/user/project') + ).rejects.toBeInstanceOf(MarkdownDocumentListingCapacityError) + }) + + it('requires reconnect instead of using an unbounded old-relay fallback', async () => { + const request = vi + .fn() + .mockRejectedValue( + Object.assign(new Error('missing'), { code: JsonRpcErrorCode.MethodNotFound }) + ) + + await expect( + requestSshMarkdownDocumentPaths({ request } as never, '/home/user/project') + ).rejects.toThrow('Reconnect the SSH target') + expect(request).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/providers/ssh-markdown-document-listing.ts b/src/main/providers/ssh-markdown-document-listing.ts new file mode 100644 index 00000000000..9061b98aac6 --- /dev/null +++ b/src/main/providers/ssh-markdown-document-listing.ts @@ -0,0 +1,27 @@ +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import { isMethodNotFoundError } from '../ssh/ssh-filesystem-stream-reader' +import { RelayErrorCode } from '../ssh/relay-protocol' +import { MarkdownDocumentListingCapacityError } from '../../shared/markdown-document-listing-limits' + +export async function requestSshMarkdownDocumentPaths( + mux: SshChannelMultiplexer, + rootPath: string +): Promise { + try { + return (await mux.request('fs.listMarkdownDocuments', { rootPath })) as string[] + } catch (error) { + if ( + error instanceof Error && + 'code' in error && + (error as { code?: unknown }).code === RelayErrorCode.MarkdownDocumentListingCapacity + ) { + throw new MarkdownDocumentListingCapacityError() + } + if (isMethodNotFoundError(error)) { + throw new Error( + 'Remote Markdown link discovery is unavailable. Reconnect the SSH target and retry.' + ) + } + throw error + } +} diff --git a/src/main/providers/ssh-pty-live-roster.test.ts b/src/main/providers/ssh-pty-live-roster.test.ts new file mode 100644 index 00000000000..aac1d5adcf4 --- /dev/null +++ b/src/main/providers/ssh-pty-live-roster.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_SSH_PTY_LIVE_ROSTER_ENTRIES, + MAX_SSH_PTY_LIVE_ROSTER_ID_BYTES, + SshPtyLiveRoster +} from './ssh-pty-live-roster' +import { MAX_SSH_APP_PTY_ID_BYTES } from './ssh-pty-wire-admission' + +describe('SshPtyLiveRoster', () => { + it('preserves ordinary listing and exit behavior', () => { + const roster = new SshPtyLiveRoster() + const listing = roster.beginListing() + + roster.reconcileListing(listing, ['pty-1']) + expect(roster.has('pty-1')).toBe(true) + + roster.recordExit('pty-1') + expect(roster.has('pty-1')).toBe(false) + }) + + it('gives a locally spawned PTY one stale-list grace', () => { + const roster = new SshPtyLiveRoster() + roster.recordSpawn('pty-new') + + roster.reconcileListing(roster.beginListing(), []) + expect(roster.has('pty-new')).toBe(true) + + roster.reconcileListing(roster.beginListing(), []) + expect(roster.has('pty-new')).toBe(false) + }) + + it('keeps a PTY observed while a listing is in flight', () => { + const roster = new SshPtyLiveRoster() + const listing = roster.beginListing() + + roster.recordNotification('pty-new') + roster.reconcileListing(listing, []) + + expect(roster.has('pty-new')).toBe(true) + }) + + it('ignores an older listing after a newer listing reconciles', () => { + const roster = new SshPtyLiveRoster() + roster.recordNotification('pty-live') + const older = roster.beginListing() + const newer = roster.beginListing() + + roster.reconcileListing(newer, ['pty-live']) + roster.reconcileListing(older, []) + + expect(roster.has('pty-live')).toBe(true) + }) + + it('does not resurrect an exit with an in-flight stale listing', () => { + const roster = new SshPtyLiveRoster() + roster.recordNotification('pty-exited') + const listing = roster.beginListing() + + roster.recordExit('pty-exited') + roster.reconcileListing(listing, ['pty-exited']) + + expect(roster.has('pty-exited')).toBe(false) + }) + + it('caps missed-exit unique-id churn and evicts the oldest live row', () => { + const roster = new SshPtyLiveRoster() + const total = MAX_SSH_PTY_LIVE_ROSTER_ENTRIES + 50 + + for (let index = 0; index < total; index += 1) { + roster.recordNotification(`pty-${index}`) + } + + expect(roster.has('pty-0')).toBe(false) + expect(roster.has(`pty-${total - MAX_SSH_PTY_LIVE_ROSTER_ENTRIES}`)).toBe(true) + expect(roster.has(`pty-${total - 1}`)).toBe(true) + }) + + it('does not let exit fences evict supported live entries', () => { + const roster = new SshPtyLiveRoster() + for (let index = 0; index < 50; index += 1) { + roster.recordNotification(`live-${index}`) + } + for (let index = 0; index < MAX_SSH_PTY_LIVE_ROSTER_ENTRIES * 2; index += 1) { + roster.recordExit(`gone-${index}`) + } + + for (let index = 0; index < 50; index += 1) { + expect(roster.has(`live-${index}`)).toBe(true) + } + }) + + it('rejects oversized ids and caps aggregate retained id bytes', () => { + const roster = new SshPtyLiveRoster() + const oversized = 'x'.repeat(MAX_SSH_APP_PTY_ID_BYTES + 1) + roster.recordNotification(oversized) + expect(roster.has(oversized)).toBe(false) + + const payload = 'x'.repeat(16 * 1024 - 32) + const total = Math.floor(MAX_SSH_PTY_LIVE_ROSTER_ID_BYTES / Buffer.byteLength(payload)) + 2 + for (let index = 0; index < total; index += 1) { + roster.recordNotification(`${index}:${payload}`) + } + + expect(roster.has(`0:${payload}`)).toBe(false) + expect(roster.has(`${total - 1}:${payload}`)).toBe(true) + }) + + it('ignores a listing response that completes after clear', () => { + const roster = new SshPtyLiveRoster() + const listing = roster.beginListing() + + roster.clear() + roster.reconcileListing(listing, ['pty-stale']) + + expect(roster.has('pty-stale')).toBe(false) + }) + + it('does not resurrect an exit after its tombstone is evicted', () => { + const roster = new SshPtyLiveRoster() + roster.recordSpawn('pty-reused') + const stale = roster.beginListing() + roster.recordExit('pty-reused') + for (let index = 0; index < MAX_SSH_PTY_LIVE_ROSTER_ENTRIES; index += 1) { + roster.recordExit(`gone-${index}`) + } + + roster.reconcileListing(stale, ['pty-reused']) + expect(roster.has('pty-reused')).toBe(false) + + const fresh = roster.beginListing() + roster.reconcileListing(fresh, ['pty-reused']) + expect(roster.has('pty-reused')).toBe(true) + }) +}) diff --git a/src/main/providers/ssh-pty-live-roster.ts b/src/main/providers/ssh-pty-live-roster.ts new file mode 100644 index 00000000000..cafbdb53e5e --- /dev/null +++ b/src/main/providers/ssh-pty-live-roster.ts @@ -0,0 +1,155 @@ +import { admittedSshAppPtyIdBytes } from './ssh-pty-wire-admission' + +export const MAX_SSH_PTY_LIVE_ROSTER_ENTRIES = 256 +export const MAX_SSH_PTY_LIVE_ROSTER_ID_BYTES = 2 * 1024 * 1024 + +export type SshPtyListingToken = { + sequence: number + observationRevision: number +} + +type RosterEntry = { + idBytes: number + live: boolean + missingListingGrace: boolean + observationRevision: number +} + +export class SshPtyLiveRoster { + private readonly entries = new Map() + private observationRevision = 0 + private listingSequence = 0 + private latestReconciledListing = 0 + private retainedIdBytes = 0 + private staleListingFenceRevision = 0 + + beginListing(): SshPtyListingToken { + return { + sequence: ++this.listingSequence, + observationRevision: this.observationRevision + } + } + + recordSpawn(id: string): void { + this.record(id, true) + } + + recordNotification(id: string): void { + this.record(id, false) + } + + recordExit(id: string): void { + const idBytes = admittedSshAppPtyIdBytes(id) + if (idBytes === null) { + return + } + this.setEntry(id, { + idBytes, + live: false, + missingListingGrace: false, + observationRevision: ++this.observationRevision + }) + } + + reconcileListing(token: SshPtyListingToken, listedIds: readonly string[]): void { + if (token.sequence <= this.latestReconciledListing) { + return + } + this.latestReconciledListing = token.sequence + if (token.observationRevision < this.staleListingFenceRevision) { + return + } + const listed = new Set() + for (const id of listedIds) { + if (listed.size < MAX_SSH_PTY_LIVE_ROSTER_ENTRIES && admittedSshAppPtyIdBytes(id) !== null) { + listed.add(id) + } + } + + for (const [id, entry] of this.entries) { + if (!entry.live || listed.has(id) || entry.observationRevision > token.observationRevision) { + continue + } + if (entry.missingListingGrace) { + entry.missingListingGrace = false + entry.observationRevision = ++this.observationRevision + continue + } + this.removeEntry(id) + } + + for (const id of listed) { + const current = this.entries.get(id) + // Why: an exit newer than this listing request must fence its stale response. + if (current && !current.live && current.observationRevision > token.observationRevision) { + continue + } + this.record(id, false) + } + } + + has(id: string): boolean { + return this.entries.get(id)?.live === true + } + + clear(): void { + this.entries.clear() + this.retainedIdBytes = 0 + // Why: responses from listings already in flight must not repopulate a disposed provider. + this.latestReconciledListing = this.listingSequence + } + + private record(id: string, missingListingGrace: boolean): void { + const idBytes = admittedSshAppPtyIdBytes(id) + if (idBytes === null) { + return + } + this.setEntry(id, { + idBytes, + live: true, + missingListingGrace, + observationRevision: ++this.observationRevision + }) + } + + private setEntry(id: string, entry: RosterEntry): void { + this.removeEntry(id) + this.entries.set(id, entry) + this.retainedIdBytes += entry.idBytes + this.capEntries() + } + + private capEntries(): void { + while ( + this.entries.size > MAX_SSH_PTY_LIVE_ROSTER_ENTRIES || + this.retainedIdBytes > MAX_SSH_PTY_LIVE_ROSTER_ID_BYTES + ) { + let inactive: string | undefined + for (const [id, entry] of this.entries) { + if (!entry.live) { + inactive = id + break + } + } + const oldest = this.entries.keys().next().value as string | undefined + const evictedId = inactive ?? oldest ?? '' + const evicted = this.entries.get(evictedId) + if (evicted && !evicted.live) { + this.staleListingFenceRevision = Math.max( + this.staleListingFenceRevision, + evicted.observationRevision + ) + } + this.removeEntry(evictedId) + } + } + + private removeEntry(id: string): void { + const entry = this.entries.get(id) + if (!entry) { + return + } + this.retainedIdBytes -= entry.idBytes + this.entries.delete(id) + } +} diff --git a/src/main/providers/ssh-pty-notification-routing.ts b/src/main/providers/ssh-pty-notification-routing.ts deleted file mode 100644 index 63721877f98..00000000000 --- a/src/main/providers/ssh-pty-notification-routing.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' -import { isPtyIncarnationId } from '../../shared/pty-incarnation' -import type { - SshPtyDataCallback, - SshPtyExitCallback, - SshPtyReplayCallback -} from './ssh-pty-provider-contract' - -export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback } - -export function subscribeSshPtyNotifications(args: { - mux: SshChannelMultiplexer - toAppPtyId: (id: string) => string - dataListeners: Set - replayListeners: Set - exitListeners: Set - livePtyIds: Set - recordExit: (relayPtyId: string, incarnationId: unknown) => void -}): () => void { - return args.mux.onNotification((method, params) => { - const id = args.toAppPtyId(params.id as string) - if (method === 'pty.exit') { - args.recordExit(params.id as string, params.incarnationId) - args.livePtyIds.delete(id) - for (const listener of args.exitListeners) { - listener({ - id, - code: params.code as number, - ...(isPtyIncarnationId(params.incarnationId) - ? { incarnationId: params.incarnationId } - : {}) - }) - } - return - } - if (method !== 'pty.data' && method !== 'pty.replay') { - return - } - args.livePtyIds.add(id) - if (method === 'pty.replay') { - for (const listener of args.replayListeners) { - listener({ id, data: params.data as string }) - } - return - } - for (const listener of args.dataListeners) { - listener({ - id, - data: params.data as string, - ...(typeof params.rawLength === 'number' ? { sequenceChars: params.rawLength } : {}), - ...(params.transformed === true ? { transformed: true } : {}), - ...(typeof params.seq === 'number' ? { seq: params.seq } : {}) - }) - } - }) -} diff --git a/src/main/providers/ssh-pty-output-delivery-credit.test.ts b/src/main/providers/ssh-pty-output-delivery-credit.test.ts new file mode 100644 index 00000000000..e3df99902c9 --- /dev/null +++ b/src/main/providers/ssh-pty-output-delivery-credit.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, it, vi } from 'vitest' +import { + MAX_SSH_PTY_CREDIT_SEGMENTS, + MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE, + MAX_SSH_PTY_CREDIT_STATE_ID_BYTES, + MAX_SSH_PTY_CREDIT_STATES, + SshPtyOutputDeliveryCredit +} from './ssh-pty-output-delivery-credit' +import { + MAX_SSH_PTY_DELIVERY_TOKEN_BYTES, + MAX_SSH_RELAY_PTY_ID_BYTES +} from './ssh-pty-wire-admission' + +function retainedStates(credit: SshPtyOutputDeliveryCredit): Map { + return Reflect.get(credit, 'states') as Map +} + +describe('SshPtyOutputDeliveryCredit', () => { + it('defers tokenized credit until the captured owner acknowledges renderer progress', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + let upstreamCredit: { charCount: number; acknowledge(chars: number): void } | undefined + + credit.ingest( + { + id: 'pty-1', + data: 'output', + rawLength: 10, + deliveryToken: 'delivery-1' + }, + (output) => { + upstreamCredit = output.upstreamCredit + } + ) + + expect(notifyAck).not.toHaveBeenCalled() + upstreamCredit?.acknowledge(4) + upstreamCredit?.acknowledge(20) + + expect(notifyAck).toHaveBeenNthCalledWith(1, { + id: 'pty-1', + charCount: 4, + deliveryToken: 'delivery-1' + }) + expect(notifyAck).toHaveBeenNthCalledWith(2, { + id: 'pty-1', + charCount: 6, + deliveryToken: 'delivery-1' + }) + }) + + it('acknowledges a later dropped segment without consuming earlier visible credit', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + let first: { acknowledge(chars: number): void } | undefined + let second: { acknowledge(chars: number): void } | undefined + credit.ingest({ id: 'pty-1', data: 'first', deliveryToken: 'delivery-1' }, (output) => { + first = output.upstreamCredit + }) + credit.ingest({ id: 'pty-1', data: 'second', deliveryToken: 'delivery-1' }, (output) => { + second = output.upstreamCredit + }) + + second?.acknowledge(6) + first?.acknowledge(5) + + expect(notifyAck.mock.calls).toEqual([ + [{ id: 'pty-1', charCount: 6, deliveryToken: 'delivery-1' }], + [{ id: 'pty-1', charCount: 5, deliveryToken: 'delivery-1' }] + ]) + }) + + it('walks token segments in order for legacy provider acknowledgement calls', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + credit.ingest({ id: 'pty-1', data: 'first', deliveryToken: 'token-1' }, () => {}) + credit.ingest({ id: 'pty-1', data: 'second', deliveryToken: 'token-1' }, () => {}) + + credit.acknowledgeLegacy('pty-1', 11) + + expect(notifyAck.mock.calls).toEqual([ + [{ id: 'pty-1', charCount: 5, deliveryToken: 'token-1' }], + [{ id: 'pty-1', charCount: 6, deliveryToken: 'token-1' }] + ]) + }) + + it('drops stale credit when attach rotates the delivery token', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + let stale: { acknowledge(chars: number): void } | undefined + let current: { acknowledge(chars: number): void } | undefined + credit.ingest({ id: 'pty-1', data: 'stale', deliveryToken: 'before-attach' }, (output) => { + stale = output.upstreamCredit + }) + credit.ingest({ id: 'pty-1', data: 'current', deliveryToken: 'after-attach' }, (output) => { + current = output.upstreamCredit + }) + + current?.acknowledge(7) + stale?.acknowledge(5) + credit.ingest({ id: 'pty-1', data: 'next', deliveryToken: 'after-attach' }, () => {}) + credit.acknowledgeLegacy('pty-1', 4) + + expect(notifyAck.mock.calls).toEqual([ + [{ id: 'pty-1', charCount: 7, deliveryToken: 'after-attach' }], + [{ id: 'pty-1', charCount: 4, deliveryToken: 'after-attach' }] + ]) + }) + + it('preserves legacy no-token acknowledgements', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + credit.ingest({ id: 'pty-1', data: 'legacy' }, () => {}) + + credit.acknowledgeLegacy('pty-1', 6) + + expect(notifyAck).toHaveBeenCalledWith({ id: 'pty-1', charCount: 6 }) + }) + + it('falls back to delivered length when raw length is invalid', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + let upstreamCredit: { acknowledge(chars: number): void } | undefined + credit.ingest( + { + id: 'pty-1', + data: 'output', + rawLength: Number.POSITIVE_INFINITY, + deliveryToken: 'delivery-2' + }, + (output) => { + upstreamCredit = output.upstreamCredit + } + ) + + upstreamCredit?.acknowledge(100) + + expect(notifyAck).toHaveBeenCalledWith({ + id: 'pty-1', + charCount: 6, + deliveryToken: 'delivery-2' + }) + }) + + it('makes captured credit inert after exit or disposal', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + let exited: { acknowledge(chars: number): void } | undefined + let disposed: { acknowledge(chars: number): void } | undefined + credit.ingest({ id: 'pty-1', data: 'exit', deliveryToken: 'delivery-3' }, (output) => { + exited = output.upstreamCredit + }) + credit.release('pty-1') + exited?.acknowledge(4) + credit.ingest({ id: 'pty-2', data: 'dispose', deliveryToken: 'delivery-4' }, (output) => { + disposed = output.upstreamCredit + }) + + credit.dispose() + disposed?.acknowledge(7) + + expect(notifyAck).not.toHaveBeenCalled() + }) + + it('returns delivery credit when listener ingestion throws', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + + expect(() => + credit.ingest({ id: 'pty-1', data: 'output', deliveryToken: 'delivery-5' }, () => { + throw new Error('listener failed') + }) + ).toThrow('listener failed') + expect(notifyAck).toHaveBeenCalledWith({ + id: 'pty-1', + charCount: 6, + deliveryToken: 'delivery-5' + }) + }) + + it('rejects oversized ids and delivery tokens before retention', () => { + const deliver = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(vi.fn()) + + credit.ingest( + { + id: 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES + 1), + data: 'output', + deliveryToken: 'token' + }, + deliver + ) + credit.ingest( + { + id: 'pty-1', + data: 'output', + deliveryToken: 'x'.repeat(MAX_SSH_PTY_DELIVERY_TOKEN_BYTES + 1) + }, + deliver + ) + + expect(deliver).not.toHaveBeenCalled() + expect(retainedStates(credit)).toHaveLength(0) + }) + + it('caps unique-id credit state churn and returns evicted credit', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + for (let index = 0; index < MAX_SSH_PTY_CREDIT_STATES + 50; index += 1) { + credit.ingest({ id: `pty-${index}`, data: 'x', deliveryToken: `token-${index}` }, () => {}) + } + + const states = retainedStates(credit) + expect(states.size).toBe(MAX_SSH_PTY_CREDIT_STATES) + expect(states.has('pty-0')).toBe(false) + expect(states.has(`pty-${MAX_SSH_PTY_CREDIT_STATES + 49}`)).toBe(true) + expect(notifyAck).toHaveBeenCalledWith({ + id: 'pty-0', + charCount: 1, + deliveryToken: 'token-0' + }) + }) + + it('caps aggregate credit-state id bytes', () => { + const credit = new SshPtyOutputDeliveryCredit(vi.fn()) + const suffix = 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES - 16) + for (let index = 0; index < MAX_SSH_PTY_CREDIT_STATES; index += 1) { + credit.ingest( + { id: `pty-${index}:${suffix}`, data: 'x', deliveryToken: `token-${index}` }, + () => {} + ) + } + + const retainedBytes = [...retainedStates(credit).keys()].reduce( + (total, id) => total + Buffer.byteLength(id), + 0 + ) + expect(retainedBytes).toBeLessThanOrEqual(MAX_SSH_PTY_CREDIT_STATE_ID_BYTES) + expect(retainedStates(credit).size).toBeLessThan(MAX_SSH_PTY_CREDIT_STATES) + }) + + it('caps retained segments per state and in aggregate', () => { + const credit = new SshPtyOutputDeliveryCredit(vi.fn()) + const perStateTotal = MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE + 50 + for (let index = 0; index < perStateTotal; index += 1) { + credit.ingest({ id: 'pty-one', data: 'x', deliveryToken: 'token-one' }, () => {}) + } + expect(retainedStates(credit).get('pty-one')?.segments).toHaveLength( + MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE + ) + + for (let state = 0; state < 8; state += 1) { + for (let segment = 0; segment < MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE; segment += 1) { + credit.ingest({ id: `pty-${state}`, data: 'x', deliveryToken: `token-${state}` }, () => {}) + } + } + + const retained = [...retainedStates(credit).values()].reduce( + (total, state) => total + state.segments.length, + 0 + ) + expect(retained).toBe(MAX_SSH_PTY_CREDIT_SEGMENTS) + }) + + it('returns unretained direct credit when state churn evicts its owner', () => { + const notifyAck = vi.fn() + const credit = new SshPtyOutputDeliveryCredit(notifyAck) + for (let index = 0; index <= MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE; index += 1) { + credit.ingest({ id: 'pty-direct', data: 'x', deliveryToken: 'token-direct' }, () => {}) + } + for (let index = 0; index < MAX_SSH_PTY_CREDIT_STATES; index += 1) { + credit.ingest({ id: `pty-${index}`, data: 'x', deliveryToken: `token-${index}` }, () => {}) + } + + expect(notifyAck).toHaveBeenCalledWith({ + id: 'pty-direct', + charCount: MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE + 1, + deliveryToken: 'token-direct' + }) + }) +}) diff --git a/src/main/providers/ssh-pty-output-delivery-credit.ts b/src/main/providers/ssh-pty-output-delivery-credit.ts new file mode 100644 index 00000000000..1c0c40ad007 --- /dev/null +++ b/src/main/providers/ssh-pty-output-delivery-credit.ts @@ -0,0 +1,280 @@ +import type { PtyDataUpstreamCredit } from './pty-provider-events' +import { + admittedSshRelayPtyIdBytes, + isAdmittedSshDeliveryToken, + isAdmittedSshRelayPtyId +} from './ssh-pty-wire-admission' + +export const MAX_SSH_PTY_CREDIT_STATES = 256 +export const MAX_SSH_PTY_CREDIT_STATE_ID_BYTES = 512 * 1024 +export const MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE = 4096 +export const MAX_SSH_PTY_CREDIT_SEGMENTS = 16_384 + +export type SshPtyOutputDelivery = { + relayId: string + data: string + rawLength?: number + transformed: boolean + seq?: number + upstreamCredit?: PtyDataUpstreamCredit +} + +type AckParams = { + id: string + charCount: number + deliveryToken?: string +} + +export class SshPtyOutputDeliveryCredit { + private readonly states = new Map() + private disposed = false + private retainedStateIdBytes = 0 + private retainedSegments = 0 + + constructor(private readonly notifyAck: (params: AckParams) => void) {} + + ingest(params: Record, deliver: (output: SshPtyOutputDelivery) => void): void { + const relayId = params.id + const data = params.data + if (!isAdmittedSshRelayPtyId(relayId) || typeof data !== 'string') { + return + } + if (params.deliveryToken !== undefined && !isAdmittedSshDeliveryToken(params.deliveryToken)) { + return + } + const deliveryToken = params.deliveryToken as string | undefined + const rawLength = + typeof params.rawLength === 'number' && + Number.isSafeInteger(params.rawLength) && + params.rawLength >= 0 + ? params.rawLength + : undefined + const seq = + typeof params.seq === 'number' && Number.isSafeInteger(params.seq) && params.seq >= 0 + ? params.seq + : undefined + const charCount = rawLength ?? data.length + const upstreamCredit = this.createCredit(relayId, charCount, deliveryToken) + try { + deliver({ + relayId, + data, + ...(rawLength === undefined ? {} : { rawLength }), + transformed: params.transformed === true, + ...(seq === undefined ? {} : { seq }), + ...(upstreamCredit ? { upstreamCredit } : {}) + }) + } catch (error) { + upstreamCredit?.acknowledge(charCount) + throw error + } + } + + acknowledgeLegacy(relayId: string, charCount: number): void { + if ( + !isAdmittedSshRelayPtyId(relayId) || + !Number.isFinite(charCount) || + charCount <= 0 || + this.disposed + ) { + return + } + const state = this.states.get(relayId) + if (!state) { + this.notifyAck({ id: relayId, charCount: Math.floor(charCount) }) + return + } + let remaining = Math.floor(charCount) + while (remaining > 0 && state.segments.length > 0) { + const segment = state.segments[0] + const acknowledged = Math.min(remaining, segment.remainingChars) + this.acknowledgeSegment(state, segment, acknowledged) + remaining -= acknowledged + } + if (remaining > 0 && !state.tokenized) { + this.notifyAck({ id: relayId, charCount: remaining }) + } + } + + release(relayId: string): void { + const state = this.states.get(relayId) + if (!state) { + return + } + this.removeState(state, false) + } + + dispose(): void { + this.disposed = true + for (const state of this.states.values()) { + state.active = false + this.clearSegments(state) + } + this.states.clear() + this.retainedStateIdBytes = 0 + } + + private createCredit( + relayId: string, + charCount: number, + deliveryToken: string | undefined + ): PtyDataUpstreamCredit | undefined { + if (this.disposed || charCount <= 0 || !deliveryToken) { + return undefined + } + let existing = this.states.get(relayId) + // Why: pty.attach rotates the relay token; stale renderer credit from the prior + // generation must not pin every later acknowledged segment behind it. + if (existing && existing.deliveryToken !== deliveryToken) { + this.removeState(existing, false) + existing = undefined + } + const state = + existing?.active === true + ? existing + : { + relayId, + idBytes: admittedSshRelayPtyIdBytes(relayId) ?? 0, + active: true, + tokenized: true, + deliveryToken, + segments: [], + directOutstandingChars: 0 + } + if (!existing) { + this.retainedStateIdBytes += state.idBytes + } else { + this.states.delete(relayId) + } + this.states.set(relayId, state) + this.capStates() + if ( + state.segments.length >= MAX_SSH_PTY_CREDIT_SEGMENTS_PER_STATE || + this.retainedSegments >= MAX_SSH_PTY_CREDIT_SEGMENTS + ) { + return this.createDirectCredit(state, charCount) + } + const segment: PtyCreditSegment = { + deliveryToken, + remainingChars: charCount + } + state.segments.push(segment) + this.retainedSegments++ + return { + charCount, + acknowledge: (requestedChars) => { + if (!Number.isFinite(requestedChars) || requestedChars <= 0) { + return + } + this.acknowledgeSegment(state, segment, Math.floor(requestedChars)) + } + } + } + + private acknowledgeSegment( + state: PtyCreditState, + segment: PtyCreditSegment, + requestedChars: number + ): void { + if (this.disposed || !state.active || requestedChars <= 0 || segment.remainingChars <= 0) { + return + } + const charCount = Math.min(requestedChars, segment.remainingChars) + segment.remainingChars -= charCount + this.notifyAck({ + id: state.relayId, + charCount, + ...(segment.deliveryToken ? { deliveryToken: segment.deliveryToken } : {}) + }) + while (state.segments[0]?.remainingChars === 0) { + state.segments.shift() + this.retainedSegments-- + } + } + + private createDirectCredit(state: PtyCreditState, charCount: number): PtyDataUpstreamCredit { + let remainingChars = charCount + state.directOutstandingChars += charCount + return { + charCount, + acknowledge: (requestedChars) => { + if ( + this.disposed || + !state.active || + state.deliveryToken.length === 0 || + !Number.isFinite(requestedChars) || + requestedChars <= 0 || + remainingChars <= 0 + ) { + return + } + const acknowledged = Math.min(Math.floor(requestedChars), remainingChars) + remainingChars -= acknowledged + state.directOutstandingChars -= acknowledged + this.notifyAck({ + id: state.relayId, + charCount: acknowledged, + deliveryToken: state.deliveryToken + }) + } + } + } + + private capStates(): void { + while ( + this.states.size > MAX_SSH_PTY_CREDIT_STATES || + this.retainedStateIdBytes > MAX_SSH_PTY_CREDIT_STATE_ID_BYTES + ) { + const oldest = this.states.values().next().value as PtyCreditState | undefined + if (!oldest) { + return + } + this.removeState(oldest, true) + } + } + + private removeState(state: PtyCreditState, returnOutstanding: boolean): void { + state.active = false + if (returnOutstanding) { + const outstandingChars = + state.directOutstandingChars + + state.segments.reduce((total, segment) => total + segment.remainingChars, 0) + if (outstandingChars > 0) { + try { + this.notifyAck({ + id: state.relayId, + charCount: outstandingChars, + deliveryToken: state.deliveryToken + }) + } catch { + // Best effort: eviction must still enforce the local memory bound. + } + } + } + state.directOutstandingChars = 0 + this.clearSegments(state) + if (this.states.delete(state.relayId)) { + this.retainedStateIdBytes -= state.idBytes + } + } + + private clearSegments(state: PtyCreditState): void { + this.retainedSegments -= state.segments.length + state.segments.length = 0 + } +} + +type PtyCreditSegment = { + deliveryToken?: string + remainingChars: number +} + +type PtyCreditState = { + relayId: string + idBytes: number + active: boolean + tokenized: boolean + deliveryToken: string + segments: PtyCreditSegment[] + directOutstandingChars: number +} diff --git a/src/main/providers/ssh-pty-provider-contract.ts b/src/main/providers/ssh-pty-provider-contract.ts index 9043133b03c..34d46a9148d 100644 --- a/src/main/providers/ssh-pty-provider-contract.ts +++ b/src/main/providers/ssh-pty-provider-contract.ts @@ -1,4 +1,5 @@ import type { PtyIncarnationId } from '../../shared/pty-incarnation' +import type { PtyDataEvent } from './pty-provider-events' export type RemoteCliBridgeEnv = { binDir: string @@ -8,13 +9,7 @@ export type RemoteCliBridgeEnv = { pathDelimiter?: ':' | ';' } -export type SshPtyDataCallback = (payload: { - id: string - data: string - sequenceChars?: number - transformed?: boolean - seq?: number -}) => void +export type SshPtyDataCallback = (payload: PtyDataEvent) => void export type SshPtyReplayCallback = (payload: { id: string; data: string }) => void export type SshPtyExitCallback = (payload: { id: string diff --git a/src/main/providers/ssh-pty-provider-exit-race.test.ts b/src/main/providers/ssh-pty-provider-exit-race.test.ts index 12616e4d133..bca158029eb 100644 --- a/src/main/providers/ssh-pty-provider-exit-race.test.ts +++ b/src/main/providers/ssh-pty-provider-exit-race.test.ts @@ -76,3 +76,34 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch', isReattach: true }) }) + +it('does not treat a malformed exit incarnation as a wildcard spawn race', async () => { + const mux = { + request: vi.fn(), + notify: vi.fn(), + onNotification: vi.fn(), + dispose: vi.fn(), + isDisposed: vi.fn().mockReturnValue(false) + } + const provider = new SshPtyProvider('conn-1', mux as never) + const exitListener = vi.fn() + provider.onExit(exitListener) + mux.request.mockImplementation(async (method: string) => { + if (method === 'pty.spawn') { + const notify = mux.onNotification.mock.calls[0]?.[0] + notify?.('pty.exit', { + id: 'pty-raced', + code: 0, + incarnationId: 'x'.repeat(129) + }) + return { id: 'pty-raced', incarnationId: 'incarnation-current' } + } + return undefined + }) + + await expect(provider.spawn({ cols: 80, rows: 24 })).resolves.toMatchObject({ + id: 'ssh:conn-1@@pty-raced', + incarnationId: 'incarnation-current' + }) + expect(exitListener).not.toHaveBeenCalled() +}) diff --git a/src/main/providers/ssh-pty-provider-notifications.ts b/src/main/providers/ssh-pty-provider-notifications.ts new file mode 100644 index 00000000000..7d6524f0818 --- /dev/null +++ b/src/main/providers/ssh-pty-provider-notifications.ts @@ -0,0 +1,124 @@ +import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import type { + SshPtyDataCallback, + SshPtyExitCallback, + SshPtyReplayCallback +} from './ssh-pty-provider-contract' +import { SshPtyOutputDeliveryCredit } from './ssh-pty-output-delivery-credit' +import { isAdmittedSshRelayPtyId } from './ssh-pty-wire-admission' + +export class SshPtyProviderNotifications { + private readonly dataListeners = new Set() + private readonly replayListeners = new Set() + private readonly exitListeners = new Set() + private readonly outputDeliveryCredit: SshPtyOutputDeliveryCredit + private unsubscribe: (() => void) | null + + constructor( + mux: SshChannelMultiplexer, + private readonly toAppPtyId: (relayId: string) => string, + recordNotification: (relayId: string) => void, + recordExit: (relayId: string, incarnationId: unknown) => void + ) { + this.outputDeliveryCredit = new SshPtyOutputDeliveryCredit((params) => { + mux.notify('pty.ackData', params) + }) + this.unsubscribe = mux.onNotification((method, params) => { + const relayId = params.id + if (!isAdmittedSshRelayPtyId(relayId)) { + return + } + switch (method) { + case 'pty.data': + this.emitData(params, recordNotification) + break + case 'pty.replay': { + if (typeof params.data !== 'string') { + return + } + recordNotification(relayId) + for (const listener of this.replayListeners) { + listener({ + id: this.toAppPtyId(relayId), + data: params.data + }) + } + break + } + case 'pty.exit': { + const code = params.code + const incarnationId = params.incarnationId + if ( + typeof code !== 'number' || + !Number.isSafeInteger(code) || + (incarnationId !== undefined && !isPtyIncarnationId(incarnationId)) + ) { + return + } + this.outputDeliveryCredit.release(relayId) + recordExit(relayId, incarnationId) + for (const listener of this.exitListeners) { + listener({ + id: this.toAppPtyId(relayId), + code, + ...(isPtyIncarnationId(incarnationId) ? { incarnationId } : {}) + }) + } + break + } + } + }) + } + + private emitData( + params: Record, + recordNotification: (relayId: string) => void + ): void { + this.outputDeliveryCredit.ingest(params, (output) => { + recordNotification(output.relayId) + if (this.dataListeners.size === 0) { + output.upstreamCredit?.acknowledge(output.upstreamCredit.charCount) + return + } + for (const listener of this.dataListeners) { + listener({ + id: this.toAppPtyId(output.relayId), + data: output.data, + ...(output.rawLength === undefined ? {} : { sequenceChars: output.rawLength }), + ...(output.transformed ? { transformed: true } : {}), + ...(output.seq === undefined ? {} : { seq: output.seq }), + ...(output.upstreamCredit ? { upstreamCredit: output.upstreamCredit } : {}) + }) + } + }) + } + + acknowledgeLegacy(relayId: string, charCount: number): void { + this.outputDeliveryCredit.acknowledgeLegacy(relayId, charCount) + } + + onData(callback: SshPtyDataCallback): () => void { + this.dataListeners.add(callback) + return () => this.dataListeners.delete(callback) + } + + onReplay(callback: SshPtyReplayCallback): () => void { + this.replayListeners.add(callback) + return () => this.replayListeners.delete(callback) + } + + onExit(callback: SshPtyExitCallback): () => void { + this.exitListeners.add(callback) + return () => this.exitListeners.delete(callback) + } + + dispose(): void { + this.unsubscribe?.() + this.unsubscribe = null + this.dataListeners.clear() + this.replayListeners.clear() + this.exitListeners.clear() + this.outputDeliveryCredit.dispose() + } +} diff --git a/src/main/providers/ssh-pty-provider-process-events.test.ts b/src/main/providers/ssh-pty-provider-process-events.test.ts index 2d033aa40d3..4a2cf30c6b0 100644 --- a/src/main/providers/ssh-pty-provider-process-events.test.ts +++ b/src/main/providers/ssh-pty-provider-process-events.test.ts @@ -1,5 +1,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SshPtyProvider } from './ssh-pty-provider' +import { + MAX_SSH_PTY_PROCESS_CWD_BYTES, + MAX_SSH_PTY_PROCESS_LIST_BYTES, + MAX_SSH_PTY_PROCESS_LIST_ENTRIES, + MAX_SSH_PTY_PROCESS_LIST_OWNERS, + MAX_SSH_PTY_PROCESS_OWNERS_PER_ENTRY, + MAX_SSH_PTY_PROCESS_TITLE_BYTES +} from './ssh-agent-session-process-list' +import { MAX_SSH_RELAY_PTY_ID_BYTES } from './ssh-pty-wire-admission' type MockMultiplexer = { request: ReturnType @@ -89,6 +98,71 @@ describe('SshPtyProvider process listings and events', () => { ]) }) + it('drops unknown process and owner fields instead of retaining remote payloads', async () => { + const unknownPayload = 'x'.repeat(1024 * 1024) + mux.request.mockResolvedValue([ + { + id: 'pty-1', + incarnationId: 'incarnation-1', + cwd: '/home', + title: 'codex', + unknownPayload, + agentSessionOwners: [ + { + claim: { + digestVersion: 1, + keyId: 'key', + identityDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + worktreeScopeDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + agent: 'codex', + unknownPayload + }, + generation: 'generation-1', + phase: 'live', + ptyId: 'pty-1', + surface: { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed', + unknownPayload + }, + unknownPayload + } + ] + } + ]) + + await expect(provider.listProcesses()).resolves.toEqual([ + { + id: scopedPty1, + incarnationId: 'incarnation-1', + cwd: '/home', + title: 'codex', + agentSessionOwners: [ + { + claim: { + digestVersion: 1, + keyId: 'key', + identityDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + worktreeScopeDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + agent: 'codex' + }, + generation: 'generation-1', + phase: 'live', + ptyId: scopedPty1, + surface: { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed' + } + } + ] + } + ]) + }) + it('rejects recovered claimed owners without PTY incarnation proof', async () => { mux.request.mockResolvedValue([ { @@ -121,6 +195,126 @@ describe('SshPtyProvider process listings and events', () => { await expect(provider.listProcesses()).rejects.toThrow('agent_session_ownership_unknown') }) + it('rejects a non-array process list before mapping ids', async () => { + mux.request.mockResolvedValue({ id: 'pty-1' }) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + }) + + it('rejects an oversized process list before mapping ids', async () => { + const id = { + toString: vi.fn(() => 'pty-never-map') + } + mux.request.mockResolvedValue( + Array.from({ length: MAX_SSH_PTY_PROCESS_LIST_ENTRIES + 1 }, () => ({ + id, + cwd: '/home', + title: 'shell' + })) + ) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + expect(id.toString).not.toHaveBeenCalled() + }) + + it('rejects oversized owner arrays before mapping owners', async () => { + mux.request.mockResolvedValue([ + { + id: 'pty-1', + cwd: '/home', + title: 'shell', + agentSessionOwners: Array.from( + { length: MAX_SSH_PTY_PROCESS_OWNERS_PER_ENTRY + 1 }, + () => ({}) + ) + } + ]) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + }) + + it('rejects owner amplification spread across otherwise valid sessions', async () => { + const owner = { + claim: { + digestVersion: 1, + keyId: 'key', + identityDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + worktreeScopeDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + agent: 'codex' + }, + generation: 'generation-1', + phase: 'live', + ptyId: 'pty-1', + surface: { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed' + } + } + const ownersPerSession = MAX_SSH_PTY_PROCESS_OWNERS_PER_ENTRY + const sessionCount = Math.floor(MAX_SSH_PTY_PROCESS_LIST_OWNERS / ownersPerSession) + 1 + mux.request.mockResolvedValue( + Array.from({ length: sessionCount }, (_, sessionIndex) => { + const id = `pty-${sessionIndex}` + return { + id, + incarnationId: 'incarnation-1', + cwd: '/home', + title: 'shell', + agentSessionOwners: Array.from({ length: ownersPerSession }, (_, ownerIndex) => ({ + ...owner, + generation: `generation-${sessionIndex}-${ownerIndex}`, + ptyId: id + })) + } + }) + ) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + }) + + it.each([ + { + field: 'id', + value: 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES + 1) + }, + { + field: 'cwd', + value: 'x'.repeat(MAX_SSH_PTY_PROCESS_CWD_BYTES + 1) + }, + { + field: 'title', + value: 'x'.repeat(MAX_SSH_PTY_PROCESS_TITLE_BYTES + 1) + } + ])('rejects an oversized $field before mapping', async ({ field, value }) => { + mux.request.mockResolvedValue([{ id: 'pty-1', cwd: '/home', title: 'shell', [field]: value }]) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + }) + + it('rejects process-list strings above the aggregate byte budget', async () => { + const cwd = 'x'.repeat(MAX_SSH_PTY_PROCESS_CWD_BYTES) + const count = Math.floor(MAX_SSH_PTY_PROCESS_LIST_BYTES / Buffer.byteLength(cwd)) + 1 + mux.request.mockResolvedValue( + Array.from({ length: count }, (_, index) => ({ + id: `pty-${index}`, + cwd, + title: 'shell' + })) + ) + + await expect(provider.listProcesses()).rejects.toThrow('invalid_ssh_pty_process_list') + }) + + it('rejects an oversized spawn id before retention', async () => { + const oversized = 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES + 1) + mux.request.mockResolvedValue({ id: oversized }) + + await expect(provider.spawn({ cols: 80, rows: 24 })).rejects.toThrow('invalid_ssh_pty_id') + expect(provider.hasPty(`ssh:conn-1@@${oversized}`)).toBe(false) + }) + it('forwards data, replay, and incarnation-aware exit notifications', () => { const dataHandler = vi.fn() const replayHandler = vi.fn() @@ -133,7 +327,9 @@ describe('SshPtyProvider process listings and events', () => { notify('pty.data', { id: 'pty-1', data: 'output' }) notify('pty.data', { id: 'pty-1', data: '', rawLength: 9, seq: 9, transformed: true }) notify('pty.replay', { id: 'pty-1', data: 'buffered output' }) + expect(provider.hasPty(scopedPty1)).toBe(true) notify('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' }) + expect(provider.hasPty(scopedPty1)).toBe(false) expect(dataHandler).toHaveBeenNthCalledWith(1, { id: scopedPty1, data: 'output' }) expect(dataHandler).toHaveBeenNthCalledWith(2, { @@ -151,6 +347,51 @@ describe('SshPtyProvider process listings and events', () => { }) }) + it('drops malformed notification ids before routing or retention', () => { + const dataHandler = vi.fn() + const replayHandler = vi.fn() + const exitHandler = vi.fn() + provider.onData(dataHandler) + provider.onReplay(replayHandler) + provider.onExit(exitHandler) + const notify = mux.onNotification.mock.calls[0][0] + const oversized = 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES + 1) + + notify('pty.data', { id: 42, data: 'output', deliveryToken: 'token' }) + notify('pty.replay', { id: oversized, data: 'replay' }) + notify('pty.exit', { id: oversized, code: 0 }) + + expect(dataHandler).not.toHaveBeenCalled() + expect(replayHandler).not.toHaveBeenCalled() + expect(exitHandler).not.toHaveBeenCalled() + expect(provider.hasPty(`ssh:conn-1@@${oversized}`)).toBe(false) + expect(mux.notify).not.toHaveBeenCalled() + }) + + it('drops malformed notification payloads before liveness or exit-race mutation', () => { + const dataHandler = vi.fn() + const replayHandler = vi.fn() + const exitHandler = vi.fn() + provider.onData(dataHandler) + provider.onReplay(replayHandler) + provider.onExit(exitHandler) + const notify = mux.onNotification.mock.calls[0][0] + + notify('pty.data', { id: 'pty-bad-data', data: 42, deliveryToken: 'token' }) + notify('pty.replay', { id: 'pty-bad-replay', data: { nested: 'output' } }) + notify('pty.exit', { id: 'pty-bad-code', code: '0', incarnationId: 'incarnation-1' }) + notify('pty.exit', { id: 'pty-bad-incarnation', code: 0, incarnationId: 'x'.repeat(129) }) + + expect(dataHandler).not.toHaveBeenCalled() + expect(replayHandler).not.toHaveBeenCalled() + expect(exitHandler).not.toHaveBeenCalled() + expect(provider.hasPty('ssh:conn-1@@pty-bad-data')).toBe(false) + expect(provider.hasPty('ssh:conn-1@@pty-bad-replay')).toBe(false) + expect(provider.hasPty('ssh:conn-1@@pty-bad-code')).toBe(false) + expect(provider.hasPty('ssh:conn-1@@pty-bad-incarnation')).toBe(false) + expect(mux.notify).not.toHaveBeenCalled() + }) + it('supports listener removal, fanout, and connection namespaces', () => { const removed = vi.fn() const first = vi.fn() diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index f204369c9ef..a21b97f6319 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -32,6 +32,16 @@ describe('SshPtyProvider', () => { provider = new SshPtyProvider('conn-1', mux as never) }) + function emitRelayNotification(method: string, params: Record): void { + const callback = mux.onNotification.mock.calls[0]?.[0] as + | ((method: string, params: Record) => void) + | undefined + if (!callback) { + throw new Error('Notification listener was not registered') + } + callback(method, params) + } + it('returns the connectionId', () => { expect(provider.getConnectionId()).toBe('conn-1') }) @@ -821,6 +831,47 @@ describe('SshPtyProvider', () => { expect(mux.notify).toHaveBeenCalledWith('pty.ackData', { id: 'pty-1', charCount: 1024 }) }) + it('defers tokenized relay output credit until downstream acknowledgement', () => { + const events: Parameters[0]>[0][] = [] + provider.onData((event) => { + events.push(event) + }) + + emitRelayNotification('pty.data', { + id: 'pty-1', + data: 'clean', + rawLength: 12, + deliveryToken: 'delivery-1' + }) + + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ id: scopedPty1, data: 'clean', sequenceChars: 12 }) + expect(mux.notify).not.toHaveBeenCalled() + + events[0]?.upstreamCredit?.acknowledge(12) + + expect(mux.notify).toHaveBeenCalledOnce() + expect(mux.notify).toHaveBeenCalledWith('pty.ackData', { + id: 'pty-1', + charCount: 12, + deliveryToken: 'delivery-1' + }) + }) + + it('restores legacy acknowledgements after tokenized PTY exit', () => { + emitRelayNotification('pty.data', { + id: 'pty-1', + data: 'output', + deliveryToken: 'delivery-3' + }) + emitRelayNotification('pty.exit', { id: 'pty-1', code: 0 }) + mux.notify.mockClear() + + provider.acknowledgeDataEvent(scopedPty1, 6) + + expect(mux.notify).toHaveBeenCalledWith('pty.ackData', { id: 'pty-1', charCount: 6 }) + }) + it('hasChildProcesses sends request and returns result', async () => { mux.request.mockResolvedValue(true) const result = await provider.hasChildProcesses(scopedPty1) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 02611e79d15..a5fe9bc3d0a 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -8,7 +8,6 @@ import type { SshPtyExitCallback, SshPtyReplayCallback } from './ssh-pty-provider-contract' -import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing' import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation' import { assertSshAgentSessionCreateResult, @@ -23,6 +22,9 @@ import { import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request' import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' +import { SshPtyProviderNotifications } from './ssh-pty-provider-notifications' +import { SshPtyLiveRoster } from './ssh-pty-live-roster' +import { isAdmittedSshRelayPtyId } from './ssh-pty-wire-admission' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined { @@ -33,15 +35,11 @@ function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: numbe export class SshPtyProvider implements IPtyProvider { private mux: SshChannelMultiplexer private connectionId: string - private dataListeners = new Set() - private replayListeners = new Set() - private exitListeners = new Set() - private livePtyIds = new Set() - // Why: stale notification callbacks must not outlive a disconnected provider. - private unsubscribeNotifications: (() => void) | null = null + private readonly livePtys = new SshPtyLiveRoster() readonly getAppliedSize: NonNullable private readonly agentSessionCapabilities: SshAgentSessionCapabilities private spawnExitRaces = new SshPtySpawnExitRaceTracker() + private readonly notifications: SshPtyProviderNotifications constructor( connectionId: string, @@ -52,28 +50,20 @@ export class SshPtyProvider implements IPtyProvider { this.mux = mux this.agentSessionCapabilities = new SshAgentSessionCapabilities(mux) this.getAppliedSize = createSshPtyAppliedSizeReader(mux, connectionId) - - this.unsubscribeNotifications = subscribeSshPtyNotifications({ + this.notifications = new SshPtyProviderNotifications( mux, - toAppPtyId: (id) => this.toAppPtyId(id), - dataListeners: this.dataListeners, - replayListeners: this.replayListeners, - exitListeners: this.exitListeners, - livePtyIds: this.livePtyIds, - recordExit: (relayPtyId, incarnationId) => - this.spawnExitRaces.recordExit(relayPtyId, incarnationId) - }) + (relayId) => this.toAppPtyId(relayId), + (relayId) => this.livePtys.recordNotification(this.toAppPtyId(relayId)), + (relayId, incarnationId) => { + this.spawnExitRaces.recordExit(relayId, incarnationId) + this.livePtys.recordExit(this.toAppPtyId(relayId)) + } + ) } dispose(): void { - if (this.unsubscribeNotifications) { - this.unsubscribeNotifications() - this.unsubscribeNotifications = null - } - this.dataListeners.clear() - this.replayListeners.clear() - this.exitListeners.clear() - this.livePtyIds.clear() + this.livePtys.clear() + this.notifications.dispose() } getConnectionId = (): string => this.connectionId @@ -107,7 +97,7 @@ export class SshPtyProvider implements IPtyProvider { options: opts, exitRaceTracker: this.spawnExitRaces }) - this.livePtyIds.add(result.id) + this.livePtys.recordSpawn(result.id) return result } @@ -137,6 +127,9 @@ export class SshPtyProvider implements IPtyProvider { assertSshAgentSessionCreateResult(result) } const spawnResult = result as PtySpawnResult + if (!isAdmittedSshRelayPtyId(spawnResult.id)) { + throw new Error('invalid_ssh_pty_id') + } if (this.spawnExitRaces.didMatchingExitArrive(operation, spawnResult)) { // Why: relay notification can share the response batch; no controller registration may follow. throw Object.assign(new Error('agent_session_exited_during_start'), { @@ -160,7 +153,7 @@ export class SshPtyProvider implements IPtyProvider { } } const id = this.toAppPtyId(spawnResult.id) - this.livePtyIds.add(id) + this.livePtys.recordSpawn(id) return { ...spawnResult, id, @@ -239,7 +232,7 @@ export class SshPtyProvider implements IPtyProvider { }, relayTimeoutOptions(opts.deadlineMs) ) - this.livePtyIds.delete(id) + this.livePtys.recordExit(id) } async sendSignal(id: string, signal: string): Promise { @@ -268,7 +261,8 @@ export class SshPtyProvider implements IPtyProvider { } acknowledgeDataEvent(id: string, charCount: number): void { - this.mux.notify('pty.ackData', { id: this.toRelayPtyId(id), charCount }) + const relayId = this.toRelayPtyId(id) + this.notifications.acknowledgeLegacy(relayId, charCount) } async hasChildProcesses(id: string): Promise { @@ -281,6 +275,14 @@ export class SshPtyProvider implements IPtyProvider { return result as string | null } + async inspectProcess( + id: string + ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> { + return (await this.mux.request('pty.inspectProcess', { + id: this.toRelayPtyId(id) + })) as { foregroundProcess: string | null; hasChildProcesses: boolean } + } + async serialize(ids: string[]): Promise { const result = await this.mux.request('pty.serialize', { ids: ids.map((id) => this.toRelayPtyId(id)) @@ -293,20 +295,22 @@ export class SshPtyProvider implements IPtyProvider { } async listProcesses(opts?: { deadlineMs?: number }): Promise { + const listing = this.livePtys.beginListing() const result = await this.mux.request( 'pty.listProcesses', undefined, relayTimeoutOptions(opts?.deadlineMs) ) - const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => this.toAppPtyId(id)) - for (const process of processes) { - this.livePtyIds.add(process.id) - } + const processes = mapSshPtyProcessList(result, (id) => this.toAppPtyId(id)) + this.livePtys.reconcileListing( + listing, + processes.map((process) => process.id) + ) return processes } hasPty(id: string): boolean { - return this.livePtyIds.has(id) + return this.livePtys.has(id) } async getDefaultShell(): Promise { @@ -320,17 +324,14 @@ export class SshPtyProvider implements IPtyProvider { } onData(callback: SshPtyDataCallback): () => void { - this.dataListeners.add(callback) - return () => this.dataListeners.delete(callback) + return this.notifications.onData(callback) } onReplay(callback: SshPtyReplayCallback): () => void { - this.replayListeners.add(callback) - return () => this.replayListeners.delete(callback) + return this.notifications.onReplay(callback) } onExit(callback: SshPtyExitCallback): () => void { - this.exitListeners.add(callback) - return () => this.exitListeners.delete(callback) + return this.notifications.onExit(callback) } } diff --git a/src/main/providers/ssh-pty-spawn-exit-race.test.ts b/src/main/providers/ssh-pty-spawn-exit-race.test.ts new file mode 100644 index 00000000000..ceace5c6aba --- /dev/null +++ b/src/main/providers/ssh-pty-spawn-exit-race.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_PENDING_SSH_PTY_SPAWN_EXIT_OPERATIONS, + MAX_SSH_PTY_SPAWN_EXIT_BYTES_PER_OPERATION, + MAX_SSH_PTY_SPAWN_EXIT_EVENTS_PER_OPERATION, + SshPtySpawnExitRaceTracker +} from './ssh-pty-spawn-exit-race' +import { MAX_SSH_RELAY_PTY_ID_BYTES } from './ssh-pty-wire-admission' + +describe('SshPtySpawnExitRaceTracker', () => { + it('matches the same PTY incarnation without fencing a replacement', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const operation = tracker.begin() + + tracker.recordExit('pty-1', 'incarnation-old') + + expect( + tracker.didMatchingExitArrive(operation, { + id: 'pty-1', + incarnationId: 'incarnation-current' + }) + ).toBe(false) + expect( + tracker.didMatchingExitArrive(operation, { + id: 'pty-1', + incarnationId: 'incarnation-old' + }) + ).toBe(true) + }) + + it('does not turn malformed incarnation metadata into a wildcard exit', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const operation = tracker.begin() + + tracker.recordExit('pty-1', 'x'.repeat(129)) + + expect( + tracker.didMatchingExitArrive(operation, { + id: 'pty-1', + incarnationId: 'incarnation-current' + }) + ).toBe(false) + }) + + it('supports all 50 relay sessions without overflowing', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const operation = tracker.begin() + for (let index = 0; index < 50; index += 1) { + tracker.recordExit(`pty-${index}`, `incarnation-${index}`) + } + + expect( + tracker.didMatchingExitArrive(operation, { + id: 'pty-49', + incarnationId: 'incarnation-49' + }) + ).toBe(true) + expect(Reflect.get(operation, 'overflowed')).toBe(false) + }) + + it('fails closed and releases retained exits when the count cap is exceeded', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const operation = tracker.begin() + for (let index = 0; index <= MAX_SSH_PTY_SPAWN_EXIT_EVENTS_PER_OPERATION; index += 1) { + tracker.recordExit(`pty-${index}`, `incarnation-${index}`) + } + + expect(Reflect.get(operation, 'overflowed')).toBe(true) + expect(Reflect.get(operation, 'exits')).toHaveLength(0) + expect( + tracker.didMatchingExitArrive(operation, { + id: 'unseen', + incarnationId: 'unseen-incarnation' + }) + ).toBe(true) + }) + + it('fails closed when retained exit identifiers exceed the byte cap', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const operation = tracker.begin() + const suffix = 'x'.repeat(MAX_SSH_RELAY_PTY_ID_BYTES - 16) + const eventBytes = Buffer.byteLength(`pty-000:${suffix}`) + Buffer.byteLength('💥'.repeat(64)) + const eventsToOverflow = Math.floor(MAX_SSH_PTY_SPAWN_EXIT_BYTES_PER_OPERATION / eventBytes) + 1 + expect(eventsToOverflow).toBeLessThan(MAX_SSH_PTY_SPAWN_EXIT_EVENTS_PER_OPERATION) + + for (let index = 0; index < eventsToOverflow; index += 1) { + tracker.recordExit(`pty-${String(index).padStart(3, '0')}:${suffix}`, '💥'.repeat(64)) + } + + expect(Reflect.get(operation, 'overflowed')).toBe(true) + expect(Reflect.get(operation, 'retainedBytes')).toBe(0) + }) + + it('rejects before dispatch when concurrent operation tracking is saturated', () => { + const tracker = new SshPtySpawnExitRaceTracker() + const admitted = Array.from({ length: MAX_PENDING_SSH_PTY_SPAWN_EXIT_OPERATIONS }, () => + tracker.begin() + ) + + expect(() => tracker.begin()).toThrow('ssh_pty_spawn_exit_tracking_capacity') + + for (const operation of admitted) { + tracker.finish(operation) + } + }) +}) diff --git a/src/main/providers/ssh-pty-spawn-exit-race.ts b/src/main/providers/ssh-pty-spawn-exit-race.ts index 0658e3e23eb..f0f9d677496 100644 --- a/src/main/providers/ssh-pty-spawn-exit-race.ts +++ b/src/main/providers/ssh-pty-spawn-exit-race.ts @@ -1,24 +1,55 @@ import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-incarnation' +import { admittedSshRelayPtyIdBytes } from './ssh-pty-wire-admission' + +export const MAX_SSH_PTY_SPAWN_EXIT_EVENTS_PER_OPERATION = 128 +export const MAX_SSH_PTY_SPAWN_EXIT_BYTES_PER_OPERATION = 512 * 1024 +export const MAX_PENDING_SSH_PTY_SPAWN_EXIT_OPERATIONS = 128 type PendingSshPtySpawn = { exits: { relayPtyId: string; incarnationId?: PtyIncarnationId }[] + retainedBytes: number + overflowed: boolean } export class SshPtySpawnExitRaceTracker { private pending = new Set() begin(): PendingSshPtySpawn { - const operation = { exits: [] } + if (this.pending.size >= MAX_PENDING_SSH_PTY_SPAWN_EXIT_OPERATIONS) { + throw new Error('ssh_pty_spawn_exit_tracking_capacity') + } + const operation: PendingSshPtySpawn = { + exits: [], + retainedBytes: 0, + overflowed: false + } this.pending.add(operation) return operation } recordExit(relayPtyId: string, incarnationId: unknown): void { + const idBytes = admittedSshRelayPtyIdBytes(relayPtyId) + if (idBytes === null || (incarnationId !== undefined && !isPtyIncarnationId(incarnationId))) { + return + } + const retainedBytes = + idBytes + (isPtyIncarnationId(incarnationId) ? Buffer.byteLength(incarnationId, 'utf8') : 0) for (const operation of this.pending) { + if ( + operation.overflowed || + operation.exits.length >= MAX_SSH_PTY_SPAWN_EXIT_EVENTS_PER_OPERATION || + operation.retainedBytes + retainedBytes > MAX_SSH_PTY_SPAWN_EXIT_BYTES_PER_OPERATION + ) { + operation.overflowed = true + operation.exits.length = 0 + operation.retainedBytes = 0 + continue + } operation.exits.push({ relayPtyId, ...(isPtyIncarnationId(incarnationId) ? { incarnationId } : {}) }) + operation.retainedBytes += retainedBytes } } @@ -26,16 +57,21 @@ export class SshPtySpawnExitRaceTracker { operation: PendingSshPtySpawn, result: { id: string; incarnationId?: PtyIncarnationId } ): boolean { - return operation.exits.some( - (exit) => - exit.relayPtyId === result.id && - (!exit.incarnationId || - !result.incarnationId || - exit.incarnationId === result.incarnationId) + return ( + operation.overflowed || + operation.exits.some( + (exit) => + exit.relayPtyId === result.id && + (!exit.incarnationId || + !result.incarnationId || + exit.incarnationId === result.incarnationId) + ) ) } finish(operation: PendingSshPtySpawn): void { this.pending.delete(operation) + operation.exits.length = 0 + operation.retainedBytes = 0 } } diff --git a/src/main/providers/ssh-pty-wire-admission.ts b/src/main/providers/ssh-pty-wire-admission.ts new file mode 100644 index 00000000000..af7ac167fc9 --- /dev/null +++ b/src/main/providers/ssh-pty-wire-admission.ts @@ -0,0 +1,29 @@ +import { hasUnsafeProviderSessionIdChars } from '../../shared/agent-session-resume' + +export const MAX_SSH_RELAY_PTY_ID_BYTES = 4 * 1024 +export const MAX_SSH_APP_PTY_ID_BYTES = 16 * 1024 +export const MAX_SSH_PTY_DELIVERY_TOKEN_BYTES = 1024 + +function admittedStringBytes(value: unknown, maxBytes: number): number | null { + if (typeof value !== 'string' || value.length === 0 || hasUnsafeProviderSessionIdChars(value)) { + return null + } + const bytes = Buffer.byteLength(value, 'utf8') + return bytes <= maxBytes ? bytes : null +} + +export function admittedSshRelayPtyIdBytes(value: unknown): number | null { + return admittedStringBytes(value, MAX_SSH_RELAY_PTY_ID_BYTES) +} + +export function admittedSshAppPtyIdBytes(value: unknown): number | null { + return admittedStringBytes(value, MAX_SSH_APP_PTY_ID_BYTES) +} + +export function isAdmittedSshRelayPtyId(value: unknown): value is string { + return admittedSshRelayPtyIdBytes(value) !== null +} + +export function isAdmittedSshDeliveryToken(value: unknown): value is string { + return admittedStringBytes(value, MAX_SSH_PTY_DELIVERY_TOKEN_BYTES) !== null +} diff --git a/src/main/providers/types.ts b/src/main/providers/types.ts index ba12c07e1ef..822f540bba0 100644 --- a/src/main/providers/types.ts +++ b/src/main/providers/types.ts @@ -21,6 +21,7 @@ import type { GitHistoryOptions, GitHistoryResult } from '../../shared/git-histo import type { PtyStartupIngressIntent } from '../../shared/pty-startup-ingress' import type { CommitMessageDraftContext } from '../../shared/commit-message-generation' import type { WorkspaceSpaceDirectoryScanResult } from '../../shared/workspace-space-types' +import type { FilesystemPathListingProvider } from './filesystem-path-listing-provider' import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' import type { GitProviderStatusOptions } from './git-provider-status-options' @@ -29,9 +30,9 @@ import type { PtySpawnResult } from './pty-spawn-result' import type { PtyIncarnationId } from '../../shared/pty-incarnation' import type { AgentSessionExecutionClaim, - AgentSessionOwnerBinding, AgentSessionSurfaceBinding } from '../../shared/agent-session-host-authority' +import type { PtyProcessInfo } from './pty-process-info' export type { PtyBackgroundStreamEvent, @@ -114,19 +115,7 @@ export type PtySpawnOptions = { signal?: AbortSignal } -export type { PtySpawnResult } - -export type PtyProcessInfo = { - id: string - incarnationId?: PtyIncarnationId - cwd: string - title: string - /** Owning worktree when the provider can report it authoritatively. */ - worktreeId?: string - /** Trusted ORCA_TERMINAL_HANDLE exported into this PTY, when known. */ - terminalHandle?: string - agentSessionOwners?: AgentSessionOwnerBinding[] -} +export type { PtyProcessInfo, PtySpawnResult } type PtyProbeOptions = { signal?: AbortSignal } @@ -237,10 +226,13 @@ export type FileReadResult = { isBinary: boolean isImage?: boolean mimeType?: string + imageDimensions?: { width: number; height: number } } -export type IFilesystemProvider = { - readDir(dirPath: string): Promise +type FilesystemDirectoryReadOptions = { maxEntries?: number; maxRetainedBytes?: number } + +export type IFilesystemProvider = FilesystemPathListingProvider & { + readDir(dirPath: string, options?: FilesystemDirectoryReadOptions): Promise readFile(filePath: string): Promise readTerminalArtifact?( filePath: string, @@ -269,10 +261,6 @@ export type IFilesystemProvider = { copy(source: string, destination: string): Promise realpath(filePath: string): Promise search(opts: SearchOptions): Promise - listFiles( - rootPath: string, - options?: { excludePaths?: string[]; signal?: AbortSignal; maxResults?: number } - ): Promise scanWorkspaceSpace?( rootPath: string, options?: { signal?: AbortSignal } @@ -393,10 +381,7 @@ export type IGitProvider = { // ─── Provider Registry ────────────────────────────────────────────── -/** - * Routes operations to the correct provider based on connectionId. - * null/undefined connectionId = local provider. - */ +/** Routes operations by connectionId; null/undefined selects the local provider. */ export type IProviderRegistry = { getPtyProvider(connectionId: string | null | undefined): IPtyProvider getFilesystemProvider(connectionId: string | null | undefined): IFilesystemProvider diff --git a/src/main/pty/config-overlay-mirroring.test.ts b/src/main/pty/config-overlay-mirroring.test.ts new file mode 100644 index 00000000000..764c35bca9e --- /dev/null +++ b/src/main/pty/config-overlay-mirroring.test.ts @@ -0,0 +1,97 @@ +import type { Dirent } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { + CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES, + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES, + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES, + ConfigOverlayCapacityError, + ConfigOverlayEntryBudget, + _configOverlayMirroringInternals +} from './config-overlay-mirroring' + +function fileEntry(name: string): Dirent { + return { + isBlockDevice: () => false, + isCharacterDevice: () => false, + isDirectory: () => false, + isFIFO: () => false, + isFile: () => true, + isSocket: () => false, + isSymbolicLink: () => false, + name + } as unknown as Dirent +} + +describe('config overlay entry budget', () => { + it('accepts the exact entry-count limit and rejects the next streamed entry', () => { + const budget = new ConfigOverlayEntryBudget() + for (let index = 0; index < CONFIG_OVERLAY_MAX_SOURCE_ENTRIES; index += 1) { + budget.reserve('a') + } + + expect(() => budget.reserve('a')).toThrowError( + new ConfigOverlayCapacityError( + 'entries', + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + 1, + CONFIG_OVERLAY_MAX_SOURCE_ENTRIES + ) + ) + }) + + it('accepts an exact-limit entry name and rejects one byte more', () => { + new ConfigOverlayEntryBudget().reserve('a'.repeat(CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES)) + + expect(() => + new ConfigOverlayEntryBudget().reserve('a'.repeat(CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES + 1)) + ).toThrowError( + new ConfigOverlayCapacityError( + 'entry-name-bytes', + CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES + 1, + CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES + ) + ) + }) + + it('accepts the exact aggregate encoded-name limit and rejects one more name', () => { + const budget = new ConfigOverlayEntryBudget() + const name = 'a'.repeat(4_094) + expect(Buffer.byteLength(JSON.stringify(name), 'utf8')).toBe(4_096) + + for (let index = 0; index < CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES / 4_096; index += 1) { + budget.reserve(name) + } + + expect(() => budget.reserve('a')).toThrowError( + new ConfigOverlayCapacityError( + 'retained-name-bytes', + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES + 3, + CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES + ) + ) + }) + + it('stops reading immediately after the first over-limit entry', () => { + const entries = Array.from({ length: 10_000 }, (_, index) => fileEntry(`entry-${index}`)) + let reads = 0 + let visits = 0 + const directory = { + readSync() { + reads += 1 + return entries.shift() ?? null + } + } + + expect(() => + _configOverlayMirroringInternals.scanOpenDirectory( + directory, + new ConfigOverlayEntryBudget({ maxEntries: 2 }), + () => { + visits += 1 + } + ) + ).toThrow(ConfigOverlayCapacityError) + expect(reads).toBe(3) + expect(visits).toBe(2) + expect(entries).toHaveLength(9_997) + }) +}) diff --git a/src/main/pty/config-overlay-mirroring.ts b/src/main/pty/config-overlay-mirroring.ts new file mode 100644 index 00000000000..237282ef1be --- /dev/null +++ b/src/main/pty/config-overlay-mirroring.ts @@ -0,0 +1,228 @@ +import { mkdirSync, opendirSync, realpathSync, statSync, type Dirent } from 'node:fs' +import { join } from 'node:path' +import { mirrorEntry } from './overlay-mirror' + +export const CONFIG_OVERLAY_MAX_SOURCE_ENTRIES = 4_096 +export const CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES = 4 * 1_024 +export const CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES = 1_024 * 1_024 + +export type ConfigOverlayCapacityKind = 'entries' | 'entry-name-bytes' | 'retained-name-bytes' + +export type ConfigOverlayLimits = { + maxEntries: number + maxEntryNameBytes: number + maxRetainedNameBytes: number +} + +const DEFAULT_LIMITS: ConfigOverlayLimits = { + maxEntries: CONFIG_OVERLAY_MAX_SOURCE_ENTRIES, + maxEntryNameBytes: CONFIG_OVERLAY_MAX_ENTRY_NAME_BYTES, + maxRetainedNameBytes: CONFIG_OVERLAY_MAX_RETAINED_NAME_BYTES +} + +export class ConfigOverlayCapacityError extends Error { + constructor( + readonly kind: ConfigOverlayCapacityKind, + readonly observed: number, + readonly limit: number + ) { + super(`Agent config overlay ${kind} exceeded its ${limit} limit (observed ${observed})`) + this.name = 'ConfigOverlayCapacityError' + } +} + +export class ConfigOverlayEntryBudget { + private entryCount = 0 + private retainedNameBytes = 0 + private readonly limits: ConfigOverlayLimits + + constructor(requested?: Partial) { + this.limits = { + maxEntries: resolveLimit(requested?.maxEntries, DEFAULT_LIMITS.maxEntries, 'maxEntries'), + maxEntryNameBytes: resolveLimit( + requested?.maxEntryNameBytes, + DEFAULT_LIMITS.maxEntryNameBytes, + 'maxEntryNameBytes' + ), + maxRetainedNameBytes: resolveLimit( + requested?.maxRetainedNameBytes, + DEFAULT_LIMITS.maxRetainedNameBytes, + 'maxRetainedNameBytes' + ) + } + } + + reserve(name: string): void { + const nextEntryCount = this.entryCount + 1 + if (nextEntryCount > this.limits.maxEntries) { + throw new ConfigOverlayCapacityError('entries', nextEntryCount, this.limits.maxEntries) + } + + const nameBytes = Buffer.byteLength(name, 'utf8') + if (nameBytes > this.limits.maxEntryNameBytes) { + throw new ConfigOverlayCapacityError( + 'entry-name-bytes', + nameBytes, + this.limits.maxEntryNameBytes + ) + } + assertPortableEntryName(name) + + const retainedNameBytes = Buffer.byteLength(JSON.stringify(name), 'utf8') + const nextRetainedNameBytes = this.retainedNameBytes + retainedNameBytes + if (nextRetainedNameBytes > this.limits.maxRetainedNameBytes) { + throw new ConfigOverlayCapacityError( + 'retained-name-bytes', + nextRetainedNameBytes, + this.limits.maxRetainedNameBytes + ) + } + + this.entryCount = nextEntryCount + this.retainedNameBytes = nextRetainedNameBytes + } +} + +export type ConfigOverlayPlan = { + sourceDir: string + topLevelEntryNames: string[] + pluginSourceDir: string | null + pluginEntryNames: string[] +} + +export type AppliedConfigOverlayEntries = { + topLevelEntryNames: string[] + pluginEntryNames: string[] +} + +type ConfigOverlayPlanOptions = { + reservedPluginFile: string + reservedTopLevelEntryNames?: ReadonlySet +} + +function resolveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} + +function assertPortableEntryName(name: string): void { + if (name.length === 0 || name === '.' || name === '..' || /[/\\]/.test(name)) { + throw new Error('Agent config overlay contains a non-portable entry name') + } +} + +function closeDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ERR_DIR_CLOSED') { + throw error + } + } +} + +function scanOpenDirectory( + directory: Pick, 'readSync'>, + budget: ConfigOverlayEntryBudget, + visit: (entry: Dirent) => void +): void { + while (true) { + const entry = directory.readSync() + if (entry === null) { + return + } + budget.reserve(entry.name) + visit(entry) + } +} + +function scanDirectory( + path: string, + budget: ConfigOverlayEntryBudget, + visit: (entry: Dirent) => void +): void { + const directory = opendirSync(path, { bufferSize: 32 }) + try { + scanOpenDirectory(directory, budget, visit) + } finally { + closeDirectory(directory) + } +} + +export function createConfigOverlayPlan( + sourceDir: string, + options: ConfigOverlayPlanOptions, + budget = new ConfigOverlayEntryBudget() +): ConfigOverlayPlan { + const topLevelEntryNames: string[] = [] + const pluginEntryNames: string[] = [] + let pluginSourceDir: string | null = null + + scanDirectory(sourceDir, budget, (entry) => { + if (options.reservedTopLevelEntryNames?.has(entry.name)) { + return + } + + const sourcePath = join(sourceDir, entry.name) + if (entry.name === 'plugins') { + const isSymlink = entry.isSymbolicLink() + let isLinkPointingToDir = false + if (isSymlink) { + try { + isLinkPointingToDir = statSync(sourcePath).isDirectory() + } catch { + isLinkPointingToDir = false + } + } + + if ((!isSymlink && entry.isDirectory()) || isLinkPointingToDir) { + pluginSourceDir = isLinkPointingToDir ? realpathSync(sourcePath) : sourcePath + scanDirectory(pluginSourceDir, budget, (pluginEntry) => { + if (pluginEntry.name !== options.reservedPluginFile) { + pluginEntryNames.push(pluginEntry.name) + } + }) + return + } + } + + topLevelEntryNames.push(entry.name) + }) + + return { sourceDir, topLevelEntryNames, pluginSourceDir, pluginEntryNames } +} + +export function applyConfigOverlayPlan( + plan: ConfigOverlayPlan, + targetDir: string, + applied: AppliedConfigOverlayEntries = { + topLevelEntryNames: [], + pluginEntryNames: [] + } +): AppliedConfigOverlayEntries { + for (const entryName of plan.topLevelEntryNames) { + mirrorEntry(join(plan.sourceDir, entryName), join(targetDir, entryName)) + applied.topLevelEntryNames.push(entryName) + } + + if (plan.pluginSourceDir === null) { + return applied + } + + const targetPluginsDir = join(targetDir, 'plugins') + mkdirSync(targetPluginsDir, { recursive: true }) + for (const entryName of plan.pluginEntryNames) { + mirrorEntry(join(plan.pluginSourceDir, entryName), join(targetPluginsDir, entryName)) + applied.pluginEntryNames.push(entryName) + } + return applied +} + +export const _configOverlayMirroringInternals = { + scanOpenDirectory +} diff --git a/src/main/pty/overlay-mirror.test.ts b/src/main/pty/overlay-mirror.test.ts index 5499d226422..a97a049d9c4 100644 --- a/src/main/pty/overlay-mirror.test.ts +++ b/src/main/pty/overlay-mirror.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from 'node:os' import type * as NodePath from 'node:path' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' -import { safeRemoveOverlay } from './overlay-mirror' +import { + OVERLAY_REMOVE_MAX_DEPTH, + OVERLAY_REMOVE_MAX_ENTRIES, + safeRemoveOverlay, + safeRemoveTree +} from './overlay-mirror' const tempRoots: string[] = [] @@ -19,6 +24,50 @@ afterEach(() => { }) describe('safeRemoveOverlay', () => { + it('publishes finite production cleanup limits', () => { + expect(OVERLAY_REMOVE_MAX_ENTRIES).toBe(100_000) + expect(OVERLAY_REMOVE_MAX_DEPTH).toBe(256) + }) + + it('removes a tree at the exact entry limit', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-overlay-entry-limit-')) + tempRoots.push(root) + writeFileSync(join(root, 'one'), '') + writeFileSync(join(root, 'two'), '') + + expect(safeRemoveTree(root, { maxEntries: 2 })).toBe(true) + expect(existsSync(root)).toBe(false) + }) + + it('stops before visiting an entry beyond the limit', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-overlay-entry-limit-')) + tempRoots.push(root) + writeFileSync(join(root, 'one'), '') + writeFileSync(join(root, 'two'), '') + writeFileSync(join(root, 'three'), '') + + expect(safeRemoveTree(root, { maxEntries: 2 })).toBe(false) + expect(existsSync(root)).toBe(true) + }) + + it('accepts the exact directory depth and stops before descending one level deeper', async () => { + const acceptedRoot = await mkdtemp(join(tmpdir(), 'orca-overlay-depth-limit-')) + tempRoots.push(acceptedRoot) + mkdirSync(join(acceptedRoot, 'one', 'two'), { recursive: true }) + writeFileSync(join(acceptedRoot, 'one', 'two', 'leaf'), '') + + expect(safeRemoveTree(acceptedRoot, { maxDepth: 2 })).toBe(true) + expect(existsSync(acceptedRoot)).toBe(false) + + const rejectedRoot = await mkdtemp(join(tmpdir(), 'orca-overlay-depth-limit-')) + tempRoots.push(rejectedRoot) + mkdirSync(join(rejectedRoot, 'one', 'two'), { recursive: true }) + writeFileSync(join(rejectedRoot, 'one', 'two', 'leaf'), '') + + expect(safeRemoveTree(rejectedRoot, { maxDepth: 1 })).toBe(false) + expect(existsSync(rejectedRoot)).toBe(true) + }) + it('removes valid overlay children whose names start with dot-dot', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-overlay-root-')) tempRoots.push(root) diff --git a/src/main/pty/overlay-mirror.ts b/src/main/pty/overlay-mirror.ts index ec1b99d2b92..eb6206e7cf8 100644 --- a/src/main/pty/overlay-mirror.ts +++ b/src/main/pty/overlay-mirror.ts @@ -11,13 +11,27 @@ import { cpSync, linkSync, lstatSync, - readdirSync, + opendirSync, rmdirSync, symlinkSync, unlinkSync } from 'node:fs' import { isAbsolute, join, relative, resolve, sep } from 'node:path' +export const OVERLAY_REMOVE_MAX_ENTRIES = 100_000 +export const OVERLAY_REMOVE_MAX_DEPTH = 256 + +type OverlayRemoveLimits = { + maxEntries: number + maxDepth: number +} + +type OverlayRemoveState = { + entries: number + exhausted: boolean + limits: OverlayRemoveLimits +} + export function mirrorEntry(sourcePath: string, targetPath: string): void { // Why: lstatSync (not statSync) so that if the user's source dir contains // its OWN symlinks (e.g. skills symlinked from ~/.agents/skills), we mirror @@ -83,67 +97,130 @@ export function isSafeDescendCandidate(stats: { return stats.isDirectory() } -// Why: the overlay tree contains symlinks/junctions that point back into the -// user's real state dir. fs.rmSync with { recursive: true } has repeatedly -// regressed on Windows when walking NTFS junctions -- it can follow them and -// delete the *target*, destroying the user's data. Never descend into a -// symlink/junction here: for any non-real-directory entry we unlink the link -// itself; only entries that are truly directories on disk are recursed into. -export function safeRemoveTree(path: string): void { +function resolveRemoveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} + +function closeDirectory(directory: ReturnType): boolean { + try { + directory.closeSync() + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ERR_DIR_CLOSED' + } +} + +function removeTreeWithinLimits(path: string, depth: number, state: OverlayRemoveState): boolean { + if (state.exhausted) { + return false + } + let stat try { stat = lstatSync(path) - } catch { - return + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT' } // On Windows, lstat on a directory junction can report BOTH // isSymbolicLink() === true AND isDirectory() === true, so we MUST check // isSymbolicLink first -- otherwise a junction enters the recursive branch - // and readdirSync enumerates the link's target, the exact bug in #1083. + // and enumerates the link's target, the exact bug in #1083. if (!isSafeDescendCandidate(stat)) { try { unlinkSync(path) + return true } catch { // Best-effort: antivirus/indexers can hold handles briefly on Windows. // A leftover link is harmless; the next spawn rebuilds the overlay. + return false } - return } - let entries + let directory try { - entries = readdirSync(path, { withFileTypes: true }) + directory = opendirSync(path, { bufferSize: 32 }) } catch { - return + return false } - for (const entry of entries) { - const child = join(path, entry.name) - if (isSafeDescendCandidate(entry)) { - safeRemoveTree(child) - continue - } - try { - unlinkSync(child) - } catch { - // Best-effort, see above. + let completed = true + try { + while (!state.exhausted) { + const entry = directory.readSync() + if (entry === null) { + break + } + if (state.entries >= state.limits.maxEntries) { + state.exhausted = true + completed = false + break + } + state.entries += 1 + + const child = join(path, entry.name) + if (isSafeDescendCandidate(entry)) { + if (depth >= state.limits.maxDepth) { + state.exhausted = true + completed = false + break + } + completed = removeTreeWithinLimits(child, depth + 1, state) && completed + continue + } + try { + unlinkSync(child) + } catch { + completed = false + } } + } catch { + completed = false + } finally { + completed = closeDirectory(directory) && completed } try { rmdirSync(path) + return completed } catch { // Directory may be non-empty if an unlink above failed; harmless. + return false } } +// Why: the overlay tree contains symlinks/junctions that point back into the +// user's real state dir. Stream a bounded traversal and never descend into a +// link; a pathological tree is left for a later cleanup instead of growing +// an unbounded directory array or call stack in Electron's main process. +export function safeRemoveTree(path: string, requested?: Partial): boolean { + const state: OverlayRemoveState = { + entries: 0, + exhausted: false, + limits: { + maxEntries: resolveRemoveLimit( + requested?.maxEntries, + OVERLAY_REMOVE_MAX_ENTRIES, + 'maxEntries' + ), + maxDepth: resolveRemoveLimit(requested?.maxDepth, OVERLAY_REMOVE_MAX_DEPTH, 'maxDepth') + } + } + return removeTreeWithinLimits(path, 0, state) +} + // Why: last-line guard against an overlay-root constant ever being // mis-resolved. Any caller that points safeRemoveTree at a path outside its // designated overlay root is refused so a misconfiguration cannot turn into // an `rm -rf` of arbitrary user data. Logs (rather than throws) so a buggy // caller stays visible without crashing the PTY spawn. -export function safeRemoveOverlay(overlayDir: string, overlayRoot: string): void { +export function safeRemoveOverlay(overlayDir: string, overlayRoot: string): boolean { const resolvedRoot = resolve(overlayRoot) const resolvedTarget = resolve(overlayDir) const rel = relative(resolvedRoot, resolvedTarget) @@ -151,7 +228,7 @@ export function safeRemoveOverlay(overlayDir: string, overlayRoot: string): void console.warn( `[overlay-mirror] refusing to remove overlay outside root: target=${resolvedTarget} root=${resolvedRoot}` ) - return + return false } - safeRemoveTree(resolvedTarget) + return safeRemoveTree(resolvedTarget) } diff --git a/src/main/pty/shell-startup-env-memory.test.ts b/src/main/pty/shell-startup-env-memory.test.ts new file mode 100644 index 00000000000..c6452b9f20a --- /dev/null +++ b/src/main/pty/shell-startup-env-memory.test.ts @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { readFileSyncMock } = vi.hoisted(() => ({ readFileSyncMock: vi.fn() })) + +vi.mock('../../shared/node-bounded-file-reader', () => ({ + readNodeFileSyncWithinLimit: (path: string, maxBytes: number) => { + const buffer = Buffer.from(readFileSyncMock(path) as string) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: { size: buffer.byteLength } } + } +})) + +import { + __resetShellStartupEnvCache, + MAX_SHELL_STARTUP_CACHE_BYTES, + MAX_SHELL_STARTUP_CACHE_ENTRIES, + MAX_SHELL_STARTUP_ENV_VALUE_CODE_UNITS, + MAX_SHELL_STARTUP_FILE_BYTES, + readShellStartupEnvVar +} from './shell-startup-env' + +describe('shell startup environment memory bounds', () => { + const originalPlatform = process.platform + + beforeEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' }) + readFileSyncMock.mockReset() + __resetShellStartupEnvCache() + }) + + afterEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + }) + + it('accepts an exact-size startup file and skips the next byte', () => { + const assignment = 'export TARGET=/bounded\n' + readFileSyncMock.mockReturnValue( + assignment + '#'.repeat(MAX_SHELL_STARTUP_FILE_BYTES - assignment.length) + ) + expect(readShellStartupEnvVar('TARGET', '/home/alice', '/bin/zsh')).toBe('/bounded') + + __resetShellStartupEnvCache() + readFileSyncMock.mockReturnValue( + assignment + '#'.repeat(MAX_SHELL_STARTUP_FILE_BYTES - assignment.length + 1) + ) + expect(readShellStartupEnvVar('TARGET', '/home/alice', '/bin/zsh')).toBeUndefined() + }) + + it('does not retain an oversized exported value', () => { + readFileSyncMock.mockReturnValue( + `export TARGET=${'x'.repeat(MAX_SHELL_STARTUP_ENV_VALUE_CODE_UNITS + 1)}\n` + ) + + expect(readShellStartupEnvVar('TARGET', '/home/alice', '/bin/zsh')).toBeUndefined() + }) + + it('caps cache entries with LRU recovery', () => { + readFileSyncMock.mockReturnValue('') + for (let index = 0; index <= MAX_SHELL_STARTUP_CACHE_ENTRIES; index += 1) { + readShellStartupEnvVar(`TARGET_${index}`, '/home/alice', '/bin/zsh') + } + const readsBeforeRetry = readFileSyncMock.mock.calls.length + + readShellStartupEnvVar('TARGET_0', '/home/alice', '/bin/zsh') + + expect(readFileSyncMock.mock.calls.length).toBeGreaterThan(readsBeforeRetry) + }) + + it('caps aggregate cached key/value memory independently of entry count', () => { + expect(MAX_SHELL_STARTUP_CACHE_BYTES).toBe(8 * 1024 * 1024) + const value = 'x'.repeat(MAX_SHELL_STARTUP_ENV_VALUE_CODE_UNITS) + for (let index = 0; index < 65; index += 1) { + readFileSyncMock.mockReturnValue(`export TARGET_${index}=${value}\n`) + expect(readShellStartupEnvVar(`TARGET_${index}`, '/home/alice', '/bin/zsh')).toBe(value) + } + const readsBeforeRetry = readFileSyncMock.mock.calls.length + readFileSyncMock.mockReturnValue(`export TARGET_0=${value}\n`) + + readShellStartupEnvVar('TARGET_0', '/home/alice', '/bin/zsh') + + expect(readFileSyncMock.mock.calls.length).toBeGreaterThan(readsBeforeRetry) + }) +}) diff --git a/src/main/pty/shell-startup-env.test.ts b/src/main/pty/shell-startup-env.test.ts index d41e6c7fdcb..2e70e2947b8 100644 --- a/src/main/pty/shell-startup-env.test.ts +++ b/src/main/pty/shell-startup-env.test.ts @@ -10,6 +10,16 @@ vi.mock('fs', () => ({ readFileSync: readFileSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', () => ({ + readNodeFileSyncWithinLimit: (path: string, maxBytes: number) => { + const buffer = Buffer.from(readFileSyncMock(path) as string) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: { size: buffer.byteLength } } + } +})) + import { __resetShellStartupEnvCache, readShellStartupEnvVar } from './shell-startup-env' describe('readShellStartupEnvVar', () => { diff --git a/src/main/pty/shell-startup-env.ts b/src/main/pty/shell-startup-env.ts index 83a4cad1185..0908f4603c8 100644 --- a/src/main/pty/shell-startup-env.ts +++ b/src/main/pty/shell-startup-env.ts @@ -1,5 +1,12 @@ -import { existsSync, readFileSync } from 'node:fs' import { posix } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const MAX_SHELL_STARTUP_FILE_BYTES = 1024 * 1024 +export const MAX_SHELL_STARTUP_CACHE_ENTRIES = 128 +export const MAX_SHELL_STARTUP_CACHE_BYTES = 8 * 1024 * 1024 +export const MAX_SHELL_STARTUP_ENV_VALUE_CODE_UNITS = 64 * 1024 +const MAX_SHELL_STARTUP_NAME_CODE_UNITS = 256 +const MAX_SHELL_STARTUP_PATH_CODE_UNITS = 16 * 1024 // Why: only files the user's actual shell would source. Mixing zsh and bash // files breaks the "last assignment wins matches the live shell" guarantee — @@ -29,7 +36,7 @@ function parseExportedValue(content: string, name: string, home: string): string // Why: $HOME / ${HOME} / ~ expansion mimics what the live shell would // do for double-quoted and unquoted values; single-quoted is literal. const expanded = quoted === "'" ? text : expandHome(text, home) - if (expanded.length > 0) { + if (expanded.length > 0 && expanded.length <= MAX_SHELL_STARTUP_ENV_VALUE_CODE_UNITS) { lastMatch = expanded } } @@ -38,11 +45,8 @@ function parseExportedValue(content: string, name: string, home: string): string } function readStartupFile(path: string): string | null { - if (!existsSync(path)) { - return null - } try { - return readFileSync(path, 'utf8') + return readNodeFileSyncWithinLimit(path, MAX_SHELL_STARTUP_FILE_BYTES).buffer.toString('utf8') } catch { return null } @@ -121,7 +125,8 @@ function expandHome(value: string, home: string): string { .replace(/\$HOME(?![A-Za-z0-9_])/g, home) } -const cache = new Map() +const cache = new Map() +let cacheRetainedBytes = 0 /** * Best-effort static read of a single env-var assignment from the user's @@ -151,18 +156,26 @@ export function readShellStartupEnvVar( home = process.env.HOME, shell = process.env.SHELL ): string | undefined { - if (!home || process.platform === 'win32') { + if ( + !home || + home.length > MAX_SHELL_STARTUP_PATH_CODE_UNITS || + (shell?.length ?? 0) > MAX_SHELL_STARTUP_PATH_CODE_UNITS || + process.platform === 'win32' + ) { return undefined } // Why: the regex above is fixed; rejecting unsafe names is cheap defense // for the day a future caller passes something with regex metacharacters. - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + if (name.length > MAX_SHELL_STARTUP_NAME_CODE_UNITS || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { return undefined } const cacheKey = `${name}\0${home}\0${shell ?? ''}` - if (cache.has(cacheKey)) { - return cache.get(cacheKey) + const cached = cache.get(cacheKey) + if (cached) { + cache.delete(cacheKey) + cache.set(cacheKey, cached) + return cached.value } let lastMatch: string | undefined @@ -179,10 +192,28 @@ export function readShellStartupEnvVar( } } - cache.set(cacheKey, lastMatch) + cacheShellStartupValue(cacheKey, lastMatch) return lastMatch } +function cacheShellStartupValue(key: string, value: string | undefined): void { + const retainedBytes = (key.length + (value?.length ?? 0)) * 2 + while ( + cache.size >= MAX_SHELL_STARTUP_CACHE_ENTRIES || + cacheRetainedBytes + retainedBytes > MAX_SHELL_STARTUP_CACHE_BYTES + ) { + const oldestKey = cache.keys().next().value as string | undefined + if (oldestKey === undefined) { + return + } + const oldest = cache.get(oldestKey) + cache.delete(oldestKey) + cacheRetainedBytes -= oldest?.retainedBytes ?? 0 + } + cache.set(key, { retainedBytes, value }) + cacheRetainedBytes += retainedBytes +} + /** * Test-only helper to reset the per-process cache between cases. * Why: production callers never invalidate (rc files don't change at @@ -190,4 +221,5 @@ export function readShellStartupEnvVar( */ export function __resetShellStartupEnvCache(): void { cache.clear() + cacheRetainedBytes = 0 } diff --git a/src/main/rate-limits/auth-filesystem-operation.test.ts b/src/main/rate-limits/auth-filesystem-operation.test.ts index 92099fa2dcc..d27dfd381ec 100644 --- a/src/main/rate-limits/auth-filesystem-operation.test.ts +++ b/src/main/rate-limits/auth-filesystem-operation.test.ts @@ -1,5 +1,24 @@ import { describe, expect, it, vi } from 'vitest' -import { createAuthFilesystemOperation } from './auth-filesystem-operation' +import { + AuthFilesystemOperationLimitError, + AuthFilesystemOperationRegistry, + createAuthFilesystemOperation, + MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES, + MAX_AUTH_FILESYSTEM_OPERATION_WAITERS, + MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES, + MAX_QUEUED_WSL_AUTH_OPERATIONS +} from './auth-filesystem-operation' + +function deferred(): { + promise: Promise + resolve: (value: T) => void +} { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} describe('createAuthFilesystemOperation', () => { it('serializes WSL aliases by distro and drops an abandoned queued read', async () => { @@ -116,4 +135,152 @@ describe('createAuthFilesystemOperation', () => { await expect(Promise.all([ubuntuWait, debianWait])).resolves.toEqual(['ubuntu', 'debian']) expect(fedoraRaw).not.toHaveBeenCalled() }) + + it('caps retained operations, coalesces existing paths at saturation, and recovers', async () => { + const registry = new AuthFilesystemOperationRegistry() + const deferredReads = Array.from({ length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES + 1 }, () => + deferred() + ) + const operations = deferredReads + .slice(0, MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + .map((read, index) => registry.getOrCreate(`/auth/${index}`, () => read.promise)) + + expect(operations.every((operation) => operation !== null)).toBe(true) + expect(registry.size).toBe(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + expect(registry.getOrCreate('/auth/0', async () => 'duplicate')).toBe(operations[0]) + const overflowRaw = vi.fn(async () => 'overflow') + expect(registry.getOrCreate('/auth/overflow', overflowRaw)).toBeNull() + expect(overflowRaw).not.toHaveBeenCalled() + + deferredReads[0]!.resolve('first') + await expect(operations[0]!.result).resolves.toBe('first') + expect(registry.size).toBe(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES - 1) + + const recovered = registry.getOrCreate( + '/auth/recovered', + () => deferredReads[MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES]!.promise + ) + expect(recovered).not.toBeNull() + expect(registry.size).toBe(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + + for (let index = 1; index < MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES; index += 1) { + deferredReads[index]!.resolve(`read-${index}`) + } + deferredReads[MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES]!.resolve('recovered') + await Promise.all([ + ...operations.slice(1).map((operation) => operation!.result), + recovered!.result + ]) + expect(registry.size).toBe(0) + }) + + it('accepts an exact-limit auth path and rejects one byte more without starting it', async () => { + const exactPath = 'x'.repeat(MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES) + const exactRaw = vi.fn(async () => 'exact') + const exact = createAuthFilesystemOperation(exactPath, exactRaw) + const controller = new AbortController() + + await expect(exact.wait(controller.signal)).resolves.toBe('exact') + expect(exactRaw).toHaveBeenCalledOnce() + + const oversizedRaw = vi.fn(async () => 'oversized') + const oversized = createAuthFilesystemOperation(`${exactPath}x`, oversizedRaw) + await expect(oversized.wait(controller.signal)).rejects.toBeInstanceOf( + AuthFilesystemOperationLimitError + ) + expect(oversizedRaw).not.toHaveBeenCalled() + }) + + it('caps coalesced waiters and admits another after one leaves', async () => { + const read = deferred() + const operation = createAuthFilesystemOperation('/auth/shared', () => read.promise) + const controllers = Array.from( + { length: MAX_AUTH_FILESYSTEM_OPERATION_WAITERS }, + () => new AbortController() + ) + const waits = controllers.map((controller) => operation.wait(controller.signal)) + + await expect(operation.wait(new AbortController().signal)).rejects.toBeInstanceOf( + AuthFilesystemOperationLimitError + ) + + const abandoned = new Error('first waiter left') + controllers[0]!.abort(abandoned) + await expect(waits[0]).rejects.toBe(abandoned) + const recovered = operation.wait(new AbortController().signal) + + read.resolve('shared') + await expect(recovered).resolves.toBe('shared') + await expect(Promise.all(waits.slice(1))).resolves.toEqual( + Array.from({ length: MAX_AUTH_FILESYSTEM_OPERATION_WAITERS - 1 }, () => 'shared') + ) + }) + + it('caps the WSL wait queue and recovers as soon as a queued operation leaves', async () => { + const ubuntuRead = deferred() + const debianRead = deferred() + const ubuntuRaw = vi.fn(() => ubuntuRead.promise) + const debianRaw = vi.fn(() => debianRead.promise) + const activeController = new AbortController() + const ubuntu = createAuthFilesystemOperation( + '\\\\wsl$\\Ubuntu\\home\\alice\\auth.json', + ubuntuRaw + ) + const debian = createAuthFilesystemOperation( + '\\\\wsl$\\Debian\\home\\alice\\auth.json', + debianRaw + ) + const ubuntuWait = ubuntu.wait(activeController.signal) + const debianWait = debian.wait(activeController.signal) + await Promise.resolve() + await Promise.resolve() + expect(ubuntuRaw).toHaveBeenCalledOnce() + expect(debianRaw).toHaveBeenCalledOnce() + + const queuedControllers: AbortController[] = [] + const queuedWaits: Promise[] = [] + for (let index = 0; index < MAX_QUEUED_WSL_AUTH_OPERATIONS; index += 1) { + const controller = new AbortController() + const operation = createAuthFilesystemOperation( + `\\\\wsl$\\Queued-${index}\\home\\alice\\auth.json`, + async () => `queued-${index}` + ) + queuedControllers.push(controller) + queuedWaits.push(operation.wait(controller.signal)) + } + + const overflowRaw = vi.fn(async () => 'overflow') + const overflow = createAuthFilesystemOperation( + '\\\\wsl$\\Overflow\\home\\alice\\auth.json', + overflowRaw + ) + await expect(overflow.wait(new AbortController().signal)).rejects.toBeInstanceOf( + AuthFilesystemOperationLimitError + ) + expect(overflowRaw).not.toHaveBeenCalled() + + const released = new Error('release queue slot') + queuedControllers[0]!.abort(released) + await expect(queuedWaits[0]).rejects.toBe(released) + + const recoveredController = new AbortController() + const recoveredRaw = vi.fn(async () => 'recovered') + const recovered = createAuthFilesystemOperation( + '\\\\wsl$\\Recovered\\home\\alice\\auth.json', + recoveredRaw + ) + const recoveredWait = recovered.wait(recoveredController.signal) + const recoveredAbort = new Error('recovered queue entry admitted') + recoveredController.abort(recoveredAbort) + await expect(recoveredWait).rejects.toBe(recoveredAbort) + expect(recoveredRaw).not.toHaveBeenCalled() + + for (const controller of queuedControllers.slice(1)) { + controller.abort(new Error('test cleanup')) + } + await Promise.allSettled(queuedWaits.slice(1)) + ubuntuRead.resolve('ubuntu') + debianRead.resolve('debian') + await expect(Promise.all([ubuntuWait, debianWait])).resolves.toEqual(['ubuntu', 'debian']) + }) }) diff --git a/src/main/rate-limits/auth-filesystem-operation.ts b/src/main/rate-limits/auth-filesystem-operation.ts index 228234e92c0..8910f30f19d 100644 --- a/src/main/rate-limits/auth-filesystem-operation.ts +++ b/src/main/rate-limits/auth-filesystem-operation.ts @@ -1,10 +1,22 @@ import { parseWslUncPath } from '../../shared/wsl-paths' const MAX_CONCURRENT_WSL_AUTH_OPERATIONS = 2 +export const MAX_QUEUED_WSL_AUTH_OPERATIONS = 128 +export const MAX_AUTH_FILESYSTEM_OPERATION_WAITERS = 256 +export const MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES = 64 * 1024 +export const MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES = 128 +export const MAX_AUTH_FILESYSTEM_REGISTRY_PATH_BYTES = 8 * 1024 * 1024 const activeWslOperationDistros = new Set() const queuedWslOperations: QueuedWslOperation[] = [] let activeWslOperationCount = 0 +export class AuthFilesystemOperationLimitError extends Error { + constructor(message: string) { + super(message) + this.name = 'AuthFilesystemOperationLimitError' + } +} + type QueuedWslOperation = { distroKey: string neededSignal: AbortSignal @@ -70,6 +82,13 @@ function scheduleWslAuthFilesystemOperation( neededSignal: AbortSignal, operation: () => Promise ): Promise { + if (queuedWslOperations.length >= MAX_QUEUED_WSL_AUTH_OPERATIONS) { + return Promise.reject( + new AuthFilesystemOperationLimitError( + `Auth filesystem queue exceeds ${MAX_QUEUED_WSL_AUTH_OPERATIONS} operations` + ) + ) + } return new Promise((resolve, reject) => { const task: QueuedWslOperation = { distroKey, @@ -126,6 +145,19 @@ export type SharedAuthFilesystemOperation = { wait: (signal: AbortSignal) => Promise } +function rejectedAuthFilesystemOperation( + error: AuthFilesystemOperationLimitError +): SharedAuthFilesystemOperation { + const result = Promise.reject(error) + void result.catch(() => undefined) + return { + result, + wait(signal) { + return Promise.reject(signal.aborted ? getAbortReason(signal) : error) + } + } +} + /** * Shares one raw operation with all callers for an auth path. WSL paths also * serialize by normalized distro because one stuck UNC request per account can @@ -135,8 +167,15 @@ export function createAuthFilesystemOperation( authPath: string, operation: () => Promise ): SharedAuthFilesystemOperation { + if (Buffer.byteLength(authPath, 'utf8') > MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES) { + return rejectedAuthFilesystemOperation( + new AuthFilesystemOperationLimitError( + `Auth filesystem path exceeds ${MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES} bytes` + ) + ) + } const neededController = new AbortController() - const waiters = new Set() + let waiterCount = 0 let settled = false const result = scheduleAuthFilesystemOperation(authPath, neededController.signal, operation) const markSettled = (): void => { @@ -148,14 +187,23 @@ export function createAuthFilesystemOperation( result, wait(signal) { if (signal.aborted) { - if (!settled && waiters.size === 0) { + if (!settled && waiterCount === 0) { neededController.abort(getAbortReason(signal)) } return Promise.reject(getAbortReason(signal)) } - const waiter = Symbol('auth-filesystem-waiter') - waiters.add(waiter) + if (settled) { + return result + } + if (waiterCount >= MAX_AUTH_FILESYSTEM_OPERATION_WAITERS) { + return Promise.reject( + new AuthFilesystemOperationLimitError( + `Auth filesystem operation exceeds ${MAX_AUTH_FILESYSTEM_OPERATION_WAITERS} waiters` + ) + ) + } + waiterCount += 1 let onAbort: (() => void) | null = null const aborted = new Promise((_resolve, reject) => { onAbort = () => reject(getAbortReason(signal)) @@ -165,11 +213,53 @@ export function createAuthFilesystemOperation( if (onAbort) { signal.removeEventListener('abort', onAbort) } - waiters.delete(waiter) - if (!settled && waiters.size === 0) { + waiterCount -= 1 + if (!settled && waiterCount === 0) { neededController.abort(getAbortReason(signal)) } }) } } } + +export class AuthFilesystemOperationRegistry { + private readonly operations = new Map>() + private retainedPathBytes = 0 + + get size(): number { + return this.operations.size + } + + getOrCreate( + authPath: string, + operation: () => Promise + ): SharedAuthFilesystemOperation | null { + const pathBytes = Buffer.byteLength(authPath, 'utf8') + if (pathBytes > MAX_AUTH_FILESYSTEM_OPERATION_PATH_BYTES) { + return null + } + const existing = this.operations.get(authPath) + if (existing) { + return existing + } + if ( + this.operations.size >= MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES || + pathBytes > MAX_AUTH_FILESYSTEM_REGISTRY_PATH_BYTES - this.retainedPathBytes + ) { + return null + } + + const shared = createAuthFilesystemOperation(authPath, operation) + this.operations.set(authPath, shared) + this.retainedPathBytes += pathBytes + const clear = (): void => { + if (this.operations.get(authPath) !== shared) { + return + } + this.operations.delete(authPath) + this.retainedPathBytes -= pathBytes + } + void shared.result.then(clear, clear) + return shared + } +} diff --git a/src/main/rate-limits/claude-fetcher.test.ts b/src/main/rate-limits/claude-fetcher.test.ts index 7eb218f04de..c46c9577951 100644 --- a/src/main/rate-limits/claude-fetcher.test.ts +++ b/src/main/rate-limits/claude-fetcher.test.ts @@ -15,18 +15,25 @@ import { } from '../claude-accounts/keychain' import type { ClaudeRuntimeAuthPreparation } from '../claude-accounts/runtime-auth-service' -const { netFetchMock, readFileMock, resolveProxyMock, setProxyMock, appGetPathMock } = vi.hoisted( - () => ({ - netFetchMock: vi.fn(), - readFileMock: vi.fn(), - resolveProxyMock: vi.fn(), - setProxyMock: vi.fn(), - appGetPathMock: vi.fn() - }) -) +const { + netFetchMock, + readFileMock, + readFileSyncMock, + resolveProxyMock, + setProxyMock, + appGetPathMock +} = vi.hoisted(() => ({ + netFetchMock: vi.fn(), + readFileMock: vi.fn(), + readFileSyncMock: vi.fn(), + resolveProxyMock: vi.fn(), + setProxyMock: vi.fn(), + appGetPathMock: vi.fn() +})) -vi.mock('node:fs/promises', () => ({ - readFile: readFileMock +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileSyncText: readFileSyncMock, + readIntegrationCredentialFileText: readFileMock })) vi.mock('electron', () => ({ @@ -74,6 +81,7 @@ describe('fetchClaudeRateLimits', () => { tempDir = null vi.clearAllMocks() readFileMock.mockRejectedValue(new Error('missing file')) + readFileSyncMock.mockImplementation((filePath: string) => readFileSync(filePath, 'utf8')) vi.mocked(readActiveClaudeKeychainCredentials).mockResolvedValue(null) vi.mocked(readActiveClaudeKeychainCredentialsStrict).mockResolvedValue(null) vi.mocked(readManagedClaudeKeychainCredentials).mockResolvedValue(null) @@ -744,10 +752,7 @@ describe('fetchClaudeRateLimits', () => { status: 'ok' }) - expect(readFileMock).toHaveBeenCalledWith( - join('/Users/test/.claude', '.credentials.json'), - 'utf-8' - ) + expect(readFileMock).toHaveBeenCalledWith(join('/Users/test/.claude', '.credentials.json')) expect(netFetchMock).toHaveBeenCalledWith( 'https://api.anthropic.com/api/oauth/usage', expect.objectContaining({ diff --git a/src/main/rate-limits/claude-fetcher.ts b/src/main/rate-limits/claude-fetcher.ts index 206748e1bc5..ddc1237fcde 100644 --- a/src/main/rate-limits/claude-fetcher.ts +++ b/src/main/rate-limits/claude-fetcher.ts @@ -1,6 +1,5 @@ /* eslint-disable max-lines -- Why: keep Claude credential ordering, OAuth usage fetch, and PTY fallback together so usage state can't drift across paths. */ -import { existsSync, lstatSync, readFileSync } from 'node:fs' -import { readFile } from 'node:fs/promises' +import { existsSync, lstatSync } from 'node:fs' import { homedir } from 'node:os' import path from 'node:path' import { net, session } from 'electron' @@ -42,6 +41,11 @@ import { classifyClaudeOAuthUsageError, type ClaudeUsageErrorClassification } from './claude-usage-error-classification' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' +import { + readIntegrationCredentialFileSyncText, + readIntegrationCredentialFileText +} from '../integration-credential-file' const OAUTH_USAGE_URL = 'https://api.anthropic.com/api/oauth/usage' const OAUTH_BETA_HEADER = 'oauth-2025-04-20' @@ -193,7 +197,7 @@ async function readCredentialsFromStrictKeychain( async function readFromCredentialsFile(configDir?: string): Promise { const credPath = path.join(configDir ?? path.join(homedir(), '.claude'), '.credentials.json') try { - const raw = await readFile(credPath, 'utf-8') + const raw = await readIntegrationCredentialFileText(credPath) return parseOAuthCredentialsJson(raw, 'credentials-file') } catch { return emptyOAuthCredentialReadResult() @@ -366,7 +370,7 @@ async function fetchViaOAuth(token: string, signal?: AbortSignal): Promise(res) if (signal?.aborted) { return abortedClaudeRateLimitResult() } @@ -1042,7 +1046,7 @@ function resolveOwnedWslClaudeManagedAuthPath(account: InactiveClaudeAccountInfo if ( !existsSync(markerPath) || lstatSync(markerPath).isSymbolicLink() || - readFileSync(markerPath, 'utf-8').trim() !== account.id + readIntegrationCredentialFileSyncText(markerPath).trim() !== account.id ) { return null } diff --git a/src/main/rate-limits/claude-oauth-usage-error.ts b/src/main/rate-limits/claude-oauth-usage-error.ts index 1ccb1471de6..06d404c5c30 100644 --- a/src/main/rate-limits/claude-oauth-usage-error.ts +++ b/src/main/rate-limits/claude-oauth-usage-error.ts @@ -1,3 +1,5 @@ +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' + // Why: a corrupt/hostile Retry-After must not gate usage refreshes for days. const MAX_RETRY_AFTER_MS = 24 * 60 * 60 * 1000 @@ -45,7 +47,7 @@ async function describeOAuthUsageError(res: Response): Promise { return 'Claude usage is rate limited right now.' } try { - const data = (await res.json()) as { error?: { message?: string } } + const data = await readFetchResponseJsonWithinLimit<{ error?: { message?: string } }>(res) if (typeof data.error?.message === 'string' && data.error.message.trim()) { return data.error.message } diff --git a/src/main/rate-limits/claude-pty.ts b/src/main/rate-limits/claude-pty.ts index b7a8c96dfba..054e2ce1e2a 100644 --- a/src/main/rate-limits/claude-pty.ts +++ b/src/main/rate-limits/claude-pty.ts @@ -13,6 +13,7 @@ import { getHiddenRateLimitWslCwdSetupCommands, resolveHiddenRateLimitPtyCwd } from './hidden-rate-limit-pty-cwd' +import { appendRateLimitPtyOutputTail } from './rate-limit-pty-output-tail' const PTY_TIMEOUT_MS = 25_000 const MAX_OUTPUT_LENGTH = 100_000 // 100KB buffer limit @@ -441,13 +442,10 @@ export async function fetchViaPty(options?: { }, STARTUP_DELAY_MS) const onDataDisposable = term.onData((data) => { - output += data - // Why: prevent memory exhaustion if the CLI process floods output - if (output.length > MAX_OUTPUT_LENGTH) { - output = output.slice(-MAX_OUTPUT_LENGTH) - } + const appended = appendRateLimitPtyOutputTail(output, data, MAX_OUTPUT_LENGTH) + output = appended.output - const cleanChunk = stripTerminalControlSequences(data) + const cleanChunk = stripTerminalControlSequences(appended.scannedChunk) // Why: the Claude CLI may prompt for first-run setup (trust files, // workspace directory). Auto-accept so we can reach /usage. diff --git a/src/main/rate-limits/codex-auth-presence.test.ts b/src/main/rate-limits/codex-auth-presence.test.ts index 6f49f27bfb4..d1b9ce38b86 100644 --- a/src/main/rate-limits/codex-auth-presence.test.ts +++ b/src/main/rate-limits/codex-auth-presence.test.ts @@ -15,6 +15,7 @@ vi.mock('node:os', () => ({ })) import { probeCodexAuthPresence } from './codex-auth-presence' +import { MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES } from './auth-filesystem-operation' function fsError(code: string): NodeJS.ErrnoException { return Object.assign(new Error(code), { code }) @@ -152,4 +153,50 @@ describe('probeCodexAuthPresence', () => { resolveAccess() await Promise.resolve() }) + + it('caps distinct stalled probes and admits a new path after one settles', async () => { + const accessResolvers = new Map void>() + accessMock.mockImplementation( + (path: string) => + new Promise((resolve) => { + accessResolvers.set(path, resolve) + }) + ) + const controllers = Array.from( + { length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES + 2 }, + () => new AbortController() + ) + const probes = Array.from({ length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES }, (_, index) => + probeCodexAuthPresence(`/managed/${index}`, { signal: controllers[index]!.signal }) + ) + await Promise.resolve() + await Promise.resolve() + + expect(accessMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + await expect( + probeCodexAuthPresence('/managed/overflow', { + signal: controllers[MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES]!.signal + }) + ).resolves.toBe('unavailable') + expect(accessMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + + accessResolvers.get(join('/managed/0', 'auth.json'))!() + await expect(probes[0]).resolves.toBe('present') + + const recovered = probeCodexAuthPresence('/managed/recovered', { + signal: controllers[MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES + 1]!.signal + }) + await Promise.resolve() + await Promise.resolve() + expect(accessMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES + 1) + accessResolvers.get(join('/managed/recovered', 'auth.json'))!() + await expect(recovered).resolves.toBe('present') + + for (let index = 1; index < MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES; index += 1) { + accessResolvers.get(join(`/managed/${index}`, 'auth.json'))!() + } + await expect(Promise.all(probes.slice(1))).resolves.toEqual( + Array.from({ length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES - 1 }, () => 'present') + ) + }) }) diff --git a/src/main/rate-limits/codex-auth-presence.ts b/src/main/rate-limits/codex-auth-presence.ts index 833f46a1b6f..2ad19f1aab6 100644 --- a/src/main/rate-limits/codex-auth-presence.ts +++ b/src/main/rate-limits/codex-auth-presence.ts @@ -3,12 +3,12 @@ import { homedir } from 'node:os' import { join } from 'node:path' import { parseWslUncPath } from '../../shared/wsl-paths' import { - createAuthFilesystemOperation, + AuthFilesystemOperationRegistry, type SharedAuthFilesystemOperation } from './auth-filesystem-operation' const AUTH_PRESENCE_TIMEOUT_MS = 5_000 -const authPresenceProbeByPath = new Map>() +const authPresenceProbes = new AuthFilesystemOperationRegistry() export type CodexAuthPresence = 'present' | 'absent' | 'timeout' | 'unavailable' @@ -22,15 +22,13 @@ function isMissingPathError(error: unknown): boolean { return code === 'ENOENT' || code === 'ENOTDIR' } -function getAuthPresenceProbe(authPath: string): SharedAuthFilesystemOperation { - const existing = authPresenceProbeByPath.get(authPath) - if (existing) { - return existing - } +function getAuthPresenceProbe( + authPath: string +): SharedAuthFilesystemOperation | null { // Why: aborting a Node fs promise does not necessarily cancel an already // issued UNC operation. Share the raw probe until it really settles so a // disconnected WSL home cannot accumulate native requests across polls. - const probe = createAuthFilesystemOperation(authPath, async () => { + return authPresenceProbes.getOrCreate(authPath, async () => { try { await access(authPath) return 'present' @@ -38,14 +36,6 @@ function getAuthPresenceProbe(authPath: string): SharedAuthFilesystemOperation { - if (authPresenceProbeByPath.get(authPath) === probe) { - authPresenceProbeByPath.delete(authPath) - } - } - void probe.result.then(clearProbe, clearProbe) - return probe } // Why: the background quota poller spawns the real `codex` binary to read rate @@ -68,14 +58,22 @@ export async function probeCodexAuthPresence( // Why: managed WSL homes are UNC paths. A synchronous stat can park // Electron main while Windows wakes or reconnects the distro; the race // also keeps a disconnected distro from serializing all later refreshes. - const authPresence = await getAuthPresenceProbe(authPath).wait(signal) + const authProbe = getAuthPresenceProbe(authPath) + if (!authProbe) { + return 'unavailable' + } + const authPresence = await authProbe.wait(signal) if (authPresence !== 'absent' || !parseWslUncPath(home)) { return authPresence } // Why: ENOENT on a WSL UNC auth path can mean either a missing auth file or // an unavailable distro. Only an accessible Codex home proves signed-out. - const homePresence = await getAuthPresenceProbe(home).wait(signal) + const homeProbe = getAuthPresenceProbe(home) + if (!homeProbe) { + return 'unavailable' + } + const homePresence = await homeProbe.wait(signal) return homePresence === 'present' ? 'absent' : 'unavailable' } catch { return timeoutSignal.aborted && !options.signal?.aborted ? 'timeout' : 'unavailable' diff --git a/src/main/rate-limits/codex-fetcher-auth-file-bounds.test.ts b/src/main/rate-limits/codex-fetcher-auth-file-bounds.test.ts new file mode 100644 index 00000000000..04f45dff2a0 --- /dev/null +++ b/src/main/rate-limits/codex-fetcher-auth-file-bounds.test.ts @@ -0,0 +1,100 @@ +import { mkdtempSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { MAX_INTEGRATION_CREDENTIAL_FILE_BYTES } from '../integration-credential-file' +import { consumeCodexRateLimitResetCredit } from './codex-fetcher' + +const fetchMock = vi.hoisted(() => vi.fn()) +const roots: string[] = [] + +function createCodexHome(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-codex-auth-bounds-')) + roots.push(root) + return root +} + +function successfulConsumeResponse(): Response { + return new Response(JSON.stringify({ code: 'already_redeemed' }), { + headers: { 'content-type': 'application/json' }, + status: 200 + }) +} + +beforeEach(() => { + fetchMock.mockReset() + fetchMock.mockResolvedValue(successfulConsumeResponse()) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('Codex backend auth file bounds', () => { + it('preserves normal auth JSON and backend headers', async () => { + const codexHomePath = createCodexHome() + writeFileSync( + join(codexHomePath, 'auth.json'), + JSON.stringify({ + tokens: { access_token: 'normal-token', account_id: 'normal-account' } + }) + ) + + await expect( + consumeCodexRateLimitResetCredit({ + codexHomePath, + idempotencyKey: 'normal-auth' + }) + ).resolves.toBe('alreadyRedeemed') + expect(fetchMock).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + headers: expect.objectContaining({ + Authorization: 'Bearer normal-token', + 'ChatGPT-Account-Id': 'normal-account' + }) + }) + ) + }) + + it('accepts valid auth JSON at the exact credential byte limit', async () => { + const codexHomePath = createCodexHome() + const authPath = join(codexHomePath, 'auth.json') + const json = JSON.stringify({ + tokens: { access_token: 'boundary-token', account_id: 'boundary-account' } + }) + writeFileSync( + authPath, + json + ' '.repeat(MAX_INTEGRATION_CREDENTIAL_FILE_BYTES - Buffer.byteLength(json)) + ) + + expect(statSync(authPath).size).toBe(MAX_INTEGRATION_CREDENTIAL_FILE_BYTES) + await expect( + consumeCodexRateLimitResetCredit({ + codexHomePath, + idempotencyKey: 'boundary-auth' + }) + ).resolves.toBe('alreadyRedeemed') + expect(fetchMock).toHaveBeenCalledOnce() + }) + + it('rejects an oversized sparse auth file before making a request', async () => { + const codexHomePath = createCodexHome() + const authPath = join(codexHomePath, 'auth.json') + writeFileSync(authPath, '') + truncateSync(authPath, MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + 1) + + await expect( + consumeCodexRateLimitResetCredit({ + codexHomePath, + idempotencyKey: 'oversized-auth' + }) + ).rejects.toBeInstanceOf(NodeFileReadTooLargeError) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/rate-limits/codex-fetcher-backend.test.ts b/src/main/rate-limits/codex-fetcher-backend.test.ts index 9102e54ab09..7c7cff90a4d 100644 --- a/src/main/rate-limits/codex-fetcher-backend.test.ts +++ b/src/main/rate-limits/codex-fetcher-backend.test.ts @@ -9,13 +9,19 @@ const { childSpawnMock, readFileMock, ptySpawnMock } = vi.hoisted(() => ({ })) vi.mock('node:child_process', () => ({ spawn: childSpawnMock })) -vi.mock('node:fs/promises', () => ({ readFile: readFileMock })) +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileText: readFileMock +})) vi.mock('node-pty', () => ({ spawn: ptySpawnMock })) vi.mock('./codex-auth-presence', () => ({ probeCodexAuthPresence: vi.fn(async () => 'present') })) import { consumeCodexRateLimitResetCredit, fetchCodexRateLimits } from './codex-fetcher' +import { + AuthFilesystemOperationLimitError, + MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES +} from './auth-filesystem-operation' describe('Codex backend rate-limit requests', () => { beforeEach(() => { @@ -113,7 +119,7 @@ describe('Codex backend rate-limit requests', () => { const second = fetchCodexRateLimits({ codexHomePath, signal: secondController.signal }) await vi.advanceTimersByTimeAsync(0) expect(readFileMock).toHaveBeenCalledTimes(1) - expect(readFileMock).toHaveBeenCalledWith(expect.stringContaining('auth.json'), 'utf8') + expect(readFileMock).toHaveBeenCalledWith(expect.stringContaining('auth.json')) firstController.abort() secondController.abort() @@ -154,7 +160,7 @@ describe('Codex backend rate-limit requests', () => { await vi.advanceTimersByTimeAsync(0) // Why: redeem is user-triggered, so it gets the longer redeem deadline. expect(timeout).toHaveBeenCalledWith(30_000) - expect(readFileMock).toHaveBeenCalledWith(join('/managed/deadline-home', 'auth.json'), 'utf8') + expect(readFileMock).toHaveBeenCalledWith(join('/managed/deadline-home', 'auth.json')) timeoutController.abort(deadlineError) @@ -217,4 +223,59 @@ describe('Codex backend rate-limit requests', () => { ).rejects.toThrow('Codex reset failed: HTTP 429') expect(cancelledBodies).toBe(1) }) + + it('caps distinct stalled backend auth reads and recovers after one settles', async () => { + const authResolvers = new Map void>() + readFileMock.mockImplementation( + (authPath: string) => + new Promise((resolve) => { + authResolvers.set(authPath, resolve) + }) + ) + vi.mocked(fetch).mockResolvedValue({ + ok: true, + json: async () => ({ code: 'already_redeemed' }) + } as Response) + const requests = Array.from({ length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES }, (_, index) => + consumeCodexRateLimitResetCredit({ + codexHomePath: `/managed/saturated-${index}`, + idempotencyKey: `saturated-${index}` + }) + ) + await vi.advanceTimersByTimeAsync(0) + + expect(readFileMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + await expect( + consumeCodexRateLimitResetCredit({ + codexHomePath: '/managed/overflow', + idempotencyKey: 'overflow' + }) + ).rejects.toBeInstanceOf(AuthFilesystemOperationLimitError) + expect(readFileMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES) + + const authJson = JSON.stringify({ + tokens: { access_token: 'capacity-token', account_id: 'capacity-account' } + }) + authResolvers.get(join('/managed/saturated-0', 'auth.json'))!(authJson) + await vi.advanceTimersByTimeAsync(0) + await expect(requests[0]).resolves.toBe('alreadyRedeemed') + + const recovered = consumeCodexRateLimitResetCredit({ + codexHomePath: '/managed/recovered', + idempotencyKey: 'recovered' + }) + await vi.advanceTimersByTimeAsync(0) + expect(readFileMock).toHaveBeenCalledTimes(MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES + 1) + authResolvers.get(join('/managed/recovered', 'auth.json'))!(authJson) + await vi.advanceTimersByTimeAsync(0) + await expect(recovered).resolves.toBe('alreadyRedeemed') + + for (let index = 1; index < MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES; index += 1) { + authResolvers.get(join(`/managed/saturated-${index}`, 'auth.json'))!(authJson) + } + await vi.advanceTimersByTimeAsync(0) + await expect(Promise.all(requests.slice(1))).resolves.toEqual( + Array.from({ length: MAX_AUTH_FILESYSTEM_REGISTRY_ENTRIES - 1 }, () => 'alreadyRedeemed') + ) + }) }) diff --git a/src/main/rate-limits/codex-fetcher-rpc-buffer.test.ts b/src/main/rate-limits/codex-fetcher-rpc-buffer.test.ts new file mode 100644 index 00000000000..ce6304f4e96 --- /dev/null +++ b/src/main/rate-limits/codex-fetcher-rpc-buffer.test.ts @@ -0,0 +1,123 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { childSpawnMock, resolveCodexCommandMock } = vi.hoisted(() => ({ + childSpawnMock: vi.fn(), + resolveCodexCommandMock: vi.fn(() => 'codex') +})) + +vi.mock('node:child_process', () => ({ + spawn: childSpawnMock +})) + +vi.mock('../codex-cli/command', () => ({ + resolveCodexCommand: resolveCodexCommandMock +})) + +vi.mock('./codex-auth-presence', () => ({ + probeCodexAuthPresence: vi.fn(async () => 'present') +})) + +import { fetchCodexRateLimits, MAX_RPC_RESPONSE_LINE_BYTES } from './codex-fetcher' + +function createRpcChild() { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter + stderr: EventEmitter + stdin: { write: ReturnType } + kill: ReturnType + } + child.stdout = new EventEmitter() + child.stderr = new EventEmitter() + child.stdin = { write: vi.fn() } + child.kill = vi.fn() + return child +} + +describe('Codex RPC response buffering', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('processes a valid response after tens of thousands of tiny stdout chunks', async () => { + vi.useRealTimers() + const child = createRpcChild() + childSpawnMock.mockReturnValue(child) + child.stdin.write.mockImplementation((line: string) => { + const message = JSON.parse(line) as { id?: number; method?: string } + if (message.method === 'initialize') { + setTimeout(() => { + const noise = Buffer.alloc(1_000_000, 0x78) + for (let offset = 0; offset < noise.byteLength; offset += 16) { + child.stdout.emit('data', noise.subarray(offset, offset + 16)) + } + child.stdout.emit( + 'data', + Buffer.from(`\n${JSON.stringify({ jsonrpc: '2.0', id: message.id, result: {} })}\n`) + ) + }, 0) + } + if (message.method === 'account/rateLimits/read') { + setTimeout(() => { + child.stdout.emit( + 'data', + Buffer.from( + `${JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { + rateLimits: { + primary: { usedPercent: 7 }, + secondary: { usedPercent: 12 } + }, + rateLimitResetCredits: { + availableCount: 0, + nextExpiresAt: Date.now() + 60_000 + } + } + })}\n` + ) + ) + }, 0) + } + return true + }) + + const resultPromise = fetchCodexRateLimits({ allowPtyFallback: false }) + + await expect(resultPromise).resolves.toMatchObject({ + status: 'ok', + session: { usedPercent: 7 }, + weekly: { usedPercent: 12 } + }) + }) + + it('kills the RPC process before retaining an oversized line', async () => { + vi.useRealTimers() + const child = createRpcChild() + childSpawnMock.mockReturnValue(child) + child.stdin.write.mockImplementation((line: string) => { + const message = JSON.parse(line) as { method?: string } + if (message.method === 'initialize') { + setTimeout(() => { + child.stdout.emit('data', Buffer.alloc(MAX_RPC_RESPONSE_LINE_BYTES + 1, 0x78)) + }, 0) + } + return true + }) + + const resultPromise = fetchCodexRateLimits({ allowPtyFallback: false }) + + await expect(resultPromise).resolves.toMatchObject({ + status: 'error', + error: `RPC response exceeded ${MAX_RPC_RESPONSE_LINE_BYTES} byte line limit` + }) + expect(child.kill).toHaveBeenCalledOnce() + expect(child.stdout.listenerCount('data')).toBe(0) + }) +}) diff --git a/src/main/rate-limits/codex-fetcher.test.ts b/src/main/rate-limits/codex-fetcher.test.ts index 571229188c6..0ef24af3695 100644 --- a/src/main/rate-limits/codex-fetcher.test.ts +++ b/src/main/rate-limits/codex-fetcher.test.ts @@ -13,8 +13,8 @@ vi.mock('node:child_process', () => ({ spawn: childSpawnMock })) -vi.mock('node:fs/promises', () => ({ - readFile: readFileMock +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileText: readFileMock })) vi.mock('../codex-cli/command', () => ({ @@ -461,7 +461,7 @@ describe('fetchCodexRateLimits', () => { } ] }) - expect(readFileMock).toHaveBeenCalledWith(join('/managed/codex-home', 'auth.json'), 'utf8') + expect(readFileMock).toHaveBeenCalledWith(join('/managed/codex-home', 'auth.json')) expect(fetch).toHaveBeenCalledWith( 'https://chatgpt.com/backend-api/wham/rate-limit-reset-credits', expect.objectContaining({ diff --git a/src/main/rate-limits/codex-fetcher.ts b/src/main/rate-limits/codex-fetcher.ts index dd328e007d5..18042db9c80 100644 --- a/src/main/rate-limits/codex-fetcher.ts +++ b/src/main/rate-limits/codex-fetcher.ts @@ -5,9 +5,10 @@ import type { RateLimitWindow } from '../../shared/rate-limit-types' import { spawn } from 'node:child_process' -import { readFile } from 'node:fs/promises' import { homedir } from 'node:os' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' +import { readIntegrationCredentialFileText } from '../integration-credential-file' import { join } from 'node:path' import { probeCodexAuthPresence } from './codex-auth-presence' import { resolveCodexCommand } from '../codex-cli/command' @@ -25,9 +26,12 @@ import { resolveHiddenRateLimitPtyCwd } from './hidden-rate-limit-pty-cwd' import { - createAuthFilesystemOperation, + AuthFilesystemOperationLimitError, + AuthFilesystemOperationRegistry, type SharedAuthFilesystemOperation } from './auth-filesystem-operation' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' +import { appendRateLimitPtyOutputTail } from './rate-limit-pty-output-tail' const RPC_TIMEOUT_MS = 10_000 const WSL_RPC_TIMEOUT_MS = 25_000 @@ -36,6 +40,7 @@ const BACKEND_TIMEOUT_MS = 10_000 // Why: redeeming a reset credit is an explicit user action, not a poll — allow more time for a slow backend. const REDEEM_BACKEND_TIMEOUT_MS = 30_000 const MAX_DIAGNOSTIC_OUTPUT_LENGTH = 100_000 +export const MAX_RPC_RESPONSE_LINE_BYTES = 4 * 1024 * 1024 export type FetchCodexRateLimitsOptions = { codexHomePath?: string | null @@ -134,10 +139,7 @@ type BackendAuthReadResult = | { content: string; error?: never } | { content?: never; error: unknown } -const backendAuthReadByPath = new Map< - string, - SharedAuthFilesystemOperation ->() +const backendAuthReads = new AuthFilesystemOperationRegistry() function shellQuote(value: string): string { return `'${value.replace(/'/g, "'\\''")}'` @@ -296,30 +298,22 @@ function createBackendRequestSignal( function getBackendAuthRead( authPath: string -): SharedAuthFilesystemOperation { - const existing = backendAuthReadByPath.get(authPath) - if (existing) { - return existing - } +): SharedAuthFilesystemOperation | null { // Why: Node can't cancel an in-flight UNC read; keep one read per auth path so repeated refreshes don't stack them. - const read = createAuthFilesystemOperation(authPath, () => - readFile(authPath, 'utf8').then( + return backendAuthReads.getOrCreate(authPath, () => + readIntegrationCredentialFileText(authPath).then( (content) => ({ content }), (error: unknown) => ({ error }) ) ) - backendAuthReadByPath.set(authPath, read) - const clearRead = (): void => { - if (backendAuthReadByPath.get(authPath) === read) { - backendAuthReadByPath.delete(authPath) - } - } - void read.result.then(clearRead, clearRead) - return read } async function readBackendAuth(authPath: string, signal: AbortSignal): Promise { - const result = await getBackendAuthRead(authPath).wait(signal) + const read = getBackendAuthRead(authPath) + if (!read) { + throw new AuthFilesystemOperationLimitError('Codex backend auth read capacity exceeded') + } + const result = await read.wait(signal) if ('error' in result) { throw result.error } @@ -375,7 +369,8 @@ async function fetchBackendRateLimitResetCredits( await cancelUnreadResponseBody(response) return null } - const payload = (await response.json()) as BackendRateLimitResetCreditsResponse + const payload = + await readFetchResponseJsonWithinLimit(response) return mapBackendRateLimitResetCredits(payload) ?? null } @@ -443,7 +438,8 @@ export async function consumeCodexRateLimitResetCredit(options: { await cancelUnreadResponseBody(response) throw new Error(`Codex reset failed: HTTP ${response.status}`) } - const payload = (await response.json()) as BackendConsumeRateLimitResetCreditResponse + const payload = + await readFetchResponseJsonWithinLimit(response) return mapBackendConsumeOutcome(payload.code) } @@ -523,7 +519,7 @@ async function fetchViaBackend( await cancelUnreadResponseBody(response) return null } - const payload = (await response.json()) as BackendUsageResponse + const payload = await readFetchResponseJsonWithinLimit(response) // Why: plan_type is required by Codex's RateLimitStatusPayload; reject malformed JSON so the app-server fallback still runs. if (typeof payload.plan_type !== 'string') { return null @@ -555,8 +551,8 @@ async function fetchViaRpc(options?: FetchCodexRateLimitsOptions): Promise((resolve) => { - let buffer = '' - let stderr = '' + const buffer = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() let resolved = false let rpcId = 0 @@ -603,6 +599,8 @@ async function fetchViaRpc(options?: FetchCodexRateLimitsOptions): Promise MAX_RPC_RESPONSE_LINE_BYTES - buffer.byteLength) { + settle( + { + provider: 'codex', + session: null, + weekly: null, + updatedAt: Date.now(), + error: `RPC response exceeded ${MAX_RPC_RESPONSE_LINE_BYTES} byte line limit`, + status: 'error' + }, + { kill: true } + ) + return + } + buffer.append(segment) + if (!hasNewline) { + return + } + offset = newlineIdx + 1 + const line = buffer.takeString('utf8').trim() if (!line) { continue } @@ -691,7 +710,7 @@ async function fetchViaRpc(options?: FetchCodexRateLimitsOptions): Promise MAX_DIAGNOSTIC_OUTPUT_LENGTH) { - stderr = stderr.slice(-MAX_DIAGNOSTIC_OUTPUT_LENGTH) - } + stderr.appendRetainedSuffix(chunk, MAX_DIAGNOSTIC_OUTPUT_LENGTH) } function onError(err: Error): void { @@ -746,7 +761,7 @@ async function fetchViaRpc(options?: FetchCodexRateLimitsOptions): Promise { - output += data - // Why: only recent status output is needed; cap noisy TUI output like the Claude fallback. - if (output.length > MAX_DIAGNOSTIC_OUTPUT_LENGTH) { - output = output.slice(-MAX_DIAGNOSTIC_OUTPUT_LENGTH) - } + const appended = appendRateLimitPtyOutputTail(output, data, MAX_DIAGNOSTIC_OUTPUT_LENGTH) + output = appended.output // Wait for prompt, then send /status - if (!sentStatus && />\s*$/.test(data)) { + if (!sentStatus && />\s*$/.test(appended.scannedChunk)) { sentStatus = true term.write('/status\r') return diff --git a/src/main/rate-limits/gemini-cli-oauth-extractor-bounds.test.ts b/src/main/rate-limits/gemini-cli-oauth-extractor-bounds.test.ts new file mode 100644 index 00000000000..d982e15d0ab --- /dev/null +++ b/src/main/rate-limits/gemini-cli-oauth-extractor-bounds.test.ts @@ -0,0 +1,204 @@ +import { mkdirSync, mkdtempSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import type * as FsPromises from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const opendirMock = vi.hoisted(() => vi.fn()) + +vi.mock('node:fs/promises', async (importOriginal) => ({ + ...(await importOriginal()), + opendir: opendirMock +})) + +import { + extractGeminiOAuthCredentialsFromBundleDir, + findGeminiPackageRoot, + MAX_GEMINI_CLI_BUNDLE_ENTRIES, + MAX_GEMINI_CLI_BUNDLE_FILES, + MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES, + MAX_GEMINI_CLI_PACKAGE_JSON_BYTES, + readGeminiOAuthCredentialsFile +} from './gemini-cli-oauth-extractor' + +const roots: string[] = [] + +function createRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), 'orca-gemini-oauth-bounds-')) + roots.push(root) + return root +} + +function useDirectoryEntries(names: string[]): { + close: ReturnType + read: ReturnType +} { + let index = 0 + const directory = { + close: vi.fn(async () => undefined), + read: vi.fn(async () => { + const name = names[index] + index += 1 + return name === undefined ? null : { name } + }) + } + opendirMock.mockResolvedValueOnce(directory) + return directory +} + +function oauthSource(clientId: string, clientSecret: string): string { + return [ + `const OAUTH_CLIENT_ID = '${clientId}'`, + `const OAUTH_CLIENT_SECRET = "${clientSecret}"` + ].join('\n') +} + +beforeEach(() => { + opendirMock.mockReset() +}) + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('Gemini CLI OAuth extraction bounds', () => { + it('preserves credentials from a normal installed source file', async () => { + const sourcePath = path.join(createRoot(), 'oauth2.js') + writeFileSync(sourcePath, oauthSource('normal-client', 'normal-secret')) + + await expect(readGeminiOAuthCredentialsFile(sourcePath)).resolves.toEqual({ + clientId: 'normal-client', + clientSecret: 'normal-secret' + }) + }) + + it('skips an oversized sparse installed source file', async () => { + const sourcePath = path.join(createRoot(), 'oauth2.js') + writeFileSync(sourcePath, '') + truncateSync(sourcePath, MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES + 1) + + await expect(readGeminiOAuthCredentialsFile(sourcePath)).resolves.toBeNull() + }) + + it('accepts package metadata at the exact byte limit', async () => { + const packageRoot = createRoot() + const packagePath = path.join(packageRoot, 'package.json') + const manifest = JSON.stringify({ name: '@google/gemini-cli' }) + writeFileSync( + packagePath, + manifest + ' '.repeat(MAX_GEMINI_CLI_PACKAGE_JSON_BYTES - Buffer.byteLength(manifest)) + ) + + expect(statSync(packagePath).size).toBe(MAX_GEMINI_CLI_PACKAGE_JSON_BYTES) + await expect(findGeminiPackageRoot(path.join(packageRoot, 'bin', 'gemini'))).resolves.toBe( + packageRoot + ) + }) + + it('skips oversized sparse package metadata and keeps walking', async () => { + const packageRoot = createRoot() + const nestedRoot = path.join(packageRoot, 'nested') + mkdirSync(path.join(nestedRoot, 'bin'), { recursive: true }) + writeFileSync(path.join(packageRoot, 'package.json'), manifestForGemini()) + const oversizedManifest = path.join(nestedRoot, 'package.json') + writeFileSync(oversizedManifest, '') + truncateSync(oversizedManifest, MAX_GEMINI_CLI_PACKAGE_JSON_BYTES + 1) + + await expect(findGeminiPackageRoot(path.join(nestedRoot, 'bin', 'gemini'))).resolves.toBe( + packageRoot + ) + }) + + it('streams bundle entries and continues after an oversized sparse chunk', async () => { + const packageRoot = createRoot() + const bundleDir = path.join(packageRoot, 'bundle') + mkdirSync(bundleDir) + const oversizedPath = path.join(bundleDir, 'old.js') + writeFileSync(oversizedPath, '') + truncateSync(oversizedPath, MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES + 1) + writeFileSync(path.join(bundleDir, 'current.js'), oauthSource('bundle-client', 'bundle-secret')) + const directory = useDirectoryEntries(['old.js', 'current.js']) + + await expect(extractGeminiOAuthCredentialsFromBundleDir(packageRoot)).resolves.toEqual({ + clientId: 'bundle-client', + clientSecret: 'bundle-secret' + }) + expect(directory.read).toHaveBeenCalledTimes(2) + expect(directory.close).toHaveBeenCalledOnce() + }) + + it('admits a credential chunk at the exact directory entry limit', async () => { + const packageRoot = createRoot() + const bundleDir = path.join(packageRoot, 'bundle') + mkdirSync(bundleDir) + writeFileSync(path.join(bundleDir, 'credentials.js'), oauthSource('last-client', 'last-secret')) + const names = [ + ...Array.from({ length: MAX_GEMINI_CLI_BUNDLE_ENTRIES - 1 }, (_, index) => `${index}.txt`), + 'credentials.js' + ] + const directory = useDirectoryEntries(names) + + await expect(extractGeminiOAuthCredentialsFromBundleDir(packageRoot)).resolves.toEqual({ + clientId: 'last-client', + clientSecret: 'last-secret' + }) + expect(directory.read).toHaveBeenCalledTimes(MAX_GEMINI_CLI_BUNDLE_ENTRIES) + }) + + it('does not retain or inspect a directory entry beyond the cap', async () => { + const packageRoot = createRoot() + const bundleDir = path.join(packageRoot, 'bundle') + mkdirSync(bundleDir) + writeFileSync(path.join(bundleDir, 'too-late.js'), oauthSource('late-client', 'late-secret')) + const names = [ + ...Array.from({ length: MAX_GEMINI_CLI_BUNDLE_ENTRIES }, (_, index) => `${index}.txt`), + 'too-late.js' + ] + const directory = useDirectoryEntries(names) + + await expect(extractGeminiOAuthCredentialsFromBundleDir(packageRoot)).resolves.toBeNull() + expect(directory.read).toHaveBeenCalledTimes(MAX_GEMINI_CLI_BUNDLE_ENTRIES) + }) + + it('admits a credential chunk at the exact JavaScript file limit', async () => { + const packageRoot = createRoot() + const bundleDir = path.join(packageRoot, 'bundle') + mkdirSync(bundleDir) + writeFileSync(path.join(bundleDir, 'credentials.js'), oauthSource('file-client', 'file-secret')) + const names = [ + ...Array.from( + { length: MAX_GEMINI_CLI_BUNDLE_FILES - 1 }, + (_, index) => `missing-${index}.js` + ), + 'credentials.js' + ] + const directory = useDirectoryEntries(names) + + await expect(extractGeminiOAuthCredentialsFromBundleDir(packageRoot)).resolves.toEqual({ + clientId: 'file-client', + clientSecret: 'file-secret' + }) + expect(directory.read).toHaveBeenCalledTimes(MAX_GEMINI_CLI_BUNDLE_FILES) + }) + + it('does not inspect a JavaScript file beyond the file cap', async () => { + const packageRoot = createRoot() + const bundleDir = path.join(packageRoot, 'bundle') + mkdirSync(bundleDir) + writeFileSync(path.join(bundleDir, 'too-late.js'), oauthSource('late-client', 'late-secret')) + const names = [ + ...Array.from({ length: MAX_GEMINI_CLI_BUNDLE_FILES }, (_, index) => `missing-${index}.js`), + 'too-late.js' + ] + const directory = useDirectoryEntries(names) + + await expect(extractGeminiOAuthCredentialsFromBundleDir(packageRoot)).resolves.toBeNull() + expect(directory.read).toHaveBeenCalledTimes(MAX_GEMINI_CLI_BUNDLE_FILES) + }) +}) + +function manifestForGemini(): string { + return JSON.stringify({ name: '@google/gemini-cli' }) +} diff --git a/src/main/rate-limits/gemini-cli-oauth-extractor.ts b/src/main/rate-limits/gemini-cli-oauth-extractor.ts index 80e841e7adf..512398a2e8e 100644 --- a/src/main/rate-limits/gemini-cli-oauth-extractor.ts +++ b/src/main/rate-limits/gemini-cli-oauth-extractor.ts @@ -1,10 +1,17 @@ import { exec } from 'node:child_process' -import { access, readdir, readFile, realpath } from 'node:fs/promises' +import { access, opendir, realpath } from 'node:fs/promises' import { promisify } from 'node:util' import { homedir } from 'node:os' import path from 'node:path' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' const execAsync = promisify(exec) +const MAX_BINARY_LOOKUP_OUTPUT_BYTES = 64 * 1024 +export const MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES = 32 * 1024 * 1024 +export const MAX_GEMINI_CLI_PACKAGE_JSON_BYTES = 1024 * 1024 +export const MAX_GEMINI_CLI_BUNDLE_ENTRIES = 4_096 +export const MAX_GEMINI_CLI_BUNDLE_FILES = 512 +export const MAX_GEMINI_CLI_BUNDLE_BYTES = 128 * 1024 * 1024 async function fileExists(filePath: string): Promise { try { @@ -21,8 +28,13 @@ const OAUTH2_SUBPATH = path.join('dist', 'src', 'code_assist', 'oauth2.js') async function resolveGeminiBinary(): Promise { const whichCmd = process.platform === 'win32' ? 'where gemini' : 'which gemini' try { - const { stdout } = await execAsync(whichCmd, { encoding: 'utf-8' }) - const fromPath = stdout.trim().split(/\r?\n/)[0] + const { stdout } = await execAsync(whichCmd, { + encoding: 'utf-8', + maxBuffer: MAX_BINARY_LOOKUP_OUTPUT_BYTES + }) + const trimmedOutput = stdout.trim() + const firstLineEnd = trimmedOutput.search(/\r?\n/) + const fromPath = trimmedOutput.slice(0, firstLineEnd === -1 ? undefined : firstLineEnd) if (fromPath && (await fileExists(fromPath))) { return fromPath } @@ -69,17 +81,35 @@ function parseOAuthCredentials(content: string): { clientId: string; clientSecre return null } -async function tryReadCredentials( - filePath: string -): Promise<{ clientId: string; clientSecret: string } | null> { +type GeminiOAuthSourceRead = { + credentials: { clientId: string; clientSecret: string } | null + bytesRead: number +} + +async function readOAuthSource( + filePath: string, + maxBytes = MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES +): Promise { try { - const content = await readFile(filePath, 'utf-8') - return parseOAuthCredentials(content) + const { buffer } = await readNodeFileWithinLimit( + filePath, + Math.min(MAX_GEMINI_CLI_OAUTH_SOURCE_BYTES, maxBytes) + ) + return { + credentials: parseOAuthCredentials(buffer.toString('utf8')), + bytesRead: buffer.length + } } catch { return null } } +export async function readGeminiOAuthCredentialsFile( + filePath: string +): Promise<{ clientId: string; clientSecret: string } | null> { + return (await readOAuthSource(filePath))?.credentials ?? null +} + // Why: these are the known stable layouts for every major Gemini CLI install method. // Checking explicit paths is fast and avoids walking the entire directory tree. async function extractFromKnownPaths( @@ -131,7 +161,7 @@ async function extractFromKnownPaths( ] for (const candidate of candidates) { - const creds = await tryReadCredentials(path.normalize(candidate)) + const creds = await readGeminiOAuthCredentialsFile(path.normalize(candidate)) if (creds) { return creds } @@ -143,50 +173,79 @@ async function extractFromKnownPaths( // Why: newer Gemini CLI versions (>=0.38) ship everything bundled into hash-named // chunks with no oauth2.js source file. Scanning the bundle dir for the credential // constants is the only reliable fallback for those installs. -async function extractFromBundleDir( +export async function extractGeminiOAuthCredentialsFromBundleDir( geminiCliPackageRoot: string ): Promise<{ clientId: string; clientSecret: string } | null> { const bundleDir = path.join(geminiCliPackageRoot, 'bundle') - if (!(await fileExists(bundleDir))) { - return null - } - - let entries: string[] + let directory: Awaited> try { - entries = (await readdir(bundleDir)).filter((f) => f.endsWith('.js')) + directory = await opendir(bundleDir, { bufferSize: 32 }) } catch { return null } - for (const entry of entries) { - const creds = await tryReadCredentials(path.join(bundleDir, entry)) - if (creds) { - return creds + let inspectedEntries = 0 + let inspectedFiles = 0 + let inspectedBytes = 0 + try { + while ( + inspectedEntries < MAX_GEMINI_CLI_BUNDLE_ENTRIES && + inspectedFiles < MAX_GEMINI_CLI_BUNDLE_FILES + ) { + const entry = await directory.read() + if (!entry) { + return null + } + inspectedEntries += 1 + if (!entry.name.endsWith('.js')) { + continue + } + inspectedFiles += 1 + const remainingBytes = MAX_GEMINI_CLI_BUNDLE_BYTES - inspectedBytes + if (remainingBytes === 0) { + return null + } + const source = await readOAuthSource(path.join(bundleDir, entry.name), remainingBytes) + if (!source) { + continue + } + if (source.bytesRead > MAX_GEMINI_CLI_BUNDLE_BYTES - inspectedBytes) { + return null + } + inspectedBytes += source.bytesRead + if (source.credentials) { + return source.credentials + } + } + return null + } catch { + return null + } finally { + try { + await directory.close() + } catch { + // Cleanup failure must not mask credential discovery. } } - - return null } // Resolves the gemini-cli package root directory by walking up the directory // tree from the real binary path, looking for package.json with the right name, // or the global Node layout under lib/node_modules. -async function findGeminiPackageRoot(realGeminiPath: string): Promise { +export async function findGeminiPackageRoot(realGeminiPath: string): Promise { const MAX_ASCENTS = 8 let current = path.dirname(realGeminiPath) for (let i = 0; i <= MAX_ASCENTS; i++) { const pkgJson = path.join(current, 'package.json') - if (await fileExists(pkgJson)) { - try { - const raw = await readFile(pkgJson, 'utf-8') - const pkg = JSON.parse(raw) as { name?: string } - if (pkg.name === '@google/gemini-cli') { - return current - } - } catch { - // malformed package.json — keep walking + try { + const { buffer } = await readNodeFileWithinLimit(pkgJson, MAX_GEMINI_CLI_PACKAGE_JSON_BYTES) + const pkg = JSON.parse(buffer.toString('utf8')) as { name?: string } + if (pkg.name === '@google/gemini-cli') { + return current } + } catch { + // Missing, malformed, or oversized package.json — keep walking. } // Global Node layout: /lib/node_modules/@google/gemini-cli @@ -245,14 +304,14 @@ export async function extractOAuthClientCredentials(): Promise<{ const packageRoot = await findGeminiPackageRoot(realPath) if (packageRoot) { const fromSource = - (await tryReadCredentials( + (await readGeminiOAuthCredentialsFile( path.join(packageRoot, 'node_modules', '@google', 'gemini-cli-core', OAUTH2_SUBPATH) - )) ?? (await tryReadCredentials(path.join(packageRoot, OAUTH2_SUBPATH))) + )) ?? (await readGeminiOAuthCredentialsFile(path.join(packageRoot, OAUTH2_SUBPATH))) if (fromSource) { return fromSource } - const fromBundle = await extractFromBundleDir(packageRoot) + const fromBundle = await extractGeminiOAuthCredentialsFromBundleDir(packageRoot) if (fromBundle) { return fromBundle } diff --git a/src/main/rate-limits/gemini-oauth-sources.ts b/src/main/rate-limits/gemini-oauth-sources.ts index 386c8b281f3..b28f4383b29 100644 --- a/src/main/rate-limits/gemini-oauth-sources.ts +++ b/src/main/rate-limits/gemini-oauth-sources.ts @@ -1,8 +1,10 @@ -import { readFile, writeFile, rename } from 'node:fs/promises' +import { writeFile, rename } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' import { net } from 'electron' import { extractOAuthClientCredentials } from './gemini-cli-oauth-extractor' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' +import { readIntegrationCredentialFileText } from '../integration-credential-file' const API_TIMEOUT_MS = 10_000 const OAUTH_CREDS_PATH = path.join(homedir(), '.gemini', 'oauth_creds.json') @@ -39,7 +41,7 @@ export async function readAuthJson(): Promise { for (const candidate of candidates) { try { - const raw = await readFile(candidate, 'utf-8') + const raw = await readIntegrationCredentialFileText(candidate) return JSON.parse(raw) as AuthJson } catch (err) { if (err && typeof err === 'object' && 'code' in err && err.code === 'ENOENT') { @@ -54,7 +56,7 @@ export async function readAuthJson(): Promise { export async function readGeminiCredentials(): Promise { try { - const raw = await readFile(OAUTH_CREDS_PATH, 'utf-8') + const raw = await readIntegrationCredentialFileText(OAUTH_CREDS_PATH) const parsed = JSON.parse(raw) as unknown if ( parsed && @@ -110,11 +112,11 @@ export async function refreshAccessToken( return { accessToken: null, newRefreshToken: null } } - const data = (await res.json()) as { + const data = await readFetchResponseJsonWithinLimit<{ access_token?: string refresh_token?: string expires_in?: number - } + }>(res) return { accessToken: typeof data.access_token === 'string' ? data.access_token : null, newRefreshToken: typeof data.refresh_token === 'string' ? data.refresh_token : null, @@ -137,7 +139,7 @@ export async function loadProjectId(accessToken: string): Promise { throw new Error(`Failed to load Gemini project ID (HTTP ${res.status})`) } - const data = (await res.json()) as { cloudaicompanionProject?: string } + const data = await readFetchResponseJsonWithinLimit<{ cloudaicompanionProject?: string }>(res) if (typeof data.cloudaicompanionProject !== 'string') { throw new Error('Gemini project ID not found in API response') } diff --git a/src/main/rate-limits/gemini-usage-fetcher.fallback.test.ts b/src/main/rate-limits/gemini-usage-fetcher.fallback.test.ts index fddd0a77d59..b2acf274e70 100644 --- a/src/main/rate-limits/gemini-usage-fetcher.fallback.test.ts +++ b/src/main/rate-limits/gemini-usage-fetcher.fallback.test.ts @@ -23,13 +23,15 @@ vi.mock('./gemini-cli-oauth-extractor', () => ({ })) vi.mock('node:fs/promises', () => ({ - readFile: readFileMock, // Why: saveGeminiCredentials is exercised on the refresh path. The atomic // tmp+rename write has no observable side effect in these tests, so the // stubs just resolve. writeFile: vi.fn().mockResolvedValue(undefined), rename: vi.fn().mockResolvedValue(undefined) })) +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileText: readFileMock +})) vi.mock('electron', () => ({ net: { fetch: netFetchMock } diff --git a/src/main/rate-limits/gemini-usage-fetcher.test.ts b/src/main/rate-limits/gemini-usage-fetcher.test.ts index e8415c2e07e..16f90a338f8 100644 --- a/src/main/rate-limits/gemini-usage-fetcher.test.ts +++ b/src/main/rate-limits/gemini-usage-fetcher.test.ts @@ -22,10 +22,12 @@ vi.mock('./gemini-cli-oauth-extractor', () => ({ })) vi.mock('node:fs/promises', () => ({ - readFile: readFileMock, writeFile: vi.fn().mockResolvedValue(undefined), rename: vi.fn().mockResolvedValue(undefined) })) +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileText: readFileMock +})) vi.mock('electron', () => ({ net: { fetch: netFetchMock } })) import { fetchGeminiRateLimits } from './gemini-usage-fetcher' diff --git a/src/main/rate-limits/gemini-usage-fetcher.ts b/src/main/rate-limits/gemini-usage-fetcher.ts index 1f214b4b5eb..d437456d8c0 100644 --- a/src/main/rate-limits/gemini-usage-fetcher.ts +++ b/src/main/rate-limits/gemini-usage-fetcher.ts @@ -1,5 +1,6 @@ import { net } from 'electron' import type { ProviderRateLimits } from '../../shared/rate-limit-types' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' import { loadProjectId, readAuthJson, @@ -63,7 +64,7 @@ async function fetchQuota(accessToken: string, projectId: string): Promise(res) const buckets = deduplicateBuckets( parseQuotaResponse(data).map((b) => ({ ...buildRateLimitBucket(b), modelId: b.modelId })) ) diff --git a/src/main/rate-limits/grok-auth-file-bounds.test.ts b/src/main/rate-limits/grok-auth-file-bounds.test.ts new file mode 100644 index 00000000000..a26e39d8700 --- /dev/null +++ b/src/main/rate-limits/grok-auth-file-bounds.test.ts @@ -0,0 +1,75 @@ +import { mkdtempSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { MAX_INTEGRATION_CREDENTIAL_FILE_BYTES } from '../integration-credential-file' +import { readGrokAuthSession } from './grok-auth' + +const roots: string[] = [] + +function createGrokHome(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-grok-auth-bounds-')) + roots.push(root) + vi.stubEnv('GROK_HOME', root) + return root +} + +afterEach(() => { + vi.unstubAllEnvs() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('Grok auth file bounds', () => { + it('preserves a normal Grok auth session', () => { + const root = createGrokHome() + writeFileSync( + join(root, 'auth.json'), + JSON.stringify({ + 'https://auth.x.ai::client': { + key: 'normal-token', + email: 'alice@example.com' + } + }) + ) + + expect(readGrokAuthSession()).toMatchObject({ + status: 'ok', + session: { + accessToken: 'normal-token', + email: 'alice@example.com' + } + }) + }) + + it('accepts valid auth JSON at the exact credential byte limit', () => { + const root = createGrokHome() + const prefix = '{"https://auth.x.ai":{"key":"' + const suffix = '"}}' + const tokenLength = MAX_INTEGRATION_CREDENTIAL_FILE_BYTES - Buffer.byteLength(prefix + suffix) + const authPath = join(root, 'auth.json') + writeFileSync(authPath, `${prefix}${'x'.repeat(tokenLength)}${suffix}`) + + expect(statSync(authPath).size).toBe(MAX_INTEGRATION_CREDENTIAL_FILE_BYTES) + const result = readGrokAuthSession() + expect(result.status).toBe('ok') + if (result.status === 'ok') { + expect(result.session.accessToken).toHaveLength(tokenLength) + expect(result.session.accessToken.at(0)).toBe('x') + expect(result.session.accessToken.at(-1)).toBe('x') + } + }) + + it('rejects an oversized sparse auth file without loading its payload', () => { + const root = createGrokHome() + const authPath = join(root, 'auth.json') + writeFileSync(authPath, '') + truncateSync(authPath, MAX_INTEGRATION_CREDENTIAL_FILE_BYTES + 1) + + expect(readGrokAuthSession()).toEqual({ + status: 'error', + error: 'Unable to read Grok auth file' + }) + }) +}) diff --git a/src/main/rate-limits/grok-auth.test.ts b/src/main/rate-limits/grok-auth.test.ts index 7bf8f9e62b3..3c133a6bb73 100644 --- a/src/main/rate-limits/grok-auth.test.ts +++ b/src/main/rate-limits/grok-auth.test.ts @@ -1,21 +1,40 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as NodeFs from 'node:fs' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const authFile = vi.hoisted<{ contents: string | null; error: Error | null }>(() => ({ + contents: null, + error: null +})) + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), + existsSync: vi.fn(() => authFile.contents !== null || authFile.error !== null) +})) + +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileSyncText: vi.fn(() => { + if (authFile.error) { + throw authFile.error + } + if (authFile.contents === null) { + throw new Error('Auth file is missing') + } + return authFile.contents + }) +})) + +import { readGrokAuthSession } from './grok-auth' describe('readGrokAuthSession', () => { - afterEach(() => { - vi.resetModules() - vi.doUnmock('node:fs') + beforeEach(() => { + authFile.contents = null + authFile.error = null }) - it('redacts filesystem paths from auth read failures', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => { - throw new Error( - 'EACCES: permission denied, open /Users/brennanbenson/private/.grok/auth.json' - ) - }) - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('redacts filesystem paths from auth read failures', () => { + authFile.error = new Error( + 'EACCES: permission denied, open /Users/brennanbenson/private/.grok/auth.json' + ) expect(readGrokAuthSession()).toEqual({ status: 'error', @@ -23,22 +42,16 @@ describe('readGrokAuthSession', () => { }) }) - it('treats a token-less auth file as signed out, not an error', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => JSON.stringify({ 'https://auth.x.ai::client': { user_id: 'u1' } })) - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('treats a token-less auth file as signed out, not an error', () => { + authFile.contents = JSON.stringify({ + 'https://auth.x.ai::client': { user_id: 'u1' } + }) expect(readGrokAuthSession()).toEqual({ status: 'missing' }) }) - it('reports malformed auth JSON without parser details', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => '{') - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('reports malformed auth JSON without parser details', () => { + authFile.contents = '{' expect(readGrokAuthSession()).toEqual({ status: 'error', @@ -48,29 +61,23 @@ describe('readGrokAuthSession', () => { it.each(['https://auth.x.ai', 'https://auth.x.ai::client'])( 'prefers the %s issuer entry over an earlier alternate issuer', - async (preferredIssuer) => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => - JSON.stringify({ - 'https://stale.example.com::client': { - key: 'stale-token', - user_id: 'stale-user', - email: 'stale@example.com', - expires_at: '2099-01-01T00:00:00.000Z' - }, - [preferredIssuer]: { - key: 'live-token', - user_id: 'live-user', - email: 'live@example.com', - team_id: 'team-1', - expires_at: '2099-06-01T00:00:00.000Z', - oidc_client_id: 'client-1' - } - }) - ) - })) - const { readGrokAuthSession } = await import('./grok-auth') + (preferredIssuer) => { + authFile.contents = JSON.stringify({ + 'https://stale.example.com::client': { + key: 'stale-token', + user_id: 'stale-user', + email: 'stale@example.com', + expires_at: '2099-01-01T00:00:00.000Z' + }, + [preferredIssuer]: { + key: 'live-token', + user_id: 'live-user', + email: 'live@example.com', + team_id: 'team-1', + expires_at: '2099-06-01T00:00:00.000Z', + oidc_client_id: 'client-1' + } + }) expect(readGrokAuthSession()).toEqual({ status: 'ok', @@ -86,21 +93,15 @@ describe('readGrokAuthSession', () => { } ) - it('falls back to the first tokenized entry when no auth.x.ai key exists', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => - JSON.stringify({ - 'https://alternate.example.com::client': { - key: 'alt-token', - user_id: 'alt-user', - email: 'alt@example.com', - expires_at: '2099-01-01T00:00:00.000Z' - } - }) - ) - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('falls back to the first tokenized entry when no auth.x.ai key exists', () => { + authFile.contents = JSON.stringify({ + 'https://alternate.example.com::client': { + key: 'alt-token', + user_id: 'alt-user', + email: 'alt@example.com', + expires_at: '2099-01-01T00:00:00.000Z' + } + }) expect(readGrokAuthSession()).toEqual({ status: 'ok', @@ -115,23 +116,17 @@ describe('readGrokAuthSession', () => { }) }) - it('skips an expired auth.x.ai client entry when a fresh one follows it', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => - JSON.stringify({ - 'https://auth.x.ai::old-client': { - key: 'expired-token', - expires_at: '2020-01-01T00:00:00.000Z' - }, - 'https://auth.x.ai::current-client': { - key: 'fresh-token', - expires_at: '2099-01-01T00:00:00.000Z' - } - }) - ) - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('skips an expired auth.x.ai client entry when a fresh one follows it', () => { + authFile.contents = JSON.stringify({ + 'https://auth.x.ai::old-client': { + key: 'expired-token', + expires_at: '2020-01-01T00:00:00.000Z' + }, + 'https://auth.x.ai::current-client': { + key: 'fresh-token', + expires_at: '2099-01-01T00:00:00.000Z' + } + }) expect(readGrokAuthSession()).toMatchObject({ status: 'ok', @@ -139,17 +134,11 @@ describe('readGrokAuthSession', () => { }) }) - it('does not resurrect an alternate issuer when an auth.x.ai entry is tokenless', async () => { - vi.doMock('node:fs', () => ({ - existsSync: vi.fn(() => true), - readFileSync: vi.fn(() => - JSON.stringify({ - 'https://alternate.example.com::client': { key: 'stale-token' }, - 'https://auth.x.ai::client': { user_id: 'signed-out-user' } - }) - ) - })) - const { readGrokAuthSession } = await import('./grok-auth') + it('does not resurrect an alternate issuer when an auth.x.ai entry is tokenless', () => { + authFile.contents = JSON.stringify({ + 'https://alternate.example.com::client': { key: 'stale-token' }, + 'https://auth.x.ai::client': { user_id: 'signed-out-user' } + }) expect(readGrokAuthSession()).toEqual({ status: 'missing' }) }) diff --git a/src/main/rate-limits/grok-auth.ts b/src/main/rate-limits/grok-auth.ts index e055c022975..5f2d5708121 100644 --- a/src/main/rate-limits/grok-auth.ts +++ b/src/main/rate-limits/grok-auth.ts @@ -1,6 +1,7 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' import { resolveGrokHomeDir } from '../../shared/grok-session-paths' +import { readIntegrationCredentialFileSyncText } from '../integration-credential-file' // Why: when GROK_HOME is set, auth.json must be the same path Grok CLI uses. export function getGrokHome(): string { @@ -88,7 +89,7 @@ export function readGrokAuthSession(): GrokAuthReadResult { return { status: 'missing' } } try { - const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + const parsed: unknown = JSON.parse(readIntegrationCredentialFileSyncText(path)) if (typeof parsed !== 'object' || parsed === null) { return { status: 'error', error: 'Grok auth file is invalid' } } diff --git a/src/main/rate-limits/grok-fetcher.test.ts b/src/main/rate-limits/grok-fetcher.test.ts index 63d7bc5cfcd..115620352a3 100644 --- a/src/main/rate-limits/grok-fetcher.test.ts +++ b/src/main/rate-limits/grok-fetcher.test.ts @@ -11,8 +11,11 @@ vi.mock('electron', () => ({ })) vi.mock('node:fs', () => ({ - existsSync: () => authState.file !== null, - readFileSync: () => { + existsSync: () => authState.file !== null +})) + +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileSyncText: () => { if (authState.readError) { throw authState.readError } diff --git a/src/main/rate-limits/grok-fetcher.ts b/src/main/rate-limits/grok-fetcher.ts index 366c75c3835..cd50412b38c 100644 --- a/src/main/rate-limits/grok-fetcher.ts +++ b/src/main/rate-limits/grok-fetcher.ts @@ -10,6 +10,7 @@ import { type GrokAuthReadResult, type GrokAuthSession } from './grok-auth' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' // Why: billing URL and headers must match Grok CLI or xAI rejects the request. const GROK_CLI_PROXY_BASE = @@ -211,7 +212,7 @@ async function fetchBillingData( result: result('error', `Grok usage request failed (HTTP ${res.status})`) } } - const data: unknown = await res.json() + const data = await readFetchResponseJsonWithinLimit(res) return { kind: 'data', data: typeof data === 'object' && data !== null ? (data as GrokBillingResponse) : {} diff --git a/src/main/rate-limits/hidden-pty-cleanup.test.ts b/src/main/rate-limits/hidden-pty-cleanup.test.ts index 43e8453fa0f..dc6e9a75e6b 100644 --- a/src/main/rate-limits/hidden-pty-cleanup.test.ts +++ b/src/main/rate-limits/hidden-pty-cleanup.test.ts @@ -2,6 +2,8 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { cleanupHiddenRateLimitPty, getActiveHiddenRateLimitPtyCount, + HiddenRateLimitPtyCapacityError, + MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS, registerHiddenRateLimitPty } from './hidden-pty-cleanup' @@ -132,4 +134,29 @@ describe('cleanupHiddenRateLimitPty', () => { expect(getActiveHiddenRateLimitPtyCount()).toBe(0) }) + + it('caps active hidden PTYs, tears down overflow, and recovers after disposal', () => { + setPlatform('darwin') + const registrations = Array.from({ length: MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS }, () => + registerHiddenRateLimitPty({ kill: vi.fn(), destroy: vi.fn() }) + ) + expect(getActiveHiddenRateLimitPtyCount()).toBe(MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS) + + const overflowKill = vi.fn() + const overflow = { kill: overflowKill, destroy: vi.fn() } + expect(() => registerHiddenRateLimitPty(overflow)).toThrow(HiddenRateLimitPtyCapacityError) + expect(overflowKill).toHaveBeenCalledOnce() + expect(overflow.destroy).toHaveBeenCalledOnce() + expect(getActiveHiddenRateLimitPtyCount()).toBe(MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS) + + registrations[0]!.dispose() + const recovered = registerHiddenRateLimitPty({ kill: vi.fn(), destroy: vi.fn() }) + expect(getActiveHiddenRateLimitPtyCount()).toBe(MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS) + + for (const registration of registrations.slice(1)) { + registration.dispose() + } + recovered.dispose() + expect(getActiveHiddenRateLimitPtyCount()).toBe(0) + }) }) diff --git a/src/main/rate-limits/hidden-pty-cleanup.ts b/src/main/rate-limits/hidden-pty-cleanup.ts index 068b937ee84..0b95f85b83f 100644 --- a/src/main/rate-limits/hidden-pty-cleanup.ts +++ b/src/main/rate-limits/hidden-pty-cleanup.ts @@ -7,9 +7,24 @@ type Disposable = { dispose: () => void } +export const MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS = 16 const activeHiddenRateLimitPtys = new Set() +export class HiddenRateLimitPtyCapacityError extends Error { + constructor() { + super(`Hidden rate-limit PTY capacity exceeds ${MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS}`) + this.name = 'HiddenRateLimitPtyCapacityError' + } +} + export function registerHiddenRateLimitPty(term: HiddenPty): Disposable { + if ( + !activeHiddenRateLimitPtys.has(term) && + activeHiddenRateLimitPtys.size >= MAX_ACTIVE_HIDDEN_RATE_LIMIT_PTYS + ) { + cleanupHiddenRateLimitPty(term, [], { kill: true }) + throw new HiddenRateLimitPtyCapacityError() + } activeHiddenRateLimitPtys.add(term) return { dispose: () => { diff --git a/src/main/rate-limits/kimi-fetcher.test.ts b/src/main/rate-limits/kimi-fetcher.test.ts index 58f11dfdb03..2ef7a137996 100644 --- a/src/main/rate-limits/kimi-fetcher.test.ts +++ b/src/main/rate-limits/kimi-fetcher.test.ts @@ -12,7 +12,12 @@ vi.mock('electron', () => ({ vi.mock('node:fs', () => ({ existsSync: () => fsState.credentials !== null, - readFileSync: () => { + writeFileSync: () => {}, + renameSync: () => {} +})) + +vi.mock('../integration-credential-file', () => ({ + readIntegrationCredentialFileSyncText: () => { if (fsState.readError) { throw fsState.readError } @@ -20,9 +25,7 @@ vi.mock('node:fs', () => ({ throw new Error('ENOENT') } return fsState.credentials - }, - writeFileSync: () => {}, - renameSync: () => {} + } })) vi.mock('node:os', () => ({ homedir: () => '/home/test' })) diff --git a/src/main/rate-limits/kimi-fetcher.ts b/src/main/rate-limits/kimi-fetcher.ts index 778b90b796c..29321b69fc5 100644 --- a/src/main/rate-limits/kimi-fetcher.ts +++ b/src/main/rate-limits/kimi-fetcher.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import { net } from 'electron' @@ -7,6 +7,8 @@ import type { RateLimitWindow, UsageRateLimitMetadata } from '../../shared/rate-limit-types' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' +import { readIntegrationCredentialFileSyncText } from '../integration-credential-file' // Why: Kimi Code's managed coding plan exposes subscription usage at // `${base}/usages` (see packages/oauth/src/managed-usage.ts in the CLI bundle). @@ -58,7 +60,7 @@ function readCredentials(): CredentialsReadResult { return { status: 'missing' } } try { - const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + const parsed: unknown = JSON.parse(readIntegrationCredentialFileSyncText(path)) const credentials = parseCredentials(parsed) return credentials ? { status: 'ok', credentials } @@ -275,7 +277,7 @@ export async function fetchKimiRateLimits(): Promise { if (!res.ok) { return result('error', `Kimi usage request failed (HTTP ${res.status})`) } - const data: unknown = await res.json() + const data = await readFetchResponseJsonWithinLimit(res) return mapUsageResponse(typeof data === 'object' && data !== null ? data : {}) } catch (err) { return result('error', err instanceof Error ? err.message : 'Kimi usage request failed') diff --git a/src/main/rate-limits/minimax-fetcher.ts b/src/main/rate-limits/minimax-fetcher.ts index 20348feee72..3d26b799cf9 100644 --- a/src/main/rate-limits/minimax-fetcher.ts +++ b/src/main/rate-limits/minimax-fetcher.ts @@ -1,4 +1,5 @@ import type { ProviderRateLimits, RateLimitWindow } from '../../shared/rate-limit-types' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' import { extractMiniMaxCookieValue, fetchMiniMaxWithManualCookieHeader, @@ -244,7 +245,7 @@ export async function fetchMiniMaxRateLimits( } let payload: MiniMaxUsageResponse try { - payload = (await fetchResult.response.json()) as MiniMaxUsageResponse + payload = await readFetchResponseJsonWithinLimit(fetchResult.response) } catch (error) { const message = error instanceof Error ? error.message : 'Invalid MiniMax usage response' return makeError(redactMiniMaxSecret(message), 'parse') diff --git a/src/main/rate-limits/minimax-request-context.test.ts b/src/main/rate-limits/minimax-request-context.test.ts index 9b01b4ff013..45ff8ff88e1 100644 --- a/src/main/rate-limits/minimax-request-context.test.ts +++ b/src/main/rate-limits/minimax-request-context.test.ts @@ -27,6 +27,7 @@ import { getUniqueMiniMaxCookieNames, logMiniMaxFetchFailure, makeMiniMaxRequestHeaders, + MINIMAX_COOKIE_WRITE_CONCURRENCY, MINIMAX_USAGE_ENDPOINT, normalizeMiniMaxCookieHeader, redactMiniMaxSecret @@ -164,7 +165,7 @@ describe('makeMiniMaxRequestHeaders', () => { describe('fetchMiniMaxWithSessionCookieJar', () => { beforeEach(() => { clearStorageDataMock.mockClear() - cookiesSetMock.mockClear() + cookiesSetMock.mockReset().mockResolvedValue(undefined) netFetchMock.mockReset() sessionFromPartitionMock.mockClear() sessionFromPartitionMock.mockImplementation(() => ({ @@ -276,6 +277,42 @@ describe('fetchMiniMaxWithSessionCookieJar', () => { ) }) + it.each([ + ['at the limit', MINIMAX_COOKIE_WRITE_CONCURRENCY], + ['above the limit', MINIMAX_COOKIE_WRITE_CONCURRENCY + 1] + ])('bounds session cookie writes %s', async (_, count) => { + netFetchMock.mockResolvedValueOnce({ ok: true, status: 200 }) + let active = 0 + let peak = 0 + const releases: (() => void)[] = [] + cookiesSetMock.mockImplementation(async () => { + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + }) + + const fetchResult = fetchMiniMaxWithSessionCookieJar({ + cookie: Array.from({ length: count }, (_, index) => `cookie_${index}=value`).join('; '), + endpoint: MINIMAX_USAGE_ENDPOINT, + groupId: null, + signal: new AbortController().signal + }) + await vi.waitFor(() => + expect(cookiesSetMock).toHaveBeenCalledTimes( + Math.min(count, MINIMAX_COOKIE_WRITE_CONCURRENCY) + ) + ) + if (count > MINIMAX_COOKIE_WRITE_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(cookiesSetMock).toHaveBeenCalledTimes(count)) + } + releases.splice(0).forEach((release) => release()) + + await fetchResult + expect(peak).toBe(Math.min(count, MINIMAX_COOKIE_WRITE_CONCURRENCY)) + }) + it('reports the transport name as session-cookie-jar on success', async () => { netFetchMock.mockResolvedValueOnce({ ok: true, @@ -303,7 +340,7 @@ describe('fetchMiniMaxWithSessionCookieJar', () => { describe('fetchMiniMaxWithManualCookieHeader', () => { beforeEach(() => { clearStorageDataMock.mockClear() - cookiesSetMock.mockClear() + cookiesSetMock.mockReset().mockResolvedValue(undefined) netFetchMock.mockReset() sessionFromPartitionMock.mockClear() sessionFromPartitionMock.mockImplementation(() => ({ diff --git a/src/main/rate-limits/minimax-request-context.ts b/src/main/rate-limits/minimax-request-context.ts index 34dcd47130b..47fce7018ce 100644 --- a/src/main/rate-limits/minimax-request-context.ts +++ b/src/main/rate-limits/minimax-request-context.ts @@ -1,4 +1,5 @@ import { session, type Session } from 'electron' +import { mapSettledWithConcurrency } from '../../shared/map-with-concurrency' export const MINIMAX_USAGE_ENDPOINT = 'https://platform.minimax.io/v1/api/openplatform/coding_plan/remains' @@ -6,6 +7,7 @@ export const MINIMAX_USAGE_ENDPOINT = const MINIMAX_ORIGIN = 'https://platform.minimax.io' const MINIMAX_REFERER = 'https://platform.minimax.io/console/usage' const MINIMAX_SESSION_PARTITION = 'orca-minimax-rate-limit-fetch' +export const MINIMAX_COOKIE_WRITE_CONCURRENCY = 8 const SENSITIVE_COOKIE_NAMES = new Set([ '_token', '_twpid', @@ -122,8 +124,10 @@ export async function fetchMiniMaxWithSessionCookieJar(args: { const cookiePairs = parseCookiePairs(args.cookie) try { await clearMiniMaxSessionCookieJarForSession(miniMaxSession) - await Promise.all( - cookiePairs.map((pair) => + const writes = await mapSettledWithConcurrency( + cookiePairs, + MINIMAX_COOKIE_WRITE_CONCURRENCY, + (pair) => miniMaxSession.cookies.set({ url: MINIMAX_ORIGIN, name: pair.name, @@ -131,8 +135,13 @@ export async function fetchMiniMaxWithSessionCookieJar(args: { secure: true, path: '/' }) - ) ) + const failedWrite = writes.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failedWrite) { + throw failedWrite.reason + } const headers = makeMiniMaxRequestHeaders(args.groupId) return { response: await miniMaxSession.fetch(args.endpoint, { diff --git a/src/main/rate-limits/opencode-go-usage-fetcher.ts b/src/main/rate-limits/opencode-go-usage-fetcher.ts index 934d1e72b42..b0713507dbd 100644 --- a/src/main/rate-limits/opencode-go-usage-fetcher.ts +++ b/src/main/rate-limits/opencode-go-usage-fetcher.ts @@ -8,6 +8,7 @@ import { OPENCODE_BASE_URL } from './opencode-go-request-session' import { parseSubscriptionFromPageText } from './opencode-go-page-scraper' +import { readFetchResponseTextWithinLimit } from '../lib/fetch-response-body' const OPENCODE_SERVER_URL = 'https://opencode.ai/_server' const API_TIMEOUT_MS = 15_000 @@ -201,7 +202,7 @@ async function fetchOpenCodeGoRateLimitsWithSession( } } - const workspacesText = await workspacesRes.text() + const workspacesText = await readFetchResponseTextWithinLimit(workspacesRes) ids = parseWorkspaceIds(workspacesText) } catch (err) { const message = err instanceof Error ? err.message : 'Unknown error' @@ -251,7 +252,7 @@ async function fetchOpenCodeGoRateLimitsWithSession( continue } - const pageText = await pageRes.text() + const pageText = await readFetchResponseTextWithinLimit(pageRes) const parsed = parseSubscriptionFromPageText(pageText) if (parsed) { const monthly = diff --git a/src/main/rate-limits/rate-limit-pty-output-tail.test.ts b/src/main/rate-limits/rate-limit-pty-output-tail.test.ts new file mode 100644 index 00000000000..672c890b493 --- /dev/null +++ b/src/main/rate-limits/rate-limit-pty-output-tail.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest' +import { appendRateLimitPtyOutputTail } from './rate-limit-pty-output-tail' + +describe('appendRateLimitPtyOutputTail', () => { + it('preserves ordinary output exactly', () => { + const first = appendRateLimitPtyOutputTail('', 'first', 16) + const second = appendRateLimitPtyOutputTail(first.output, ' second', 16) + + expect(second).toEqual({ + output: 'first second', + scannedChunk: ' second' + }) + }) + + it('keeps the exact newest characters across chunks', () => { + const first = appendRateLimitPtyOutputTail('', '123456', 8) + const second = appendRateLimitPtyOutputTail(first.output, '789tail', 8) + + expect(second.output).toBe('6789tail') + }) + + it('copies only a bounded suffix from one oversized PTY chunk', () => { + const result = appendRateLimitPtyOutputTail('old', `HEAD${'x'.repeat(1_000_000)}TAIL`, 16) + + expect(result.output).toHaveLength(16) + expect(result.output).toBe(result.scannedChunk) + expect(result.output).toBe(`${'x'.repeat(12)}TAIL`) + expect(result.output).not.toContain('HEAD') + }) + + it('preserves UTF-16 code units at the retained boundary', () => { + const result = appendRateLimitPtyOutputTail('', `${'x'.repeat(20)}😀`, 2) + + expect(result.output).toBe('😀') + }) +}) diff --git a/src/main/rate-limits/rate-limit-pty-output-tail.ts b/src/main/rate-limits/rate-limit-pty-output-tail.ts new file mode 100644 index 00000000000..2e3a608aeb2 --- /dev/null +++ b/src/main/rate-limits/rate-limit-pty-output-tail.ts @@ -0,0 +1,33 @@ +export type RateLimitPtyOutputAppend = { + output: string + scannedChunk: string +} + +export function appendRateLimitPtyOutputTail( + existing: string, + chunk: string, + maxChars: number +): RateLimitPtyOutputAppend { + if (!Number.isSafeInteger(maxChars) || maxChars < 0) { + throw new RangeError('Rate-limit PTY output limit must be a non-negative safe integer') + } + if (maxChars === 0) { + return { output: '', scannedChunk: '' } + } + + const scannedChunk = chunk.length > maxChars ? copyUtf16Suffix(chunk, maxChars) : chunk + if (scannedChunk.length >= maxChars) { + return { output: scannedChunk, scannedChunk } + } + const existingBudget = maxChars - scannedChunk.length + const existingTail = existing.length > existingBudget ? existing.slice(-existingBudget) : existing + return { + output: `${existingTail}${scannedChunk}`, + scannedChunk + } +} + +function copyUtf16Suffix(value: string, maxChars: number): string { + // Why: a sliced string may retain the oversized PTY chunk's backing store; this bounded round trip detaches it. + return Buffer.from(value.slice(-maxChars), 'utf16le').toString('utf16le') +} diff --git a/src/main/rate-limits/service.test.ts b/src/main/rate-limits/service.test.ts index f9b113927f0..4a6539075a9 100644 --- a/src/main/rate-limits/service.test.ts +++ b/src/main/rate-limits/service.test.ts @@ -5,7 +5,12 @@ Keeping them in one file makes the ordering contract reviewable as a unit. */ import { beforeEach, describe, expect, it, vi } from 'vitest' import { EventEmitter } from 'node:events' import type { ProviderRateLimits } from '../../shared/rate-limit-types' -import { RateLimitService } from './service' +import { + MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES, + MAX_INACTIVE_RATE_LIMIT_ACCOUNTS, + RateLimitFetchCycleCapacityError, + RateLimitService +} from './service' import { fetchClaudeRateLimits, fetchManagedAccountUsage } from './claude-fetcher' import { fetchCodexRateLimits } from './codex-fetcher' import { fetchGeminiRateLimits } from './gemini-usage-fetcher' @@ -1220,6 +1225,60 @@ describe('RateLimitService', () => { expect(fetchCodexRateLimits).toHaveBeenCalledTimes(2) }) + it('shares one idle promise across a burst of queued refresh waiters and recovers', async () => { + const service = new RateLimitService() + const internals = service as unknown as { + fetchIdlePromise: Promise | null + isFetching: boolean + resolveFetchIdleWaiters: () => void + waitForFetchIdle: () => Promise + } + internals.isFetching = true + const shared = internals.waitForFetchIdle() + + for (let index = 0; index < 10_000; index += 1) { + expect(internals.waitForFetchIdle()).toBe(shared) + } + expect(internals.fetchIdlePromise).toBe(shared) + + internals.isFetching = false + internals.resolveFetchIdleWaiters() + await expect(shared).resolves.toBeUndefined() + expect(internals.fetchIdlePromise).toBeNull() + + internals.isFetching = true + const recovered = internals.waitForFetchIdle() + expect(recovered).not.toBe(shared) + internals.isFetching = false + internals.resolveFetchIdleWaiters() + await expect(recovered).resolves.toBeUndefined() + }) + + it('caps active fetch controllers and recovers when a cycle finishes', () => { + const service = new RateLimitService() + const internals = service as unknown as { + activeFetchAbortControllers: Set + beginFetchCycle: () => AbortController + finishFetchCycle: (controller: AbortController) => void + } + const controllers = Array.from({ length: MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES }, () => + internals.beginFetchCycle() + ) + + expect(internals.activeFetchAbortControllers.size).toBe(MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES) + expect(() => internals.beginFetchCycle()).toThrow(RateLimitFetchCycleCapacityError) + + internals.finishFetchCycle(controllers[0]!) + const recovered = internals.beginFetchCycle() + expect(internals.activeFetchAbortControllers.size).toBe(MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES) + + for (const controller of controllers.slice(1)) { + internals.finishFetchCycle(controller) + } + internals.finishFetchCycle(recovered) + expect(internals.activeFetchAbortControllers.size).toBe(0) + }) + it('publishes non-Grok provider results before a slow Grok fetch completes', async () => { const service = new RateLimitService() const grok = deferred() @@ -1747,6 +1806,24 @@ describe('RateLimitService', () => { await firstFetch }) + it('caps inactive Codex preview state at the account admission limit', async () => { + const service = new RateLimitService() + const accounts = Array.from({ length: MAX_INACTIVE_RATE_LIMIT_ACCOUNTS + 1 }, (_, index) => ({ + id: `account-${index}`, + managedHomePath: `/tmp/account-${index}/home` + })) + service.setInactiveCodexAccountsResolver(() => accounts) + vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 50, Date.now())) + + await service.fetchInactiveCodexAccountsOnOpen() + + expect(fetchCodexRateLimits).toHaveBeenCalledTimes(MAX_INACTIVE_RATE_LIMIT_ACCOUNTS) + expect(service.getState().inactiveCodexAccounts).toHaveLength(MAX_INACTIVE_RATE_LIMIT_ACCOUNTS) + expect(service.getState().inactiveCodexAccounts.at(-1)?.accountId).toBe( + `account-${MAX_INACTIVE_RATE_LIMIT_ACCOUNTS - 1}` + ) + }) + it('keeps sibling inactive Codex preview fetches alive when one account is evicted', async () => { const service = new RateLimitService() const accountFetch = deferred() diff --git a/src/main/rate-limits/service.ts b/src/main/rate-limits/service.ts index 610e6542a68..40bea38156d 100644 --- a/src/main/rate-limits/service.ts +++ b/src/main/rate-limits/service.ts @@ -91,6 +91,16 @@ const RATE_LIMITED_STALE_THRESHOLD_MS = 24 * 60 * 60 * 1000 const LIVE_CLAUDE_INGEST_DEDUPE_MS = 30 * 1000 const INACTIVE_FETCH_DEBOUNCE_MS = 60 * 1000 // 60 seconds — debounce fetch-on-open const DEFERRED_STARTUP_ACTIVE_REFRESH_MS = 1000 +export const MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES = 8 +export const MAX_INACTIVE_RATE_LIMIT_ACCOUNTS = 256 +export const MAX_RATE_LIMIT_ACCOUNT_ID_BYTES = 1024 + +export class RateLimitFetchCycleCapacityError extends Error { + constructor() { + super(`Active rate-limit fetch cycles exceed ${MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES}`) + this.name = 'RateLimitFetchCycleCapacityError' + } +} // Why: inactive account arrays are derived from provider caches on demand in getState()/pushToRenderer(). type InternalRateLimitState = { @@ -111,6 +121,36 @@ function normalizePollingInterval(ms: number): number { return Math.min(MAX_POLL_MS, Math.max(MIN_POLL_MS, ms)) } +function boundedInactiveAccounts(accounts: T[]): T[] { + const bounded: T[] = [] + const inspectedCount = Math.min(accounts.length, MAX_INACTIVE_RATE_LIMIT_ACCOUNTS) + for (let index = 0; index < inspectedCount; index += 1) { + const account = accounts[index] + if (account && Buffer.byteLength(account.id, 'utf8') <= MAX_RATE_LIMIT_ACCOUNT_ID_BYTES) { + bounded.push(account) + } + } + return bounded +} + +function hasBoundedInactiveAccount( + accounts: T[], + accountId: string +): boolean { + const inspectedCount = Math.min(accounts.length, MAX_INACTIVE_RATE_LIMIT_ACCOUNTS) + for (let index = 0; index < inspectedCount; index += 1) { + const account = accounts[index] + if ( + account && + Buffer.byteLength(account.id, 'utf8') <= MAX_RATE_LIMIT_ACCOUNT_ID_BYTES && + account.id === accountId + ) { + return true + } + } + return false +} + function isSystemDefaultClaudeAuth( authPreparation: ClaudeRuntimeAuthPreparation | undefined ): boolean { @@ -187,7 +227,8 @@ export class RateLimitService { private claudeOnlyFetchQueued = false private grokOnlyFetchQueued = false private activeFetchAbortControllers = new Set() - private fetchIdleResolvers: (() => void)[] = [] + private fetchIdlePromise: Promise | null = null + private resolveFetchIdlePromise: (() => void) | null = null private codexFetchGeneration = 0 private claudeFetchGeneration = 0 // Why: statusline ingest must attribute live windows to the selected account without re-running the side-effectful auth sync per post. @@ -494,7 +535,7 @@ export class RateLimitService { if (this.inactiveClaudeFetching.size > 0) { return } - const accounts = this.inactiveClaudeAccountsResolver?.() ?? [] + const accounts = boundedInactiveAccounts(this.inactiveClaudeAccountsResolver?.() ?? []) if (accounts.length === 0) { return } @@ -571,7 +612,7 @@ export class RateLimitService { if (this.inactiveCodexFetching.size > 0) { return } - const accounts = this.inactiveCodexAccountsResolver?.() ?? [] + const accounts = boundedInactiveAccounts(this.inactiveCodexAccountsResolver?.() ?? []) if (accounts.length === 0) { return } @@ -651,20 +692,18 @@ export class RateLimitService { } private isCurrentInactiveClaudeAccount(accountId: string): boolean { - return (this.inactiveClaudeAccountsResolver?.() ?? []).some( - (account) => account.id === accountId - ) + return hasBoundedInactiveAccount(this.inactiveClaudeAccountsResolver?.() ?? [], accountId) } private isCurrentInactiveCodexAccount(accountId: string): boolean { - return (this.inactiveCodexAccountsResolver?.() ?? []).some( - (account) => account.id === accountId - ) + return hasBoundedInactiveAccount(this.inactiveCodexAccountsResolver?.() ?? [], accountId) } private pruneInactiveClaudeState(): void { const currentIds = new Set( - (this.inactiveClaudeAccountsResolver?.() ?? []).map((account) => account.id) + boundedInactiveAccounts(this.inactiveClaudeAccountsResolver?.() ?? []).map( + (account) => account.id + ) ) for (const accountId of this.inactiveClaudeCache.keys()) { if (!currentIds.has(accountId)) { @@ -680,7 +719,9 @@ export class RateLimitService { private pruneInactiveCodexState(): void { const currentIds = new Set( - (this.inactiveCodexAccountsResolver?.() ?? []).map((account) => account.id) + boundedInactiveAccounts(this.inactiveCodexAccountsResolver?.() ?? []).map( + (account) => account.id + ) ) for (const accountId of this.inactiveCodexCache.keys()) { if (!currentIds.has(accountId)) { @@ -1122,9 +1163,10 @@ export class RateLimitService { return Promise.resolve() } // Why: explicit-refresh callers must await the queued follow-up cycle when a poll is in flight, else the UI stops spinning early. - return new Promise((resolve) => { - this.fetchIdleResolvers.push(resolve) + this.fetchIdlePromise ??= new Promise((resolve) => { + this.resolveFetchIdlePromise = resolve }) + return this.fetchIdlePromise } private resolveFetchIdleWaiters(): void { @@ -1137,14 +1179,16 @@ export class RateLimitService { ) { return } - const resolvers = this.fetchIdleResolvers - this.fetchIdleResolvers = [] - for (const resolve of resolvers) { - resolve() - } + const resolve = this.resolveFetchIdlePromise + this.fetchIdlePromise = null + this.resolveFetchIdlePromise = null + resolve?.() } private beginFetchCycle(): AbortController { + if (this.activeFetchAbortControllers.size >= MAX_ACTIVE_RATE_LIMIT_FETCH_CYCLES) { + throw new RateLimitFetchCycleCapacityError() + } const controller = new AbortController() this.activeFetchAbortControllers.add(controller) return controller @@ -1181,11 +1225,10 @@ export class RateLimitService { } private resolveAndClearFetchIdleWaiters(): void { - const resolvers = this.fetchIdleResolvers - this.fetchIdleResolvers = [] - for (const resolve of resolvers) { - resolve() - } + const resolve = this.resolveFetchIdlePromise + this.fetchIdlePromise = null + this.resolveFetchIdlePromise = null + resolve?.() } private isSameCodexTarget( diff --git a/src/main/remote-agent-trust-presets.test.ts b/src/main/remote-agent-trust-presets.test.ts index 226d689a537..92770a09086 100644 --- a/src/main/remote-agent-trust-presets.test.ts +++ b/src/main/remote-agent-trust-presets.test.ts @@ -14,6 +14,7 @@ vi.mock('./providers/ssh-filesystem-dispatch', () => ({ })) const { markRemoteAgentWorkspaceTrusted } = await import('./remote-agent-trust-presets') +const { MAX_AGENT_STATE_FILE_BYTES } = await import('./agent-state-file-reader') function makeFsProvider(overrides: Record = {}) { return { @@ -144,6 +145,27 @@ describe('markRemoteAgentWorkspaceTrusted', () => { }) }) + it('rejects oversized remote config without overwriting it', async () => { + const fsProvider = makeFsProvider({ + readFile: vi.fn(async () => ({ + content: 'x'.repeat(MAX_AGENT_STATE_FILE_BYTES + 1), + isBinary: false + })) + }) + mocks.getSshFilesystemProvider.mockReturnValue(fsProvider) + + await expect( + markRemoteAgentWorkspaceTrusted({ + preset: 'codex', + connectionId: 'ssh-1', + workspacePath: '/repo/worktree' + }) + ).rejects.toThrow('Remote agent state exceeds') + + expect(fsProvider.createDir).not.toHaveBeenCalled() + expect(fsProvider.writeFile).not.toHaveBeenCalled() + }) + it('does nothing when the SSH home cannot be resolved safely', async () => { const fsProvider = makeFsProvider() mocks.getActiveMultiplexer.mockReturnValue({ diff --git a/src/main/remote-agent-trust-presets.ts b/src/main/remote-agent-trust-presets.ts index a88aaceae58..d7ccf96a975 100644 --- a/src/main/remote-agent-trust-presets.ts +++ b/src/main/remote-agent-trust-presets.ts @@ -7,6 +7,14 @@ import { isWindowsAbsolutePathLike, normalizeRuntimePathSeparators } from '../shared/cross-platform-path' +import { MAX_AGENT_STATE_FILE_BYTES } from './agent-state-file-reader' + +class RemoteAgentStateFileTooLargeError extends Error { + constructor() { + super(`Remote agent state exceeds ${MAX_AGENT_STATE_FILE_BYTES} bytes`) + this.name = 'RemoteAgentStateFileTooLargeError' + } +} export async function markRemoteAgentWorkspaceTrusted(args: { preset: AgentTrustPreset @@ -69,12 +77,27 @@ async function readRemoteTextFile( ): Promise { try { const result = await fsProvider.readFile(filePath) - return result.isBinary ? '' : result.content - } catch { + if (result.isBinary) { + return '' + } + if (Buffer.byteLength(result.content, 'utf8') > MAX_AGENT_STATE_FILE_BYTES) { + throw new RemoteAgentStateFileTooLargeError() + } + return result.content + } catch (error) { + if (error instanceof RemoteAgentStateFileTooLargeError) { + throw error + } return '' } } +function assertRemoteAgentStateWithinLimit(content: string): void { + if (Buffer.byteLength(content, 'utf8') > MAX_AGENT_STATE_FILE_BYTES) { + throw new RemoteAgentStateFileTooLargeError() + } +} + async function markRemoteCodexProjectTrusted( fsProvider: IFilesystemProvider, remoteHome: string, @@ -91,6 +114,7 @@ async function markRemoteCodexProjectTrusted( if (updated === existing) { return } + assertRemoteAgentStateWithinLimit(updated) await fsProvider.createDir(codexDir) await fsProvider.writeFile(configPath, updated) } @@ -113,10 +137,9 @@ async function markRemoteCursorWorkspaceTrusted( // Missing marker: write the same shape the local trust preset writes. } await fsProvider.createDir(trustDir) - await fsProvider.writeFile( - trustFile, - `${JSON.stringify({ trustedAt: new Date().toISOString(), workspacePath }, null, 2)}\n` - ) + const content = `${JSON.stringify({ trustedAt: new Date().toISOString(), workspacePath }, null, 2)}\n` + assertRemoteAgentStateWithinLimit(content) + await fsProvider.writeFile(trustFile, content) } async function markRemoteCopilotFolderTrusted( @@ -143,6 +166,8 @@ async function markRemoteCopilotFolderTrusted( return } config.trustedFolders = [...existing.filter((entry) => typeof entry === 'string'), workspacePath] + const content = `${JSON.stringify(config, null, 2)}\n` + assertRemoteAgentStateWithinLimit(content) await fsProvider.createDir(configDir) - await fsProvider.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`) + await fsProvider.writeFile(configPath, content) } diff --git a/src/main/repo-git-remote-identity-enrichment.test.ts b/src/main/repo-git-remote-identity-enrichment.test.ts index 81a8d2dbd55..c30fff555f5 100644 --- a/src/main/repo-git-remote-identity-enrichment.test.ts +++ b/src/main/repo-git-remote-identity-enrichment.test.ts @@ -5,6 +5,9 @@ import { detectGitRemoteIdentity } from './repo-git-remote-identity' import { enrichMissingRepoGitRemoteIdentities, flushRepoGitRemoteIdentityEnrichmentForTests, + getRepoGitRemoteIdentityNegativeCacheSizeForTests, + REPO_LOCATION_CACHE_KEY_MAX_BYTES, + REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES, resetRepoGitRemoteIdentityEnrichmentForTests } from './repo-git-remote-identity-enrichment' @@ -120,6 +123,44 @@ describe('enrichMissingRepoGitRemoteIdentities', () => { expect(detectGitRemoteIdentity).toHaveBeenCalledTimes(1) }) + it('bounds negative results across churned repo locations', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + vi.mocked(detectGitRemoteIdentity).mockResolvedValue(null) + const repo = makeRepo() + const store = makeStore(repo) + + for (let index = 0; index <= REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES; index++) { + repo.path = `/workspace/repo-${index}` + enrichMissingRepoGitRemoteIdentities(store) + await flushRepoGitRemoteIdentityEnrichmentForTests() + } + + expect(getRepoGitRemoteIdentityNegativeCacheSizeForTests()).toBe( + REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES + ) + repo.path = '/workspace/repo-0' + enrichMissingRepoGitRemoteIdentities(store) + await flushRepoGitRemoteIdentityEnrichmentForTests() + expect(detectGitRemoteIdentity).toHaveBeenCalledTimes( + REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES + 2 + ) + }) + + it('does not retain oversized repo-location keys', async () => { + vi.mocked(detectGitRemoteIdentity).mockResolvedValue(null) + const repo = makeRepo({ path: `/${'x'.repeat(REPO_LOCATION_CACHE_KEY_MAX_BYTES)}` }) + const store = makeStore(repo) + + enrichMissingRepoGitRemoteIdentities(store) + await flushRepoGitRemoteIdentityEnrichmentForTests() + enrichMissingRepoGitRemoteIdentities(store) + await flushRepoGitRemoteIdentityEnrichmentForTests() + + expect(detectGitRemoteIdentity).toHaveBeenCalledTimes(2) + expect(getRepoGitRemoteIdentityNegativeCacheSizeForTests()).toBe(0) + }) + it('does not write stale identity data after the repo path changes', async () => { const probe = deferred() vi.mocked(detectGitRemoteIdentity).mockReturnValue(probe.promise) diff --git a/src/main/repo-git-remote-identity-enrichment.ts b/src/main/repo-git-remote-identity-enrichment.ts index 166b4d0bec8..46952f8d0db 100644 --- a/src/main/repo-git-remote-identity-enrichment.ts +++ b/src/main/repo-git-remote-identity-enrichment.ts @@ -1,7 +1,11 @@ import type { Repo } from '../shared/types' import { detectGitRemoteIdentity } from './repo-git-remote-identity' +import { getRepoLocationCacheKey } from './repo-location-cache-key' + +export { REPO_LOCATION_CACHE_KEY_MAX_BYTES } from './repo-location-cache-key' const NO_IDENTITY_RETRY_TTL_MS = 5 * 60 * 1000 +export const REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES = 512 type RepoIdentityStore = { getRepos(): Repo[] @@ -16,8 +20,25 @@ type EnrichmentOptions = { const inFlightProbesByLocation = new Map>() const noIdentityRetryAfterByLocation = new Map() -function getRepoLocationKey(repo: Pick): string { - return `${repo.connectionId ?? 'local'}\0${repo.path}` +function pruneNoIdentityRetryCache(now: number): void { + for (const [locationKey, retryAfter] of noIdentityRetryAfterByLocation) { + if (retryAfter <= now) { + noIdentityRetryAfterByLocation.delete(locationKey) + } + } + while (noIdentityRetryAfterByLocation.size > REPO_IDENTITY_NEGATIVE_CACHE_MAX_ENTRIES) { + const oldestLocation = noIdentityRetryAfterByLocation.keys().next().value + if (oldestLocation === undefined) { + break + } + noIdentityRetryAfterByLocation.delete(oldestLocation) + } +} + +function rememberNoIdentityRetry(locationKey: string, retryAfter: number): void { + noIdentityRetryAfterByLocation.delete(locationKey) + noIdentityRetryAfterByLocation.set(locationKey, retryAfter) + pruneNoIdentityRetryCache(Date.now()) } function getCurrentRepo(store: RepoIdentityStore, id: string): Repo | undefined { @@ -35,12 +56,14 @@ function isSameUnenrichedRepo(snapshot: Repo, current: Repo | undefined): boolea } async function enrichRepoGitRemoteIdentity(store: RepoIdentityStore, repo: Repo): Promise { - const locationKey = getRepoLocationKey(repo) - const retryAfter = noIdentityRetryAfterByLocation.get(locationKey) ?? 0 - if (retryAfter > Date.now()) { + const locationKey = getRepoLocationCacheKey(repo) + const now = Date.now() + pruneNoIdentityRetryCache(now) + const retryAfter = locationKey ? (noIdentityRetryAfterByLocation.get(locationKey) ?? 0) : 0 + if (retryAfter > now) { return false } - const inFlight = inFlightProbesByLocation.get(locationKey) + const inFlight = locationKey ? inFlightProbesByLocation.get(locationKey) : undefined if (inFlight) { return inFlight } @@ -49,22 +72,28 @@ async function enrichRepoGitRemoteIdentity(store: RepoIdentityStore, repo: Repo) if (!identity) { // Why: repos without a parseable remote are common; cache misses briefly so // list calls stay cheap while still allowing recent remote changes to land. - noIdentityRetryAfterByLocation.set(locationKey, Date.now() + NO_IDENTITY_RETRY_TTL_MS) + if (locationKey) { + rememberNoIdentityRetry(locationKey, Date.now() + NO_IDENTITY_RETRY_TTL_MS) + } return false } - noIdentityRetryAfterByLocation.delete(locationKey) + if (locationKey) { + noIdentityRetryAfterByLocation.delete(locationKey) + } const current = getCurrentRepo(store, repo.id) if (!isSameUnenrichedRepo(repo, current)) { return false } return !!store.updateRepo(repo.id, { gitRemoteIdentity: identity }) })().finally(() => { - if (inFlightProbesByLocation.get(locationKey) === probe) { + if (locationKey && inFlightProbesByLocation.get(locationKey) === probe) { inFlightProbesByLocation.delete(locationKey) } }) - inFlightProbesByLocation.set(locationKey, probe) + if (locationKey) { + inFlightProbesByLocation.set(locationKey, probe) + } return probe } @@ -105,3 +134,7 @@ export function resetRepoGitRemoteIdentityEnrichmentForTests(): void { inFlightProbesByLocation.clear() noIdentityRetryAfterByLocation.clear() } + +export function getRepoGitRemoteIdentityNegativeCacheSizeForTests(): number { + return noIdentityRetryAfterByLocation.size +} diff --git a/src/main/repo-git-username-enrichment.test.ts b/src/main/repo-git-username-enrichment.test.ts index bcecb8a2340..bc48ee2bb8c 100644 --- a/src/main/repo-git-username-enrichment.test.ts +++ b/src/main/repo-git-username-enrichment.test.ts @@ -11,6 +11,9 @@ vi.mock('./git/git-username', () => ({ import { enrichRepoGitUsernames, flushRepoGitUsernameEnrichmentForTests, + getRepoGitUsernameAttemptCountForTests, + REPO_LOCATION_CACHE_KEY_MAX_BYTES, + REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES, resetRepoGitUsernameEnrichmentForTests } from './repo-git-username-enrichment' @@ -82,6 +85,37 @@ describe('enrichRepoGitUsernames', () => { expect(resolveLocalGitUsernameDetailedMock).toHaveBeenCalledTimes(1) }) + it('bounds remembered attempts across churned repo locations', async () => { + const repos = Array.from({ length: REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES + 1 }, (_, index) => + makeRepo({ id: `r${index}`, path: `C:/repos/${index}` }) + ) + const store = makeStore(repos) + + enrichRepoGitUsernames(store) + await flushRepoGitUsernameEnrichmentForTests() + + expect(getRepoGitUsernameAttemptCountForTests()).toBe(REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES) + enrichRepoGitUsernames(store) + await flushRepoGitUsernameEnrichmentForTests() + expect(resolveLocalGitUsernameDetailedMock).toHaveBeenCalledTimes( + REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES + 2 + ) + }) + + it('does not retain oversized repo-location keys', async () => { + const store = makeStore([ + makeRepo({ path: `C:/${'x'.repeat(REPO_LOCATION_CACHE_KEY_MAX_BYTES)}` }) + ]) + + enrichRepoGitUsernames(store) + await flushRepoGitUsernameEnrichmentForTests() + enrichRepoGitUsernames(store) + await flushRepoGitUsernameEnrichmentForTests() + + expect(resolveLocalGitUsernameDetailedMock).toHaveBeenCalledTimes(2) + expect(getRepoGitUsernameAttemptCountForTests()).toBe(0) + }) + it('keeps persisted usernames on a non-authoritative empty resolution', async () => { resolveLocalGitUsernameDetailedMock.mockResolvedValue(resolved('', false)) const store = makeStore([makeRepo()]) diff --git a/src/main/repo-git-username-enrichment.ts b/src/main/repo-git-username-enrichment.ts index c5ae5c6d7b9..1054e6ae3b5 100644 --- a/src/main/repo-git-username-enrichment.ts +++ b/src/main/repo-git-username-enrichment.ts @@ -1,5 +1,8 @@ import type { Repo } from '../shared/types' import { resolveLocalGitUsernameDetailed } from './git/git-username' +import { getRepoLocationCacheKey } from './repo-location-cache-key' + +export { REPO_LOCATION_CACHE_KEY_MAX_BYTES } from './repo-location-cache-key' type RepoUsernameStore = { getRepos(): Repo[] @@ -10,6 +13,7 @@ type EnrichmentOptions = { onChanged?: () => void } +export const REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES = 512 // Why: resolution spawns git (and possibly gh) subprocesses, so run it at most // once per repo location per app session — hydrateRepo serves the persisted // value in between, and a relaunch picks up config changes. @@ -17,8 +21,16 @@ const attemptedLocations = new Set() let enrichmentInFlight: Promise | null = null let rerunRequested = false -function getRepoLocationKey(repo: Pick): string { - return `${repo.connectionId ?? 'local'}\0${repo.path}` +function rememberAttemptedLocation(locationKey: string): void { + attemptedLocations.delete(locationKey) + attemptedLocations.add(locationKey) + while (attemptedLocations.size > REPO_GIT_USERNAME_ATTEMPT_MAX_ENTRIES) { + const oldestLocation = attemptedLocations.values().next().value + if (oldestLocation === undefined) { + break + } + attemptedLocations.delete(oldestLocation) + } } async function enrichRepoGitUsernamesInBackground( @@ -31,11 +43,14 @@ async function enrichRepoGitUsernamesInBackground( // Why: SSH repo paths are remote; local git cannot inspect them. The // SSH username path (getSshGitUsername) stays caller-driven. !repo.connectionId && - !attemptedLocations.has(getRepoLocationKey(repo)) + !attemptedLocations.has(getRepoLocationCacheKey(repo) ?? '') ) let changed = false for (const repo of candidates) { - attemptedLocations.add(getRepoLocationKey(repo)) + const locationKey = getRepoLocationCacheKey(repo) + if (locationKey) { + rememberAttemptedLocation(locationKey) + } const { username, authoritative } = await resolveLocalGitUsernameDetailed(repo.path) // Why: a non-authoritative '' means a probe timed out and says nothing // about the account — keep the persisted value. An authoritative result @@ -93,3 +108,7 @@ export function resetRepoGitUsernameEnrichmentForTests(): void { enrichmentInFlight = null rerunRequested = false } + +export function getRepoGitUsernameAttemptCountForTests(): number { + return attemptedLocations.size +} diff --git a/src/main/repo-icon-autodetect.test.ts b/src/main/repo-icon-autodetect.test.ts index c491f344422..647ebf8359f 100644 --- a/src/main/repo-icon-autodetect.test.ts +++ b/src/main/repo-icon-autodetect.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { afterEach, describe, expect, it } from 'vitest' @@ -10,6 +10,16 @@ const PNG_1X1_BASE64 = const tempDirs: string[] = [] +function pngHeader(width: number, height: number): Buffer { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(height, 20) + return bytes +} + async function makeTempRepoDir(): Promise { const dir = await mkdtemp(join(tmpdir(), 'orca-repo-icon-')) tempDirs.push(dir) @@ -52,6 +62,13 @@ describe('detectRepoIcon', () => { }) }) + it('ignores a repo-local raster dimension bomb', async () => { + const repoPath = await makeTempRepoDir() + await writeFile(join(repoPath, 'favicon.png'), pngHeader(32_769, 1)) + + await expect(detectRepoIcon({ repoPath, kind: 'folder' })).resolves.toBeUndefined() + }) + it('resolves declared icon hrefs from project source files', async () => { const repoPath = await makeTempRepoDir() await writeFile(join(repoPath, 'index.html'), '') @@ -91,10 +108,9 @@ describe('detectRepoIcon', () => { it('skips oversized source files when looking for declared icon hrefs', async () => { const repoPath = await makeTempRepoDir() - await writeFile( - join(repoPath, 'index.html'), - `${'x'.repeat(256 * 1024 + 1)}` - ) + const sourcePath = join(repoPath, 'index.html') + await writeFile(sourcePath, '') + await truncate(sourcePath, 256 * 1024 + 1) await mkdir(join(repoPath, 'public', 'brand'), { recursive: true }) await writeFile( join(repoPath, 'public', 'brand', 'icon.png'), diff --git a/src/main/repo-icon-autodetect.ts b/src/main/repo-icon-autodetect.ts index 233721a8086..94fdc2bbd1b 100644 --- a/src/main/repo-icon-autodetect.ts +++ b/src/main/repo-icon-autodetect.ts @@ -1,50 +1,13 @@ -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import type { GitHubRepositoryIdentity, RepoKind } from '../shared/types' -import { - faviconUrlFromWebsite, - githubAvatarIcon, - MAX_REPO_ICON_UPLOAD_BYTES, - type RepoIcon -} from '../shared/repo-icon' +import { faviconUrlFromWebsite, githubAvatarIcon, type RepoIcon } from '../shared/repo-icon' import { getRepoSlug, getRepoUpstream } from './github/client' import { getSshFilesystemProvider } from './providers/ssh-filesystem-dispatch' import type { IFilesystemProvider } from './providers/types' import { detectGitRemoteIdentity } from './repo-git-remote-identity' -import { iconHrefCandidates } from './repo-icon-href-candidates' import { joinWorktreeRelativePath } from './runtime/runtime-relative-paths' - -const REPO_ICON_FILE_CANDIDATES = [ - 'favicon.png', - 'public/favicon.png', - 'app/favicon.png', - 'app/icon.png', - 'src/favicon.png', - 'src/app/icon.png', - 'assets/favicon.png', - 'assets/icon.png', - 'static/favicon.png', - 'logo.png', - 'public/logo.png' -] - -const REPO_ICON_SOURCE_FILE_CANDIDATES = [ - 'index.html', - 'public/index.html', - 'app/routes/__root.tsx', - 'src/routes/__root.tsx', - 'app/root.tsx', - 'src/root.tsx', - 'src/index.html' -] - -// Why: repo icon detection runs while adding repos; declared-icon probing should -// not read large app entrypoints just to find a small favicon href. -const MAX_REPO_ICON_SOURCE_BYTES = 256 * 1024 - -const LINK_ICON_HTML_RE = - /]*\brel=["'](?:icon|shortcut icon)["'])(?=[^>]*\bhref=["']([^"'?]+))[^>]*>/i -const LINK_ICON_OBJECT_RE = - /(?=[^}]*\brel\s*:\s*["'](?:icon|shortcut icon)["'])(?=[^}]*\bhref\s*:\s*["']([^"'?]+))[^}]*/i +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' +import { detectLocalRepoPngIcon, detectRemoteRepoPngIcon } from './repo-icon-file-detection' const WEBSITE_HOSTS_TO_SKIP = new Set([ 'github.com', @@ -54,21 +17,6 @@ const WEBSITE_HOSTS_TO_SKIP = new Set([ 'bitbucket.org', 'www.bitbucket.org' ]) - -function isPngBuffer(buffer: Buffer): boolean { - return ( - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a - ) -} - function shouldUseWebsiteFavicon(rawUrl: string): boolean { try { const url = new URL(rawUrl.includes('://') ? rawUrl : `https://${rawUrl}`) @@ -78,140 +26,6 @@ function shouldUseWebsiteFavicon(rawUrl: string): boolean { } } -function extractIconHref(source: string): string | null { - return source.match(LINK_ICON_HTML_RE)?.[1] ?? source.match(LINK_ICON_OBJECT_RE)?.[1] ?? null -} - -async function readLocalPngIcon(repoPath: string, relativePath: string): Promise { - const filePath = joinWorktreeRelativePath(repoPath, relativePath) - const info = await stat(filePath) - if (!info.isFile() || info.size > MAX_REPO_ICON_UPLOAD_BYTES) { - return null - } - const buffer = await readFile(filePath) - if (!isPngBuffer(buffer)) { - return null - } - return { - type: 'image', - src: `data:image/png;base64,${buffer.toString('base64')}`, - source: 'file', - label: relativePath - } -} - -async function readRemotePngIcon( - repoPath: string, - fsProvider: IFilesystemProvider, - relativePath: string -): Promise { - const filePath = joinWorktreeRelativePath(repoPath, relativePath) - const info = await fsProvider.stat(filePath) - if (info.type !== 'file' || info.size > MAX_REPO_ICON_UPLOAD_BYTES) { - return null - } - const result = await fsProvider.readFile(filePath) - if (!result.isBinary || result.mimeType !== 'image/png' || !result.content) { - return null - } - const buffer = Buffer.from(result.content, 'base64') - if (!isPngBuffer(buffer)) { - return null - } - return { - type: 'image', - src: `data:image/png;base64,${buffer.toString('base64')}`, - source: 'file', - label: relativePath - } -} - -async function detectLocalPngIcon(repoPath: string): Promise { - for (const relativePath of REPO_ICON_FILE_CANDIDATES) { - try { - const icon = await readLocalPngIcon(repoPath, relativePath) - if (icon) { - return icon - } - } catch { - // Try the next conventional icon path. - } - } - for (const sourceFile of REPO_ICON_SOURCE_FILE_CANDIDATES) { - try { - const sourcePath = joinWorktreeRelativePath(repoPath, sourceFile) - const sourceInfo = await stat(sourcePath) - if (!sourceInfo.isFile() || sourceInfo.size > MAX_REPO_ICON_SOURCE_BYTES) { - continue - } - const source = await readFile(sourcePath, 'utf8') - const href = extractIconHref(source) - if (!href) { - continue - } - for (const relativePath of iconHrefCandidates(href, sourceFile)) { - try { - const icon = await readLocalPngIcon(repoPath, relativePath) - if (icon) { - return icon - } - } catch { - // Try the next href resolution. - } - } - } catch { - // Try the next source file. - } - } - return null -} - -async function detectRemotePngIcon( - repoPath: string, - fsProvider: IFilesystemProvider -): Promise { - for (const relativePath of REPO_ICON_FILE_CANDIDATES) { - try { - const icon = await readRemotePngIcon(repoPath, fsProvider, relativePath) - if (icon) { - return icon - } - } catch { - // Try the next conventional icon path. - } - } - for (const sourceFile of REPO_ICON_SOURCE_FILE_CANDIDATES) { - try { - const sourcePath = joinWorktreeRelativePath(repoPath, sourceFile) - const sourceInfo = await fsProvider.stat(sourcePath) - if (sourceInfo.type !== 'file' || sourceInfo.size > MAX_REPO_ICON_SOURCE_BYTES) { - continue - } - const result = await fsProvider.readFile(sourcePath) - if (result.isBinary) { - continue - } - const href = extractIconHref(result.content) - if (!href) { - continue - } - for (const relativePath of iconHrefCandidates(href, sourceFile)) { - try { - const icon = await readRemotePngIcon(repoPath, fsProvider, relativePath) - if (icon) { - return icon - } - } catch { - // Try the next href resolution. - } - } - } catch { - // Try the next source file. - } - } - return null -} - function packageHomepageIcon(packageJson: unknown): RepoIcon | null { if (!packageJson || typeof packageJson !== 'object') { return null @@ -231,7 +45,11 @@ async function detectLocalPackageHomepageIcon(repoPath: string): Promise 128 * 1024) { return null } - return packageHomepageIcon(JSON.parse(await readFile(packageJsonPath, 'utf8'))) + const packageRead = await readNodeFileWithinLimit(packageJsonPath, 128 * 1024) + if (!packageRead.stats.isFile()) { + return null + } + return packageHomepageIcon(JSON.parse(packageRead.buffer.toString('utf8'))) } catch { return null } @@ -251,6 +69,9 @@ async function detectRemotePackageHomepageIcon( if (result.isBinary) { return null } + if (Buffer.byteLength(result.content, 'utf8') > 128 * 1024) { + return null + } return packageHomepageIcon(JSON.parse(result.content)) } catch { return null @@ -285,8 +106,8 @@ export async function detectRepoIcon({ try { const fsProvider = connectionId ? getSshFilesystemProvider(connectionId) : undefined const fileIcon = fsProvider - ? await detectRemotePngIcon(repoPath, fsProvider) - : await detectLocalPngIcon(repoPath) + ? await detectRemoteRepoPngIcon(repoPath, fsProvider) + : await detectLocalRepoPngIcon(repoPath) if (fileIcon) { return fileIcon } diff --git a/src/main/repo-icon-file-detection.ts b/src/main/repo-icon-file-detection.ts new file mode 100644 index 00000000000..6e97e19c9b5 --- /dev/null +++ b/src/main/repo-icon-file-detection.ts @@ -0,0 +1,190 @@ +import { stat } from 'node:fs/promises' +import { MAX_REPO_ICON_UPLOAD_BYTES, type RepoIcon } from '../shared/repo-icon' +import { assertRasterImagePreviewWithinLimits } from '../shared/raster-image-preview-limits' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' +import type { IFilesystemProvider } from './providers/types' +import { iconHrefCandidates } from './repo-icon-href-candidates' +import { joinWorktreeRelativePath } from './runtime/runtime-relative-paths' + +const REPO_ICON_FILE_CANDIDATES = [ + 'favicon.png', + 'public/favicon.png', + 'app/favicon.png', + 'app/icon.png', + 'src/favicon.png', + 'src/app/icon.png', + 'assets/favicon.png', + 'assets/icon.png', + 'static/favicon.png', + 'logo.png', + 'public/logo.png' +] + +const REPO_ICON_SOURCE_FILE_CANDIDATES = [ + 'index.html', + 'public/index.html', + 'app/routes/__root.tsx', + 'src/routes/__root.tsx', + 'app/root.tsx', + 'src/root.tsx', + 'src/index.html' +] + +// Why: repo icon detection runs while adding repos; declared-icon probing should +// not read large app entrypoints just to find a small favicon href. +const MAX_REPO_ICON_SOURCE_BYTES = 256 * 1024 + +const LINK_ICON_HTML_RE = + /]*\brel=["'](?:icon|shortcut icon)["'])(?=[^>]*\bhref=["']([^"'?]+))[^>]*>/i +const LINK_ICON_OBJECT_RE = + /(?=[^}]*\brel\s*:\s*["'](?:icon|shortcut icon)["'])(?=[^}]*\bhref\s*:\s*["']([^"'?]+))[^}]*/i +const PNG_SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex') + +function isPngBuffer(buffer: Buffer): boolean { + return buffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) +} + +function extractIconHref(source: string): string | null { + return source.match(LINK_ICON_HTML_RE)?.[1] ?? source.match(LINK_ICON_OBJECT_RE)?.[1] ?? null +} + +async function readLocalPngIcon(repoPath: string, relativePath: string): Promise { + const filePath = joinWorktreeRelativePath(repoPath, relativePath) + const info = await stat(filePath) + if (!info.isFile() || info.size > MAX_REPO_ICON_UPLOAD_BYTES) { + return null + } + const { buffer, stats } = await readNodeFileWithinLimit(filePath, MAX_REPO_ICON_UPLOAD_BYTES) + if (!stats.isFile() || !isPngBuffer(buffer)) { + return null + } + assertRasterImagePreviewWithinLimits(buffer, 'image/png') + return { + type: 'image', + src: `data:image/png;base64,${buffer.toString('base64')}`, + source: 'file', + label: relativePath + } +} + +async function readRemotePngIcon( + repoPath: string, + fsProvider: IFilesystemProvider, + relativePath: string +): Promise { + const filePath = joinWorktreeRelativePath(repoPath, relativePath) + const info = await fsProvider.stat(filePath) + if (info.type !== 'file' || info.size > MAX_REPO_ICON_UPLOAD_BYTES) { + return null + } + const result = await fsProvider.readFile(filePath) + if (!result.isBinary || result.mimeType !== 'image/png' || !result.content) { + return null + } + if (result.content.length > Math.ceil((MAX_REPO_ICON_UPLOAD_BYTES * 4) / 3) + 4) { + return null + } + const buffer = Buffer.from(result.content, 'base64') + if (buffer.byteLength > MAX_REPO_ICON_UPLOAD_BYTES || !isPngBuffer(buffer)) { + return null + } + assertRasterImagePreviewWithinLimits(buffer, 'image/png') + return { + type: 'image', + src: `data:image/png;base64,${buffer.toString('base64')}`, + source: 'file', + label: relativePath + } +} + +export async function detectLocalRepoPngIcon(repoPath: string): Promise { + for (const relativePath of REPO_ICON_FILE_CANDIDATES) { + try { + const icon = await readLocalPngIcon(repoPath, relativePath) + if (icon) { + return icon + } + } catch { + // Try the next conventional icon path. + } + } + for (const sourceFile of REPO_ICON_SOURCE_FILE_CANDIDATES) { + try { + const sourcePath = joinWorktreeRelativePath(repoPath, sourceFile) + const sourceInfo = await stat(sourcePath) + if (!sourceInfo.isFile() || sourceInfo.size > MAX_REPO_ICON_SOURCE_BYTES) { + continue + } + const sourceRead = await readNodeFileWithinLimit(sourcePath, MAX_REPO_ICON_SOURCE_BYTES) + if (!sourceRead.stats.isFile()) { + continue + } + const href = extractIconHref(sourceRead.buffer.toString('utf8')) + if (!href) { + continue + } + for (const relativePath of iconHrefCandidates(href, sourceFile)) { + try { + const icon = await readLocalPngIcon(repoPath, relativePath) + if (icon) { + return icon + } + } catch { + // Try the next href resolution. + } + } + } catch { + // Try the next source file. + } + } + return null +} + +export async function detectRemoteRepoPngIcon( + repoPath: string, + fsProvider: IFilesystemProvider +): Promise { + for (const relativePath of REPO_ICON_FILE_CANDIDATES) { + try { + const icon = await readRemotePngIcon(repoPath, fsProvider, relativePath) + if (icon) { + return icon + } + } catch { + // Try the next conventional icon path. + } + } + for (const sourceFile of REPO_ICON_SOURCE_FILE_CANDIDATES) { + try { + const sourcePath = joinWorktreeRelativePath(repoPath, sourceFile) + const sourceInfo = await fsProvider.stat(sourcePath) + if (sourceInfo.type !== 'file' || sourceInfo.size > MAX_REPO_ICON_SOURCE_BYTES) { + continue + } + const result = await fsProvider.readFile(sourcePath) + if ( + result.isBinary || + Buffer.byteLength(result.content, 'utf8') > MAX_REPO_ICON_SOURCE_BYTES + ) { + continue + } + const href = extractIconHref(result.content) + if (!href) { + continue + } + for (const relativePath of iconHrefCandidates(href, sourceFile)) { + try { + const icon = await readRemotePngIcon(repoPath, fsProvider, relativePath) + if (icon) { + return icon + } + } catch { + // Try the next href resolution. + } + } + } catch { + // Try the next source file. + } + } + return null +} diff --git a/src/main/repo-location-cache-key.ts b/src/main/repo-location-cache-key.ts new file mode 100644 index 00000000000..5efea35198a --- /dev/null +++ b/src/main/repo-location-cache-key.ts @@ -0,0 +1,23 @@ +import { measureUtf8ByteLength } from '../shared/utf8-byte-limits' + +export const REPO_LOCATION_CACHE_KEY_MAX_BYTES = 4 * 1024 + +export function getRepoLocationCacheKey(repo: { + path: string + connectionId?: string | null +}): string | null { + const connectionId = repo.connectionId ?? 'local' + const connectionBytes = measureUtf8ByteLength(connectionId, { + stopAfterBytes: REPO_LOCATION_CACHE_KEY_MAX_BYTES - 1 + }) + if (connectionBytes.exceededLimit) { + return null + } + const pathBytes = measureUtf8ByteLength(repo.path, { + stopAfterBytes: REPO_LOCATION_CACHE_KEY_MAX_BYTES - connectionBytes.byteLength - 1 + }) + if (pathBytes.exceededLimit) { + return null + } + return `${connectionId}\0${repo.path}` +} diff --git a/src/main/runtime-environment-focus-self-heal.test.ts b/src/main/runtime-environment-focus-self-heal.test.ts deleted file mode 100644 index edce1d6902b..00000000000 --- a/src/main/runtime-environment-focus-self-heal.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import type { GlobalSettings } from '../shared/types' -import type { KnownRuntimeEnvironment } from '../shared/runtime-environments' -import { - clearActiveRuntimeEnvironmentFocusIfMatches, - selfHealRuntimeEnvironmentFocus -} from './runtime-environment-focus-self-heal' - -function environment( - id: string, - source?: KnownRuntimeEnvironment['source'] -): KnownRuntimeEnvironment { - return { - id, - name: id, - createdAt: 0, - updatedAt: 0, - lastUsedAt: null, - runtimeId: null, - ...(source ? { source } : {}), - endpoints: [ - { - id: `ws-${id}`, - kind: 'websocket', - label: 'WebSocket', - endpoint: 'ws://127.0.0.1:6768', - deviceToken: 'token', - publicKeyB64: 'key' - } - ], - preferredEndpointId: `ws-${id}` - } -} - -function makeStore(activeRuntimeEnvironmentId: string | null | undefined) { - const settings: Pick = {} - if (activeRuntimeEnvironmentId !== undefined) { - settings.activeRuntimeEnvironmentId = activeRuntimeEnvironmentId - } - const updateSettings = vi.fn((updates: Pick) => { - settings.activeRuntimeEnvironmentId = updates.activeRuntimeEnvironmentId - return settings - }) - return { - store: { - getSettings: () => settings, - updateSettings - }, - updateSettings - } -} - -describe('runtime environment focus self-heal', () => { - it('keeps a focus id that resolves to a user-managed environment', () => { - const { store, updateSettings } = makeStore('env-1') - - selfHealRuntimeEnvironmentFocus({ - store, - userDataPath: '/user-data', - listKnownEnvironments: () => [environment('env-1')] - }) - - expect(updateSettings).not.toHaveBeenCalled() - }) - - it('clears a dangling focus id and logs one diagnostic line', () => { - const { store, updateSettings } = makeStore('missing-env') - const log = vi.fn() - - selfHealRuntimeEnvironmentFocus({ - store, - userDataPath: '/user-data', - listKnownEnvironments: () => [environment('env-1')], - log - }) - - expect(updateSettings).toHaveBeenCalledWith({ activeRuntimeEnvironmentId: null }) - expect(log).toHaveBeenCalledTimes(1) - expect(log.mock.calls[0][0]).toContain('missing-env') - }) - - it('clears an ephemeral-VM focus id after restart', () => { - const { store, updateSettings } = makeStore('vm-env') - const log = vi.fn() - - selfHealRuntimeEnvironmentFocus({ - store, - userDataPath: '/user-data', - listKnownEnvironments: () => [environment('vm-env', 'ephemeral-vm')], - log - }) - - expect(updateSettings).toHaveBeenCalledWith({ activeRuntimeEnvironmentId: null }) - expect(log).toHaveBeenCalledTimes(1) - }) - - it('leaves null and absent focus settings untouched', () => { - const nullCase = makeStore(null) - const absentCase = makeStore(undefined) - const listKnownEnvironments = vi.fn(() => [environment('env-1')]) - - selfHealRuntimeEnvironmentFocus({ - store: nullCase.store, - userDataPath: '/user-data', - listKnownEnvironments - }) - selfHealRuntimeEnvironmentFocus({ - store: absentCase.store, - userDataPath: '/user-data', - listKnownEnvironments - }) - - expect(nullCase.updateSettings).not.toHaveBeenCalled() - expect(absentCase.updateSettings).not.toHaveBeenCalled() - expect(listKnownEnvironments).not.toHaveBeenCalled() - }) - - it('normalizes an empty persisted id to null without reading the registry', () => { - const { store, updateSettings } = makeStore('') - const listKnownEnvironments = vi.fn(() => [environment('env-1')]) - - selfHealRuntimeEnvironmentFocus({ - store, - userDataPath: '/user-data', - listKnownEnvironments - }) - - expect(updateSettings).toHaveBeenCalledWith({ activeRuntimeEnvironmentId: null }) - expect(listKnownEnvironments).not.toHaveBeenCalled() - }) - - it('fails soft when the registry cannot be read', () => { - const { store, updateSettings } = makeStore('env-1') - - selfHealRuntimeEnvironmentFocus({ - store, - userDataPath: '/user-data', - listKnownEnvironments: () => { - throw new Error('invalid registry') - } - }) - - expect(updateSettings).not.toHaveBeenCalled() - }) - - it('clears the active focus on matching in-process removal with listener notification', () => { - const { store, updateSettings } = makeStore('env-1') - - clearActiveRuntimeEnvironmentFocusIfMatches(store, 'env-1') - - expect(updateSettings).toHaveBeenCalledWith( - { activeRuntimeEnvironmentId: null }, - { notifyListeners: true } - ) - }) -}) diff --git a/src/main/runtime-environment-focus-self-heal.ts b/src/main/runtime-environment-focus-self-heal.ts deleted file mode 100644 index a748f4794e2..00000000000 --- a/src/main/runtime-environment-focus-self-heal.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { GlobalSettings } from '../shared/types' -import { listEnvironments } from '../shared/runtime-environment-store' -import { - isUserManagedRuntimeEnvironment, - type KnownRuntimeEnvironment -} from '../shared/runtime-environments' - -type RuntimeEnvironmentFocusStore = { - getSettings: () => Pick - updateSettings: ( - updates: Pick, - options?: { notifyListeners?: boolean } - ) => unknown -} - -type SelfHealRuntimeEnvironmentFocusArgs = { - store: RuntimeEnvironmentFocusStore - userDataPath: string - listKnownEnvironments?: (userDataPath: string) => KnownRuntimeEnvironment[] - log?: (message: string) => void -} - -function logClearedFocus(log: ((message: string) => void) | undefined, reason: string): void { - const writeLog = log ?? console.info - writeLog(`[runtime-environment-focus] cleared active runtime environment: ${reason}`) -} - -export function clearActiveRuntimeEnvironmentFocusIfMatches( - store: RuntimeEnvironmentFocusStore, - environmentId: string -): void { - if (store.getSettings().activeRuntimeEnvironmentId !== environmentId) { - return - } - store.updateSettings({ activeRuntimeEnvironmentId: null }, { notifyListeners: true }) -} - -export function selfHealRuntimeEnvironmentFocus({ - store, - userDataPath, - listKnownEnvironments = listEnvironments, - log -}: SelfHealRuntimeEnvironmentFocusArgs): void { - const activeRuntimeEnvironmentId = store.getSettings().activeRuntimeEnvironmentId - if (activeRuntimeEnvironmentId === undefined || activeRuntimeEnvironmentId === null) { - return - } - - if (activeRuntimeEnvironmentId.trim() === '') { - store.updateSettings({ activeRuntimeEnvironmentId: null }) - logClearedFocus(log, 'empty persisted id') - return - } - - let environments: KnownRuntimeEnvironment[] - try { - environments = listKnownEnvironments(userDataPath) - } catch { - // Why: an unreadable registry must not clear a possibly-valid focus; keep - // it and let a later launch heal once the registry reads again. - return - } - - const focusedEnvironment = environments.find((entry) => entry.id === activeRuntimeEnvironmentId) - if (focusedEnvironment && isUserManagedRuntimeEnvironment(focusedEnvironment)) { - return - } - - store.updateSettings({ activeRuntimeEnvironmentId: null }) - logClearedFocus(log, `dangling id ${activeRuntimeEnvironmentId}`) -} diff --git a/src/main/runtime/agent-session-claim-identity.test.ts b/src/main/runtime/agent-session-claim-identity.test.ts index 39392ea9e72..c5407ca303a 100644 --- a/src/main/runtime/agent-session-claim-identity.test.ts +++ b/src/main/runtime/agent-session-claim-identity.test.ts @@ -1,7 +1,11 @@ +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { canonicalizeAgentSessionIdentity, - createEphemeralAgentSessionClaimSigner + createEphemeralAgentSessionClaimSigner, + loadAgentSessionClaimSigner } from './agent-session-claim-identity' describe('agent session claim identity', () => { @@ -40,4 +44,17 @@ describe('agent session claim identity', () => { canonicalizeAgentSessionIdentity('blank', { key: 'session_id', id: 'session-1' }) ).toThrow('agent_session_identity_required') }) + + it('rejects an oversized persisted coordination key before reading it', () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-claim-key-')) + try { + const descriptor = openSync(join(directory, 'agent-session-authority.key'), 'w') + ftruncateSync(descriptor, 1024 * 1024 * 1024) + closeSync(descriptor) + + expect(() => loadAgentSessionClaimSigner(directory, 'profile-1')).toThrow() + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) }) diff --git a/src/main/runtime/agent-session-claim-identity.ts b/src/main/runtime/agent-session-claim-identity.ts index 2bc276f87e5..0d302723e68 100644 --- a/src/main/runtime/agent-session-claim-identity.ts +++ b/src/main/runtime/agent-session-claim-identity.ts @@ -1,13 +1,5 @@ import { createHash, createHmac, randomBytes } from 'node:crypto' -import { - closeSync, - mkdirSync, - openSync, - readFileSync, - realpathSync, - statSync, - writeFileSync -} from 'node:fs' +import { closeSync, mkdirSync, openSync, realpathSync, statSync, writeFileSync } from 'node:fs' import { dirname, isAbsolute, join, normalize } from 'node:path' import { AGENT_SESSION_CLAIM_DIGEST_VERSION, @@ -20,6 +12,7 @@ import { type AgentProviderSessionMetadata, type ResumableTuiAgent } from '../../shared/agent-session-resume' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' const COORDINATION_KEY_BYTES = 32 const COORDINATION_KEY_FILE = 'agent-session-authority.key' @@ -146,7 +139,7 @@ export function loadAgentSessionClaimSigner( mkdirSync(dirname(keyPath), { recursive: true }) let key: Buffer try { - key = readFileSync(keyPath) + key = readNodeFileSyncWithinLimit(keyPath, COORDINATION_KEY_BYTES).buffer } catch { const candidate = randomBytes(COORDINATION_KEY_BYTES) let fd: number | null = null @@ -155,7 +148,7 @@ export function loadAgentSessionClaimSigner( writeFileSync(fd, candidate) key = candidate } catch { - key = readFileSync(keyPath) + key = readNodeFileSyncWithinLimit(keyPath, COORDINATION_KEY_BYTES).buffer } finally { if (fd !== null) { closeSync(fd) diff --git a/src/main/runtime/claude-agent-teams-shim-env.test.ts b/src/main/runtime/claude-agent-teams-shim-env.test.ts index f79c9d150cf..1c3e43cb2ba 100644 --- a/src/main/runtime/claude-agent-teams-shim-env.test.ts +++ b/src/main/runtime/claude-agent-teams-shim-env.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { chmod, mkdtemp, readFile, rm, stat, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' @@ -25,6 +25,19 @@ describe('claude agent teams shim env', () => { await expect(readFile(join(root, 'tmux'), 'utf8')).resolves.toContain('agent-teams-tmux "$@"') }) + it('replaces a large sparse shim without reading its payload', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-agent-teams-large-shim-')) + roots.push(root) + await ensureClaudeAgentTeamsShimDir(root) + const shimPath = join(root, 'tmux') + await truncate(shimPath, 256 * 1024 * 1024) + + await ensureClaudeAgentTeamsShimDir(root) + + expect((await stat(shimPath)).size).toBeLessThan(64 * 1024) + await expect(readFile(shimPath, 'utf8')).resolves.toContain('agent-teams-tmux "$@"') + }) + it('builds native shim env only for direct Claude commands', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-agent-teams-cli-')) roots.push(root) diff --git a/src/main/runtime/claude-agent-teams-shim-env.ts b/src/main/runtime/claude-agent-teams-shim-env.ts index 6b6eeea94ba..639aa707376 100644 --- a/src/main/runtime/claude-agent-teams-shim-env.ts +++ b/src/main/runtime/claude-agent-teams-shim-env.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { chmod, mkdir, rename, rm, writeFile } from 'node:fs/promises' import { accessSync, constants, existsSync } from 'node:fs' import { homedir } from 'node:os' import { delimiter, dirname, join } from 'node:path' @@ -8,6 +8,7 @@ import { isDirectClaudeCommand, type ClaudeAgentTeamsMode } from '../../shared/claude-agent-teams-tmux-compat' +import { nodeFileContentsEqual } from '../../shared/node-file-content-equality' import { getOrcaCliCommandNameForPlatform } from '../../shared/orca-cli-command-name' export type ClaudeAgentTeamsLaunchPlan = { @@ -136,7 +137,7 @@ function windowsShimScript(): string { async function writeIfChanged(path: string, content: string): Promise { try { - if ((await readFile(path, 'utf8')) === content) { + if (await nodeFileContentsEqual(path, content)) { return } } catch { diff --git a/src/main/runtime/client-session-tab-selection-bounds.test.ts b/src/main/runtime/client-session-tab-selection-bounds.test.ts new file mode 100644 index 00000000000..4f4285684c0 --- /dev/null +++ b/src/main/runtime/client-session-tab-selection-bounds.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest' +import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' +import type { PersistedMobileClientTabSelection } from '../../shared/types' +import { ClientSessionTabSelectionStore } from './client-session-tab-selection' +import { + isMobileTabSelectionIdRetainable, + MOBILE_TAB_SELECTION_ID_MAX_BYTES, + MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT, + MOBILE_TAB_SELECTION_MAX_CLIENTS, + MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE, + MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT, + mobileTabSelectionRetainedBytes, + normalizePersistedMobileClientTabSelections +} from './client-session-tab-selection-persistence' + +function selection( + activeTabIdByGroupId: Readonly> = {} +): PersistedMobileClientTabSelection { + return { activeTabId: 'browser-1', activeGroupId: 'group-1', activeTabIdByGroupId } +} + +function snapshot(worktree: string): RuntimeMobileSessionTabsResult { + return { + worktree, + publicationEpoch: 'renderer:1', + snapshotVersion: 1, + activeGroupId: 'group-1', + activeTabId: 'browser-1', + activeTabType: 'browser', + tabGroups: [{ id: 'group-1', activeTabId: 'browser-1', tabOrder: ['browser-1'] }], + tabs: [ + { + type: 'browser', + id: 'browser-1', + browserWorkspaceId: 'browser-workspace', + browserPageId: 'page-1', + title: 'Browser', + url: 'about:blank', + loading: false, + canGoBack: false, + canGoForward: false, + isActive: true + } + ] + } +} + +describe('client session-tab selection retention bounds', () => { + it('keeps the newest persisted clients without changing accepted selections', () => { + const clients = Object.fromEntries( + Array.from({ length: MOBILE_TAB_SELECTION_MAX_CLIENTS + 2 }, (_, index) => [ + `device-${index}`, + { 'wt-1': selection() } + ]) + ) + + const normalized = normalizePersistedMobileClientTabSelections(clients) + + expect(Object.keys(normalized)).toHaveLength(MOBILE_TAB_SELECTION_MAX_CLIENTS) + expect(Object.keys(normalized).at(0)).toBe('device-2') + expect(normalized[`device-${MOBILE_TAB_SELECTION_MAX_CLIENTS + 1}`]?.['wt-1']).toEqual( + selection() + ) + }) + + it('keeps the newest worktrees and group selections within each client', () => { + const groups = Object.fromEntries( + Array.from({ length: MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE + 2 }, (_, index) => [ + `group-${index}`, + `tab-${index}` + ]) + ) + const worktrees = Object.fromEntries( + Array.from({ length: MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + 2 }, (_, index) => [ + `wt-${index}`, + selection(index === MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + 1 ? groups : {}) + ]) + ) + + const normalized = normalizePersistedMobileClientTabSelections({ 'device-1': worktrees }) + const retainedWorktrees = normalized['device-1']! + const retainedGroups = + retainedWorktrees[`wt-${MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + 1}`]! + .activeTabIdByGroupId + + expect(Object.keys(retainedWorktrees)).toHaveLength( + MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + ) + expect(Object.keys(retainedWorktrees).at(0)).toBe('wt-2') + expect(Object.keys(retainedGroups)).toHaveLength(MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE) + expect(Object.keys(retainedGroups).at(0)).toBe('group-2') + }) + + it('bounds clients and worktrees created during the current runtime', () => { + const clientStore = new ClientSessionTabSelectionStore() + for (let index = 0; index <= MOBILE_TAB_SELECTION_MAX_CLIENTS; index++) { + clientStore.activate(snapshot('wt-1'), `device-${index}`, 'browser-1') + } + const retainedClients = clientStore.serialize() + + expect(Object.keys(retainedClients)).toHaveLength(MOBILE_TAB_SELECTION_MAX_CLIENTS) + expect(retainedClients['device-0']).toBeUndefined() + expect(retainedClients[`device-${MOBILE_TAB_SELECTION_MAX_CLIENTS}`]).toBeDefined() + + const worktreeStore = new ClientSessionTabSelectionStore() + for (let index = 0; index <= MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT; index++) { + worktreeStore.activate(snapshot(`wt-${index}`), 'device-1', 'browser-1') + } + const retainedWorktrees = worktreeStore.serialize()['device-1']! + + expect(Object.keys(retainedWorktrees)).toHaveLength( + MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + ) + expect(retainedWorktrees['wt-0']).toBeUndefined() + expect(retainedWorktrees[`wt-${MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT}`]).toBeDefined() + }) + + it('rejects relay-sized client and worktree ids instead of retaining them', () => { + const oversizedId = 'x'.repeat(MOBILE_TAB_SELECTION_ID_MAX_BYTES + 1) + expect(isMobileTabSelectionIdRetainable(oversizedId)).toBe(false) + expect( + normalizePersistedMobileClientTabSelections({ + [oversizedId]: { 'wt-1': selection() }, + 'device-1': { [oversizedId]: selection() } + }) + ).toEqual({}) + + const store = new ClientSessionTabSelectionStore() + const ordinarySnapshot = snapshot('wt-1') + expect(store.activate(ordinarySnapshot, oversizedId, 'browser-1')).toBe(ordinarySnapshot) + const oversizedWorktreeSnapshot = snapshot(oversizedId) + expect(store.activate(oversizedWorktreeSnapshot, 'device-1', 'browser-1')).toBe( + oversizedWorktreeSnapshot + ) + expect(store.serialize()).toEqual({}) + }) + + it('retains newest worktree selections within the per-client byte budget', () => { + const fixedSizeId = (prefix: string, index: number): string => { + const label = `${prefix}-${index}-` + return `${label}${'x'.repeat(MOBILE_TAB_SELECTION_ID_MAX_BYTES - label.length)}` + } + const groups = Object.fromEntries( + Array.from({ length: MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE }, (_, index) => [ + fixedSizeId('group', index), + fixedSizeId('tab', index) + ]) + ) + const normalized = normalizePersistedMobileClientTabSelections({ + 'device-1': Object.fromEntries( + Array.from({ length: 10 }, (_, index) => [`wt-${index}`, selection(groups)]) + ) + }) + const retained = normalized['device-1']! + const retainedBytes = Object.entries(retained).reduce( + (total, [worktreeId, value]) => total + mobileTabSelectionRetainedBytes(worktreeId, value), + 0 + ) + + expect(retainedBytes).toBeLessThanOrEqual(MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT) + expect(retained['wt-0']).toBeUndefined() + expect(retained['wt-9']).toBeDefined() + }) +}) diff --git a/src/main/runtime/client-session-tab-selection-persistence.ts b/src/main/runtime/client-session-tab-selection-persistence.ts index 358144cb49d..d8f72add49e 100644 --- a/src/main/runtime/client-session-tab-selection-persistence.ts +++ b/src/main/runtime/client-session-tab-selection-persistence.ts @@ -2,6 +2,105 @@ import type { PersistedMobileClientTabSelection, PersistedMobileClientTabSelections } from '../../shared/types' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MOBILE_TAB_SELECTION_MAX_CLIENTS = 64 +export const MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT = 512 +export const MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE = 128 +export const MOBILE_TAB_SELECTION_ID_MAX_BYTES = 4 * 1024 +export const MOBILE_TAB_SELECTION_MAX_BYTES_PER_SELECTION = 64 * 1024 +export const MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT = 256 * 1024 + +export function isMobileTabSelectionIdRetainable(value: string): boolean { + return !measureUtf8ByteLength(value, { + stopAfterBytes: MOBILE_TAB_SELECTION_ID_MAX_BYTES + }).exceededLimit +} + +function mobileTabSelectionStringBytes(value: string | null): number { + return value ? measureUtf8ByteLength(value).byteLength : 0 +} + +export function mobileTabSelectionRetainedBytes( + worktreeId: string, + selection: PersistedMobileClientTabSelection +): number { + let bytes = + mobileTabSelectionStringBytes(worktreeId) + + mobileTabSelectionStringBytes(selection.activeTabId) + + mobileTabSelectionStringBytes(selection.activeGroupId) + for (const [groupId, tabId] of Object.entries(selection.activeTabIdByGroupId)) { + bytes += mobileTabSelectionStringBytes(groupId) + mobileTabSelectionStringBytes(tabId) + } + return bytes +} + +function newestOwnEntries( + record: Record, + maxEntries: number, + acceptKey: (key: string) => boolean = () => true +): [string, unknown][] { + const entries: [string, unknown][] = [] + let nextReplacementIndex = 0 + for (const key in record) { + if (!Object.prototype.hasOwnProperty.call(record, key)) { + continue + } + if (!acceptKey(key)) { + continue + } + const entry: [string, unknown] = [key, record[key]] + if (entries.length < maxEntries) { + entries.push(entry) + } else { + entries[nextReplacementIndex] = entry + nextReplacementIndex = (nextReplacementIndex + 1) % maxEntries + } + } + return nextReplacementIndex === 0 + ? entries + : [...entries.slice(nextReplacementIndex), ...entries.slice(0, nextReplacementIndex)] +} + +export function boundMobileClientTabSelectionGroups( + selection: PersistedMobileClientTabSelection +): PersistedMobileClientTabSelection { + const activeTabId = + selection.activeTabId && isMobileTabSelectionIdRetainable(selection.activeTabId) + ? selection.activeTabId + : null + const activeGroupId = + selection.activeGroupId && isMobileTabSelectionIdRetainable(selection.activeGroupId) + ? selection.activeGroupId + : null + let retainedBytes = + mobileTabSelectionStringBytes(activeTabId) + mobileTabSelectionStringBytes(activeGroupId) + const retainedGroups: [string, string][] = [] + const candidates = newestOwnEntries( + selection.activeTabIdByGroupId, + MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE, + isMobileTabSelectionIdRetainable + ).filter( + (entry): entry is [string, string] => + typeof entry[1] === 'string' && isMobileTabSelectionIdRetainable(entry[1]) + ) + for (let index = candidates.length - 1; index >= 0; index--) { + const entry = candidates[index]! + const entryBytes = + mobileTabSelectionStringBytes(entry[0]) + mobileTabSelectionStringBytes(entry[1]) + if (retainedBytes + entryBytes > MOBILE_TAB_SELECTION_MAX_BYTES_PER_SELECTION) { + break + } + retainedBytes += entryBytes + retainedGroups.push(entry) + } + return { + ...selection, + activeTabId, + activeGroupId, + activeTabIdByGroupId: Object.fromEntries(retainedGroups.toReversed()) + } +} function normalizeClientSessionTabSelection( raw: unknown @@ -10,16 +109,28 @@ function normalizeClientSessionTabSelection( return null } const candidate = raw as Partial - const activeTabId = typeof candidate.activeTabId === 'string' ? candidate.activeTabId : null - const activeGroupId = typeof candidate.activeGroupId === 'string' ? candidate.activeGroupId : null + const activeTabId = + typeof candidate.activeTabId === 'string' && + isMobileTabSelectionIdRetainable(candidate.activeTabId) + ? candidate.activeTabId + : null + const activeGroupId = + typeof candidate.activeGroupId === 'string' && + isMobileTabSelectionIdRetainable(candidate.activeGroupId) + ? candidate.activeGroupId + : null const activeTabIdByGroupId: Record = {} if ( typeof candidate.activeTabIdByGroupId === 'object' && candidate.activeTabIdByGroupId && !Array.isArray(candidate.activeTabIdByGroupId) ) { - for (const [groupId, tabId] of Object.entries(candidate.activeTabIdByGroupId)) { - if (typeof tabId === 'string') { + for (const [groupId, tabId] of newestOwnEntries( + candidate.activeTabIdByGroupId as Record, + MOBILE_TAB_SELECTION_MAX_GROUPS_PER_WORKTREE, + isMobileTabSelectionIdRetainable + )) { + if (typeof tabId === 'string' && isMobileTabSelectionIdRetainable(tabId)) { activeTabIdByGroupId[groupId] = tabId } } @@ -27,7 +138,7 @@ function normalizeClientSessionTabSelection( if (!activeTabId && !activeGroupId && Object.keys(activeTabIdByGroupId).length === 0) { return null } - return { activeTabId, activeGroupId, activeTabIdByGroupId } + return boundMobileClientTabSelectionGroups({ activeTabId, activeGroupId, activeTabIdByGroupId }) } // Why: this state comes off disk (and, for remote runtimes, another machine); a bad payload must degrade to "no selection", not throw. @@ -38,7 +149,11 @@ export function normalizePersistedMobileClientTabSelections( if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) { return normalized } - for (const [clientNavigationId, selectionsByWorktree] of Object.entries(raw)) { + for (const [clientNavigationId, selectionsByWorktree] of newestOwnEntries( + raw as Record, + MOBILE_TAB_SELECTION_MAX_CLIENTS, + isMobileTabSelectionIdRetainable + )) { if ( typeof selectionsByWorktree !== 'object' || selectionsByWorktree === null || @@ -46,15 +161,29 @@ export function normalizePersistedMobileClientTabSelections( ) { continue } - const entries: Record = {} - for (const [worktreeId, selection] of Object.entries(selectionsByWorktree)) { + const entries = new Map() + let retainedBytes = 0 + for (const [worktreeId, selection] of newestOwnEntries( + selectionsByWorktree as Record, + MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT, + isMobileTabSelectionIdRetainable + )) { const normalizedSelection = normalizeClientSessionTabSelection(selection) if (normalizedSelection) { - entries[worktreeId] = normalizedSelection + entries.set(worktreeId, normalizedSelection) + retainedBytes += mobileTabSelectionRetainedBytes(worktreeId, normalizedSelection) + while (retainedBytes > MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT) { + const oldest = entries.entries().next() + if (oldest.done) { + break + } + entries.delete(oldest.value[0]) + retainedBytes -= mobileTabSelectionRetainedBytes(oldest.value[0], oldest.value[1]) + } } } - if (Object.keys(entries).length > 0) { - normalized[clientNavigationId] = entries + if (entries.size > 0) { + normalized[clientNavigationId] = Object.fromEntries(entries) } } return normalized diff --git a/src/main/runtime/client-session-tab-selection-retention.ts b/src/main/runtime/client-session-tab-selection-retention.ts new file mode 100644 index 00000000000..0dd7672a49d --- /dev/null +++ b/src/main/runtime/client-session-tab-selection-retention.ts @@ -0,0 +1,64 @@ +import type { ClientSessionTabSelection } from './client-session-tab-selection' +import { + isMobileTabSelectionIdRetainable, + MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT, + MOBILE_TAB_SELECTION_MAX_CLIENTS, + MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT, + mobileTabSelectionRetainedBytes +} from './client-session-tab-selection-persistence' + +export type StoredClientSessionTabSelection = { + selection: ClientSessionTabSelection + revision: number + shouldPersist: boolean +} + +export function getOrCreateClientTabSelectionWorktrees( + statesByClient: Map>, + clientNavigationId: string +): Map { + let statesByWorktree = statesByClient.get(clientNavigationId) + if (!statesByWorktree) { + if (statesByClient.size >= MOBILE_TAB_SELECTION_MAX_CLIENTS) { + const oldestClientId = statesByClient.keys().next().value + if (oldestClientId !== undefined) { + statesByClient.delete(oldestClientId) + } + } + statesByWorktree = new Map() + statesByClient.set(clientNavigationId, statesByWorktree) + } + return statesByWorktree +} + +export function rememberClientTabSelectionWorktree( + statesByWorktree: Map, + worktreeId: string, + state: StoredClientSessionTabSelection +): void { + if (!isMobileTabSelectionIdRetainable(worktreeId)) { + return + } + if ( + !statesByWorktree.has(worktreeId) && + statesByWorktree.size >= MOBILE_TAB_SELECTION_MAX_WORKTREES_PER_CLIENT + ) { + const oldestWorktreeId = statesByWorktree.keys().next().value + if (oldestWorktreeId !== undefined) { + statesByWorktree.delete(oldestWorktreeId) + } + } + statesByWorktree.set(worktreeId, state) + let retainedBytes = 0 + for (const [retainedWorktreeId, retainedState] of statesByWorktree) { + retainedBytes += mobileTabSelectionRetainedBytes(retainedWorktreeId, retainedState.selection) + } + while (retainedBytes > MOBILE_TAB_SELECTION_MAX_BYTES_PER_CLIENT) { + const oldest = statesByWorktree.entries().next() + if (oldest.done) { + break + } + statesByWorktree.delete(oldest.value[0]) + retainedBytes -= mobileTabSelectionRetainedBytes(oldest.value[0], oldest.value[1].selection) + } +} diff --git a/src/main/runtime/client-session-tab-selection.ts b/src/main/runtime/client-session-tab-selection.ts index 7c8f84f88d2..5af3bd48cf7 100644 --- a/src/main/runtime/client-session-tab-selection.ts +++ b/src/main/runtime/client-session-tab-selection.ts @@ -3,7 +3,16 @@ import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' import type { PersistedMobileClientTabSelections } from '../../shared/types' -import { normalizePersistedMobileClientTabSelections } from './client-session-tab-selection-persistence' +import { + boundMobileClientTabSelectionGroups, + isMobileTabSelectionIdRetainable, + normalizePersistedMobileClientTabSelections +} from './client-session-tab-selection-persistence' +import { + getOrCreateClientTabSelectionWorktrees, + rememberClientTabSelectionWorktree, + type StoredClientSessionTabSelection +} from './client-session-tab-selection-retention' export type ClientSessionTabSelection = { activeTabId: string | null @@ -11,13 +20,6 @@ export type ClientSessionTabSelection = { activeTabIdByGroupId: Readonly> } -type StoredClientSessionTabSelection = { - selection: ClientSessionTabSelection - revision: number - // Why: listAll projects every worktree; only hydrated or user-activated selections belong on disk. - shouldPersist: boolean -} - function emptyClientSessionTabSelection(): ClientSessionTabSelection { return { activeTabId: null, activeGroupId: null, activeTabIdByGroupId: {} } } @@ -42,7 +44,7 @@ function findTabByTopLevelId( export function deriveClientSessionTabSelection( snapshot: RuntimeMobileSessionTabsResult ): ClientSessionTabSelection { - return { + return boundMobileClientTabSelectionGroups({ activeTabId: snapshot.activeTabId, activeGroupId: snapshot.activeGroupId, activeTabIdByGroupId: Object.fromEntries( @@ -50,7 +52,7 @@ export function deriveClientSessionTabSelection( group.activeTabId ? [[group.id, group.activeTabId] as const] : [] ) ?? [] ) - } + }) } export function activateClientSessionTabSelection( @@ -66,13 +68,13 @@ export function activateClientSessionTabSelection( const activeGroup = snapshot.tabGroups?.find((group) => group.tabOrder.includes(activeTopLevelTabId) ) - return { + return boundMobileClientTabSelectionGroups({ activeTabId, activeGroupId: activeGroup?.id ?? selection.activeGroupId, activeTabIdByGroupId: activeGroup ? { ...selection.activeTabIdByGroupId, [activeGroup.id]: activeTopLevelTabId } : selection.activeTabIdByGroupId - } + }) } export function projectClientSessionTabSelection( @@ -110,11 +112,11 @@ export function projectClientSessionTabSelection( : null) ?? tabGroups?.[0]?.id ?? null - const nextSelection: ClientSessionTabSelection = { + const nextSelection: ClientSessionTabSelection = boundMobileClientTabSelectionGroups({ activeTabId: activeTab?.id ?? null, activeGroupId, activeTabIdByGroupId - } + }) return { selection: nextSelection, snapshot: { @@ -137,9 +139,16 @@ export class ClientSessionTabSelectionStore { for (const [clientNavigationId, selectionsByWorktree] of Object.entries( normalizePersistedMobileClientTabSelections(persisted) )) { - const statesByWorktree = this.getStatesByWorktree(clientNavigationId) + const statesByWorktree = getOrCreateClientTabSelectionWorktrees( + this.statesByClient, + clientNavigationId + ) for (const [worktreeId, selection] of Object.entries(selectionsByWorktree)) { - statesByWorktree.set(worktreeId, { selection, revision: 0, shouldPersist: true }) + rememberClientTabSelectionWorktree(statesByWorktree, worktreeId, { + selection, + revision: 0, + shouldPersist: true + }) } } } @@ -168,25 +177,21 @@ export class ClientSessionTabSelectionStore { this.persistListener?.(this.serialize()) } - private getStatesByWorktree( - clientNavigationId: string - ): Map { - let statesByWorktree = this.statesByClient.get(clientNavigationId) - if (!statesByWorktree) { - statesByWorktree = new Map() - this.statesByClient.set(clientNavigationId, statesByWorktree) - } - return statesByWorktree - } - project( snapshot: RuntimeMobileSessionTabsResult, clientNavigationId?: string ): RuntimeMobileSessionTabsResult { - if (!clientNavigationId) { + if ( + !clientNavigationId || + !isMobileTabSelectionIdRetainable(clientNavigationId) || + !isMobileTabSelectionIdRetainable(snapshot.worktree) + ) { return snapshot } - const statesByWorktree = this.getStatesByWorktree(clientNavigationId) + const statesByWorktree = getOrCreateClientTabSelectionWorktrees( + this.statesByClient, + clientNavigationId + ) const state = statesByWorktree.get(snapshot.worktree) ?? { // Why: host focus is private navigation; a new paired device starts from deterministic topology instead of inheriting it. selection: emptyClientSessionTabSelection(), @@ -202,7 +207,7 @@ export class ClientSessionTabSelectionStore { } } const projected = projectClientSessionTabSelection(snapshot, state.selection) - statesByWorktree.set(snapshot.worktree, { + rememberClientTabSelectionWorktree(statesByWorktree, snapshot.worktree, { selection: projected.selection, revision: state.revision, shouldPersist: state.shouldPersist @@ -219,14 +224,23 @@ export class ClientSessionTabSelectionStore { clientNavigationId: string, activeTabId: string ): RuntimeMobileSessionTabsResult { - const statesByWorktree = this.getStatesByWorktree(clientNavigationId) + if ( + !isMobileTabSelectionIdRetainable(clientNavigationId) || + !isMobileTabSelectionIdRetainable(snapshot.worktree) + ) { + return snapshot + } + const statesByWorktree = getOrCreateClientTabSelectionWorktrees( + this.statesByClient, + clientNavigationId + ) const state = statesByWorktree.get(snapshot.worktree) ?? { selection: emptyClientSessionTabSelection(), revision: 0, shouldPersist: false } const nextSelection = activateClientSessionTabSelection(snapshot, state.selection, activeTabId) - statesByWorktree.set(snapshot.worktree, { + rememberClientTabSelectionWorktree(statesByWorktree, snapshot.worktree, { selection: nextSelection, revision: state.revision + 1, shouldPersist: true @@ -255,8 +269,8 @@ export class ClientSessionTabSelectionStore { if (!state) { continue } - statesByWorktree.set(newWorktreeId, state) statesByWorktree.delete(oldWorktreeId) + rememberClientTabSelectionWorktree(statesByWorktree, newWorktreeId, state) changed = state.shouldPersist || changed } if (changed) { diff --git a/src/main/runtime/device-registry.ts b/src/main/runtime/device-registry.ts index d246b1f7309..91ad94e1534 100644 --- a/src/main/runtime/device-registry.ts +++ b/src/main/runtime/device-registry.ts @@ -3,9 +3,12 @@ // compromising one device doesn't expose others. The registry is a simple // JSON file with hardened permissions matching the runtime metadata pattern. import { randomBytes, randomUUID } from 'node:crypto' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' -import { hardenExistingSecureFile, writeSecureJsonFile } from '../../shared/secure-file' +import { JsonStringifyByteLimitError } from '../../shared/node-bounded-json-stringify' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { writeSecureJsonFileWithinLimit } from '../../shared/bounded-secure-json-file' +import { hardenExistingSecureFile } from '../../shared/secure-file' import type { DeviceScope } from '../../shared/runtime-types' import { DEVICE_REGISTRY_FILENAME } from './mobile-pairing-files' import type { RelayDeviceBinding } from './relay/relay-revoke-outbox' @@ -13,6 +16,16 @@ import type { MobilePairingConnectionMode } from '../../shared/mobile-pairing-co export type { DeviceScope } +export const MAX_DEVICE_REGISTRY_FILE_BYTES = 1024 * 1024 +export const MAX_DEVICE_REGISTRY_ENTRIES = 4096 + +export class DeviceRegistryCapacityError extends Error { + constructor() { + super('Device registry exceeds its durable capacity') + this.name = 'DeviceRegistryCapacityError' + } +} + export type DeviceEntry = { deviceId: string name: string @@ -61,6 +74,9 @@ export class DeviceRegistry { name: string, scope: DeviceScope ): DeviceEntry { + if (existingDevices.length >= MAX_DEVICE_REGISTRY_ENTRIES) { + throw new DeviceRegistryCapacityError() + } const entry: DeviceEntry = { deviceId: randomUUID(), name, @@ -102,13 +118,13 @@ export class DeviceRegistry { } removeDevice(deviceId: string): boolean { - const before = this.devices.length - this.devices = this.devices.filter((d) => d.deviceId !== deviceId) - if (this.devices.length < before) { - this.save() - return true + const nextDevices = this.devices.filter((device) => device.deviceId !== deviceId) + if (nextDevices.length === this.devices.length) { + return false } - return false + this.save(nextDevices) + this.devices = nextDevices + return true } getDevice(deviceId: string): DeviceEntry | null { @@ -124,8 +140,12 @@ export class DeviceRegistry { if (!device || binding.relayDeviceId !== deviceId) { return false } + this.save( + this.devices.map((candidate) => + candidate === device ? { ...candidate, relayBinding: binding } : candidate + ) + ) device.relayBinding = binding - this.save() return true } @@ -134,8 +154,12 @@ export class DeviceRegistry { if (!device || device.scope !== 'mobile') { return false } + this.save( + this.devices.map((candidate) => + candidate === device ? { ...candidate, mobilePairingConnectionMode: mode } : candidate + ) + ) device.mobilePairingConnectionMode = mode - this.save() return true } @@ -160,8 +184,13 @@ export class DeviceRegistry { updateLastSeen(deviceId: string): void { const device = this.devices.find((d) => d.deviceId === deviceId) if (device) { - device.lastSeenAt = Date.now() - this.save() + const lastSeenAt = Date.now() + this.save( + this.devices.map((candidate) => + candidate === device ? { ...candidate, lastSeenAt } : candidate + ) + ) + device.lastSeenAt = lastSeenAt } } @@ -172,8 +201,16 @@ export class DeviceRegistry { } try { hardenExistingSecureFile(this.registryPath) - const parsed = JSON.parse(readFileSync(this.registryPath, 'utf-8')) as DeviceEntry[] - this.devices = parsed.map((device) => ({ + const parsed: unknown = JSON.parse( + readNodeFileSyncWithinLimit( + this.registryPath, + MAX_DEVICE_REGISTRY_FILE_BYTES + ).buffer.toString('utf8') + ) + if (!Array.isArray(parsed) || parsed.length > MAX_DEVICE_REGISTRY_ENTRIES) { + throw new DeviceRegistryCapacityError() + } + this.devices = (parsed as DeviceEntry[]).map((device) => ({ ...device, // Why: older registries only existed for phone pairing. Treat missing // scope as mobile so legacy device tokens do not gain new CLI powers. @@ -188,6 +225,13 @@ export class DeviceRegistry { } private save(devices: DeviceEntry[] = this.devices): void { - writeSecureJsonFile(this.registryPath, devices) + try { + writeSecureJsonFileWithinLimit(this.registryPath, devices, MAX_DEVICE_REGISTRY_FILE_BYTES) + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new DeviceRegistryCapacityError() + } + throw error + } } } diff --git a/src/main/runtime/e2ee-keypair.ts b/src/main/runtime/e2ee-keypair.ts index 3e8b3e0aa0a..228a76a2317 100644 --- a/src/main/runtime/e2ee-keypair.ts +++ b/src/main/runtime/e2ee-keypair.ts @@ -1,15 +1,19 @@ // Why: the E2EE keypair enables application-layer encryption between mobile // and desktop over plain ws://. The public key is embedded in the QR pairing // offer so the mobile client can derive a shared secret via ECDH. -import { existsSync, readFileSync, statSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' import nacl from 'tweetnacl' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' import { hardenExistingSecureFile, writeSecureJsonFile } from '../../shared/secure-file' import { E2EE_KEYPAIR_FILENAME } from './mobile-pairing-files' const KEYPAIR_FILENAME = E2EE_KEYPAIR_FILENAME const KEYPAIR_VERSION = 1 -const MAX_KEYPAIR_FILE_BYTES = 8 * 1024 +export const MAX_KEYPAIR_FILE_BYTES = 8 * 1024 +export const MAX_KEYPAIR_JSON_STRUCTURAL_TOKENS = 2048 +export const MAX_KEYPAIR_JSON_NESTING_DEPTH = 8 type KeypairFile = { v: number @@ -29,12 +33,15 @@ export function loadOrCreateE2EEKeypair(userDataPath: string): E2EEKeypair { if (existsSync(filePath)) { try { hardenExistingSecureFile(filePath) - // Why: this startup path reads synchronously; valid keypair files are - // tiny, so oversized/corrupt files should be replaced without loading. - if (statSync(filePath).size > MAX_KEYPAIR_FILE_BYTES) { - throw new Error('E2EE keypair file is too large') - } - const raw: KeypairFile = JSON.parse(readFileSync(filePath, 'utf-8')) + const serialized = readNodeFileSyncWithinLimit( + filePath, + MAX_KEYPAIR_FILE_BYTES + ).buffer.toString('utf8') + assertJsonTextStructureWithinLimits(serialized, { + structuralTokens: MAX_KEYPAIR_JSON_STRUCTURAL_TOKENS, + nestingDepth: MAX_KEYPAIR_JSON_NESTING_DEPTH + }) + const raw = JSON.parse(serialized) as KeypairFile if (raw.v === KEYPAIR_VERSION && raw.publicKeyB64 && raw.secretKeyB64) { const publicKey = Uint8Array.from(Buffer.from(raw.publicKeyB64, 'base64')) const secretKey = Uint8Array.from(Buffer.from(raw.secretKeyB64, 'base64')) diff --git a/src/main/runtime/orca-runtime-files.test.ts b/src/main/runtime/orca-runtime-files.test.ts index f92ebd6b656..3bb68dcff51 100644 --- a/src/main/runtime/orca-runtime-files.test.ts +++ b/src/main/runtime/orca-runtime-files.test.ts @@ -14,6 +14,7 @@ import type * as GitRunner from '../git/runner' const { lstatMock, openMock, + opendirMock, readdirMock, renameMock, resolveAuthorizedPathMock, @@ -29,6 +30,7 @@ const { getLocalGitOptionsForRegisteredWorktreeMock: vi.fn(), lstatMock: vi.fn(), openMock: vi.fn(), + opendirMock: vi.fn(), readdirMock: vi.fn(), renameMock: vi.fn(), resolveAuthorizedPathMock: vi.fn(), @@ -56,6 +58,10 @@ vi.mock('fs/promises', async () => { const impl = openMock.getMockImplementation() return impl ? openMock(...args) : actual.open(...args) }, + opendir: (...args: Parameters) => { + const impl = opendirMock.getMockImplementation() + return impl ? opendirMock(...args) : actual.opendir(...args) + }, readdir: readdirMock, rename: renameMock, stat: statMock @@ -97,7 +103,11 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ 'Remote connection dropped. Click Reconnect on the SSH target before retrying.' })) -import { awaitRuntimeFileWatcherUnsubscribes, RuntimeFileCommands } from './orca-runtime-files' +import { + awaitRuntimeFileWatcherUnsubscribes, + classifyRuntimeMobileDirectoryEntries, + RuntimeFileCommands +} from './orca-runtime-files' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { resetSshConnectionGenerations, @@ -105,6 +115,16 @@ import { } from '../ssh/ssh-connection-generation' import { SEARCH_TIMEOUT_MS } from '../../shared/text-search' +function pngHeader(width = 1, height = 1): Buffer { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(height, 20) + return bytes +} + type MockRuntimeSearchChild = EventEmitter & { stdout: EventEmitter & { setEncoding: ReturnType } stderr: EventEmitter @@ -202,6 +222,7 @@ describe('RuntimeFileCommands', () => { vi.useFakeTimers() lstatMock.mockReset() openMock.mockReset() + opendirMock.mockReset() readdirMock.mockReset() renameMock.mockReset() resolveAuthorizedPathMock.mockReset() @@ -212,6 +233,7 @@ describe('RuntimeFileCommands', () => { checkRgAvailableMock.mockReset() vi.mocked(getSshFilesystemProvider).mockReset() resetSshConnectionGenerations() + setSshConnectionGeneration('ssh-1', 0) getLocalGitOptionsForRegisteredWorktreeMock.mockReset() wslAwareSpawnMock.mockReset() getLocalGitOptionsForRegisteredWorktreeMock.mockReturnValue({}) @@ -298,6 +320,46 @@ describe('RuntimeFileCommands', () => { }) }) + it('validates local file explorer raster dimensions before returning base64', async () => { + const dir = await mkdtemp(join(tmpdir(), 'orca-runtime-image-preview-')) + const filePath = join(dir, 'image.png') + const content = pngHeader(640, 480) + await writeFile(filePath, content) + const { commands } = createRuntimeFileCommands({ path: dir }) + resolveAuthorizedPathMock.mockImplementation(async (value: string) => value) + statMock.mockResolvedValue({ size: content.length }) + + try { + await expect(commands.readFileExplorerPreview('id:wt-1', 'image.png')).resolves.toEqual({ + content: content.toString('base64'), + isBinary: true, + isImage: true, + mimeType: 'image/png', + imageDimensions: { width: 640, height: 480 } + }) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('rejects a local file explorer raster dimension bomb', async () => { + const dir = await mkdtemp(join(tmpdir(), 'orca-runtime-image-bomb-')) + const filePath = join(dir, 'bomb.png') + const content = pngHeader(32_769, 1) + await writeFile(filePath, content) + const { commands } = createRuntimeFileCommands({ path: dir }) + resolveAuthorizedPathMock.mockImplementation(async (value: string) => value) + statMock.mockResolvedValue({ size: content.length }) + + try { + await expect(commands.readFileExplorerPreview('id:wt-1', 'bomb.png')).rejects.toThrow( + 'Image dimensions exceed the preview safety limit' + ) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + it('leaves non-previewable binaries unavailable on mobile', async () => { const openFile = vi.fn() const { commands } = createRuntimeFileCommands({ openFile }) @@ -354,10 +416,12 @@ describe('RuntimeFileCommands', () => { it('does not follow symlinks when reading runtime-local file explorer dirs', async () => { const { commands } = createRuntimeFileCommands() resolveAuthorizedPathMock.mockResolvedValue('/repo') - readdirMock.mockResolvedValue([ - dirEntry({ name: 'README.md' }), - dirEntry({ name: 'linked-docs', directory: true, symlink: true }) - ]) + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + yield dirEntry({ name: 'README.md' }) + yield dirEntry({ name: 'linked-docs', directory: true, symlink: true }) + } + }) const result = await commands.readFileExplorerDir('id:wt-1', '') @@ -368,6 +432,46 @@ describe('RuntimeFileCommands', () => { expect(statMock).not.toHaveBeenCalledWith('/repo/linked-docs') }) + it('stops runtime-local directory enumeration at the mobile entry limit', async () => { + const { commands } = createRuntimeFileCommands() + resolveAuthorizedPathMock.mockResolvedValue('/repo') + let enumerated = 0 + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + while (enumerated < 20_000) { + enumerated += 1 + yield dirEntry({ name: 'entry' }) + } + } + }) + + await expect(commands.readFileExplorerDir('id:wt-1', '')).rejects.toThrow( + 'This folder is too large to show safely on mobile' + ) + expect(enumerated).toBe(10_001) + }) + + it('bounds runtime-local directory classification while preserving entry order', async () => { + let active = 0 + let maxActive = 0 + const classify = vi.fn(async () => { + active += 1 + maxActive = Math.max(maxActive, active) + await Promise.resolve() + active -= 1 + return true + }) + const entries = Array.from({ length: 100 }, (_, index) => + dirEntry({ name: `entry-${index}`, symlink: true }) + ) + + const result = await classifyRuntimeMobileDirectoryEntries('/repo', entries, classify) + + expect(maxActive).toBe(32) + expect(result.map((entry) => entry.name)).toEqual(entries.map((entry) => entry.name)) + expect(result.every((entry) => entry.isDirectory && entry.isSymlink)).toBe(true) + }) + it('renames a runtime-local file when destination does not exist', async () => { const { commands } = createRuntimeFileCommands() resolveAuthorizedPathMock.mockImplementation(async (p: string) => p) @@ -799,7 +903,7 @@ describe('RuntimeFileCommands', () => { tempDirs = [] }) - async function tempFile(name: string, content: string): Promise { + async function tempFile(name: string, content: string | Uint8Array): Promise { const dir = await mkdtemp(join(tmpdir(), 'orca-terminal-artifact-')) tempDirs.push(dir) const filePath = join(dir, name) @@ -1766,6 +1870,49 @@ describe('RuntimeFileCommands', () => { ).rejects.toThrow('terminal_file_grant_stale') }) + it('validates local terminal artifact raster dimensions before returning base64', async () => { + const content = pngHeader(320, 240) + const artifactPath = await tempFile('result.png', content) + const { commands } = createRuntimeFileCommands({ path: '/repo' }) + resolveAuthorizedPathMock.mockImplementation(async (value: string) => value) + + const result = await resolveTerminalArtifactPath(commands, artifactPath) + const target = absoluteFileTarget(result) + + await expect( + commands.readTerminalArtifactPreview( + 'id:wt-1', + target.grantId, + target.absolutePath, + 'client-a' + ) + ).resolves.toEqual({ + content: content.toString('base64'), + isBinary: true, + isImage: true, + mimeType: 'image/png', + imageDimensions: { width: 320, height: 240 } + }) + }) + + it('rejects a local terminal artifact raster dimension bomb', async () => { + const artifactPath = await tempFile('result.png', pngHeader(32_769, 1)) + const { commands } = createRuntimeFileCommands({ path: '/repo' }) + resolveAuthorizedPathMock.mockImplementation(async (value: string) => value) + + const result = await resolveTerminalArtifactPath(commands, artifactPath) + const target = absoluteFileTarget(result) + + await expect( + commands.readTerminalArtifactPreview( + 'id:wt-1', + target.grantId, + target.absolutePath, + 'client-a' + ) + ).rejects.toThrow('Image dimensions exceed the preview safety limit') + }) + it('rejects binary-extension terminal artifacts from the editable text path', async () => { const artifactPath = await tempFile('report.pdf', '%PDF text-looking bytes') const { commands } = createRuntimeFileCommands({ path: '/repo' }) diff --git a/src/main/runtime/orca-runtime-files.ts b/src/main/runtime/orca-runtime-files.ts index c518a4ced3d..3b7164a2c91 100644 --- a/src/main/runtime/orca-runtime-files.ts +++ b/src/main/runtime/orca-runtime-files.ts @@ -1,7 +1,7 @@ /* eslint-disable max-lines -- Why: filesystem, editor-file, and search commands share the same local/SSH path authorization rules. Keeping that IO adapter together prevents separate command paths from drifting on safety checks. */ import type { ChildProcess } from 'node:child_process' import { randomUUID } from 'node:crypto' -import { watch as watchFs } from 'node:fs' +import { watch as watchFs, type Dirent } from 'node:fs' import type { FileHandle } from 'node:fs/promises' import { chmod, @@ -10,8 +10,7 @@ import { lstat, mkdir, open, - readFile, - readdir, + opendir, rename, realpath, rm, @@ -38,6 +37,13 @@ import { resolveRuntimePath } from '../../shared/cross-platform-path' import { PhysicalExitTracker } from '../../shared/physical-exit-tracker' +import { + assertMobileFileDirectoryWithinLimit, + createMobileFileDirectoryLimitState, + MOBILE_FILE_DIRECTORY_MAX_ENTRIES, + MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES, + trackMobileFileDirectoryEntry +} from '../../shared/mobile-file-directory-limit' import type { RuntimeFileListResult, RuntimeFileOpenResult, @@ -80,6 +86,10 @@ import { WatcherProcessFailure } from '../ipc/parcel-watcher-process-failure' import { assertNoClobberRenameDestinationAvailable } from '../../shared/filesystem-rename-collision' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' import { joinWorktreeRelativePath, normalizeRuntimeRelativePath } from './runtime-relative-paths' import { rankRuntimeMobileFilePaths, @@ -88,12 +98,18 @@ import { import { beginWatcherInstall } from '../ipc/watcher-removal-gate' import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation' import { toSshExecutionHostId } from '../../shared/execution-host' +import { assertRasterImagePreviewWithinLimits } from '../../shared/raster-image-preview-limits' +import { SearchSubprocessLineAccumulator } from '../../shared/search-subprocess-lines' +import { retainRuntimeTerminalFileGrant } from './runtime-terminal-file-grant-retention' +import { assertRuntimeTextSearchAdmission } from './runtime-text-search-admission' +import { RuntimeFileWatcherAdmission } from './runtime-file-watcher-admission' const MOBILE_FILE_LIST_LIMIT = 5000 const MOBILE_FILE_PATH_SEARCH_CACHE_LIMIT = 20_000 const MOBILE_FILE_PATH_SEARCH_CACHE_ENTRIES = 8 const MOBILE_FILE_PATH_SEARCH_CACHE_TTL_MS = 30_000 const MOBILE_FILE_READ_MAX_BYTES = 512 * 1024 +const RUNTIME_MOBILE_DIRECTORY_CLASSIFICATION_CONCURRENCY = 32 const RUNTIME_PREVIEWABLE_BINARY_MAX_BYTES = 10 * 1024 * 1024 const WINDOWS_RUNTIME_FILE_WATCH_DEBOUNCE_MS = 150 export const WINDOWS_RUNTIME_FILE_WATCH_CLOSE_DEADLINE_MS = 10_000 @@ -126,6 +142,7 @@ type RuntimeFileWatcherLease = { forget(): void } const runtimeFileWatcherLeasesByOwnerAndRoot = new Map>() +const runtimeFileWatcherAdmission = new RuntimeFileWatcherAdmission() const MOBILE_BINARY_EXTENSIONS = new Set([ '.avif', '.bmp', @@ -231,7 +248,8 @@ function registerRuntimeFileWatcherRelease( rootPaths: string[], unsubscribe: () => Promise, restart: () => Promise<() => Promise>, - onRestoreError: (error: Error) => void + onRestoreError: (error: Error) => void, + releaseAdmission: () => void ): () => Promise { const keys = Array.from( new Set( @@ -252,6 +270,7 @@ function registerRuntimeFileWatcherRelease( runtimeFileWatcherLeasesByOwnerAndRoot.delete(key) } } + releaseAdmission() } const suspend = (): Promise => { if (releasePromise) { @@ -830,7 +849,10 @@ export class RuntimeFileCommands { expiresAt: Date.now() + TERMINAL_FILE_GRANT_TTL_MS, statIdentity: terminalFileStatIdentity(args.stats) } - this.terminalFileGrants.set(grant.id, grant) + this.pruneExpiredTerminalFileGrants() + retainRuntimeTerminalFileGrant(this.terminalFileGrants, grant, (id, retained) => + this.releaseTerminalFileGrant(id, retained) + ) this.scheduleTerminalFileGrantExpiry(grant) return grant } @@ -1162,21 +1184,23 @@ export class RuntimeFileCommands { if (!provider) { throw new Error(SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE) } - return provider.readDir(target.path) + const entries = await provider.readDir(target.path, { + maxEntries: MOBILE_FILE_DIRECTORY_MAX_ENTRIES, + maxRetainedBytes: MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES + }) + assertMobileFileDirectoryWithinLimit(entries) + return entries } const dirPath = await resolveAuthorizedPath(target.path, this.host.requireStore()) - const entries = await readdir(dirPath, { withFileTypes: true }) - const mapped = await Promise.all( - entries.map(async (entry) => { - const entryPath = join(dirPath, entry.name) - return { - name: entry.name, - isDirectory: await isRuntimeDirectoryEntry(entry, entryPath), - isSymlink: entry.isSymbolicLink() - } - }) - ) + const limit = createMobileFileDirectoryLimitState() + const directory = await opendir(dirPath) + const entries: Dirent[] = [] + for await (const entry of directory) { + trackMobileFileDirectoryEntry(limit, entry) + entries.push(entry) + } + const mapped = await classifyRuntimeMobileDirectoryEntries(dirPath, entries) return mapped.sort((a, b) => { if (a.isDirectory !== b.isDirectory) { return a.isDirectory ? -1 : 1 @@ -1192,6 +1216,11 @@ export class RuntimeFileCommands { signal?: AbortSignal ): Promise<() => void> { const target = await this.resolveFileExplorerPath(worktreeSelector, '') + const releaseAdmission = runtimeFileWatcherAdmission.claim( + this.host.getRuntimeId(), + target.connectionId, + target.path + ) const open = async (): Promise<{ unsubscribe: () => Promise rootPaths: string[] @@ -1229,15 +1258,30 @@ export class RuntimeFileCommands { finishInstall() } } - const initial = await open() - return registerRuntimeFileWatcherRelease( - this.host.getRuntimeId(), - target.connectionId, - initial.rootPaths, - initial.unsubscribe, - async () => (await open()).unsubscribe, - onTerminalError - ) + let initial: Awaited> + try { + initial = await open() + } catch (error) { + releaseAdmission() + throw error + } + try { + return registerRuntimeFileWatcherRelease( + this.host.getRuntimeId(), + target.connectionId, + initial.rootPaths, + initial.unsubscribe, + async () => (await open()).unsubscribe, + onTerminalError, + releaseAdmission + ) + } catch (error) { + releaseAdmission() + await trackRuntimeFileWatcherUnsubscribe(initial.rootPaths[0], initial.unsubscribe).catch( + () => undefined + ) + throw error + } } async closeFileExplorerWatchersForPath(rootPath: string, connectionId?: string): Promise { @@ -1295,23 +1339,30 @@ export class RuntimeFileCommands { const filePath = await resolveAuthorizedPath(target.path, this.host.requireStore()) const fileStats = await stat(filePath) const mimeType = RUNTIME_PREVIEWABLE_BINARY_MIME_TYPES[extname(filePath).toLowerCase()] - if (mimeType) { - if (fileStats.size > RUNTIME_PREVIEWABLE_BINARY_MAX_BYTES) { + const sizeLimit = mimeType ? RUNTIME_PREVIEWABLE_BINARY_MAX_BYTES : MOBILE_FILE_READ_MAX_BYTES + if (fileStats.size > sizeLimit) { + throw new Error('file_too_large') + } + let buffer: Buffer + try { + buffer = (await readNodeFileWithinLimit(filePath, sizeLimit)).buffer + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { throw new Error('file_too_large') } - const buffer = await readFile(filePath) + throw error + } + if (mimeType) { + const imageDimensions = assertRasterImagePreviewWithinLimits(buffer, mimeType) return { content: buffer.toString('base64'), isBinary: true, isImage: true, - mimeType + mimeType, + ...(imageDimensions ? { imageDimensions } : {}) } } - if (fileStats.size > MOBILE_FILE_READ_MAX_BYTES) { - throw new Error('file_too_large') - } - const buffer = await readFile(filePath) if (isBinaryBuffer(buffer)) { return { content: '', isBinary: true } } @@ -1738,7 +1789,12 @@ export class RuntimeFileCommands { if (!provider) { throw new Error(SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE) } - const relativePaths = await provider.listFiles(target.worktree.path) + if (!provider.listMarkdownDocuments) { + throw new Error( + 'Remote Markdown link discovery is unavailable. Reconnect the SSH target and retry.' + ) + } + const relativePaths = await provider.listMarkdownDocuments(target.worktree.path) return markdownDocumentsFromRelativePaths(target.worktree.path, relativePaths) } return listMarkdownDocuments(target.worktree.path) @@ -1788,11 +1844,12 @@ export class RuntimeFileCommands { return new Promise((resolvePromise) => { const searchKey = `${this.host.getRuntimeId()}:${authorizedRootPath}` + assertRuntimeTextSearchAdmission(this.activeRuntimeTextSearches, searchKey) const rgArgs = buildRgArgs(options.query, authorizedRootPath, options) this.activeRuntimeTextSearches.get(searchKey)?.kill() const acc = createAccumulator() - let stdoutBuffer = '' + const stdoutLines = new SearchSubprocessLineAccumulator() let resolved = false let child: ChildProcess | null = null const wslInfo = parseWslPath(authorizedRootPath) @@ -1845,13 +1902,11 @@ export class RuntimeFileCommands { child = nextChild this.activeRuntimeTextSearches.set(searchKey, nextChild) - nextChild.stdout!.setEncoding('utf-8') - const onStdoutData = (chunk: string): void => { - stdoutBuffer += chunk - const lines = stdoutBuffer.split('\n') - stdoutBuffer = lines.pop() ?? '' - for (const line of lines) { - processLine(line) + const onStdoutData = (chunk: Buffer): void => { + if (!stdoutLines.push(chunk, processLine)) { + acc.truncated = true + child?.kill() + resolveOnce() } } const onStderrData = (): void => { @@ -1859,8 +1914,9 @@ export class RuntimeFileCommands { } const onError = (): void => resolveOnce() const onClose = (): void => { - if (stdoutBuffer) { - processLine(stdoutBuffer) + const trailingLine = stdoutLines.finish() + if (trailingLine !== null) { + processLine(trailingLine) } resolveOnce() } @@ -2059,6 +2115,45 @@ async function isRuntimeDirectoryEntry( return false } +type RuntimeDirectorySourceEntry = { + name: string + isDirectory(): boolean + isSymbolicLink(): boolean +} + +type RuntimeDirectoryClassifier = ( + entry: RuntimeDirectorySourceEntry, + entryPath: string +) => Promise + +export async function classifyRuntimeMobileDirectoryEntries( + dirPath: string, + entries: readonly RuntimeDirectorySourceEntry[], + classify: RuntimeDirectoryClassifier = isRuntimeDirectoryEntry +): Promise { + const mapped: DirEntry[] = [] + for ( + let offset = 0; + offset < entries.length; + offset += RUNTIME_MOBILE_DIRECTORY_CLASSIFICATION_CONCURRENCY + ) { + const batch = entries.slice( + offset, + offset + RUNTIME_MOBILE_DIRECTORY_CLASSIFICATION_CONCURRENCY + ) + mapped.push( + ...(await Promise.all( + batch.map(async (entry) => ({ + name: entry.name, + isDirectory: await classify(entry, join(dirPath, entry.name)), + isSymlink: entry.isSymbolicLink() + })) + )) + ) + } + return mapped +} + function isBinaryBuffer(buffer: Buffer): boolean { const len = Math.min(buffer.length, 8192) for (let i = 0; i < len; i += 1) { @@ -2148,11 +2243,13 @@ async function readLocalTerminalArtifactPreviewFromHandle( handle, RUNTIME_PREVIEWABLE_BINARY_MAX_BYTES + 1 ) + const imageDimensions = assertRasterImagePreviewWithinLimits(buffer, mimeType) return { content: buffer.toString('base64'), isBinary: true, isImage: true, - mimeType + mimeType, + ...(imageDimensions ? { imageDimensions } : {}) } } diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 3c8066265e1..e982b9bb080 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -86,6 +86,8 @@ import { unregisterSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { registerSshGitProvider, unregisterSshGitProvider } from '../providers/ssh-git-dispatch' +import { inspectPtyProviderProcess } from '../providers/pty-process-inspection' +import type { IPtyProvider } from '../providers/types' import * as worktreePathComparison from '../ipc/worktree-path-comparison' import * as localWorktreeFilesystem from '../local-worktree-filesystem' import { @@ -112,6 +114,7 @@ import { const ORIGINAL_PLATFORM = process.platform const ORIGINAL_PLATFORM_DESCRIPTOR = Object.getOwnPropertyDescriptor(process, 'platform') +const TEST_MAX_HOOK_GITIGNORE_BYTES = 4 * 1024 * 1024 const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) const findExistingWorktreeSymlinkPathsMock = vi.hoisted(() => vi.fn()) const resolveLocalGitUsernameMock = vi.hoisted(() => vi.fn(async () => '')) @@ -437,6 +440,8 @@ vi.mock('../hooks', () => ({ ORCA_ROOT_PATH: '/remote/repo', ORCA_WORKTREE_PATH: worktreePath }), + MAX_HOOK_GITIGNORE_BYTES: 4 * 1024 * 1024, + MAX_ISSUE_COMMAND_BYTES: 1024 * 1024, loadHooks: vi.fn().mockReturnValue(null), runHook: vi.fn().mockResolvedValue({ success: true, output: '' }), shouldRunSetupForCreate: vi @@ -910,6 +915,7 @@ const TEST_FOLDER_WORKSPACE_PATH = '/tmp/platform' const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ const HEADLESS_LEAF_ID = '11111111-1111-4111-8111-111111111111' const HEADLESS_SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222' +const HEADLESS_THIRD_LEAF_ID = '33333333-3333-4333-8333-333333333333' function isOriginMainBaseRefProbe(args: string[]): boolean { return ( @@ -1278,6 +1284,9 @@ function createRuntimeWithSshLease( async function createExplicitAgentStatusHarness(options: { getForegroundProcess: (ptyId: string) => Promise + inspectProcess?: ( + ptyId: string + ) => Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> confirmForegroundProcess?: (ptyId: string) => Promise title?: string }): Promise<{ @@ -1307,6 +1316,7 @@ async function createExplicitAgentStatusHarness(options: { write: () => true, kill: () => true, getForegroundProcess: options.getForegroundProcess, + inspectProcess: options.inspectProcess, confirmForegroundProcess: options.confirmForegroundProcess }) runtime.attachWindow(1) @@ -3271,7 +3281,10 @@ describe('OrcaRuntimeService', () => { expect(listWorktrees).not.toHaveBeenCalled() expect(gitProvider.listWorktrees).toHaveBeenCalledWith('//Server/Share/Repo') - expect(fsProvider.readDir).toHaveBeenCalledWith('\\\\Server\\Share\\Repo\\src') + expect(fsProvider.readDir).toHaveBeenCalledWith('\\\\Server\\Share\\Repo\\src', { + maxEntries: 10_000, + maxRetainedBytes: 4 * 1024 * 1024 + }) expect(gitProvider.getStatus).toHaveBeenCalledWith('//Server/Share/Repo') }) @@ -3338,7 +3351,10 @@ describe('OrcaRuntimeService', () => { } expect(fsProvider.stat).toHaveBeenCalledWith(folderPath) - expect(fsProvider.readDir).toHaveBeenCalledWith('/srv/platform/src') + expect(fsProvider.readDir).toHaveBeenCalledWith('/srv/platform/src', { + maxEntries: 10_000, + maxRetainedBytes: 4 * 1024 * 1024 + }) expect(fsProvider.stat).toHaveBeenCalledWith('/srv/platform/src/app.ts') expect(fsProvider.readFile).toHaveBeenCalledWith('/srv/platform/src/app.ts') }) @@ -5210,6 +5226,12 @@ describe('OrcaRuntimeService', () => { listWorktrees: vi.fn().mockResolvedValue([created]) } const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('orca.yaml')) { + return { size: 7, type: 'file', mtime: 0 } + } + throw Object.assign(new Error('missing'), { code: 'ENOENT' }) + }), readFile: vi.fn().mockResolvedValue({ isBinary: false, content: 'hooks:\n' }), createDir: vi.fn().mockResolvedValue(undefined), writeFile: vi.fn().mockResolvedValue(undefined) @@ -5367,6 +5389,12 @@ describe('OrcaRuntimeService', () => { listWorktrees: vi.fn().mockResolvedValue([created]) } const fsProvider = { + stat: vi.fn(async (filePath: string) => { + if (filePath.endsWith('orca.yaml')) { + return { size: 7, type: 'file', mtime: 0 } + } + throw Object.assign(new Error('missing'), { code: 'ENOENT' }) + }), readFile: vi.fn().mockResolvedValue({ isBinary: false, content: 'hooks:\n' }), createDir: vi.fn().mockResolvedValue(undefined), writeFile: vi.fn().mockResolvedValue(undefined) @@ -5579,6 +5607,7 @@ describe('OrcaRuntimeService', () => { ] } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 32, type: 'file', mtime: 0 }), readFile: vi.fn().mockResolvedValue({ content: 'scripts:\n setup: pnpm install\n', isBinary: false @@ -5655,6 +5684,7 @@ describe('OrcaRuntimeService', () => { ] } const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 32, type: 'file', mtime: 0 }), readFile: vi.fn(async (filePath: string) => ({ content: filePath.endsWith('orca.yaml') ? 'scripts:\n setup: pnpm install\n' @@ -5700,6 +5730,49 @@ describe('OrcaRuntimeService', () => { ) }) + it('does not materialize an oversized SSH .gitignore while writing an issue command', async () => { + const remoteStore = { + ...store, + getRepos: () => [ + { + id: TEST_REPO_ID, + path: '/remote/repo', + displayName: 'repo', + badgeColor: 'blue', + addedAt: 1, + connectionId: 'ssh-1' + } + ] + } + const fsProvider = { + stat: vi.fn().mockResolvedValue({ + size: TEST_MAX_HOOK_GITIGNORE_BYTES + 1, + type: 'file', + mtime: 0 + }), + readFile: vi.fn(), + writeFile: vi.fn().mockResolvedValue(undefined), + createDir: vi.fn().mockResolvedValue(undefined) + } + registerSshFilesystemProvider('ssh-1', fsProvider as never) + const runtime = new OrcaRuntimeService(remoteStore as never) + + try { + await expect(runtime.writeRepoIssueCommand('id:repo-1', 'Ship it')).resolves.toEqual({ + ok: true + }) + } finally { + unregisterSshFilesystemProvider('ssh-1') + } + + expect(fsProvider.readFile).not.toHaveBeenCalled() + expect(fsProvider.writeFile).toHaveBeenCalledOnce() + expect(fsProvider.writeFile).toHaveBeenCalledWith( + '/remote/repo/.orca/issue-command', + 'Ship it\n' + ) + }) + it('resolves SSH issue commands from shared orca.yaml and deletes empty overrides', async () => { const remoteStore = { ...store, @@ -5719,6 +5792,7 @@ describe('OrcaRuntimeService', () => { issueCommand: 'claude -p "Fix #{{issue}}"' }) const fsProvider = { + stat: vi.fn().mockResolvedValue({ size: 48, type: 'file', mtime: 0 }), readFile: vi.fn(async (filePath: string) => { if (filePath.endsWith('.orca/issue-command')) { throw Object.assign(new Error('missing'), { code: 'ENOENT' }) @@ -7139,7 +7213,7 @@ describe('OrcaRuntimeService', () => { getRepo: (id: string) => added.find((repo) => repo.id === id) as never } const runtime = new OrcaRuntimeService(createStore as never) - const tempRoot = await mkdtemp('/tmp/orca-runtime-create-parent-') + const tempRoot = await mkdtemp(join(tmpdir(), 'orca-runtime-create-parent-')) const parentDir = join(tempRoot, 'orca', 'projects') try { const result = await runtime.createRepo(parentDir, 'first-project', 'folder') @@ -7166,7 +7240,7 @@ describe('OrcaRuntimeService', () => { getRepo: (id: string) => added.find((repo) => repo.id === id) as never } const runtime = new OrcaRuntimeService(runtimeStore as never) - const parentDir = await mkdtemp('/tmp/orca-runtime-create-root-prep-') + const parentDir = await mkdtemp(join(tmpdir(), 'orca-runtime-create-root-prep-')) try { const result = await runtime.createRepo(parentDir, 'runtime-create-root-prep', 'folder') if ('error' in result) { @@ -7180,10 +7254,11 @@ describe('OrcaRuntimeService', () => { }) it('preserves existing badgeColor on runtime createRepo dedupe', async () => { + const repoName = 'runtime-existing-create' const existing = { - id: 'runtime-existing-create', - path: '/tmp/runtime-existing-create', - displayName: 'runtime-existing-create', + id: repoName, + path: join(tmpdir(), repoName), + displayName: repoName, badgeColor: '#14b8a6', addedAt: 1, kind: 'folder' as const @@ -7194,7 +7269,7 @@ describe('OrcaRuntimeService', () => { } const runtime = new OrcaRuntimeService(colorStore as never) - const result = await runtime.createRepo('/tmp', 'runtime-existing-create', 'folder') + const result = await runtime.createRepo(tmpdir(), repoName, 'folder') expect(result).toEqual({ repo: existing }) expect(result).toHaveProperty('repo.badgeColor', '#14b8a6') @@ -10366,6 +10441,23 @@ describe('OrcaRuntimeService', () => { expect(confirmForegroundProcess).toHaveBeenCalledWith('pty-1') }) + it('preserves provider failure during completion-sensitive process inspection', async () => { + const failure = new Error('daemon unavailable') + const providerInspectProcess = vi.fn().mockRejectedValue(failure) + const provider = { inspectProcess: providerInspectProcess } as unknown as IPtyProvider + const inspectProcess = vi.fn((ptyId: string) => inspectPtyProviderProcess(provider, ptyId)) + const getForegroundProcess = vi.fn(async () => null) + const { runtime, handle } = await createExplicitAgentStatusHarness({ + getForegroundProcess, + inspectProcess + }) + + await expect(runtime.inspectTerminalProcess(handle)).rejects.toBe(failure) + expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('pty-1') + expect(providerInspectProcess).toHaveBeenCalledExactlyOnceWith('pty-1') + expect(getForegroundProcess).not.toHaveBeenCalled() + }) + it('calls foreground confirmation with its controller receiver', async () => { const getForegroundProcess = vi.fn(async () => 'powershell.exe') const confirmForegroundProcess = vi.fn( @@ -15910,6 +16002,991 @@ describe('OrcaRuntimeService', () => { expect(writes).toEqual(['still writable']) }) + it('adopts a v1.4.150-shaped agent, setup, and shell orphan as one topology transaction', async () => { + const session = { + ...getDefaultWorkspaceSession(), + activeRepoId: TEST_REPO_ID, + activeWorktreeId: TEST_WORKTREE_ID, + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const writes: [string, string][] = [] + const resize = vi.fn(() => true) + const processes = [ + ['pty-agent', 'inc-agent', 'term_agent', 'Agent'], + ['pty-setup', 'inc-setup', 'term_setup', 'Setup'], + ['pty-shell', 'inc-shell', 'term_shell', 'Shell'] + ] as const + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.setPtyController({ + write: (ptyId, data) => { + writes.push([ptyId, data]) + return true + }, + resize, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => + processes.map(([id, incarnationId, terminalHandle, title]) => ({ + id, + incarnationId, + terminalHandle, + title, + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + })) + }) + const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`) + expect(before.terminals.map((terminal) => terminal.tabId)).toEqual( + processes.map(([id]) => `pty:${id}`) + ) + const targeted = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`, 100, { + handles: ['term_setup'], + requireFreshPtyLiveness: true + }) + expect(targeted).toMatchObject({ + terminals: [expect.objectContaining({ handle: 'term_setup', ptyId: 'pty-setup' })], + totalCount: 1, + truncated: false + }) + runtime.onPtyData('pty-agent', 'legacy output\n', 1) + + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_agent', + ptyId: 'pty-agent', + incarnationId: 'stale-incarnation', + tabId: 'tab-agent', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_stale') + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + + const adopted = await runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: before.topologyRevisions?.[TEST_WORKTREE_ID] ?? 0, + activeTabId: 'tab-agent', + activeGroupId: 'legacy-group', + claims: processes.map(([ptyId, incarnationId, terminal], index) => ({ + terminal, + ptyId, + incarnationId, + tabId: ['tab-agent', 'tab-setup', 'tab-shell'][index]!, + leafId: [HEADLESS_LEAF_ID, HEADLESS_SECOND_LEAF_ID, HEADLESS_THIRD_LEAF_ID][index]! + })) + }) + + expect(adopted.adopted).toBe(true) + expect(adopted.topologyRevision).toBe(1) + expect(adopted.snapshot.tabs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + parentTabId: 'tab-agent', + leafId: HEADLESS_LEAF_ID, + title: 'Agent', + terminal: 'term_agent' + }), + expect.objectContaining({ + parentTabId: 'tab-setup', + leafId: HEADLESS_SECOND_LEAF_ID, + title: 'Setup', + terminal: 'term_setup' + }), + expect.objectContaining({ + parentTabId: 'tab-shell', + leafId: HEADLESS_THIRD_LEAF_ID, + title: 'Shell', + terminal: 'term_shell' + }) + ]) + ) + expect(adopted.snapshot.tabGroups).toEqual([ + expect.objectContaining({ + activeTabId: 'tab-agent', + tabOrder: ['tab-agent', 'tab-setup', 'tab-shell'] + }) + ]) + expect(getSession().terminalTopologyRevisionByRepoId?.[TEST_REPO_ID]).toBe(1) + + await runtime.sendTerminal('term_agent', { text: 'input' }) + await runtime.updateRemoteDesktopViewer('pty-agent', 'viewer', 'client', 132, 41) + expect(writes).toEqual([['pty-agent', 'input']]) + expect(resize).toHaveBeenCalledWith('pty-agent', 132, 41) + await expect(runtime.readTerminal('term_agent')).resolves.toMatchObject({ + tail: ['legacy output'] + }) + + const secondClient = await runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: processes.map(([ptyId, incarnationId, terminal], index) => ({ + terminal, + ptyId, + incarnationId, + tabId: ['tab-agent', 'tab-setup', 'tab-shell'][index]!, + leafId: [HEADLESS_LEAF_ID, HEADLESS_SECOND_LEAF_ID, HEADLESS_THIRD_LEAF_ID][index]! + })) + }) + expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 }) + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_agent', + ptyId: 'pty-agent', + incarnationId: 'inc-agent', + tabId: 'competing-tab', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_competing_owner') + }) + + it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => { + const session: WorkspaceSessionState = { + ...makeWorkspaceSessionWithHeadlessTerminal({ + activeTabIdByWorktree: { [TEST_WORKTREE_ID]: 'terminal-3' }, + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: 'terminal-3', + ptyId: 'pty-new', + worktreeId: TEST_WORKTREE_ID, + title: 'Terminal 3', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 3 + } + ] + }, + terminalLayoutsByTabId: { + 'terminal-3': makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: 'pty-new' }) + } + }), + tabGroups: { + [TEST_WORKTREE_ID]: [ + { + id: 'group-live', + worktreeId: TEST_WORKTREE_ID, + activeTabId: 'terminal-3', + tabOrder: ['terminal-3'] + } + ] + }, + tabGroupLayouts: { + [TEST_WORKTREE_ID]: { type: 'leaf', groupId: 'group-live' } + }, + activeGroupIdByWorktree: { [TEST_WORKTREE_ID]: 'group-live' }, + terminalPtyIncarnationsByPaneKey: { + [`terminal-3:${HEADLESS_LEAF_ID}`]: 'inc-new' + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-new', + incarnationId: 'inc-new', + terminalHandle: 'term_new', + title: 'Terminal 3', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + }, + { + id: 'pty-agent', + incarnationId: 'inc-agent', + terminalHandle: 'term_agent', + title: 'Claude', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + }, + { + id: 'pty-setup', + incarnationId: 'inc-setup', + terminalHandle: 'term_setup', + title: 'Setup', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + }, + { + id: 'pty-shell', + incarnationId: 'inc-shell', + terminalHandle: 'term_shell', + title: 'Shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + + const adopted = await runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + activeTabId: 'tab-shell', + activeGroupId: 'group-old-right', + claims: [ + { + terminal: 'term_agent', + ptyId: 'pty-agent', + incarnationId: 'inc-agent', + tabId: 'tab-agent', + leafId: HEADLESS_LEAF_ID + }, + { + terminal: 'term_setup', + ptyId: 'pty-setup', + incarnationId: 'inc-setup', + tabId: 'tab-agent', + leafId: HEADLESS_SECOND_LEAF_ID + }, + { + terminal: 'term_shell', + ptyId: 'pty-shell', + incarnationId: 'inc-shell', + tabId: 'tab-shell', + leafId: HEADLESS_THIRD_LEAF_ID + } + ], + topology: { + tabs: [ + { + tabId: 'tab-agent', + root: { + type: 'split', + direction: 'horizontal', + ratio: 0.7, + first: { type: 'leaf', leafId: HEADLESS_LEAF_ID }, + second: { type: 'leaf', leafId: HEADLESS_SECOND_LEAF_ID } + }, + activeLeafId: HEADLESS_SECOND_LEAF_ID, + expandedLeafId: null + }, + { + tabId: 'tab-shell', + root: { type: 'leaf', leafId: HEADLESS_THIRD_LEAF_ID }, + activeLeafId: HEADLESS_THIRD_LEAF_ID, + expandedLeafId: HEADLESS_THIRD_LEAF_ID + } + ], + groups: [ + { + id: 'group-old-left', + activeTabId: 'tab-agent', + tabOrder: ['tab-agent'], + recentTabIds: ['tab-agent'] + }, + { + id: 'group-old-right', + activeTabId: 'tab-shell', + tabOrder: ['tab-shell'] + } + ], + groupLayout: { + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-old-left' }, + second: { type: 'leaf', groupId: 'group-old-right' } + } + } + }) + + expect(adopted.snapshot.activeGroupId).toBe('group-old-right') + expect(adopted.snapshot.activeTabId).toBe(`tab-shell::${HEADLESS_THIRD_LEAF_ID}`) + expect(adopted.snapshot.tabGroups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'group-live', tabOrder: ['terminal-3'] }), + expect.objectContaining({ id: 'group-old-left', tabOrder: ['tab-agent'] }), + expect.objectContaining({ id: 'group-old-right', tabOrder: ['tab-shell'] }) + ]) + ) + expect(adopted.snapshot.tabGroupLayout).toMatchObject({ + type: 'split', + direction: 'vertical', + first: { type: 'leaf', groupId: 'group-live' }, + second: { + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-old-left' }, + second: { type: 'leaf', groupId: 'group-old-right' } + } + }) + expect(getSession().terminalLayoutsByTabId['tab-agent']).toMatchObject({ + root: { type: 'split', direction: 'horizontal', ratio: 0.7 }, + activeLeafId: HEADLESS_SECOND_LEAF_ID, + ptyIdsByLeafId: { + [HEADLESS_LEAF_ID]: 'pty-agent', + [HEADLESS_SECOND_LEAF_ID]: 'pty-setup' + } + }) + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID].map((tab) => tab.id)).toEqual([ + 'terminal-3', + 'tab-agent', + 'tab-shell' + ]) + }) + + it('canonicalizes an equivalent persisted worktree key without duplicating terminal topology', async () => { + const aliasWorktreeId = `${TEST_REPO_ID}::/tmp//worktree-a/` + const base = makeWorkspaceSessionWithHeadlessTerminal({ + terminalPtyIncarnationsByPaneKey: { + [`host-tab:${HEADLESS_LEAF_ID}`]: 'inc-alias' + } + }) + const session: WorkspaceSessionState = { + ...base, + activeTabIdByWorktree: { [aliasWorktreeId]: 'host-tab' }, + tabsByWorktree: { + [aliasWorktreeId]: base.tabsByWorktree[TEST_WORKTREE_ID]!.map((tab) => ({ + ...tab, + worktreeId: aliasWorktreeId + })) + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'persisted-pty', + incarnationId: 'inc-alias', + terminalHandle: 'term_alias', + title: 'Alias shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + + const adopted = await runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_alias', + ptyId: 'persisted-pty', + incarnationId: 'inc-alias', + tabId: 'host-tab', + leafId: HEADLESS_LEAF_ID + } + ] + }) + + expect(adopted).toMatchObject({ adopted: true, topologyRevision: 1 }) + expect(adopted.snapshot.worktree).toBe(TEST_WORKTREE_ID) + expect(Object.keys(getSession().tabsByWorktree)).toContain(TEST_WORKTREE_ID) + expect(Object.keys(getSession().tabsByWorktree)).not.toContain(aliasWorktreeId) + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]?.[0]?.worktreeId).toBe(TEST_WORKTREE_ID) + expect(getSession().activeTabIdByWorktree).toEqual({ [TEST_WORKTREE_ID]: 'host-tab' }) + }) + + it('keeps current-generation tab and leaf identity across a host restart', async () => { + const session = makeWorkspaceSessionWithHeadlessTerminal({ + terminalPtyIncarnationsByPaneKey: { + [`host-tab:${HEADLESS_LEAF_ID}`]: 'inc-current' + }, + terminalTopologyRevisionByRepoId: { [TEST_REPO_ID]: 4 } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + let connected = true + const writes: [string, string][] = [] + const resize = vi.fn(() => true) + const makeRuntime = (): OrcaRuntimeService => { + const runtime = new OrcaRuntimeService(runtimeStore as never) + runtime.setPtyController({ + write: (ptyId, data) => { + writes.push([ptyId, data]) + return true + }, + resize, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => + connected + ? [ + { + id: 'persisted-pty', + incarnationId: 'inc-current', + terminalHandle: 'term_current', + title: 'Current shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + : [] + }) + runtime.syncWindowGraph(0, { tabs: [], leaves: [] }) + return runtime + } + + const originalRuntime = makeRuntime() + const beforeRestart = await originalRuntime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + connected = false + const disconnected = await originalRuntime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + connected = true + const reconnected = await originalRuntime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + const restarted = makeRuntime() + const afterRestart = await restarted.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + const listed = await restarted.listTerminals(`id:${TEST_WORKTREE_ID}`) + restarted.onPtyData('persisted-pty', 'after restart\n', 1) + await restarted.sendTerminal('term_current', { text: 'input' }) + await restarted.updateRemoteDesktopViewer('persisted-pty', 'viewer', 'client', 132, 41) + + expect(beforeRestart.tabs[0]).toMatchObject({ + parentTabId: 'host-tab', + leafId: HEADLESS_LEAF_ID, + status: 'ready', + terminal: 'term_current', + title: 'Persisted Terminal' + }) + expect(afterRestart.tabs[0]).toMatchObject({ + parentTabId: 'host-tab', + leafId: HEADLESS_LEAF_ID, + status: 'ready', + terminal: 'term_current' + }) + expect(disconnected.tabs[0]).toMatchObject({ status: 'pending-handle', terminal: null }) + expect(reconnected.tabs[0]).toMatchObject({ + parentTabId: 'host-tab', + leafId: HEADLESS_LEAF_ID, + status: 'ready', + terminal: 'term_current' + }) + expect(listed.terminals[0]).toMatchObject({ + tabId: 'host-tab', + leafId: HEADLESS_LEAF_ID, + incarnationId: 'inc-current', + orphaned: false + }) + expect(listed.topologyRevisions?.[TEST_WORKTREE_ID]).toBe(4) + await expect(restarted.readTerminal('term_current')).resolves.toMatchObject({ + tail: ['after restart'] + }) + expect(writes).toEqual([['persisted-pty', 'input']]) + expect(resize).toHaveBeenCalledWith('persisted-pty', 132, 41) + }) + + it('uses topology CAS before a client can claim a still-orphaned PTY', async () => { + const session = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + terminalTopologyRevisionByRepoId: { [TEST_REPO_ID]: 7 } + } + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-cas', + incarnationId: 'inc-cas', + terminalHandle: 'term_cas', + title: 'shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 6, + claims: [ + { + terminal: 'term_cas', + ptyId: 'pty-cas', + incarnationId: 'inc-cas', + tabId: 'tab-cas', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_topology_conflict') + }) + + it('rejects connection mismatch and reused handles while allowing a WSL-owned orphan', async () => { + const makeRuntime = (): OrcaRuntimeService => { + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + }) + return new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + } + const ownerMismatch = makeRuntime() + ownerMismatch.registerPty('pty-wrong-owner', TEST_WORKTREE_ID, 'ssh-other-host') + ownerMismatch.onPtySpawned('pty-wrong-owner', 'inc-owner', { awaitsRegistration: false }) + ownerMismatch.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-wrong-owner', + incarnationId: 'inc-owner', + terminalHandle: 'term_wrong_owner', + title: 'shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + await expect( + ownerMismatch.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_wrong_owner', + ptyId: 'pty-wrong-owner', + incarnationId: 'inc-owner', + tabId: 'tab-owner', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_owner_mismatch') + + const reusedHandle = makeRuntime() + for (const [ptyId, incarnationId] of [ + ['pty-first', 'inc-first'], + ['pty-second', 'inc-second'] + ] as const) { + reusedHandle.registerPty(ptyId, TEST_WORKTREE_ID) + reusedHandle.onPtySpawned(ptyId, incarnationId, { awaitsRegistration: false }) + } + reusedHandle.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-first', + incarnationId: 'inc-first', + terminalHandle: 'term_reused', + title: 'shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + }, + { + id: 'pty-second', + incarnationId: 'inc-second', + terminalHandle: 'term_reused', + title: 'shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + await expect( + reusedHandle.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_reused', + ptyId: 'pty-second', + incarnationId: 'inc-second', + tabId: 'tab-second', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_stale') + + await withPlatform('win32', async () => { + const makeWslRuntime = (reportedWslDistro?: string | null): OrcaRuntimeService => { + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + }) + const wsl = new OrcaRuntimeService({ + ...runtimeStore, + flushOrThrow: vi.fn(), + getProjects: () => [ + { + id: 'project-wsl', + displayName: 'WSL', + badgeColor: 'blue', + sourceRepoIds: [TEST_REPO_ID], + localWindowsRuntimePreference: { kind: 'wsl', distro: 'Ubuntu' }, + createdAt: 1, + updatedAt: 1 + } + ], + getSettings: () => ({ + ...store.getSettings(), + localWindowsRuntimeDefault: { kind: 'windows-host' } + }) + } as never) + wsl.registerPty('pty-wsl', TEST_WORKTREE_ID, null, undefined, true) + wsl.onPtySpawned('pty-wsl', 'inc-wsl', { awaitsRegistration: false }) + wsl.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-wsl', + incarnationId: 'inc-wsl', + terminalHandle: 'term_wsl', + title: 'shell', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + ...(reportedWslDistro !== undefined ? { wslDistro: reportedWslDistro } : {}) + } + ] + }) + return wsl + } + const request = { + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_wsl', + ptyId: 'pty-wsl', + incarnationId: 'inc-wsl', + tabId: 'tab-wsl', + leafId: HEADLESS_LEAF_ID + } + ] + } + + await expect(makeWslRuntime('Ubuntu').adoptTerminalOrphans(request)).resolves.toMatchObject({ + adopted: true, + topologyRevision: 1 + }) + await expect(makeWslRuntime('Debian').adoptTerminalOrphans(request)).rejects.toThrow( + 'terminal_orphan_owner_mismatch' + ) + await expect(makeWslRuntime().adoptTerminalOrphans(request)).rejects.toThrow( + 'terminal_orphan_owner_mismatch' + ) + }) + }) + + it('preserves legacy pane and group topology without changing host focus', async () => { + const session = { + ...getDefaultWorkspaceSession(), + activeWorktreeId: 'other-worktree', + activeTabId: 'other-tab', + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + const processes = [ + ['pty-left', 'inc-left', 'term_left'], + ['pty-right', 'inc-right', 'term_right'], + ['pty-shell', 'inc-shell', 'term_shell'] + ] as const + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => + processes.map(([id, incarnationId, terminalHandle]) => ({ + id, + incarnationId, + terminalHandle, + title: id, + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + })) + }) + + await runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + activeTabId: 'tab-agent', + activeGroupId: 'group-left', + claims: processes.map(([ptyId, incarnationId, terminal], index) => ({ + terminal, + ptyId, + incarnationId, + tabId: index < 2 ? 'tab-agent' : 'tab-shell', + leafId: [HEADLESS_LEAF_ID, HEADLESS_SECOND_LEAF_ID, HEADLESS_THIRD_LEAF_ID][index]! + })), + topology: { + tabs: [ + { + tabId: 'tab-agent', + root: { + type: 'split', + direction: 'horizontal', + ratio: 0.35, + first: { type: 'leaf', leafId: HEADLESS_LEAF_ID }, + second: { type: 'leaf', leafId: HEADLESS_SECOND_LEAF_ID } + }, + activeLeafId: HEADLESS_SECOND_LEAF_ID, + expandedLeafId: HEADLESS_SECOND_LEAF_ID + }, + { + tabId: 'tab-shell', + root: { type: 'leaf', leafId: HEADLESS_THIRD_LEAF_ID }, + activeLeafId: HEADLESS_THIRD_LEAF_ID, + expandedLeafId: null + } + ], + groups: [ + { + id: 'group-left', + activeTabId: 'tab-agent', + tabOrder: ['tab-agent'], + recentTabIds: ['tab-agent'] + }, + { id: 'group-right', activeTabId: 'tab-shell', tabOrder: ['tab-shell'] } + ], + groupLayout: { + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-left' }, + second: { type: 'leaf', groupId: 'group-right' } + } + } + }) + + expect(getSession()).toMatchObject({ + activeWorktreeId: 'other-worktree', + activeTabId: 'other-tab', + activeTabIdByWorktree: { [TEST_WORKTREE_ID]: 'tab-agent' }, + activeGroupIdByWorktree: { [TEST_WORKTREE_ID]: 'group-left' }, + tabGroups: { + [TEST_WORKTREE_ID]: [ + { id: 'group-left', activeTabId: 'tab-agent', tabOrder: ['tab-agent'] }, + { id: 'group-right', activeTabId: 'tab-shell', tabOrder: ['tab-shell'] } + ] + }, + tabGroupLayouts: { + [TEST_WORKTREE_ID]: expect.objectContaining({ + type: 'split', + direction: 'vertical', + ratio: 0.6 + }) + }, + terminalLayoutsByTabId: { + 'tab-agent': expect.objectContaining({ + root: expect.objectContaining({ + type: 'split', + direction: 'horizontal', + ratio: 0.35 + }), + activeLeafId: HEADLESS_SECOND_LEAF_ID, + expandedLeafId: HEADLESS_SECOND_LEAF_ID + }) + } + }) + }) + + it('never lets an old handle adopt a replacement PTY incarnation', async () => { + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + }) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + let process = { + id: 'reused-pty-id', + incarnationId: 'inc-old', + terminalHandle: 'term_old', + title: 'old', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [process] + }) + await expect(runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)).resolves.toMatchObject({ + terminals: [expect.objectContaining({ handle: 'term_old', incarnationId: 'inc-old' })] + }) + + process = { ...process, incarnationId: 'inc-new', terminalHandle: 'term_new', title: 'new' } + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_old', + ptyId: process.id, + incarnationId: 'inc-new', + tabId: 'stale-tab', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_stale') + await expect(runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)).resolves.toMatchObject({ + terminals: [expect.objectContaining({ handle: 'term_new', incarnationId: 'inc-new' })] + }) + }) + + it('rejects a proposed visual surface occupied by a different PTY', async () => { + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] } + }) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'occupied-tab', + worktreeId: TEST_WORKTREE_ID, + title: 'occupied', + activeLeafId: HEADLESS_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: 'occupied-tab', + worktreeId: TEST_WORKTREE_ID, + leafId: HEADLESS_LEAF_ID, + paneRuntimeId: 1, + ptyId: 'visual-pty' + } + ] + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'orphan-pty', + incarnationId: 'inc-orphan', + terminalHandle: 'term_orphan', + title: 'orphan', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_orphan', + ptyId: 'orphan-pty', + incarnationId: 'inc-orphan', + tabId: 'occupied-tab', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_surface_occupied') + }) + + it('rejects ambiguous duplicate persisted bindings before idempotence', async () => { + const duplicateTab = (id: string) => ({ + id, + ptyId: 'duplicate-pty', + worktreeId: TEST_WORKTREE_ID, + title: id, + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [TEST_WORKTREE_ID]: [duplicateTab('duplicate-a'), duplicateTab('duplicate-b')] + }, + terminalLayoutsByTabId: { + 'duplicate-a': { + root: { type: 'leaf', leafId: HEADLESS_LEAF_ID }, + activeLeafId: HEADLESS_LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { [HEADLESS_LEAF_ID]: 'duplicate-pty' } + }, + 'duplicate-b': { + root: { type: 'leaf', leafId: HEADLESS_SECOND_LEAF_ID }, + activeLeafId: HEADLESS_SECOND_LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { [HEADLESS_SECOND_LEAF_ID]: 'duplicate-pty' } + } + }, + terminalPtyIncarnationsByPaneKey: { + [`duplicate-a:${HEADLESS_LEAF_ID}`]: 'inc-duplicate', + [`duplicate-b:${HEADLESS_SECOND_LEAF_ID}`]: 'inc-duplicate' + } + }) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'duplicate-pty', + incarnationId: 'inc-duplicate', + terminalHandle: 'term_duplicate', + title: 'duplicate', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_duplicate', + ptyId: 'duplicate-pty', + incarnationId: 'inc-duplicate', + tabId: 'duplicate-a', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).rejects.toThrow('terminal_orphan_competing_owner') + }) + it('does not adopt a discovered terminal handle already bound to another live PTY', async () => { const runtime = new OrcaRuntimeService(store) const writesByPty = new Map() @@ -17192,9 +18269,16 @@ describe('OrcaRuntimeService', () => { worktree: TEST_WORKTREE_ID, publicationEpoch: 'epoch-1', snapshotVersion: 1, - activeGroupId: null, + activeGroupId: 'group-1', activeTabId: 'tab-1::pane:1', activeTabType: 'terminal', + tabGroups: [ + { + id: 'group-1', + activeTabId: 'missing-tab', + tabOrder: ['missing-tab', 'tab-1'] + } + ], tabs: [ { type: 'terminal', @@ -17226,6 +18310,7 @@ describe('OrcaRuntimeService', () => { worktreeId: TEST_WORKTREE_ID, root: { type: 'group', + activeTabId: 'tab-1', tabs: [ { tabId: 'tab-1', @@ -29683,6 +30768,42 @@ describe('OrcaRuntimeService', () => { expect(listWorktrees).toHaveBeenCalledTimes(15) }) + it('bounds resolved worktree scanning to a fixed repo worker pool', async () => { + vi.mocked(listWorktrees).mockReset() + const repos = Array.from({ length: 40 }, (_, index) => ({ + id: `repo-${index}`, + path: `/tmp/repo-${index}`, + displayName: `repo-${index}`, + badgeColor: 'blue' as const, + addedAt: 1 + })) + const gate = deferred() + let inFlight = 0 + let peak = 0 + vi.mocked(listWorktrees).mockImplementation(async (repoPath) => { + inFlight += 1 + peak = Math.max(peak, inFlight) + await gate.promise + inFlight -= 1 + return [makeWorktreeInfo(repoPath)] + }) + const runtime = new OrcaRuntimeService({ + ...store, + getRepos: () => repos, + getRepo: (id: string) => repos.find((candidate) => candidate.id === id), + getAllWorktreeMeta: () => ({}), + getWorktreeMeta: () => undefined + } as never) + + const listing = runtime.listManagedWorktrees() + await vi.waitFor(() => expect(listWorktrees).toHaveBeenCalledTimes(8)) + expect(peak).toBe(8) + gate.resolve() + await expect(listing).resolves.toMatchObject({ totalCount: repos.length }) + expect(listWorktrees).toHaveBeenCalledTimes(repos.length) + expect(peak).toBe(8) + }) + it('worktree scan cache: shares one in-flight repo scan across concurrent consumers', async () => { vi.mocked(listWorktrees).mockClear() const pending = deferred[]>() @@ -35956,7 +37077,11 @@ describe('OrcaRuntimeService', () => { }) it('does not start Git removal when physical PTY stop cannot be proven', async () => { - const localProvider = createProviderStub(async () => []) + // A failed stop only rejects when a fresh inventory still shows the PTY + // live; keep pty-1 present so the exit cannot be proven. + const localProvider = createProviderStub(async () => [ + { id: 'pty-1', cwd: '/tmp', title: 'shell' } + ]) const runtime = new OrcaRuntimeService(store, undefined, { getLocalProvider: () => localProvider as never }) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 625a60e9a6b..396e77dc46c 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -91,11 +91,12 @@ import { GIT_FETCH_SKIP_AUTO_MAINTENANCE_CONFIG_ARGS } from '../../shared/git-fe import { createHash, randomUUID } from 'node:crypto' import { homedir } from 'node:os' import { isAbsolute, join, resolve } from 'node:path' -import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises' +import { mkdir, opendir, rm, stat } from 'node:fs/promises' import { resolveWorktreeCreateBase } from '../worktree-create-base' import { resolveWorktreeAddBaseRef } from '../../shared/worktree-base-ref' import { OrchestrationDb } from './orchestration/db' import { formatMessagesForInjection } from './orchestration/formatter' +import { OrchestrationMessageWaiterRegistry } from './orchestration-message-waiter-registry' import type { Automation, AutomationCreateInput, @@ -189,6 +190,11 @@ import { } from '../../shared/runtime-navigation' import type { SshConnectionState } from '../../shared/ssh-types' import { getPublicSshState } from './public-ssh-state' +import { + RuntimeSshRelayRecoveryGenerations, + type RuntimeSshRelayRecoveryGenerationLease +} from './runtime-ssh-relay-recovery-generations' +import { RuntimeOperationGenerations } from './runtime-operation-generations' import { closeTerminalTabInWorkspaceSession } from '../../shared/workspace-session-terminal-tab-close' import type { LinearCurrentIssueContextHints, @@ -241,6 +247,7 @@ import { import { parsePtySessionId } from '../../shared/pty-session-id-format' import { clampLinearIssueListLimit } from '../../shared/linear-issue-read-limits' import { isFolderRepo } from '../../shared/repo-kind' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' import { DEFAULT_WORKSPACE_STATUS_ID } from '../../shared/workspace-statuses' import { buildSetupRunnerCommand, @@ -368,6 +375,8 @@ import type { RuntimeTerminalFocus, RuntimeTerminalClose, RuntimeTerminalListResult, + RuntimeTerminalOrphanAdoptionRequest, + RuntimeTerminalOrphanAdoptionResult, RuntimeWorktreeTerminalSleepResult, RuntimeTerminalResolvePane, RuntimeTerminalState, @@ -426,6 +435,11 @@ import { buildHeadlessTabGroupMove, buildHeadlessTabGroupSplit } from './headless-tab-group-split-layout' +import { + hasExactTerminalOrphanGroupLayout, + mergeTerminalOrphanGroupLayout +} from './terminal-orphan-topology' +import { terminalOrphanExecutionOwnersEqual } from './terminal-orphan-owner' import { retireTerminalSurfacesFromSnapshot, type RetiredTerminalSurface @@ -552,6 +566,14 @@ import type { PRRefreshOutcome } from '../../shared/types' import { inspectSetupScriptImportCandidates } from '../../shared/setup-script-imports' +import { + readSetupScriptImportFile, + SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS +} from '../setup-script-import-file' +import { MAX_ORCA_YAML_BYTES, MAX_ORCA_YAML_CODE_UNITS } from '../../shared/orca-yaml-file-limit' +import { readFilesystemProviderBoundedText } from '../filesystem-provider-bounded-text' +import { readLocalFilesystemDirectory } from '../ipc/filesystem-directory-reader' import type { CreateHostedReviewInput, CreateHostedReviewResult, @@ -754,6 +776,8 @@ import { hasUnrecognizedOrcaYamlKeys, hasHooksFile, loadHooks, + MAX_HOOK_GITIGNORE_BYTES, + MAX_ISSUE_COMMAND_BYTES, parseOrcaYaml, readIssueCommand, runHook, @@ -1134,6 +1158,7 @@ type RuntimePtyWorktreeRecord = { lastOscTitleAt: number | null managementTitle: string | null managementTitleAt: number | null + controllerTitle: string | null title: string | null titleUpdatedAt: number | null lastOutputAt: number | null @@ -1419,6 +1444,9 @@ type RuntimePtyController = { markReversibleStops?(ptyIds: readonly string[]): () => void getCwd?(ptyId: string): Promise getForegroundProcess(ptyId: string): Promise + inspectProcess?( + ptyId: string + ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> confirmForegroundProcess?(ptyId: string): Promise hasChildProcesses?(ptyId: string): Promise clearBuffer?(ptyId: string): Promise @@ -1647,14 +1675,6 @@ type TerminalWaiter = { abortCleanup: (() => void) | null } -type MessageWaiter = { - handle: string - typeFilter: string[] | undefined - resolve: (result: void) => void - timeout: NodeJS.Timeout | null - abortCleanup: (() => void) | null -} - function omitUndefinedProperties>(value: T): Partial { return Object.fromEntries( Object.entries(value).filter(([, entry]) => entry !== undefined) @@ -2462,6 +2482,10 @@ export class OrcaRuntimeService { private handles = new Map() private handleByLeafKey = new Map() private handleByPtyId = new Map() + private controllerTerminalIdentityByPtyId = new Map< + string, + { handle: string; incarnationId: string; wslDistro?: string | null } + >() private detachedPreAllocatedLeaves = new Map() private graphSyncCallbacks: (() => void)[] = [] private waitersByHandle = new Map>() @@ -2475,7 +2499,7 @@ export class OrcaRuntimeService { private resolvedWorktreeCache: ResolvedWorktreeCache | null = null private resolvedWorktreeInFlight: ResolvedWorktreeInFlight | null = null private resolvedWorktreeGeneration = 0 - private worktreeScanGenerations = new Map() + private readonly worktreeScanGenerations = new RuntimeOperationGenerations() private worktreeScanCache = new Map() private worktreeScanInFlight = new Map() private cloneInFlightByPath = new Map>() @@ -2483,7 +2507,7 @@ export class OrcaRuntimeService { private ptyForegroundAgentRefreshes = new Map() private ptyDelayedForegroundSnapshotTitleObservations = new Map() private _orchestrationDb: OrchestrationDb | null = null - private messageWaitersByHandle = new Map>() + private readonly messageWaiters = new OrchestrationMessageWaiterRegistry() // Why: mobile clients subscribe to terminal output via terminal.subscribe. // These listeners fire on every onPtyData call, enabling real-time streaming // without polling. Keyed by ptyId for O(1) lookup per data event. @@ -2554,8 +2578,7 @@ export class OrcaRuntimeService { private providerVisibleStateByPtyId = new Map() private providerVisibleRetryAtByPtyId = new Map() private providerSnapshotsWithLiveModeTransition = new WeakSet() - private ptyLifecycleGenerationById = new Map() - private nextPtyLifecycleGeneration = 1 + private readonly ptyLifecycleGenerations = new RuntimeOperationGenerations() private recentPtyPathCandidatesById = new Map() // Why: candidates only feed mobile file-tap provenance; desktop-only // sessions skip the 3-regex extraction on every PTY chunk until a @@ -2855,7 +2878,7 @@ export class OrcaRuntimeService { | null private readonly agentSessionClaimSigner: AgentSessionClaimSigner private readonly agentSessionCreateOperations = new Map() - private sshRelayRecoveryGenerationByTargetId = new Map() + private readonly sshRelayRecoveryGenerations = new RuntimeSshRelayRecoveryGenerations() private accountServices: RuntimeAccountServices | null = null private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null private automationService: AutomationService | null = null @@ -3552,33 +3575,32 @@ export class OrcaRuntimeService { // Why: SSH state changes originate in main's ssh handlers, not in runtime // methods, so they need a public entry point onto the client-event stream. notifySshStateChanged(targetId: string, state: SshConnectionState): void { - this.bumpSshRelayRecoveryGeneration(targetId) + this.sshRelayRecoveryGenerations.invalidate(targetId) this.invalidateSshWorktreeScanCache(targetId) this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! }) } notifySshRelayReady(targetId: string): void { - const generation = this.bumpSshRelayRecoveryGeneration(targetId) - void this.publishRecoveredSshMobileSessionTabs(targetId, generation).catch((error) => { - if (this.sshRelayRecoveryGenerationByTargetId.get(targetId) !== generation) { - return - } - console.warn('[runtime] failed to publish recovered SSH session tabs', { - targetId, - error + const generationLease = this.sshRelayRecoveryGenerations.begin(targetId) + if (!generationLease) { + return + } + void this.publishRecoveredSshMobileSessionTabs(targetId, generationLease) + .catch((error) => { + if (!generationLease.isCurrent()) { + return + } + console.warn('[runtime] failed to publish recovered SSH session tabs', { + targetId, + error + }) }) - }) - } - - private bumpSshRelayRecoveryGeneration(targetId: string): number { - const generation = (this.sshRelayRecoveryGenerationByTargetId.get(targetId) ?? 0) + 1 - this.sshRelayRecoveryGenerationByTargetId.set(targetId, generation) - return generation + .finally(() => generationLease.release()) } private async publishRecoveredSshMobileSessionTabs( targetId: string, - generation: number + generationLease: RuntimeSshRelayRecoveryGenerationLease ): Promise { const repoIds = new Set( (this.store?.getRepos() ?? []) @@ -3610,7 +3632,7 @@ export class OrcaRuntimeService { }) } await this.refreshMobileSessionPtyRecords() - if (this.sshRelayRecoveryGenerationByTargetId.get(targetId) !== generation) { + if (!generationLease.isCurrent()) { return } for (const worktreeId of worktreeIds) { @@ -7141,11 +7163,29 @@ export class OrcaRuntimeService { } } - private adoptControllerTerminalHandle(ptyId: string, handle: string | undefined): void { + private adoptControllerTerminalHandle( + ptyId: string, + handle: string | undefined, + incarnationId?: string + ): void { const trimmed = handle?.trim() if (!trimmed || !trimmed.startsWith('term_')) { return } + const pty = this.ptysById.get(ptyId) + const changedIncarnation = Boolean( + incarnationId && pty?.incarnationId && incarnationId !== pty.incarnationId + ) + if (changedIncarnation) { + const priorHandle = this.handleByPtyId.get(ptyId) + this.invalidateAllHandlesForPty(ptyId) + pty!.tabId = null + pty!.paneKey = null + // Reusing an exported handle would make stale client metadata name the replacement process. + if (priorHandle === trimmed) { + return + } + } if (this.isTerminalHandleAdoptionBlocked(ptyId, trimmed)) { return } @@ -7154,6 +7194,23 @@ export class OrcaRuntimeService { this.registerPreAllocatedHandleForPty(ptyId, trimmed) } + private invalidateAllHandlesForPty(ptyId: string): void { + this.handleByPtyId.delete(ptyId) + const invalidated = new Set() + for (const [handle, record] of this.handles) { + if (record.ptyId === ptyId) { + invalidated.add(handle) + this.handles.delete(handle) + this.rejectWaitersForHandle(handle, 'terminal_handle_stale') + } + } + for (const [leafKey, handle] of this.handleByLeafKey) { + if (invalidated.has(handle)) { + this.handleByLeafKey.delete(leafKey) + } + } + } + // Why: adoption is best-effort restart recovery and must be first-wins. // Re-keying a pty that already has a handle this session would strand // waiters registered under the old handle, and provider-reported values @@ -8351,17 +8408,15 @@ export class OrcaRuntimeService { } private getPtyLifecycleGeneration(ptyId: string): number { - const existing = this.ptyLifecycleGenerationById.get(ptyId) - if (existing !== undefined) { - return existing - } - const generation = this.nextPtyLifecycleGeneration++ - this.ptyLifecycleGenerationById.set(ptyId, generation) - return generation + return this.ptyLifecycleGenerations.current(ptyId) + } + + private isPtyLifecycleGenerationCurrent(ptyId: string, generation: number): boolean { + return this.ptyLifecycleGenerations.isCurrent(ptyId, generation) } private advancePtyLifecycleGeneration(ptyId: string): void { - this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++) + this.ptyLifecycleGenerations.advance(ptyId) // Why: a provider response belongs to the process generation that issued // it; a respawn must neither reuse its frame nor join its in-flight call. this.providerBufferAcquisitionsByPtyId.delete(ptyId) @@ -9213,7 +9268,7 @@ export class OrcaRuntimeService { // Why: daemon PTYs survive an app relaunch before any renderer mounts. // Mobile still needs their retained history without navigating desktop. const snapshot = await this.ptyController?.serializeProviderBuffer?.(ptyId, opts) - if (!snapshot || this.getPtyLifecycleGeneration(ptyId) !== generation) { + if (!snapshot || !this.isPtyLifecycleGenerationCurrent(ptyId, generation)) { return null } const snapshotModeTracker = new TerminalKittyKeyboardModeTracker() @@ -9351,7 +9406,7 @@ export class OrcaRuntimeService { { scrollbackRows: 0 }, { timeoutMs: VISIBLE_TERMINAL_SNAPSHOT_TIMEOUT_MS } ) - if (!snapshot || this.getPtyLifecycleGeneration(ptyId) !== generation) { + if (!snapshot || !this.isPtyLifecycleGenerationCurrent(ptyId, generation)) { this.providerVisibleRetryAtByPtyId.set(ptyId, Date.now() + VISIBLE_TERMINAL_SNAPSHOT_RETRY_MS) return null } @@ -9365,7 +9420,7 @@ export class OrcaRuntimeService { } } const lines = await this.parseVisibleSnapshotLines(snapshot) - if (this.getPtyLifecycleGeneration(ptyId) !== generation) { + if (!this.isPtyLifecycleGenerationCurrent(ptyId, generation)) { return null } const visibleState: RuntimeVisibleTerminalState = { @@ -9391,7 +9446,7 @@ export class OrcaRuntimeService { await state.writeChain if ( this.headlessTerminals.get(ptyId) !== state || - this.getPtyLifecycleGeneration(ptyId) !== generation + !this.isPtyLifecycleGenerationCurrent(ptyId, generation) ) { return null } @@ -9626,16 +9681,15 @@ export class OrcaRuntimeService { // before the first-ever connect no longer yields candidates. for (const [ptyId, buffer] of this.recentPtyOutputById) { let candidates = this.recentPtyPathCandidatesById.get(ptyId) - const { chunks, headChunkIsPartial } = buffer.retainedChunks() - for (let index = 0; index < chunks.length; index += 1) { + buffer.forEachRetainedChunk((chunk, index, headChunkIsPartial) => { if (index === 0 && headChunkIsPartial) { // A pre-sliced over-window chunk was already extracted eagerly at // append time (while its original text was intact); replaying its // truncated remainder would mint or drop candidates spuriously. - continue + return } - candidates = appendRecentPtyPathCandidates(candidates, chunks[index]!) - } + candidates = appendRecentPtyPathCandidates(candidates, chunk) + }) if (candidates) { this.recentPtyPathCandidatesById.set(ptyId, candidates) } @@ -12417,7 +12471,7 @@ export class OrcaRuntimeService { async listTerminals( worktreeSelector?: string, limit = DEFAULT_TERMINAL_LIST_LIMIT, - opts: { requireFreshPtyLiveness?: boolean } = {} + opts: { handles?: readonly string[]; requireFreshPtyLiveness?: boolean } = {} ): Promise { if (!Number.isInteger(limit) || limit <= 0) { throw new Error('invalid_limit') @@ -12529,7 +12583,11 @@ export class OrcaRuntimeService { terminals.push(this.buildPtyTerminalSummary(pty, worktreesById)) } - const listedTerminals = terminals.slice(0, limit) + const requestedHandles = opts.handles ? new Set(opts.handles) : null + const matchingTerminals = requestedHandles + ? terminals.filter((terminal) => requestedHandles.has(terminal.handle)) + : terminals + const listedTerminals = matchingTerminals.slice(0, limit) const visualLayouts = this.buildTerminalVisualLayouts( listedTerminals, worktreesById, @@ -12539,8 +12597,480 @@ export class OrcaRuntimeService { return { terminals: listedTerminals, ...(visualLayouts.length > 0 ? { visualLayouts } : {}), - totalCount: terminals.length, - truncated: terminals.length > limit + topologyRevisions: Object.fromEntries( + [...new Set(matchingTerminals.map((terminal) => terminal.worktreeId))].map((worktreeId) => [ + worktreeId, + this.getTerminalTopologyRevision(worktreeId) + ]) + ), + totalCount: matchingTerminals.length, + truncated: matchingTerminals.length > limit + } + } + + private getTerminalTopologyRevision(worktreeId: string): number { + const repoId = getRepoIdFromWorktreeId(worktreeId) + return ( + this.store?.getWorkspaceSession?.()?.terminalTopologyRevisionByRepoId?.[repoId] ?? + this.terminalTopologyRevisionByRepoId.get(repoId) ?? + 0 + ) + } + + async adoptTerminalOrphans( + request: RuntimeTerminalOrphanAdoptionRequest + ): Promise { + if (request.claims.length === 0) { + throw new Error('terminal_orphan_claims_required') + } + const worktree = await this.resolveWorktreeSelector(request.worktree) + const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([worktree], worktree.id) + if (!livePtyIds) { + throw new Error('terminal_liveness_unavailable') + } + const store = this.store + const session = store?.getWorkspaceSession?.() + if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) { + throw new Error('workspace_session_unavailable') + } + const sessionWorktreeId = resolveTerminalSessionWorktreeId(session, worktree.id) + if (!sessionWorktreeId) { + throw new Error('terminal_orphan_competing_owner') + } + const repoId = getRepoIdFromWorktreeId(worktree.id) + const worktreeRepo = store.getRepo(repoId) + if (!worktreeRepo) { + throw new Error('terminal_orphan_owner_mismatch') + } + const worktreeConnectionId = worktreeRepo.connectionId ?? null + let worktreeWslDistro: string | null = null + if (!worktreeConnectionId) { + try { + worktreeWslDistro = + getLocalProjectWorktreeGitOptions(this.requireStore(), worktreeRepo).wslDistro ?? null + } catch { + throw new Error('terminal_orphan_owner_mismatch') + } + } + const currentRevision = this.getTerminalTopologyRevision(worktree.id) + const seenPtyIds = new Set() + const seenPaneKeys = new Set() + const validated = request.claims.map((claim) => { + const paneKey = makePaneKey(claim.tabId, claim.leafId) + if (seenPtyIds.has(claim.ptyId) || seenPaneKeys.has(paneKey)) { + throw new Error('terminal_orphan_claim_duplicate') + } + seenPtyIds.add(claim.ptyId) + seenPaneKeys.add(paneKey) + const live = this.getLivePtyForHandle(claim.terminal) + const pty = live?.pty + const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId) + if ( + !pty || + pty.ptyId !== claim.ptyId || + controllerIdentity?.handle !== claim.terminal || + controllerIdentity?.incarnationId !== claim.incarnationId || + !livePtyIds.has(claim.ptyId) || + !pty.connected || + !pty.incarnationId || + pty.incarnationId !== claim.incarnationId + ) { + throw new Error('terminal_orphan_stale') + } + if ( + !runtimeWorktreeIdsEqual(pty.worktreeId, worktree.id) || + !terminalOrphanExecutionOwnersEqual( + { connectionId: worktreeConnectionId, wslDistro: worktreeWslDistro }, + { + connectionId: pty.connectionId ?? null, + ...(controllerIdentity?.wslDistro !== undefined + ? { wslDistro: controllerIdentity.wslDistro } + : process.platform === 'win32' && !worktreeConnectionId + ? {} + : { wslDistro: null }) + } + ) + ) { + throw new Error('terminal_orphan_owner_mismatch') + } + const visualOwners = this.getLeavesForPty(claim.ptyId) + if ( + visualOwners.some( + (owner) => + !runtimeWorktreeIdsEqual(owner.worktreeId, worktree.id) || + owner.tabId !== claim.tabId || + owner.leafId !== claim.leafId + ) + ) { + throw new Error('terminal_orphan_already_visual') + } + if ((pty.tabId && pty.tabId !== claim.tabId) || (pty.paneKey && pty.paneKey !== paneKey)) { + throw new Error('terminal_orphan_competing_owner') + } + return { claim, pty, paneKey } + }) + + const persistedBindingsByPtyId = new Map() + const addPersistedBinding = ( + ptyId: string, + binding: { worktreeId: string; paneKey: string } + ): void => { + const bindings = persistedBindingsByPtyId.get(ptyId) ?? [] + bindings.push(binding) + persistedBindingsByPtyId.set(ptyId, bindings) + } + for (const [worktreeId, tabs] of Object.entries(session.tabsByWorktree)) { + for (const tab of tabs) { + const layout = session.terminalLayoutsByTabId[tab.id] + for (const [leafId, boundPtyId] of Object.entries(layout?.ptyIdsByLeafId ?? {})) { + if (boundPtyId) { + addPersistedBinding(boundPtyId, { + worktreeId, + paneKey: makePaneKey(tab.id, leafId) + }) + } + } + if (tab.ptyId && !layout) { + addPersistedBinding(tab.ptyId, { worktreeId, paneKey: tab.id }) + } + } + } + const persistedBinding = (ptyId: string): { worktreeId: string; paneKey: string } | null => { + const bindings = persistedBindingsByPtyId.get(ptyId) ?? [] + if (bindings.length > 1) { + throw new Error('terminal_orphan_competing_owner') + } + return bindings[0] ?? null + } + const isExactPersisted = validated.every(({ claim, paneKey }) => { + const binding = persistedBinding(claim.ptyId) + return ( + binding !== null && + runtimeWorktreeIdsEqual(binding.worktreeId, worktree.id) && + binding.paneKey === paneKey && + session.terminalPtyIncarnationsByPaneKey?.[paneKey] === claim.incarnationId + ) + }) + if (isExactPersisted && sessionWorktreeId === worktree.id) { + return { + adopted: false, + topologyRevision: currentRevision, + snapshot: await this.listMobileSessionTabs(`id:${worktree.id}`) + } + } + if (currentRevision !== request.expectedTopologyRevision) { + throw new Error('terminal_topology_conflict') + } + + const topologyTabsById = new Map(request.topology?.tabs.map((tab) => [tab.tabId, tab]) ?? []) + const topologyGroups = request.topology?.groups ?? [] + if (request.topology) { + const claimedLeafIdsByTabId = new Map>() + for (const { claim } of validated) { + const leafIds = claimedLeafIdsByTabId.get(claim.tabId) ?? new Set() + leafIds.add(claim.leafId) + claimedLeafIdsByTabId.set(claim.tabId, leafIds) + } + if ( + topologyTabsById.size !== request.topology.tabs.length || + topologyTabsById.size !== claimedLeafIdsByTabId.size + ) { + throw new Error('terminal_orphan_topology_invalid') + } + for (const [tabId, claimedLeafIds] of claimedLeafIdsByTabId) { + const topologyTab = topologyTabsById.get(tabId) + if (!topologyTab) { + throw new Error('terminal_orphan_topology_invalid') + } + const topologyLeafIds = new Set() + const nodes = [topologyTab.root] + let leafCount = 0 + while (nodes.length > 0) { + const node = nodes.pop()! + if (node.type === 'leaf') { + leafCount += 1 + topologyLeafIds.add(node.leafId) + } else { + nodes.push(node.first, node.second) + } + } + if ( + leafCount !== topologyLeafIds.size || + topologyLeafIds.size !== claimedLeafIds.size || + [...topologyLeafIds].some((leafId) => !claimedLeafIds.has(leafId)) || + !topologyLeafIds.has(topologyTab.activeLeafId) || + (topologyTab.expandedLeafId !== null && !topologyLeafIds.has(topologyTab.expandedLeafId)) + ) { + throw new Error('terminal_orphan_topology_invalid') + } + } + const seenGroupIds = new Set() + const groupedTabIds = new Set() + for (const group of topologyGroups) { + if (seenGroupIds.has(group.id) || !group.tabOrder.includes(group.activeTabId)) { + throw new Error('terminal_orphan_topology_invalid') + } + seenGroupIds.add(group.id) + for (const tabId of group.tabOrder) { + if (!topologyTabsById.has(tabId) || groupedTabIds.has(tabId)) { + throw new Error('terminal_orphan_topology_invalid') + } + groupedTabIds.add(tabId) + } + if (group.recentTabIds?.some((tabId) => !group.tabOrder.includes(tabId))) { + throw new Error('terminal_orphan_topology_invalid') + } + } + if (groupedTabIds.size !== topologyTabsById.size) { + throw new Error('terminal_orphan_topology_invalid') + } + if (request.topology.groupLayout) { + if (!hasExactTerminalOrphanGroupLayout(request.topology.groupLayout, seenGroupIds)) { + throw new Error('terminal_orphan_topology_invalid') + } + } + } + + for (const { claim, paneKey } of validated) { + const existingBinding = persistedBinding(claim.ptyId) + if ( + existingBinding && + (!runtimeWorktreeIdsEqual(existingBinding.worktreeId, worktree.id) || + existingBinding.paneKey !== paneKey) + ) { + throw new Error('terminal_orphan_competing_owner') + } + const proposedPtyId = + session.terminalLayoutsByTabId[claim.tabId]?.ptyIdsByLeafId?.[claim.leafId] + if (proposedPtyId && proposedPtyId !== claim.ptyId) { + throw new Error('terminal_orphan_surface_occupied') + } + const graphOwner = this.leaves.get(this.getLeafKey(claim.tabId, claim.leafId)) + if ( + graphOwner && + (graphOwner.ptyId !== claim.ptyId || + !runtimeWorktreeIdsEqual(graphOwner.worktreeId, worktree.id)) + ) { + throw new Error('terminal_orphan_surface_occupied') + } + if ( + Object.entries(session.tabsByWorktree).some( + ([ownerWorktreeId, tabs]) => + !runtimeWorktreeIdsEqual(ownerWorktreeId, worktree.id) && + tabs.some((tab) => tab.id === claim.tabId) + ) + ) { + throw new Error('terminal_orphan_surface_occupied') + } + if (session.terminalSurfaceTombstonesByPaneKey?.[paneKey]) { + throw new Error('terminal_orphan_surface_retired') + } + for (const snapshot of this.mobileSessionTabsByWorktree.values()) { + const surfaceOwner = snapshot.tabs.find( + (tab): tab is RuntimeMobileSessionTerminalTab => + tab.type === 'terminal' && + tab.parentTabId === claim.tabId && + tab.leafId === claim.leafId + ) + if ( + surfaceOwner && + (snapshot.worktree !== worktree.id || surfaceOwner.ptyId !== claim.ptyId) + ) { + throw new Error('terminal_orphan_surface_occupied') + } + const owner = snapshot.tabs.find( + (tab): tab is RuntimeMobileSessionTerminalTab => + tab.type === 'terminal' && tab.ptyId === claim.ptyId + ) + if ( + owner && + (snapshot.worktree !== worktree.id || + owner.parentTabId !== claim.tabId || + owner.leafId !== claim.leafId) + ) { + throw new Error('terminal_orphan_competing_owner') + } + } + } + + const next = structuredClone(session) + canonicalizeTerminalSessionWorktreeId(next, sessionWorktreeId, worktree.id) + const existingTabs = next.tabsByWorktree[worktree.id] ?? [] + const tabsById = new Map(existingTabs.map((tab) => [tab.id, tab])) + for (const { claim, pty, paneKey } of validated) { + let tab = tabsById.get(claim.tabId) + if (!tab) { + const title = + getLatestPtyTitle(pty) ?? pty.controllerTitle ?? `Terminal ${tabsById.size + 1}` + tab = { + id: claim.tabId, + ptyId: claim.ptyId, + worktreeId: worktree.id, + title, + defaultTitle: title, + customTitle: null, + color: null, + sortOrder: tabsById.size, + createdAt: Date.now(), + pendingActivationSpawn: true + } + tabsById.set(claim.tabId, tab) + } + const existingLayout = next.terminalLayoutsByTabId[claim.tabId] + const topologyTab = topologyTabsById.get(claim.tabId) + next.terminalLayoutsByTabId[claim.tabId] = topologyTab + ? { + ...existingLayout, + root: topologyTab.root, + activeLeafId: topologyTab.activeLeafId, + expandedLeafId: topologyTab.expandedLeafId, + ptyIdsByLeafId: { + ...existingLayout?.ptyIdsByLeafId, + [claim.leafId]: claim.ptyId + } + } + : existingLayout + ? { + ...existingLayout, + root: this.collectPersistedTerminalLeafIds(existingLayout).includes(claim.leafId) + ? existingLayout.root + : existingLayout.root === null + ? { type: 'leaf', leafId: claim.leafId } + : { + type: 'split', + direction: 'vertical', + first: existingLayout.root, + second: { type: 'leaf', leafId: claim.leafId } + }, + ptyIdsByLeafId: { + ...existingLayout.ptyIdsByLeafId, + [claim.leafId]: claim.ptyId + } + } + : { + root: { type: 'leaf', leafId: claim.leafId }, + activeLeafId: claim.leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [claim.leafId]: claim.ptyId } + } + next.terminalPtyIncarnationsByPaneKey = { + ...next.terminalPtyIncarnationsByPaneKey, + [paneKey]: claim.incarnationId + } + } + const adoptedTabIds = [...new Set(validated.map(({ claim }) => claim.tabId))] + next.tabsByWorktree[worktree.id] = [...tabsById.values()] + const activeTabId = + request.activeTabId && tabsById.has(request.activeTabId) + ? request.activeTabId + : (adoptedTabIds[0] ?? null) + const existingGroups = next.tabGroups?.[worktree.id] ?? [] + const targetGroupId = + (request.activeGroupId && existingGroups.some((group) => group.id === request.activeGroupId) + ? request.activeGroupId + : existingGroups[0]?.id) ?? + request.activeGroupId ?? + randomUUID() + const proposedGroups = topologyGroups.map((group) => ({ + ...group, + worktreeId: worktree.id + })) + const groups = + existingGroups.length === 0 && proposedGroups.length > 0 + ? proposedGroups + : existingGroups.length > 0 + ? existingGroups + .map((group) => { + const proposed = proposedGroups.find((candidate) => candidate.id === group.id) + const tabOrder = proposed + ? [ + ...group.tabOrder.filter((tabId) => !adoptedTabIds.includes(tabId)), + ...proposed.tabOrder + ] + : group.id === targetGroupId && proposedGroups.length === 0 + ? [...new Set([...group.tabOrder, ...adoptedTabIds])] + : group.tabOrder.filter((tabId) => !adoptedTabIds.includes(tabId)) + return { + ...group, + tabOrder, + activeTabId: proposed + ? proposed.activeTabId + : group.id === targetGroupId && activeTabId + ? activeTabId + : group.activeTabId && tabOrder.includes(group.activeTabId) + ? group.activeTabId + : (tabOrder[0] ?? null), + ...(proposed?.recentTabIds ? { recentTabIds: proposed.recentTabIds } : {}) + } + }) + .concat( + proposedGroups.filter( + (proposed) => !existingGroups.some((group) => group.id === proposed.id) + ) + ) + : [{ id: targetGroupId, worktreeId: worktree.id, activeTabId, tabOrder: adoptedTabIds }] + const retainedGroups = groups.filter((group) => group.tabOrder.length > 0) + next.tabGroups = { + ...next.tabGroups, + [worktree.id]: retainedGroups + } + const mergedGroupLayout = mergeTerminalOrphanGroupLayout({ + existingLayout: next.tabGroupLayouts?.[worktree.id], + existingGroupIds: existingGroups.map((group) => group.id), + proposedLayout: request.topology?.groupLayout, + proposedGroupIds: proposedGroups.map((group) => group.id), + mergedGroupIds: retainedGroups.map((group) => group.id) + }) + if (mergedGroupLayout) { + next.tabGroupLayouts = { + ...next.tabGroupLayouts, + [worktree.id]: mergedGroupLayout + } + } + const activeGroup = + (request.activeGroupId + ? retainedGroups.find( + (group) => + group.id === request.activeGroupId && + (!activeTabId || group.tabOrder.includes(activeTabId)) + ) + : undefined) ?? + retainedGroups.find((group) => activeTabId && group.tabOrder.includes(activeTabId)) ?? + retainedGroups[0]! + const convergedActiveTabId = + activeTabId && activeGroup.tabOrder.includes(activeTabId) + ? activeTabId + : activeGroup.activeTabId + next.activeTabIdByWorktree = { + ...next.activeTabIdByWorktree, + ...(convergedActiveTabId ? { [worktree.id]: convergedActiveTabId } : {}) + } + next.activeGroupIdByWorktree = { + ...next.activeGroupIdByWorktree, + [worktree.id]: activeGroup.id + } + const persisted = advanceTerminalTopologyRevision(next, worktree.id) + try { + store.setWorkspaceSession(persisted) + store.flushOrThrow() + } catch (error) { + store.setWorkspaceSession(session) + throw error + } + for (const { claim, pty, paneKey } of validated) { + pty.tabId = claim.tabId + pty.paneKey = paneKey + } + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktree.id, { + force: true, + allowAttachedWindow: true, + onlyRuntimeOwnedTerminals: true + }) + this.notifyMobileSessionTabsChanged(worktree.id) + return { + adopted: true, + topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1, + snapshot: await this.listMobileSessionTabs(`id:${worktree.id}`) } } @@ -12643,7 +13173,10 @@ export class OrcaRuntimeService { return { type: 'group', groupId: group.id, - activeTabId: group.activeTabId, + activeTabId: + group.activeTabId && tabs.some((tab) => tab.tabId === group.activeTabId) + ? group.activeTabId + : (tabs[0]?.tabId ?? null), tabs } }) @@ -14566,21 +15099,7 @@ export class OrcaRuntimeService { if (!dirStat.isDirectory()) { throw new Error(`${dirPath} is not a directory`) } - const entries = await readdir(dirPath, { withFileTypes: true }) - const mapped = entries - .filter((entry) => entry.name !== '.' && entry.name !== '..') - .map((entry) => ({ - name: entry.name, - isDirectory: entry.isDirectory(), - isSymlink: entry.isSymbolicLink() - })) - mapped.sort((a, b) => { - if (a.isDirectory !== b.isDirectory) { - return a.isDirectory ? -1 : 1 - } - return a.name.localeCompare(b.name) - }) - return { resolvedPath: dirPath, entries: mapped } + return { resolvedPath: dirPath, entries: await readLocalFilesystemDirectory(dirPath) } } async isGitAvailable(): Promise { @@ -14853,9 +15372,13 @@ export class OrcaRuntimeService { if (!existingStat.isDirectory()) { return { error: `"${trimmedName}" already exists at this location and is not a folder.` } } - const entries = await readdir(targetPath) - if (entries.length > 0) { - return { error: `"${trimmedName}" already exists at this location and is not empty.` } + const directory = await opendir(targetPath) + try { + if ((await directory.read()) !== null) { + return { error: `"${trimmedName}" already exists at this location and is not empty.` } + } + } finally { + await directory.close().catch(() => {}) } } else { await mkdir(targetPath, { recursive: false }) @@ -15186,24 +15709,36 @@ export class OrcaRuntimeService { } const repo = await this.resolveRepoSelector(repoSelector) this.store.removeProject(repo.id) - this.terminalTopologyRevisionByRepoId.delete(repo.id) - this.invalidateResolvedWorktreeCache() - this.invalidateWorktreeScanCacheForRepo(repo.id) + this.notifyRepoStoreChanged(repo.id) invalidateAuthorizedRootsCache() - this.notifyReposChanged() return { removed: true } } + notifyRepoStoreChanged(repoId: string): void { + const removed = !this.store?.getRepo(repoId) + if (removed) { + this.terminalTopologyRevisionByRepoId.delete(repoId) + } + this.invalidateResolvedWorktreeCache() + this.invalidateWorktreeScanCacheForRepo(repoId) + if (removed) { + this.worktreeScanGenerations.forget(repoId) + } + this.notifyReposChanged() + } + async inspectTerminalProcess( terminalSelector: string ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> { - const leaf = this.resolveLeafForHandle(terminalSelector) + const leaf = this.resolveLiveLeafForHandle(terminalSelector) if (!leaf?.ptyId || !this.ptyController) { - return { foregroundProcess: null, hasChildProcesses: false } + throw new Error('terminal_gone') + } + if (this.ptyController.inspectProcess) { + return this.ptyController.inspectProcess(leaf.ptyId) } const foregroundProcess = await this.ptyController.getForegroundProcess(leaf.ptyId) - const hasChildProcesses = - (await this.ptyController.hasChildProcesses?.(leaf.ptyId).catch(() => false)) ?? false + const hasChildProcesses = (await this.ptyController.hasChildProcesses?.(leaf.ptyId)) ?? false return { foregroundProcess, hasChildProcesses } } @@ -16577,8 +17112,12 @@ export class OrcaRuntimeService { } } try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repo.path, 'orca.yaml')) - const hooks = result.isBinary ? null : parseOrcaYaml(result.content) + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repo.path, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + const hooks = result.kind === 'text' ? parseOrcaYaml(result.content) : null return { hasHooksFile: Boolean(hooks), hooks, @@ -16626,11 +17165,19 @@ export class OrcaRuntimeService { return { hasHooks: false, hooks: null, mayNeedUpdate: false } } try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repo.path, 'orca.yaml')) - if (result.isBinary) { + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repo.path, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + if (result.kind === 'binary') { return { hasHooks: false, hooks: null, mayNeedUpdate: false } } - return { hasHooks: true, hooks: parseOrcaYaml(result.content), mayNeedUpdate: false } + return { + hasHooks: true, + hooks: result.kind === 'text' ? parseOrcaYaml(result.content) : null, + mayNeedUpdate: false + } } catch { return { hasHooks: false, hooks: null, mayNeedUpdate: false } } @@ -16659,15 +17206,18 @@ export class OrcaRuntimeService { return null } try { - const result = await fsProvider.readFile(filePath) - return result.isBinary ? null : result.content + const result = await readFilesystemProviderBoundedText(fsProvider, filePath, { + maxBytes: SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + maxCodeUnits: SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS + }) + return result.kind === 'text' ? result.content : null } catch { return null } } try { - return await readFile(filePath, 'utf-8') + return await readSetupScriptImportFile(filePath) } catch (error) { if (!isENOENT(error)) { console.warn('[runtime] Failed to inspect setup script import candidate:', error) @@ -16725,11 +17275,11 @@ export class OrcaRuntimeService { issueCommandPath: string ): Promise { try { - const result = await fsProvider.readFile(issueCommandPath) - if (result.isBinary) { - return null - } - return result.content.trim() || null + const result = await readFilesystemProviderBoundedText(fsProvider, issueCommandPath, { + maxBytes: MAX_ISSUE_COMMAND_BYTES, + maxCodeUnits: MAX_ISSUE_COMMAND_BYTES + }) + return result.kind === 'text' ? result.content.trim() || null : null } catch { return null } @@ -16740,8 +17290,12 @@ export class OrcaRuntimeService { repoPath: string ): Promise { try { - const result = await fsProvider.readFile(joinWorktreeRelativePath(repoPath, 'orca.yaml')) - if (result.isBinary) { + const result = await readFilesystemProviderBoundedText( + fsProvider, + joinWorktreeRelativePath(repoPath, 'orca.yaml'), + { maxBytes: MAX_ORCA_YAML_BYTES, maxCodeUnits: MAX_ORCA_YAML_CODE_UNITS } + ) + if (result.kind !== 'text') { return null } return parseOrcaYaml(result.content)?.issueCommand?.trim() || null @@ -16787,9 +17341,12 @@ export class OrcaRuntimeService { options: { required?: boolean } = {} ): Promise { const gitignorePath = joinWorktreeRelativePath(repoPath, '.gitignore') - let result: Awaited> + let result: Awaited> try { - result = await fsProvider.readFile(gitignorePath) + result = await readFilesystemProviderBoundedText(fsProvider, gitignorePath, { + maxBytes: MAX_HOOK_GITIGNORE_BYTES, + maxCodeUnits: MAX_HOOK_GITIGNORE_BYTES + }) } catch (error) { if (!isENOENT(error)) { if (options.required) { @@ -16808,9 +17365,13 @@ export class OrcaRuntimeService { } return } - if (result.isBinary) { + if (result.kind !== 'text') { if (options.required) { - throw new Error('Remote .gitignore is binary; cannot verify .orca is ignored') + throw new Error( + result.kind === 'binary' + ? 'Remote .gitignore is binary; cannot verify .orca is ignored' + : 'Remote .gitignore exceeds the supported size limit' + ) } return } @@ -24065,8 +24626,10 @@ export class OrcaRuntimeService { getAgentLaunchPlatformForRepo(repo, projectRuntimeByRepoId.get(repo.id)) ]) ) - const perRepoWorktrees = await Promise.all( - repos.map(async (repo) => { + const perRepoWorktrees = await mapWithConcurrency( + repos, + RESOLVED_WORKTREE_REPO_CONCURRENCY, + async (repo) => { if (isFolderRepo(repo)) { return listRuntimeFolderWorkspaces(this.requireStore(), repo).map((worktree) => ({ ...worktree, @@ -24123,7 +24686,7 @@ export class OrcaRuntimeService { comment: merged.comment } }) - }) + } ) const worktrees = projectResolvedWorktreeLineage( perRepoWorktrees.flat(), @@ -24187,7 +24750,7 @@ export class OrcaRuntimeService { projectRuntimeByRepoId?: ReadonlyMap ): Promise { const now = Date.now() - const generation = this.worktreeScanGenerations.get(repo.id) ?? 0 + const generation = this.worktreeScanGenerations.current(repo.id) const projectRuntime = projectRuntimeByRepoId ? projectRuntimeByRepoId.get(repo.id) : !repo.connectionId @@ -24218,7 +24781,7 @@ export class OrcaRuntimeService { const result = await promise if ( result.ok && - generation === (this.worktreeScanGenerations.get(repo.id) ?? 0) && + this.worktreeScanGenerations.isCurrent(repo.id, generation) && this.worktreeScanInFlight.get(repo.id)?.promise === promise ) { this.worktreeScanCache.set(repo.id, { @@ -24298,7 +24861,7 @@ export class OrcaRuntimeService { } private invalidateWorktreeScanCacheForRepo(repoId: string): void { - this.worktreeScanGenerations.set(repoId, (this.worktreeScanGenerations.get(repoId) ?? 0) + 1) + this.worktreeScanGenerations.advance(repoId) this.worktreeScanCache.delete(repoId) this.worktreeScanInFlight.delete(repoId) } @@ -24309,7 +24872,7 @@ export class OrcaRuntimeService { repos.filter((repo) => repo.connectionId === targetId).map((repo) => repo.id) ) for (const repoId of affectedRepoIds) { - this.worktreeScanGenerations.set(repoId, (this.worktreeScanGenerations.get(repoId) ?? 0) + 1) + this.worktreeScanGenerations.advance(repoId) this.worktreeScanCache.delete(repoId) this.worktreeScanInFlight.delete(repoId) } @@ -24394,6 +24957,7 @@ export class OrcaRuntimeService { lastOscTitleAt: null, managementTitle: null, managementTitleAt: null, + controllerTitle: null, title: state.title ?? null, titleUpdatedAt: titleObservedAt, lastOutputAt: state.lastOutputAt ?? null, @@ -24519,13 +25083,56 @@ export class OrcaRuntimeService { return null } const sessions = sessionsResult.value + const controllerIdentityByPtyId = new Map< + string, + { handle: string; incarnationId: string; wslDistro?: string | null } + >() + const ptyIdByControllerHandle = new Map() + const ambiguousControllerPtyIds = new Set() + for (const session of sessions) { + const handle = session.terminalHandle?.trim() + const incarnationId = session.incarnationId?.trim() + if (!handle?.startsWith('term_') || !incarnationId) { + continue + } + const priorPtyId = ptyIdByControllerHandle.get(handle) + if (priorPtyId && priorPtyId !== session.id) { + ambiguousControllerPtyIds.add(priorPtyId) + ambiguousControllerPtyIds.add(session.id) + controllerIdentityByPtyId.delete(priorPtyId) + continue + } + if (controllerIdentityByPtyId.has(session.id)) { + ambiguousControllerPtyIds.add(session.id) + controllerIdentityByPtyId.delete(session.id) + continue + } + ptyIdByControllerHandle.set(handle, session.id) + controllerIdentityByPtyId.set(session.id, { + handle, + incarnationId, + ...(session.wslDistro !== undefined ? { wslDistro: session.wslDistro } : {}) + }) + } + for (const ptyId of ambiguousControllerPtyIds) { + controllerIdentityByPtyId.delete(ptyId) + } + this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings( this.store?.getWorkspaceSession?.() ) + const persistedSurfaceByPtyId = indexPersistedPtySurfaceBindings( + this.store?.getWorkspaceSession?.() + ) const allLivePtyIds = new Set(sessions.map((session) => session.id)) const selectedLivePtyIds = new Set() for (const session of sessions) { - this.adoptControllerTerminalHandle(session.id, session.terminalHandle) + const controllerIdentity = controllerIdentityByPtyId.get(session.id) + this.adoptControllerTerminalHandle( + session.id, + controllerIdentity?.handle ?? session.terminalHandle, + controllerIdentity?.incarnationId ?? session.incarnationId + ) const persistedWorktreeId = persistedWorktreeIdByPtyId.get(session.id) const providerWorktree = resolvedWorktrees.find( (worktree) => session.worktreeId && runtimeWorktreeIdsEqual(worktree.id, session.worktreeId) @@ -24564,10 +25171,23 @@ export class OrcaRuntimeService { continue } if (worktreeId) { - this.recordPtyWorktree(session.id, worktreeId, { + const persistedSurface = persistedSurfaceByPtyId.get(session.id) + const restoresExactSurface = + persistedSurface && + session.incarnationId && + persistedSurface.incarnationId === session.incarnationId && + runtimeWorktreeIdsEqual(persistedSurface.worktreeId, worktreeId) + const pty = this.recordPtyWorktree(session.id, worktreeId, { connected: true, - ...(session.incarnationId ? { incarnationId: session.incarnationId } : {}) + ...(session.incarnationId ? { incarnationId: session.incarnationId } : {}), + ...(session.wslDistro !== undefined + ? { isWsl: Boolean(session.wslDistro), wslDistro: session.wslDistro } + : {}), + ...(restoresExactSurface + ? { tabId: persistedSurface.tabId, paneKey: persistedSurface.paneKey } + : {}) }) + pty.controllerTitle = session.title?.trim() || null } // Why: fire-and-forget so this listing hot path doesn't serialize a relay round-trip per session and a throw can't abort the sweep below. this.refreshPtyForegroundAgent(session.id) @@ -24730,6 +25350,7 @@ export class OrcaRuntimeService { this.terminalFileUriHostnameByPtyId.delete(ptyId) this.wslDistroByPtyId.delete(ptyId) this.clearAgentRowSnapshotsForPty(ptyId) + this.ptyLifecycleGenerations.forget(ptyId) const handle = this.handleByPtyId.get(ptyId) if (handle) { // Why: pruning can remove a PTY without onPtyExit firing; release this leader's agent team so it doesn't leak. @@ -24805,9 +25426,12 @@ export class OrcaRuntimeService { const worktree = worktreesById.get(leaf.worktreeId) const tab = this.tabs.get(leaf.tabId) ?? null + const pty = leaf.ptyId ? this.ptysById.get(leaf.ptyId) : undefined return { handle: this.issueHandle(leaf), ptyId: leaf.ptyId, + incarnationId: pty?.incarnationId ?? null, + orphaned: false, worktreeId: leaf.worktreeId, worktreePath: worktree?.path ?? '', branch: worktree?.branch ?? '', @@ -26069,84 +26693,14 @@ export class OrcaRuntimeService { // Why: wake blocking orchestration.check --wait calls on this handle so they return the new message immediately instead of polling. notifyMessageArrived(handle: string, messageType?: string): void { - const waiters = this.messageWaitersByHandle.get(handle) - if (!waiters || waiters.size === 0) { - return - } - for (const waiter of [...waiters]) { - // Why: don't wake a coordinator waiting for worker_done/escalation on heartbeat noise it would misread as idleness. - if (messageType && waiter.typeFilter && !waiter.typeFilter.includes(messageType)) { - continue - } - this.resolveMessageWaiter(waiter) - } + this.messageWaiters.notify(handle, messageType) } waitForMessage( handle: string, options?: { typeFilter?: string[]; timeoutMs?: number; signal?: AbortSignal } ): Promise { - return new Promise((resolve) => { - const timeoutMs = options?.timeoutMs ?? MESSAGE_WAIT_DEFAULT_TIMEOUT_MS - - const waiter: MessageWaiter = { - handle, - typeFilter: options?.typeFilter, - resolve, - timeout: null, - abortCleanup: null - } - - // Why: on caller abort (RPC socket closed — design doc §3.1), resolve now to release the long-poll slot instead of waiting out timeoutMs. - const signal = options?.signal - const onAbort = (): void => { - this.removeMessageWaiter(waiter) - resolve() - } - if (signal) { - if (signal.aborted) { - resolve() - return - } - waiter.abortCleanup = () => signal.removeEventListener('abort', onAbort) - signal.addEventListener('abort', onAbort, { once: true }) - } - - waiter.timeout = setTimeout(() => { - this.removeMessageWaiter(waiter) - resolve() - }, timeoutMs) - - let waiters = this.messageWaitersByHandle.get(handle) - if (!waiters) { - waiters = new Set() - this.messageWaitersByHandle.set(handle, waiters) - } - waiters.add(waiter) - }) - } - - private resolveMessageWaiter(waiter: MessageWaiter): void { - this.removeMessageWaiter(waiter) - waiter.resolve() - } - - private removeMessageWaiter(waiter: MessageWaiter): void { - if (waiter.timeout) { - clearTimeout(waiter.timeout) - waiter.timeout = null - } - if (waiter.abortCleanup) { - waiter.abortCleanup() - waiter.abortCleanup = null - } - const waiters = this.messageWaitersByHandle.get(waiter.handle) - if (waiters) { - waiters.delete(waiter) - if (waiters.size === 0) { - this.messageWaitersByHandle.delete(waiter.handle) - } - } + return this.messageWaiters.wait(handle, options) } private buildPtyTerminalSummary( @@ -26155,14 +26709,18 @@ export class OrcaRuntimeService { ): RuntimeTerminalSummary { const worktree = worktreesById.get(pty.worktreeId) + const pane = parsePaneKey(pty.paneKey ?? '') + const orphaned = !pty.tabId || !pane || pane.tabId !== pty.tabId return { handle: this.issuePtyHandle(pty), ptyId: pty.ptyId, + incarnationId: pty.incarnationId, + orphaned, worktreeId: pty.worktreeId, worktreePath: worktree?.path ?? '', branch: worktree?.branch ?? '', - tabId: `pty:${pty.ptyId}`, - leafId: `pty:${pty.ptyId}`, + tabId: orphaned ? `pty:${pty.ptyId}` : pty.tabId!, + leafId: orphaned ? `pty:${pty.ptyId}` : pane.leafId, title: getLatestPtyTitle(pty), connected: pty.connected, writable: pty.connected, @@ -29655,6 +30213,7 @@ const WORKTREE_SCAN_CACHE_TTL_MS = 30_000 // these (crash-cluster diagnostics, 2026-07). const WORKTREE_SCAN_AGENT_SCRATCH_TTL_MS = 5 * 60_000 const RESOLVED_WORKTREE_REPO_TIMEOUT_MS = 5000 +const RESOLVED_WORKTREE_REPO_CONCURRENCY = 8 export function resolveWorktreeScanCacheTtlMs(repo: Pick): number { return !repo.connectionId && isAgentScratchRepoRootPath(repo.path) @@ -31101,7 +31660,6 @@ async function assertTerminalInputWithinLimitWithYield(text: string | undefined) const TUI_IDLE_DEFAULT_TIMEOUT_MS = 5 * 60 * 1000 const TUI_IDLE_POLL_INTERVAL_MS = 2000 const TUI_IDLE_QUIESCENCE_MS = 3000 -const MESSAGE_WAIT_DEFAULT_TIMEOUT_MS = 2 * 60 * 1000 const EXPLICIT_IDLE_TITLE_RE = /(^|\s)(ready|idle|done)(\s|$|[.!?])/i const CLAUDE_IDLE_PREFIX = '\u2733' const GEMINI_IDLE_PREFIX = '\u25c7' @@ -31425,6 +31983,59 @@ function runtimeWorktreeIdentityKey(worktreeId: string): string { : worktreeId } +function resolveTerminalSessionWorktreeId( + session: WorkspaceSessionState, + targetWorktreeId: string +): string | null { + const keyedWorktreeIds = new Set([ + ...Object.keys(session.tabsByWorktree), + ...Object.keys(session.tabGroups ?? {}), + ...Object.keys(session.tabGroupLayouts ?? {}), + ...Object.keys(session.activeTabIdByWorktree ?? {}), + ...Object.keys(session.activeGroupIdByWorktree ?? {}) + ]) + const matches = [...keyedWorktreeIds].filter((worktreeId) => + runtimeWorktreeIdsEqual(worktreeId, targetWorktreeId) + ) + return matches.length > 1 ? null : (matches[0] ?? targetWorktreeId) +} + +function canonicalizeTerminalSessionWorktreeId( + session: WorkspaceSessionState, + sourceWorktreeId: string, + targetWorktreeId: string +): void { + if (sourceWorktreeId === targetWorktreeId) { + return + } + const tabs = session.tabsByWorktree[sourceWorktreeId] ?? [] + delete session.tabsByWorktree[sourceWorktreeId] + session.tabsByWorktree[targetWorktreeId] = tabs.map((tab) => ({ + ...tab, + worktreeId: targetWorktreeId + })) + + const groups = session.tabGroups?.[sourceWorktreeId] + if (groups) { + delete session.tabGroups![sourceWorktreeId] + session.tabGroups![targetWorktreeId] = groups.map((group) => ({ + ...group, + worktreeId: targetWorktreeId + })) + } + for (const keyedState of [ + session.tabGroupLayouts, + session.activeTabIdByWorktree, + session.activeGroupIdByWorktree + ]) { + if (!keyedState || !Object.hasOwn(keyedState, sourceWorktreeId)) { + continue + } + keyedState[targetWorktreeId] = keyedState[sourceWorktreeId] as never + delete keyedState[sourceWorktreeId] + } +} + function inferWorktreeIdFromPtyId(ptyId: string): string | null { return parsePtySessionId(ptyId).worktreeId } @@ -31461,6 +32072,49 @@ function indexPersistedPtyWorktreeBindings( return worktreeIdByPtyId } +function indexPersistedPtySurfaceBindings( + session: WorkspaceSessionState | null | undefined +): ReadonlyMap< + string, + { worktreeId: string; tabId: string; paneKey: string; incarnationId: string } +> { + const bindingByPtyId = new Map< + string, + { worktreeId: string; tabId: string; paneKey: string; incarnationId: string } + >() + const ambiguousPtyIds = new Set() + for (const [worktreeId, tabs] of Object.entries(session?.tabsByWorktree ?? {})) { + for (const tab of tabs) { + for (const [leafId, ptyId] of Object.entries( + session?.terminalLayoutsByTabId[tab.id]?.ptyIdsByLeafId ?? {} + )) { + if (!ptyId || ambiguousPtyIds.has(ptyId)) { + continue + } + const paneKey = makePaneKey(tab.id, leafId) + const incarnationId = session?.terminalPtyIncarnationsByPaneKey?.[paneKey] + if (!incarnationId) { + continue + } + const binding = { worktreeId, tabId: tab.id, paneKey, incarnationId } + const existing = bindingByPtyId.get(ptyId) + if ( + existing && + (existing.worktreeId !== worktreeId || + existing.paneKey !== paneKey || + existing.incarnationId !== incarnationId) + ) { + bindingByPtyId.delete(ptyId) + ambiguousPtyIds.add(ptyId) + continue + } + bindingByPtyId.set(ptyId, binding) + } + } + } + return bindingByPtyId +} + function setsEqual(a: ReadonlySet, b: ReadonlySet): boolean { if (a.size !== b.size) { return false diff --git a/src/main/runtime/orchestration-message-waiter-registry.test.ts b/src/main/runtime/orchestration-message-waiter-registry.test.ts new file mode 100644 index 00000000000..ee8ed7afb70 --- /dev/null +++ b/src/main/runtime/orchestration-message-waiter-registry.test.ts @@ -0,0 +1,235 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + MAX_ORCHESTRATION_MESSAGE_WAITERS, + MAX_ORCHESTRATION_MESSAGE_WAITERS_PER_HANDLE, + MAX_ORCHESTRATION_MESSAGE_WAITER_HANDLE_BYTES, + MAX_ORCHESTRATION_MESSAGE_WAITER_RETAINED_HANDLE_BYTES, + OrchestrationMessageWaiterLimitError, + OrchestrationMessageWaiterRegistry +} from './orchestration-message-waiter-registry' + +const BOUNDS = { + maxWaiters: 3, + maxWaitersPerHandle: 2, + maxHandleBytes: 8, + maxRetainedHandleBytes: 10 +} + +describe('OrchestrationMessageWaiterRegistry', () => { + afterEach(() => { + vi.useRealTimers() + }) + + it('caps global waiters and admits again after settlement', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const first = registry.wait('a') + const second = registry.wait('b') + const third = registry.wait('c') + + expect(() => registry.wait('d')).toThrowError(expect.objectContaining({ reason: 'global' })) + registry.notify('a') + await first + const replacement = registry.wait('d') + expect(registry.evidence()).toEqual({ + waiters: 3, + handles: 3, + retainedHandleBytes: 3 + }) + + registry.close() + await Promise.all([second, third, replacement]) + }) + + it('caps waiters per handle without blocking another handle', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const first = registry.wait('same') + const second = registry.wait('same') + + expect(() => registry.wait('same')).toThrowError( + expect.objectContaining({ reason: 'per-handle' }) + ) + const other = registry.wait('other') + expect(registry.evidence()).toEqual({ + waiters: 3, + handles: 2, + retainedHandleBytes: 9 + }) + + registry.close() + await Promise.all([first, second, other]) + }) + + it('rejects oversized handles without retaining them', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + + expect(() => registry.wait('🌊🌊🌊')).toThrow(OrchestrationMessageWaiterLimitError) + expect(() => registry.wait('🌊🌊🌊')).toThrowError( + expect.objectContaining({ reason: 'handle-bytes' }) + ) + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + + const controller = new AbortController() + controller.abort() + await expect(registry.wait('🌊🌊🌊', { signal: controller.signal })).resolves.toBeUndefined() + }) + + it('accounts for each unique handle key once and caps aggregate key bytes', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const first = registry.wait('123456') + const sameHandle = registry.wait('123456') + const second = registry.wait('abcd') + + expect(registry.evidence()).toEqual({ + waiters: 3, + handles: 2, + retainedHandleBytes: 10 + }) + registry.notify('abcd') + await second + expect(() => registry.wait('abcde')).toThrowError( + expect.objectContaining({ reason: 'retained-handle-bytes' }) + ) + + registry.close() + await Promise.all([first, sameHandle]) + }) + + it('releases all key accounting under unique-handle churn', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + + for (let index = 0; index < 10_000; index += 1) { + const handle = String(index) + const wait = registry.wait(handle) + registry.notify(handle) + await wait + } + + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + }) + + it('preserves type-filter wake behavior and deduplicates filters', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + let statusResolved = false + let workerResolved = false + const status = registry.wait('same', { + typeFilter: ['status', 'status'] + }) + const worker = registry.wait('same', { + typeFilter: ['worker_done'] + }) + void status.then(() => { + statusResolved = true + }) + void worker.then(() => { + workerResolved = true + }) + + expect(registry.notify('same', 'status')).toBe(1) + await status + expect(statusResolved).toBe(true) + expect(workerResolved).toBe(false) + expect(registry.evidence()).toEqual({ + waiters: 1, + handles: 1, + retainedHandleBytes: 4 + }) + + expect(registry.notify('same')).toBe(1) + await worker + }) + + it('removes timeout and abort state after notification', async () => { + vi.useFakeTimers() + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + const wait = registry.wait('a', { signal: controller.signal, timeoutMs: 100 }) + + registry.notify('a') + await wait + expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function)) + expect(vi.getTimerCount()).toBe(0) + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + }) + + it('cleans up exactly on timeout', async () => { + vi.useFakeTimers() + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + const wait = registry.wait('abc', { signal: controller.signal, timeoutMs: 100 }) + + await vi.advanceTimersByTimeAsync(100) + await wait + expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function)) + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + }) + + it('cleans up exactly on abort and does not register an already-aborted signal', async () => { + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + const wait = registry.wait('abc', { signal: controller.signal }) + + controller.abort() + await wait + expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function)) + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + + const alreadyAborted = new AbortController() + alreadyAborted.abort() + const addListener = vi.spyOn(alreadyAborted.signal, 'addEventListener') + await registry.wait('abc', { signal: alreadyAborted.signal }) + expect(addListener).not.toHaveBeenCalled() + expect(registry.evidence().waiters).toBe(0) + }) + + it('settles every waiter and releases all accounting on close', async () => { + vi.useFakeTimers() + const registry = new OrchestrationMessageWaiterRegistry(BOUNDS) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + const first = registry.wait('first') + const sameHandle = registry.wait('first', { signal: controller.signal }) + const second = registry.wait('other') + + registry.close() + await Promise.all([first, sameHandle, second]) + expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function)) + expect(vi.getTimerCount()).toBe(0) + expect(registry.evidence()).toEqual({ + waiters: 0, + handles: 0, + retainedHandleBytes: 0 + }) + await expect(registry.wait('later')).resolves.toBeUndefined() + expect(registry.evidence().waiters).toBe(0) + }) + + it('publishes explicit production bounds', () => { + expect(MAX_ORCHESTRATION_MESSAGE_WAITERS).toBe(1_024) + expect(MAX_ORCHESTRATION_MESSAGE_WAITERS_PER_HANDLE).toBe(64) + expect(MAX_ORCHESTRATION_MESSAGE_WAITER_HANDLE_BYTES).toBe(64 * 1024) + expect(MAX_ORCHESTRATION_MESSAGE_WAITER_RETAINED_HANDLE_BYTES).toBe(1024 * 1024) + }) +}) diff --git a/src/main/runtime/orchestration-message-waiter-registry.ts b/src/main/runtime/orchestration-message-waiter-registry.ts new file mode 100644 index 00000000000..56f40f9a1e3 --- /dev/null +++ b/src/main/runtime/orchestration-message-waiter-registry.ts @@ -0,0 +1,235 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MAX_ORCHESTRATION_MESSAGE_WAITERS = 1_024 +export const MAX_ORCHESTRATION_MESSAGE_WAITERS_PER_HANDLE = 64 +export const MAX_ORCHESTRATION_MESSAGE_WAITER_HANDLE_BYTES = 64 * 1024 +export const MAX_ORCHESTRATION_MESSAGE_WAITER_RETAINED_HANDLE_BYTES = 1024 * 1024 + +const DEFAULT_MESSAGE_WAIT_TIMEOUT_MS = 2 * 60 * 1000 + +export type OrchestrationMessageWaiterBounds = { + maxWaiters: number + maxWaitersPerHandle: number + maxHandleBytes: number + maxRetainedHandleBytes: number +} + +type MessageWaiterBucket = { + handle: string + handleBytes: number + waiters: Set +} + +type MessageWaiter = { + bucket: MessageWaiterBucket + typeFilter: Set | undefined + resolve: () => void + timeout: ReturnType | null + signal: AbortSignal | undefined + onAbort: () => void + active: boolean +} + +const DEFAULT_BOUNDS: OrchestrationMessageWaiterBounds = { + maxWaiters: MAX_ORCHESTRATION_MESSAGE_WAITERS, + maxWaitersPerHandle: MAX_ORCHESTRATION_MESSAGE_WAITERS_PER_HANDLE, + maxHandleBytes: MAX_ORCHESTRATION_MESSAGE_WAITER_HANDLE_BYTES, + maxRetainedHandleBytes: MAX_ORCHESTRATION_MESSAGE_WAITER_RETAINED_HANDLE_BYTES +} + +export type OrchestrationMessageWaiterLimitReason = + | 'global' + | 'per-handle' + | 'handle-bytes' + | 'retained-handle-bytes' + +export class OrchestrationMessageWaiterLimitError extends Error { + constructor( + readonly reason: OrchestrationMessageWaiterLimitReason, + readonly limit: number + ) { + super(getLimitErrorMessage(reason, limit)) + this.name = 'OrchestrationMessageWaiterLimitError' + } +} + +export class OrchestrationMessageWaiterRegistry { + private readonly waitersByHandle = new Map() + private waiterCount = 0 + private retainedHandleBytes = 0 + private closed = false + + constructor( + private readonly bounds: OrchestrationMessageWaiterBounds = DEFAULT_BOUNDS, + private readonly defaultTimeoutMs = DEFAULT_MESSAGE_WAIT_TIMEOUT_MS + ) { + if ( + !Number.isSafeInteger(bounds.maxWaiters) || + bounds.maxWaiters < 1 || + !Number.isSafeInteger(bounds.maxWaitersPerHandle) || + bounds.maxWaitersPerHandle < 1 || + !Number.isSafeInteger(bounds.maxHandleBytes) || + bounds.maxHandleBytes < 1 || + !Number.isSafeInteger(bounds.maxRetainedHandleBytes) || + bounds.maxRetainedHandleBytes < 1 + ) { + throw new RangeError('Orchestration message waiter bounds must be positive integers') + } + } + + wait( + handle: string, + options: { typeFilter?: string[]; timeoutMs?: number; signal?: AbortSignal } = {} + ): Promise { + if (options.signal?.aborted || this.closed) { + return Promise.resolve() + } + + const bucket = this.admit(handle) + const typeFilter = options.typeFilter ? new Set(options.typeFilter) : undefined + + return new Promise((resolve) => { + let waiter!: MessageWaiter + const onAbort = (): void => this.settle(waiter) + waiter = { + bucket, + typeFilter, + resolve, + timeout: null, + signal: options.signal, + onAbort, + active: true + } + this.retain(waiter) + options.signal?.addEventListener('abort', onAbort, { once: true }) + if (options.signal?.aborted) { + this.settle(waiter) + return + } + waiter.timeout = setTimeout( + () => this.settle(waiter), + options.timeoutMs ?? this.defaultTimeoutMs + ) + }) + } + + notify(handle: string, messageType?: string): number { + const bucket = this.waitersByHandle.get(handle) + if (!bucket) { + return 0 + } + + let notified = 0 + for (const waiter of Array.from(bucket.waiters)) { + if (messageType && waiter.typeFilter && !waiter.typeFilter.has(messageType)) { + continue + } + this.settle(waiter) + notified += 1 + } + return notified + } + + close(): void { + if (this.closed) { + return + } + this.closed = true + for (const bucket of this.waitersByHandle.values()) { + for (const waiter of bucket.waiters) { + this.settle(waiter) + } + } + } + + evidence(): { waiters: number; handles: number; retainedHandleBytes: number } { + return { + waiters: this.waiterCount, + handles: this.waitersByHandle.size, + retainedHandleBytes: this.retainedHandleBytes + } + } + + private admit(handle: string): MessageWaiterBucket { + const existing = this.waitersByHandle.get(handle) + if (existing) { + if (this.waiterCount >= this.bounds.maxWaiters) { + throw new OrchestrationMessageWaiterLimitError('global', this.bounds.maxWaiters) + } + if (existing.waiters.size >= this.bounds.maxWaitersPerHandle) { + throw new OrchestrationMessageWaiterLimitError( + 'per-handle', + this.bounds.maxWaitersPerHandle + ) + } + return existing + } + + const measurement = measureUtf8ByteLength(handle, { + stopAfterBytes: this.bounds.maxHandleBytes + }) + if (measurement.exceededLimit) { + throw new OrchestrationMessageWaiterLimitError('handle-bytes', this.bounds.maxHandleBytes) + } + if (this.waiterCount >= this.bounds.maxWaiters) { + throw new OrchestrationMessageWaiterLimitError('global', this.bounds.maxWaiters) + } + if (measurement.byteLength > this.bounds.maxRetainedHandleBytes - this.retainedHandleBytes) { + throw new OrchestrationMessageWaiterLimitError( + 'retained-handle-bytes', + this.bounds.maxRetainedHandleBytes + ) + } + return { + handle, + handleBytes: measurement.byteLength, + waiters: new Set() + } + } + + private retain(waiter: MessageWaiter): void { + const { bucket } = waiter + if (bucket.waiters.size === 0) { + this.waitersByHandle.set(bucket.handle, bucket) + this.retainedHandleBytes += bucket.handleBytes + } + bucket.waiters.add(waiter) + this.waiterCount += 1 + } + + private settle(waiter: MessageWaiter): void { + if (!waiter.active) { + return + } + waiter.active = false + if (waiter.timeout !== null) { + clearTimeout(waiter.timeout) + waiter.timeout = null + } + waiter.signal?.removeEventListener('abort', waiter.onAbort) + const { bucket } = waiter + if (bucket.waiters.delete(waiter)) { + this.waiterCount -= 1 + } + if (bucket.waiters.size === 0 && this.waitersByHandle.delete(bucket.handle)) { + this.retainedHandleBytes -= bucket.handleBytes + } + waiter.resolve() + } +} + +function getLimitErrorMessage( + reason: OrchestrationMessageWaiterLimitReason, + limit: number +): string { + if (reason === 'handle-bytes') { + return `Orchestration message wait handle exceeds ${limit} UTF-8 bytes.` + } + if (reason === 'per-handle') { + return `Orchestration message wait capacity reached for this terminal (${limit}); retry shortly.` + } + if (reason === 'retained-handle-bytes') { + return `Orchestration message wait handle capacity reached (${limit} UTF-8 bytes); retry shortly.` + } + return `Orchestration message wait capacity reached (${limit}); retry shortly.` +} diff --git a/src/main/runtime/orchestration/coordinator.ts b/src/main/runtime/orchestration/coordinator.ts index d0e56b7e40a..7c640588387 100644 --- a/src/main/runtime/orchestration/coordinator.ts +++ b/src/main/runtime/orchestration/coordinator.ts @@ -3,6 +3,12 @@ import type { OrchestrationDb } from './db' import type { MessageRow, TaskRow, CoordinatorStatus } from './types' import { buildDispatchPreamble } from './preamble' import { reconcileLifecycleMessage } from './lifecycle-reconciliation' +import { + orchestrationRowRetainedUtf8Bytes, + ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES, + ORCHESTRATION_QUERY_MAX_ROWS, + parseOrchestrationJson +} from './query-retention' export type CoordinatorRuntime = { sendTerminalAgentPrompt(handle: string, prompt: string): Promise @@ -66,6 +72,7 @@ type CoordinatorState = { completedTasks: string[] failedTasks: string[] escalations: MessageRow[] + escalationRetainedBytes: number } const DEFAULT_POLL_MS = 2000 @@ -74,6 +81,23 @@ const MAX_CONCURRENT_DEFAULT = 4 // Why: 10 min = documented heartbeat cadence (5 min) × 2, so one missed heartbeat is the earliest a dispatch can look stale. const HUNG_THRESHOLD_MS = 10 * 60 * 1000 +function mergeCoordinatorTaskIds(...sources: string[][]): string[] { + const retained: string[] = [] + const seen = new Set() + for (const source of sources) { + for (const id of source) { + if (!seen.has(id)) { + seen.add(id) + retained.push(id) + if (retained.length >= ORCHESTRATION_QUERY_MAX_ROWS) { + return retained + } + } + } + } + return retained +} + export class Coordinator { private db: OrchestrationDb private runtime: CoordinatorRuntime @@ -100,7 +124,8 @@ export class Coordinator { phase: 'decomposing', completedTasks: [], failedTasks: [], - escalations: [] + escalations: [], + escalationRetainedBytes: 0 } } @@ -152,14 +177,12 @@ export class Coordinator { } // Why: an early stop leaves tasks incomplete, so the run counts as failed. - const tasks = this.db.listTasks() - const allDone = tasks.every((t) => t.status === 'completed' || t.status === 'failed') - const failedTasks = [ - ...new Set([ - ...this.state.failedTasks, - ...tasks.filter((task) => task.status === 'failed').map((task) => task.id) - ]) - ] + const counts = this.db.getTaskStatusCounts() + const allDone = counts.total === counts.completed + counts.failed + const failedTasks = mergeCoordinatorTaskIds( + this.state.failedTasks, + this.db.listTaskIdsByStatus('failed') + ) const finalStatus = this.stopped || failedTasks.length > 0 || !allDone ? 'failed' : 'completed' this.db.updateCoordinatorRun(runId, finalStatus) @@ -185,13 +208,13 @@ export class Coordinator { // Why: decomposition isn't implemented yet — tasks must be pre-created before run(); AI-driven decomposition is a future phase. private async decompose(): Promise { this.state.phase = 'decomposing' - const existing = this.db.listTasks() - if (existing.length === 0) { + const taskCount = this.db.getTaskStatusCounts().total + if (taskCount === 0) { throw new Error( 'No tasks found. Create tasks with orchestration.taskCreate before running the coordinator.' ) } - this.opts.onLog(`Found ${existing.length} tasks in DAG`) + this.opts.onLog(`Found ${taskCount} tasks in DAG`) this.state.phase = 'dispatching' } @@ -207,13 +230,12 @@ export class Coordinator { // Why: warn only, never auto-fail — a false positive (slow but correct worker) costs more than a false negative (hung worker holding a slot); see R6 of DESIGN_DOC_PREAMBLE_FIX.md. private warnStaleDispatches(): void { const thresholdIso = new Date(Date.now() - HUNG_THRESHOLD_MS).toISOString() - const stale = this.db.getStaleDispatches(thresholdIso) - for (const ctx of stale) { + this.db.forEachStaleDispatch(thresholdIso, (ctx) => { const minutes = Math.round(HUNG_THRESHOLD_MS / 60000) this.opts.onLog( `Warning: worker ${ctx.assignee_handle ?? ''} on task ${ctx.task_id} has not sent a heartbeat in ~${minutes} min (dispatch ${ctx.id})` ) - } + }) } private processMessages(): void { @@ -252,7 +274,10 @@ export class Coordinator { private handleLifecycleMessage(msg: MessageRow): void { const result = reconcileLifecycleMessage(this.db, msg, this.opts.onLog) if (result.action === 'completed') { - if (!this.state.completedTasks.includes(result.taskId)) { + if ( + this.state.completedTasks.length < ORCHESTRATION_QUERY_MAX_ROWS && + !this.state.completedTasks.includes(result.taskId) + ) { this.state.completedTasks.push(result.taskId) } } @@ -260,13 +285,24 @@ export class Coordinator { private handleEscalation(msg: MessageRow): void { this.opts.onLog(`Escalation from ${msg.from_handle}: ${msg.subject}`) - this.state.escalations.push(msg) + const messageBytes = orchestrationRowRetainedUtf8Bytes(msg) + if ( + this.state.escalations.length < ORCHESTRATION_QUERY_MAX_ROWS && + this.state.escalationRetainedBytes + messageBytes <= + ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES + ) { + this.state.escalations.push(msg) + this.state.escalationRetainedBytes += messageBytes + } let taskId: string | undefined if (msg.payload) { try { - const payload = JSON.parse(msg.payload) - taskId = payload.taskId + const payload = parseOrchestrationJson(msg.payload) + if (payload && typeof payload === 'object') { + const rawTaskId = (payload as { taskId?: unknown }).taskId + taskId = typeof rawTaskId === 'string' ? rawTaskId : undefined + } } catch { // Escalation without structured payload — log subject as context } @@ -291,7 +327,12 @@ export class Coordinator { if (updated?.status === 'circuit_broken') { this.opts.onLog(`Task ${taskId} circuit broken after repeated failures`) this.db.updateTaskStatus(taskId, 'failed', `Circuit broken: ${msg.subject}`) - this.state.failedTasks.push(taskId) + if ( + this.state.failedTasks.length < ORCHESTRATION_QUERY_MAX_ROWS && + !this.state.failedTasks.includes(taskId) + ) { + this.state.failedTasks.push(taskId) + } } else { this.opts.onLog(`Task ${taskId} will be retried (failure ${updated?.failure_count ?? 0}/3)`) } @@ -303,7 +344,10 @@ export class Coordinator { let payload: { taskId?: string; question?: string; options?: string[] } = {} if (msg.payload) { try { - payload = JSON.parse(msg.payload) + const parsed = parseOrchestrationJson(msg.payload) + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + payload = parsed as typeof payload + } } catch { return } @@ -329,14 +373,7 @@ export class Coordinator { private processDecisionGates(): void { // Why: the coordinator never auto-resolves gates (humans do, via orchestration.gateResolve) — that would defeat them as approval checkpoints. - const pendingGates = this.db.listGates({ status: 'pending' }) - for (const gate of pendingGates) { - const task = this.db.getTask(gate.task_id) - if (task && task.status !== 'blocked') { - // Why: gate exists but task isn't blocked — re-block to restore the invariant. - this.db.updateTaskStatus(gate.task_id, 'blocked') - } - } + this.db.restorePendingGateTaskStatuses() } private async dispatchReadyTasks(): Promise { @@ -346,8 +383,8 @@ export class Coordinator { return } - const dispatched = this.db.listTasks({ status: 'dispatched' }) - let slotsAvailable = this.opts.maxConcurrent - dispatched.length + const dispatchedCount = this.db.getTaskStatusCounts().dispatched + let slotsAvailable = this.opts.maxConcurrent - dispatchedCount if (slotsAvailable <= 0) { return } @@ -437,10 +474,9 @@ export class Coordinator { }) // Why: surface a since-resolved decision gate's outcome to the worker via the preamble. - const gates = this.db.listGates({ taskId: task.id, status: 'resolved' }) let gateContext = '' - if (gates.length > 0) { - const latest = gates.at(-1)! + const latest = this.db.getLatestGate({ taskId: task.id, status: 'resolved' }) + if (latest) { gateContext = `\n\n--- DECISION GATE RESOLVED ---\nQuestion: ${latest.question}\nResolution: ${latest.resolution}\n---\n` } @@ -452,7 +488,12 @@ export class Coordinator { err instanceof Error ? err.message : String(err) ) if (updated?.status === 'circuit_broken') { - this.state.failedTasks.push(task.id) + if ( + this.state.failedTasks.length < ORCHESTRATION_QUERY_MAX_ROWS && + !this.state.failedTasks.includes(task.id) + ) { + this.state.failedTasks.push(task.id) + } } throw err } @@ -464,13 +505,11 @@ export class Coordinator { private async getAvailableTerminals(): Promise { try { const result = await this.runtime.listTerminals(this.opts.worktree) - const dispatched = this.db.listTasks({ status: 'dispatched' }) const busyHandles = new Set() - for (const task of dispatched) { - const ctx = this.db.getDispatchContext(task.id) - if (ctx?.assignee_handle) { - busyHandles.add(ctx.assignee_handle) + for (const terminal of result.terminals) { + if (this.db.getActiveDispatchForTerminal(terminal.handle)) { + busyHandles.add(terminal.handle) } } @@ -490,25 +529,22 @@ export class Coordinator { } private checkConvergence(): boolean { - const tasks = this.db.listTasks() - if (tasks.length === 0) { + const counts = this.db.getTaskStatusCounts() + if (counts.total === 0) { return true } - const allDone = tasks.every((t) => t.status === 'completed' || t.status === 'failed') + const allDone = counts.total === counts.completed + counts.failed if (allDone) { this.state.phase = 'done' return true } // Why: no active tasks but some blocked → dependencies can never be satisfied (stuck). - const active = tasks.filter( - (t) => t.status === 'ready' || t.status === 'dispatched' || t.status === 'pending' - ) - const blocked = tasks.filter((t) => t.status === 'blocked') - if (active.length === 0 && blocked.length > 0) { + const activeCount = counts.ready + counts.dispatched + counts.pending + if (activeCount === 0 && counts.blocked > 0) { this.opts.onLog( - `Stuck: ${blocked.length} tasks blocked with no active tasks. Resolve decision gates to continue.` + `Stuck: ${counts.blocked} tasks blocked with no active tasks. Resolve decision gates to continue.` ) } diff --git a/src/main/runtime/orchestration/db-query-retention.test.ts b/src/main/runtime/orchestration/db-query-retention.test.ts new file mode 100644 index 00000000000..44068bfcace --- /dev/null +++ b/src/main/runtime/orchestration/db-query-retention.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' +import { + orchestrationRowRetainedUtf8Bytes, + ORCHESTRATION_JSON_STRUCTURE_LIMITS, + ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES, + ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES, + ORCHESTRATION_QUERY_MAX_ROWS, + ORCHESTRATION_WRITE_MAX_ITEMS, + ORCHESTRATION_WRITE_MAX_UTF8_BYTES, + parseOrchestrationJson +} from './query-retention' + +function sqliteFor(db: OrchestrationDb): Database.Database { + return (db as unknown as { db: Database.Database }).db +} + +function paneKey(index: number, tab = `tab_${index}`): string { + return `${tab}:00000000-0000-4000-8000-${index.toString(16).padStart(12, '0')}` +} + +describe('OrchestrationDb query retention', () => { + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + }) + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + it('returns unread rows in stable bounded pages that can be drained successively', () => { + const d = createDb() + const inserted = Array.from({ length: ORCHESTRATION_QUERY_MAX_ROWS + 2 }, (_, index) => + d.insertMessage({ from: 'sender', to: 'worker', subject: `message-${index}` }) + ) + + const first = d.getUnreadMessages('worker') + expect(first.map((row) => row.id)).toEqual( + inserted.slice(0, ORCHESTRATION_QUERY_MAX_ROWS).map((row) => row.id) + ) + + d.markAsRead(first.map((row) => row.id)) + expect(d.getUnreadMessages('worker').map((row) => row.id)).toEqual( + inserted.slice(ORCHESTRATION_QUERY_MAX_ROWS).map((row) => row.id) + ) + }) + + it('drains undelivered rows successively without replaying the first page', () => { + const d = createDb() + const inserted = Array.from({ length: ORCHESTRATION_QUERY_MAX_ROWS + 1 }, (_, index) => + d.insertMessage({ from: 'sender', to: 'worker', subject: `delivery-${index}` }) + ) + + const first = d.getUndeliveredUnreadMessages('worker') + d.markAsDelivered(first.map((row) => row.id)) + + expect(d.getUndeliveredUnreadMessages('worker').map((row) => row.id)).toEqual([ + inserted.at(-1)!.id + ]) + }) + + it('caps aggregate retained bytes and resumes at the first omitted message', () => { + const d = createDb() + const body = 'x'.repeat(500 * 1024) + const inserted = Array.from({ length: 20 }, (_, index) => + d.insertMessage({ from: 'sender', to: 'worker', subject: `large-${index}`, body }) + ) + + const first = d.getUnreadMessages('worker') + const retainedBytes = first.reduce( + (total, row) => total + orchestrationRowRetainedUtf8Bytes(row), + 0 + ) + expect(first.length).toBeGreaterThan(0) + expect(first.length).toBeLessThan(inserted.length) + expect(retainedBytes).toBeLessThanOrEqual(ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES) + + d.markAsRead(first.map((row) => row.id)) + expect(d.getUnreadMessages('worker')[0]?.id).toBe(inserted[first.length].id) + }) + + it('rejects new oversized rows and skips oversized legacy rows without starving later rows', () => { + const d = createDb() + const exactSubject = 'x'.repeat(ORCHESTRATION_WRITE_MAX_UTF8_BYTES - 2) + expect(d.insertMessage({ from: 'a', to: 'b', subject: exactSubject }).subject).toBe( + exactSubject + ) + expect(() => d.insertMessage({ from: 'a', to: 'b', subject: `${exactSubject}x` })).toThrow( + /orchestration limit/ + ) + + sqliteFor(d) + .prepare( + `INSERT INTO messages (id, from_handle, to_handle, subject, body) + VALUES (?, 'a', 'legacy', 'oversized', ?)` + ) + .run('msg_oversized', 'z'.repeat(ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES + 1)) + const later = d.insertMessage({ from: 'a', to: 'legacy', subject: 'later-valid-row' }) + + expect(d.getMessageById('msg_oversized')).toBeUndefined() + expect(d.getUnreadMessages('legacy').map((row) => row.id)).toEqual([later.id]) + }) + + it('keeps exact task counts and promotes dependencies beyond the returned page', () => { + const d = createDb() + const parent = d.createTask({ spec: 'parent' }) + const children = Array.from({ length: ORCHESTRATION_QUERY_MAX_ROWS + 1 }, (_, index) => + d.createTask({ spec: `child-${index}`, deps: [parent.id] }) + ) + + expect(d.listTasks()).toHaveLength(ORCHESTRATION_QUERY_MAX_ROWS) + expect(d.getTaskStatusCounts()).toMatchObject({ + total: ORCHESTRATION_QUERY_MAX_ROWS + 2, + pending: ORCHESTRATION_QUERY_MAX_ROWS + 1, + ready: 1 + }) + + d.updateTaskStatus(parent.id, 'completed') + + expect(d.getTask(children.at(-1)!.id)?.status).toBe('ready') + expect(d.getTaskStatusCounts()).toMatchObject({ + total: ORCHESTRATION_QUERY_MAX_ROWS + 2, + completed: 1, + ready: ORCHESTRATION_QUERY_MAX_ROWS + 1 + }) + }) + + it('rejects dependency amplification and promotes valid rows after oversized legacy DAG data', () => { + const d = createDb() + const parent = d.createTask({ spec: 'parent' }) + expect(() => + d.createTask({ + spec: 'too many deps', + deps: Array.from({ length: ORCHESTRATION_WRITE_MAX_ITEMS + 1 }, () => parent.id) + }) + ).toThrow(/item orchestration limit/) + + sqliteFor(d) + .prepare( + `INSERT INTO tasks (id, spec, status, deps) + VALUES ('task_legacy_amplified', 'legacy', 'pending', ?)` + ) + .run( + JSON.stringify(Array.from({ length: ORCHESTRATION_WRITE_MAX_ITEMS + 1 }, () => parent.id)) + ) + const valid = d.createTask({ spec: 'valid', deps: [parent.id] }) + + d.updateTaskStatus(parent.id, 'completed') + + expect(d.getTask('task_legacy_amplified')?.status).toBe('pending') + expect(d.getTask(valid.id)?.status).toBe('ready') + }) + + it('streams active pane and stale-dispatch scans beyond the returned page', () => { + const d = createDb() + const contexts = Array.from({ length: ORCHESTRATION_QUERY_MAX_ROWS + 1 }, (_, index) => { + const task = d.createTask({ spec: `task-${index}` }) + return d.createDispatchContext(task.id, `worker-${index}`, paneKey(index)) + }) + const duplicateTask = d.createTask({ spec: 'duplicate pane' }) + + expect(() => + d.createDispatchContext( + duplicateTask.id, + 'reminted-worker', + paneKey(ORCHESTRATION_QUERY_MAX_ROWS, 'reminted-tab') + ) + ).toThrow(/already has an active dispatch/) + + sqliteFor(d) + .prepare( + "UPDATE dispatch_contexts SET dispatched_at = '2020-01-01 00:00:00', last_heartbeat_at = NULL" + ) + .run() + const threshold = '2021-01-01T00:00:00.000Z' + expect(d.getStaleDispatches(threshold)).toHaveLength(ORCHESTRATION_QUERY_MAX_ROWS) + + const visited: string[] = [] + d.forEachStaleDispatch(threshold, (row) => { + visited.push(row.id) + }) + expect(visited).toEqual(contexts.map((row) => row.id)) + }) + + it('repairs every pending-gate task even when the gate list is capped', () => { + const d = createDb() + const rejectedTask = d.createTask({ spec: 'reject amplified options' }) + expect(() => + d.createGate({ + taskId: rejectedTask.id, + question: 'too many', + options: Array.from({ length: ORCHESTRATION_WRITE_MAX_ITEMS + 1 }, () => '') + }) + ).toThrow(/item orchestration limit/) + const tasks = Array.from({ length: ORCHESTRATION_QUERY_MAX_ROWS + 1 }, (_, index) => { + const task = d.createTask({ spec: `gated-${index}` }) + d.createGate({ taskId: task.id, question: `question-${index}` }) + return task + }) + + expect(d.listGates({ status: 'pending' })).toHaveLength(ORCHESTRATION_QUERY_MAX_ROWS) + sqliteFor(d).prepare("UPDATE tasks SET status = 'ready'").run() + + d.restorePendingGateTaskStatuses() + + expect(d.getTask(tasks.at(-1)!.id)?.status).toBe('blocked') + expect(d.getTaskStatusCounts()).toMatchObject({ + blocked: ORCHESTRATION_QUERY_MAX_ROWS + 1, + ready: 1 + }) + }) +}) + +describe('orchestration JSON structure admission', () => { + it('admits the exact structural-token limit and rejects limit +1', () => { + const exactValues = ORCHESTRATION_JSON_STRUCTURE_LIMITS.structuralTokens - 1 + const exact = `[${Array.from({ length: exactValues }, () => '0').join(',')}]` + const over = `${exact.slice(0, -1)},0]` + + expect(parseOrchestrationJson(exact)).toHaveLength(exactValues) + expect(() => parseOrchestrationJson(over)).toThrow(/JSON structure exceeds/) + }) + + it('admits the exact nesting limit and rejects limit +1', () => { + const depth = ORCHESTRATION_JSON_STRUCTURE_LIMITS.nestingDepth + const exact = `${'['.repeat(depth)}0${']'.repeat(depth)}` + const over = `[${exact}]` + + expect(parseOrchestrationJson(exact)).toBeDefined() + expect(() => parseOrchestrationJson(over)).toThrow(/JSON nesting exceeds/) + }) +}) diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index b287cb0182c..b8cb36bd5c8 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -16,6 +16,80 @@ import type { } from './types' import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display' import { parsePaneKey } from '../../../shared/stable-pane-id' +import { + assertOrchestrationStringListFits, + assertOrchestrationWriteFits, + clampOrchestrationQueryLimit, + ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES, + ORCHESTRATION_QUERY_MAX_ROWS, + ORCHESTRATION_WRITE_MAX_ITEMS, + parseOrchestrationJson, + retainOrchestrationQueryRows, + truncateOrchestrationDiagnostic +} from './query-retention' + +function retainedTextBytesSql(columns: string[]): string { + return columns.map((column) => `length(CAST(COALESCE(${column}, '') AS BLOB))`).join(' + ') +} + +const MESSAGE_ROW_BYTES_SQL = retainedTextBytesSql([ + 'messages.id', + 'messages.from_handle', + 'messages.to_handle', + 'messages.subject', + 'messages.body', + 'messages.type', + 'messages.priority', + 'messages.thread_id', + 'messages.payload', + 'messages.created_at', + 'messages.delivered_at', + 'messages.sender_pane_key' +]) +const TASK_ROW_BYTES_SQL = retainedTextBytesSql([ + 'tasks.id', + 'tasks.parent_id', + 'tasks.created_by_terminal_handle', + 'tasks.task_title', + 'tasks.display_name', + 'tasks.spec', + 'tasks.status', + 'tasks.deps', + 'tasks.result', + 'tasks.created_at', + 'tasks.completed_at' +]) +const DISPATCH_ROW_BYTES_SQL = retainedTextBytesSql([ + 'dispatch_contexts.id', + 'dispatch_contexts.task_id', + 'dispatch_contexts.assignee_handle', + 'dispatch_contexts.assignee_pane_key', + 'dispatch_contexts.status', + 'dispatch_contexts.last_failure', + 'dispatch_contexts.dispatched_at', + 'dispatch_contexts.completed_at', + 'dispatch_contexts.created_at', + 'dispatch_contexts.last_heartbeat_at' +]) +const GATE_ROW_BYTES_SQL = retainedTextBytesSql([ + 'decision_gates.id', + 'decision_gates.task_id', + 'decision_gates.question', + 'decision_gates.options', + 'decision_gates.status', + 'decision_gates.resolution', + 'decision_gates.created_at', + 'decision_gates.resolved_at' +]) +const COORDINATOR_RUN_ROW_BYTES_SQL = retainedTextBytesSql([ + 'coordinator_runs.id', + 'coordinator_runs.spec', + 'coordinator_runs.status', + 'coordinator_runs.coordinator_handle', + 'coordinator_runs.created_at', + 'coordinator_runs.completed_at' +]) +const INTERNAL_SCAN_BATCH_ROWS = 64 // Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. function isEquivalentPaneKey(a: string, b: string): boolean { @@ -41,14 +115,29 @@ export type { CoordinatorRun } +export type TaskStatusCounts = Record & { total: number } + function generateId(prefix: string): string { return `${prefix}_${randomBytes(6).toString('hex')}` } +function uniqueMessageTypes(types: MessageType[] | undefined): MessageType[] { + const unique: MessageType[] = [] + for (const type of types ?? []) { + if (!unique.includes(type)) { + unique.push(type) + if (unique.length === 8) { + break + } + } + } + return unique +} + function addLifecycleRejectionMarker(payload: string | null, reason: string): string { let parsed: Record = {} try { - const value: unknown = payload ? JSON.parse(payload) : {} + const value: unknown = payload ? parseOrchestrationJson(payload) : {} if (value && typeof value === 'object' && !Array.isArray(value)) { parsed = value as Record } @@ -106,6 +195,24 @@ export class OrchestrationDb { this.migrate() } + private readRows( + sql: string, + params: Database.BindValue[] = [], + requestedLimit?: number + ): T[] { + const limit = clampOrchestrationQueryLimit(requestedLimit) + if (limit === 0) { + return [] + } + const rows = this.db.prepare(`${sql} LIMIT ?`).iterate(...params, limit) as Iterable + return retainOrchestrationQueryRows(rows, limit) + } + + private countRows(sql: string, params: Database.BindValue[] = []): number { + const row = this.db.prepare(sql).get(...params) as { count: number } + return row.count + } + private createTables(): void { this.db.exec(` CREATE TABLE IF NOT EXISTS messages ( @@ -333,6 +440,17 @@ export class OrchestrationDb { payload?: string senderPaneKey?: string }): MessageRow { + assertOrchestrationWriteFits('Message', [ + msg.from, + msg.to, + msg.subject, + msg.body, + msg.type, + msg.priority, + msg.threadId, + msg.payload, + msg.senderPaneKey + ]) const id = generateId('msg') const stmt = this.db.prepare(` INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key) @@ -350,26 +468,37 @@ export class OrchestrationDb { msg.payload ?? null, msg.senderPaneKey ?? null ) - return exposeMessageTimestamps( - this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow - ) + return this.getMessageById(id)! } getUnreadMessages(toHandle: string, types?: MessageType[]): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND type IN (${placeholders}) ORDER BY sequence` - ) - .all(toHandle, ...types) as MessageRow[] - ) + const params: Database.BindValue[] = [toHandle] + let typeFilter = '' + const retainedTypes = uniqueMessageTypes(types) + if (retainedTypes.length > 0) { + const placeholders = retainedTypes.map(() => '?').join(',') + typeFilter = ` AND type IN (${placeholders})` + params.push(...retainedTypes) } return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? AND read = 0 ORDER BY sequence') - .all(toHandle) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0${typeFilter} + AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence`, + params + ) + ) + } + + countUnreadMessages(toHandle: string, types?: MessageType[]): number { + const retainedTypes = uniqueMessageTypes(types) + const placeholders = retainedTypes.map(() => '?').join(',') + const typeFilter = retainedTypes.length > 0 ? ` AND type IN (${placeholders})` : '' + return this.countRows( + `SELECT COUNT(*) AS count FROM messages + WHERE to_handle = ? AND read = 0${typeFilter}`, + [toHandle, ...retainedTypes] ) } @@ -380,8 +509,11 @@ export class OrchestrationDb { } const originalBody = message.body ? `\n\nOriginal body:\n${message.body}` : '' - const body = `Orca rejected this ${message.type}: ${reason}${originalBody}` - const payload = addLifecycleRejectionMarker(message.payload, reason) + const boundedReason = truncateOrchestrationDiagnostic(reason) + const body = truncateOrchestrationDiagnostic( + `Orca rejected this ${message.type}: ${boundedReason}${originalBody}` + ) + const payload = addLifecycleRejectionMarker(message.payload, boundedReason) // Why: rejected lifecycle signals stay auditable but must not reach read paths as actionable completion/liveness events. this.db .prepare( @@ -395,116 +527,133 @@ export class OrchestrationDb { // Why: delivered_at IS NULL filter — push-on-idle delivers each row at most once; read (set only by check) wouldn't prevent replay. getUndeliveredUnreadMessages(toHandle: string, types?: MessageType[]): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL AND type IN (${placeholders}) ORDER BY sequence` - ) - .all(toHandle, ...types) as MessageRow[] - ) + const params: Database.BindValue[] = [toHandle] + let typeFilter = '' + const retainedTypes = uniqueMessageTypes(types) + if (retainedTypes.length > 0) { + const placeholders = retainedTypes.map(() => '?').join(',') + typeFilter = ` AND type IN (${placeholders})` + params.push(...retainedTypes) } return exposeMessageListTimestamps( - this.db - .prepare( - 'SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL ORDER BY sequence' - ) - .all(toHandle) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL${typeFilter} + AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence`, + params + ) ) } getAllMessages(toHandle: string, limit = 20): MessageRow[] { return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') - .all(toHandle, limit) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE to_handle = ? + AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence DESC`, + [toHandle], + limit + ) ) } getMessageById(id: string): MessageRow | undefined { - const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as - | MessageRow - | undefined + const message = this.db + .prepare( + `SELECT * FROM messages + WHERE id = ? AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) + .get(id) as MessageRow | undefined return message ? exposeMessageTimestamps(message) : undefined } markAsRead(ids: string[]): void { - if (ids.length === 0) { - return - } - const placeholders = ids.map(() => '?').join(',') - this.db.prepare(`UPDATE messages SET read = 1 WHERE id IN (${placeholders})`).run(...ids) + this.updateMessageIds(ids, 'read = 1') } // Why: use datetime('now') so delivered_at matches the space-format UTC shape of the table's other timestamps for correct ordering (§3.2). markAsDelivered(ids: string[]): void { - if (ids.length === 0) { - return - } - const placeholders = ids.map(() => '?').join(',') - this.db - .prepare(`UPDATE messages SET delivered_at = datetime('now') WHERE id IN (${placeholders})`) - .run(...ids) + this.updateMessageIds(ids, "delivered_at = datetime('now')") } markAsReadAndDelivered(ids: string[]): void { - if (ids.length === 0) { - return - } - const placeholders = ids.map(() => '?').join(',') // Why: superseded lifecycle messages stay in history but must not be consumed or injected after their dispatch finished. - this.db - .prepare( - `UPDATE messages SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) WHERE id IN (${placeholders})` - ) - .run(...ids) + this.updateMessageIds(ids, "read = 1, delivered_at = COALESCE(delivered_at, datetime('now'))") + } + + private updateMessageIds(ids: string[], assignments: string): void { + for (let start = 0; start < ids.length; start += ORCHESTRATION_QUERY_MAX_ROWS) { + const batch = ids.slice(start, start + ORCHESTRATION_QUERY_MAX_ROWS) + const placeholders = batch.map(() => '?').join(',') + this.db + .prepare(`UPDATE messages SET ${assignments} WHERE id IN (${placeholders})`) + .run(...batch) + } } getInbox(limit = 20): MessageRow[] { return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?') - .all(limit) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence DESC`, + [], + limit + ) ) } + countInbox(): number { + return this.countRows('SELECT COUNT(*) AS count FROM messages') + } + // Why: read-only history for a handle — returns every message regardless of read/delivered state, never flips the read bit (§3.3). getAllMessagesForHandle(toHandle: string, limit = 100, types?: MessageType[]): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` - ) - .all(toHandle, ...types, limit) as MessageRow[] - ) + const params: Database.BindValue[] = [toHandle] + let typeFilter = '' + const retainedTypes = uniqueMessageTypes(types) + if (retainedTypes.length > 0) { + const placeholders = retainedTypes.map(() => '?').join(',') + typeFilter = ` AND type IN (${placeholders})` + params.push(...retainedTypes) } return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') - .all(toHandle, limit) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE to_handle = ?${typeFilter} + AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence DESC`, + params, + limit + ) + ) + } + + countAllMessagesForHandle(toHandle: string, types?: MessageType[]): number { + const retainedTypes = uniqueMessageTypes(types) + const placeholders = retainedTypes.map(() => '?').join(',') + const typeFilter = retainedTypes.length > 0 ? ` AND type IN (${placeholders})` : '' + return this.countRows( + `SELECT COUNT(*) AS count FROM messages WHERE to_handle = ?${typeFilter}`, + [toHandle, ...retainedTypes] ) } // Why: ask wait-loop read — to_handle filter shows only replies to the worker; afterSequence resumes past its own outbound ask. getThreadMessagesFor(threadId: string, toHandle: string, afterSequence?: number): MessageRow[] { - if (afterSequence !== undefined) { - return exposeMessageListTimestamps( - this.db - .prepare( - 'SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? AND sequence > ? ORDER BY sequence ASC' - ) - .all(threadId, toHandle, afterSequence) as MessageRow[] - ) - } + const sequenceFilter = afterSequence === undefined ? '' : ' AND sequence > ?' + const params: Database.BindValue[] = + afterSequence === undefined ? [threadId, toHandle] : [threadId, toHandle, afterSequence] return exposeMessageListTimestamps( - this.db - .prepare( - 'SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? ORDER BY sequence ASC' - ) - .all(threadId, toHandle) as MessageRow[] + this.readRows( + `SELECT * FROM messages + WHERE thread_id = ? AND to_handle = ?${sequenceFilter} + AND (${MESSAGE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY sequence ASC`, + params + ) ) } @@ -519,14 +668,32 @@ export class OrchestrationDb { createdByTerminalHandle?: string }): TaskRow { const id = generateId('task') - const depsJson = JSON.stringify(task.deps ?? []) - const hasDeps = (task.deps ?? []).length > 0 + const deps = task.deps ?? [] + assertOrchestrationStringListFits('Task dependencies', deps) + assertOrchestrationWriteFits('Task', [ + task.spec, + task.taskTitle, + task.displayName, + task.parentId, + task.createdByTerminalHandle, + ...deps + ]) + const depsJson = JSON.stringify(deps) + const hasDeps = deps.length > 0 const status: TaskStatus = hasDeps ? 'pending' : 'ready' const display = buildOrchestrationTaskDisplayMetadata({ spec: task.spec, taskTitle: task.taskTitle, displayName: task.displayName }) + assertOrchestrationWriteFits('Task', [ + task.spec, + display.taskTitle, + display.displayName, + depsJson, + task.parentId, + task.createdByTerminalHandle + ]) this.db .prepare( 'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' @@ -541,25 +708,35 @@ export class OrchestrationDb { status, depsJson ) - return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow + return this.getTask(id)! } getTask(id: string): TaskRow | undefined { - return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined + return this.db + .prepare( + `SELECT * FROM tasks + WHERE id = ? AND (${TASK_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) + .get(id) as TaskRow | undefined } listTasks(filter?: { status?: TaskStatus; ready?: boolean }): TaskRow[] { - if (filter?.ready) { - return this.db - .prepare("SELECT * FROM tasks WHERE status = 'ready' ORDER BY created_at") - .all() as TaskRow[] - } - if (filter?.status) { - return this.db - .prepare('SELECT * FROM tasks WHERE status = ? ORDER BY created_at') - .all(filter.status) as TaskRow[] - } - return this.db.prepare('SELECT * FROM tasks ORDER BY created_at').all() as TaskRow[] + const status = filter?.ready ? 'ready' : filter?.status + const statusFilter = status ? 'status = ? AND ' : '' + return this.readRows( + `SELECT * FROM tasks + WHERE ${statusFilter}(${TASK_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY created_at, rowid`, + status ? [status] : [] + ) + } + + countTasks(filter?: { status?: TaskStatus; ready?: boolean }): number { + const status = filter?.ready ? 'ready' : filter?.status + return this.countRows( + `SELECT COUNT(*) AS count FROM tasks${status ? ' WHERE status = ?' : ''}`, + status ? [status] : [] + ) } // Why: LEFT JOIN keeps non-dispatched tasks (NULL assignee); the MAX(rowid) subquery matches getDispatchContext's most-recent-active-dispatch semantics. @@ -570,20 +747,25 @@ export class OrchestrationDb { const whereClauses: string[] = [] const params: Database.BindValue[] = [] if (filter?.ready) { - whereClauses.push("t.status = 'ready'") + whereClauses.push("tasks.status = 'ready'") } else if (filter?.status) { - whereClauses.push('t.status = ?') + whereClauses.push('tasks.status = ?') params.push(filter.status) } + whereClauses.push( + `(${TASK_ROW_BYTES_SQL} + + length(CAST(COALESCE(d.assignee_handle, '') AS BLOB)) + + length(CAST(COALESCE(d.id, '') AS BLOB))) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) const where = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '' const sql = ` SELECT - t.*, + tasks.*, d.assignee_handle AS assignee_handle, d.id AS dispatch_id - FROM tasks t + FROM tasks LEFT JOIN ( - SELECT dc.* + SELECT dc.id, dc.task_id, dc.assignee_handle FROM dispatch_contexts dc INNER JOIN ( SELECT task_id, MAX(rowid) AS max_rowid @@ -591,17 +773,23 @@ export class OrchestrationDb { WHERE status IN ('pending', 'dispatched') GROUP BY task_id ) latest ON latest.task_id = dc.task_id AND latest.max_rowid = dc.rowid - ) d ON d.task_id = t.id + ) d ON d.task_id = tasks.id ${where} - ORDER BY t.created_at + ORDER BY tasks.created_at, tasks.rowid ` - return this.db.prepare(sql).all(...params) as (TaskRow & { + return this.readRows< + TaskRow & { + assignee_handle: string | null + dispatch_id: string | null + } + >(sql, params) as (TaskRow & { assignee_handle: string | null dispatch_id: string | null })[] } updateTaskStatus(id: string, status: TaskStatus, result?: string): TaskRow | undefined { + assertOrchestrationWriteFits('Task result', [result]) const completedAt = status === 'completed' || status === 'failed' ? new Date().toISOString() : null this.db @@ -620,26 +808,81 @@ export class OrchestrationDb { // Why: runs in the status-update transaction, so a completed task never leaves its ready children unpromoted. private promoteReadyTasks(completedTaskId: string): void { - const candidates = this.db - .prepare("SELECT * FROM tasks WHERE status = 'pending'") - .all() as TaskRow[] - - for (const task of candidates) { - const deps: string[] = JSON.parse(task.deps) - if (!deps.includes(completedTaskId)) { - continue + let afterRowId = 0 + while (true) { + const candidates = this.readRows<{ rowid: number; id: string; deps: string }>( + `SELECT rowid, id, deps FROM tasks + WHERE status = 'pending' AND rowid > ? + AND (${retainedTextBytesSql(['tasks.id', 'tasks.deps'])}) + <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + AND CASE WHEN json_valid(deps) + THEN json_type(deps) = 'array' + AND json_array_length(deps) <= ${ORCHESTRATION_WRITE_MAX_ITEMS} + ELSE 0 + END + ORDER BY rowid`, + [afterRowId], + INTERNAL_SCAN_BATCH_ROWS + ) + if (candidates.length === 0) { + return } + afterRowId = candidates.at(-1)!.rowid - const allDepsCompleted = deps.every((depId) => { - const dep = this.getTask(depId) - return dep?.status === 'completed' - }) - if (allDepsCompleted) { - this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id) + for (const task of candidates) { + let parsedDeps: unknown + try { + parsedDeps = parseOrchestrationJson(task.deps) + } catch { + continue + } + if ( + !Array.isArray(parsedDeps) || + parsedDeps.length > ORCHESTRATION_WRITE_MAX_ITEMS || + !parsedDeps.every((dependency) => typeof dependency === 'string') + ) { + continue + } + const deps = parsedDeps + if (!deps.includes(completedTaskId)) { + continue + } + const allDepsCompleted = deps.every((depId) => this.getTask(depId)?.status === 'completed') + if (allDepsCompleted) { + this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id) + } } } } + getTaskStatusCounts(): TaskStatusCounts { + const row = this.db + .prepare( + `SELECT + COUNT(*) AS total, + COALESCE(SUM(status = 'pending'), 0) AS pending, + COALESCE(SUM(status = 'ready'), 0) AS ready, + COALESCE(SUM(status = 'dispatched'), 0) AS dispatched, + COALESCE(SUM(status = 'completed'), 0) AS completed, + COALESCE(SUM(status = 'failed'), 0) AS failed, + COALESCE(SUM(status = 'blocked'), 0) AS blocked + FROM tasks` + ) + .get() as TaskStatusCounts + return row + } + + listTaskIdsByStatus(status: TaskStatus): string[] { + const rows = this.readRows<{ id: string }>( + `SELECT id FROM tasks + WHERE status = ? + AND length(CAST(id AS BLOB)) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY created_at, rowid`, + [status] + ) + return rows.map((row) => row.id) + } + // ── Dispatch Contexts ── createDispatchContext( @@ -648,6 +891,7 @@ export class OrchestrationDb { // Why: pane key is the remint-stable identity behind the handle — lets worker_done ownership survive handle reissue. assigneePaneKey?: string ): DispatchContextRow { + assertOrchestrationWriteFits('Dispatch context', [taskId, assigneeHandle, assigneePaneKey]) const task = this.getTask(taskId) if (!task) { throw new Error(`Task not found: ${taskId}`) @@ -682,21 +926,28 @@ export class OrchestrationDb { this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId) - return this.db - .prepare('SELECT * FROM dispatch_contexts WHERE id = ?') - .get(id) as DispatchContextRow + return this.getDispatchContextById(id)! } getDispatchContext(taskId: string): DispatchContextRow | undefined { return this.db - .prepare('SELECT * FROM dispatch_contexts WHERE task_id = ? ORDER BY rowid DESC LIMIT 1') + .prepare( + `SELECT * FROM dispatch_contexts + WHERE task_id = ? + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY rowid DESC LIMIT 1` + ) .get(taskId) as DispatchContextRow | undefined } getDispatchContextById(dispatchId: string): DispatchContextRow | undefined { - return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as - | DispatchContextRow - | undefined + return this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE id = ? + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) + .get(dispatchId) as DispatchContextRow | undefined } getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined { @@ -723,7 +974,10 @@ export class OrchestrationDb { ): DispatchContextRow | undefined { const byHandle = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') LIMIT 1" + `SELECT * FROM dispatch_contexts + WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + LIMIT 1` ) .get(assigneeHandle) as DispatchContextRow | undefined if (byHandle) { @@ -736,22 +990,33 @@ export class OrchestrationDb { const actives = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')" + `SELECT id, assignee_pane_key FROM dispatch_contexts + WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched') + AND (${retainedTextBytesSql([ + 'dispatch_contexts.id', + 'dispatch_contexts.assignee_pane_key' + ])}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY rowid` ) - .all() as DispatchContextRow[] + .iterate() as Iterable<{ id: string; assignee_pane_key: string }> + let matchingId: string | undefined for (const row of actives) { - if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) { - return row + if (isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) { + matchingId = row.id + break } } - return undefined + return matchingId ? this.getDispatchContextById(matchingId) : undefined } getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined { return this.db .prepare( - 'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1' + `SELECT * FROM dispatch_contexts + WHERE assignee_handle = ? + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY rowid DESC LIMIT 1` ) .get(handle) as DispatchContextRow | undefined } @@ -767,9 +1032,9 @@ export class OrchestrationDb { completeActiveDispatchForTask(taskId: string): void { const active = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" + "SELECT id FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" ) - .get(taskId) as DispatchContextRow | undefined + .get(taskId) as { id: string } | undefined if (active) { this.completeDispatch(active.id) } @@ -778,14 +1043,15 @@ export class OrchestrationDb { failActiveDispatchForTask(taskId: string, error: string): DispatchContextRow | undefined { const active = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" + "SELECT id FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" ) - .get(taskId) as DispatchContextRow | undefined + .get(taskId) as { id: string } | undefined return active ? this.failDispatch(active.id, error) : undefined } // Why: only bump status='dispatched' — a zombie heartbeat from a finished dispatch would mask a hung retry from the stale detector (§5.3.4). recordHeartbeat(dispatchId: string, at: string): void { + assertOrchestrationWriteFits('Dispatch heartbeat', [dispatchId, at]) this.db .prepare( "UPDATE dispatch_contexts SET last_heartbeat_at = ? WHERE id = ? AND status = 'dispatched'" @@ -795,21 +1061,37 @@ export class OrchestrationDb { // Why: dispatched_at grace skips workers still within their first heartbeat interval; julianday() vs raw-TEXT compare avoids misflagging space-format timestamps as stale (#8452). getStaleDispatches(thresholdIso: string): DispatchContextRow[] { - return this.db + return this.readRows( + `SELECT * FROM dispatch_contexts + WHERE status = 'dispatched' + AND dispatched_at IS NOT NULL + AND julianday(dispatched_at) < julianday(?) + AND (last_heartbeat_at IS NULL OR julianday(last_heartbeat_at) < julianday(?)) + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY rowid`, + [thresholdIso, thresholdIso] + ) + } + + forEachStaleDispatch(thresholdIso: string, visit: (row: DispatchContextRow) => void): void { + const rows = this.db .prepare( `SELECT * FROM dispatch_contexts WHERE status = 'dispatched' AND dispatched_at IS NOT NULL AND julianday(dispatched_at) < julianday(?) - AND (last_heartbeat_at IS NULL OR julianday(last_heartbeat_at) < julianday(?))` + AND (last_heartbeat_at IS NULL OR julianday(last_heartbeat_at) < julianday(?)) + AND (${DISPATCH_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY rowid` ) - .all(thresholdIso, thresholdIso) as DispatchContextRow[] + .iterate(thresholdIso, thresholdIso) as Iterable + for (const row of rows) { + visit(row) + } } failDispatch(ctxId: string, error: string): DispatchContextRow | undefined { - const ctx = this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as - | DispatchContextRow - | undefined + const ctx = this.getDispatchContextById(ctxId) if (!ctx) { return undefined } @@ -821,22 +1103,24 @@ export class OrchestrationDb { .prepare( 'UPDATE dispatch_contexts SET status = ?, failure_count = ?, last_failure = ? WHERE id = ?' ) - .run(newStatus, newFailureCount, error, ctxId) + .run(newStatus, newFailureCount, truncateOrchestrationDiagnostic(error), ctxId) // Why: back to 'ready' not 'pending' — 'pending' would strand it since promoteReadyTasks only runs when a dep completes. const taskStatus: TaskStatus = newStatus === 'circuit_broken' ? 'failed' : 'ready' this.db.prepare('UPDATE tasks SET status = ? WHERE id = ?').run(taskStatus, ctx.task_id) - return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as - | DispatchContextRow - | undefined + return this.getDispatchContextById(ctxId) } // ── Decision Gates ── createGate(gate: { taskId: string; question: string; options?: string[] }): DecisionGateRow { const id = generateId('gate') - const optionsJson = JSON.stringify(gate.options ?? []) + const options = gate.options ?? [] + assertOrchestrationStringListFits('Decision gate options', options) + assertOrchestrationWriteFits('Decision gate', [gate.taskId, gate.question, ...options]) + const optionsJson = JSON.stringify(options) + assertOrchestrationWriteFits('Decision gate', [gate.taskId, gate.question, optionsJson]) this.db .prepare('INSERT INTO decision_gates (id, task_id, question, options) VALUES (?, ?, ?, ?)') .run(id, gate.taskId, gate.question, optionsJson) @@ -844,13 +1128,12 @@ export class OrchestrationDb { this.completeActiveDispatchForTask(gate.taskId) this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(gate.taskId) - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as DecisionGateRow + return this.getGate(id)! } resolveGate(gateId: string, resolution: string): DecisionGateRow | undefined { - const gate = this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined + assertOrchestrationWriteFits('Decision gate resolution', [gateId, resolution]) + const gate = this.getGate(gateId) if (!gate) { return undefined } @@ -864,9 +1147,7 @@ export class OrchestrationDb { // Why: set to 'ready' (not the previous status) so the coordinator re-dispatches the worker with the resolution context. this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(gate.task_id) - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined + return this.getGate(gateId) } timeoutGate(gateId: string): DecisionGateRow | undefined { @@ -875,38 +1156,81 @@ export class OrchestrationDb { "UPDATE decision_gates SET status = 'timeout', resolved_at = datetime('now') WHERE id = ?" ) .run(gateId) - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined + return this.getGate(gateId) } listGates(filter?: { taskId?: string; status?: GateStatus }): DecisionGateRow[] { + const clauses = [`(${GATE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}`] + const params: Database.BindValue[] = [] if (filter?.taskId && filter?.status) { - return this.db - .prepare( - 'SELECT * FROM decision_gates WHERE task_id = ? AND status = ? ORDER BY created_at' - ) - .all(filter.taskId, filter.status) as DecisionGateRow[] + clauses.push('task_id = ?', 'status = ?') + params.push(filter.taskId, filter.status) + } else if (filter?.taskId) { + clauses.push('task_id = ?') + params.push(filter.taskId) + } else if (filter?.status) { + clauses.push('status = ?') + params.push(filter.status) } + return this.readRows( + `SELECT * FROM decision_gates + WHERE ${clauses.join(' AND ')} + ORDER BY created_at, rowid`, + params + ) + } + + countGates(filter?: { taskId?: string; status?: GateStatus }): number { + const clauses: string[] = [] + const params: Database.BindValue[] = [] if (filter?.taskId) { - return this.db - .prepare('SELECT * FROM decision_gates WHERE task_id = ? ORDER BY created_at') - .all(filter.taskId) as DecisionGateRow[] + clauses.push('task_id = ?') + params.push(filter.taskId) } if (filter?.status) { - return this.db - .prepare('SELECT * FROM decision_gates WHERE status = ? ORDER BY created_at') - .all(filter.status) as DecisionGateRow[] + clauses.push('status = ?') + params.push(filter.status) } - return this.db - .prepare('SELECT * FROM decision_gates ORDER BY created_at') - .all() as DecisionGateRow[] + return this.countRows( + `SELECT COUNT(*) AS count FROM decision_gates${ + clauses.length > 0 ? ` WHERE ${clauses.join(' AND ')}` : '' + }`, + params + ) } getGate(id: string): DecisionGateRow | undefined { - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as - | DecisionGateRow - | undefined + return this.db + .prepare( + `SELECT * FROM decision_gates + WHERE id = ? AND (${GATE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) + .get(id) as DecisionGateRow | undefined + } + + getLatestGate(filter: { taskId: string; status: GateStatus }): DecisionGateRow | undefined { + return this.db + .prepare( + `SELECT * FROM decision_gates + WHERE task_id = ? AND status = ? + AND (${GATE_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY created_at DESC, rowid DESC LIMIT 1` + ) + .get(filter.taskId, filter.status) as DecisionGateRow | undefined + } + + restorePendingGateTaskStatuses(): void { + this.db + .prepare( + `UPDATE tasks SET status = 'blocked' + WHERE status <> 'blocked' + AND EXISTS ( + SELECT 1 FROM decision_gates + WHERE decision_gates.task_id = tasks.id + AND decision_gates.status = 'pending' + )` + ) + .run() } // ── Coordinator Runs ── @@ -916,19 +1240,24 @@ export class OrchestrationDb { coordinatorHandle: string pollIntervalMs?: number }): CoordinatorRun { + assertOrchestrationWriteFits('Coordinator run', [run.spec, run.coordinatorHandle]) const id = generateId('run') this.db .prepare( "INSERT INTO coordinator_runs (id, spec, status, coordinator_handle, poll_interval_ms) VALUES (?, ?, 'running', ?, ?)" ) .run(id, run.spec, run.coordinatorHandle, run.pollIntervalMs ?? 2000) - return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as CoordinatorRun + return this.getCoordinatorRun(id)! } getCoordinatorRun(id: string): CoordinatorRun | undefined { - return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as - | CoordinatorRun - | undefined + return this.db + .prepare( + `SELECT * FROM coordinator_runs + WHERE id = ? + AND (${COORDINATOR_RUN_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES}` + ) + .get(id) as CoordinatorRun | undefined } updateCoordinatorRun(id: string, status: CoordinatorStatus): CoordinatorRun | undefined { @@ -945,7 +1274,10 @@ export class OrchestrationDb { getActiveCoordinatorRun(): CoordinatorRun | undefined { return this.db .prepare( - "SELECT * FROM coordinator_runs WHERE status = 'running' ORDER BY created_at DESC LIMIT 1" + `SELECT * FROM coordinator_runs + WHERE status = 'running' + AND (${COORDINATOR_RUN_ROW_BYTES_SQL}) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + ORDER BY created_at DESC, rowid DESC LIMIT 1` ) .get() as CoordinatorRun | undefined } @@ -953,22 +1285,31 @@ export class OrchestrationDb { // ── Queries for Coordinator ── getIdleTerminals(excludeHandles: string[] = []): string[] { - const active = this.db + const rows = this.db .prepare( - "SELECT DISTINCT assignee_handle FROM dispatch_contexts WHERE status IN ('pending', 'dispatched')" + `WITH handles(handle) AS ( + SELECT to_handle FROM messages + UNION + SELECT from_handle FROM messages + ) + SELECT handle FROM handles + WHERE length(CAST(handle AS BLOB)) <= ${ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES} + AND NOT EXISTS ( + SELECT 1 FROM dispatch_contexts + WHERE dispatch_contexts.assignee_handle = handles.handle + AND dispatch_contexts.status IN ('pending', 'dispatched') + ) + ORDER BY handle` ) - .all() as { assignee_handle: string }[] - const busyHandles = new Set(active.map((r) => r.assignee_handle)) - for (const h of excludeHandles) { - busyHandles.add(h) - } - // Return handles from message history that aren't busy - const allHandles = this.db - .prepare( - 'SELECT DISTINCT to_handle FROM messages UNION SELECT DISTINCT from_handle FROM messages' - ) - .all() as { to_handle: string }[] - return [...new Set(allHandles.map((r) => r.to_handle))].filter((h) => !busyHandles.has(h)) + .iterate() as Iterable<{ handle: string }> + const filtered = (function* (): Iterable<{ handle: string }> { + for (const row of rows) { + if (!excludeHandles.includes(row.handle)) { + yield row + } + } + })() + return retainOrchestrationQueryRows(filtered).map((row) => row.handle) } // ── Lifecycle ── diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.ts index 96917babaf6..0485ebdeba8 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.ts @@ -1,6 +1,7 @@ import type { OrchestrationDb } from './db' import type { MessageRow } from './types' import { parsePaneKey } from '../../../shared/stable-pane-id' +import { parseOrchestrationJson } from './query-retention' // Why: the tab half can change on pane break-out, while opaque legacy keys // have no safe equivalence beyond exact equality. @@ -53,8 +54,10 @@ function parseObjectPayload(msg: MessageRow, onInvalidJson: () => void): Record< } try { - const parsed: unknown = JSON.parse(msg.payload) - return parsed && typeof parsed === 'object' ? (parsed as Record) : {} + const parsed = parseOrchestrationJson(msg.payload) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} } catch { onInvalidJson() return {} diff --git a/src/main/runtime/orchestration/query-retention.ts b/src/main/runtime/orchestration/query-retention.ts new file mode 100644 index 00000000000..b7e479c8c97 --- /dev/null +++ b/src/main/runtime/orchestration/query-retention.ts @@ -0,0 +1,124 @@ +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from '../../../shared/json-text-structure-limit' +import { measureUtf8ByteLength } from '../../../shared/utf8-byte-limits' + +export const ORCHESTRATION_QUERY_MAX_ROWS = 256 +export const ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES = 2 * 1024 * 1024 +export const ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES = 8 * 1024 * 1024 +export const ORCHESTRATION_WRITE_MAX_UTF8_BYTES = 512 * 1024 +export const ORCHESTRATION_WRITE_MAX_ITEMS = 4096 +export const ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES = 1024 +export const ORCHESTRATION_JSON_STRUCTURE_LIMITS: JsonTextStructureLimits = { + structuralTokens: 64 * 1024, + nestingDepth: 64 +} + +export function parseOrchestrationJson(content: string): unknown { + assertJsonTextStructureWithinLimits(content, ORCHESTRATION_JSON_STRUCTURE_LIMITS) + return JSON.parse(content) as unknown +} + +export function clampOrchestrationQueryLimit( + requested: number | undefined, + fallback = ORCHESTRATION_QUERY_MAX_ROWS +): number { + if (requested === undefined || !Number.isFinite(requested)) { + return Math.min(fallback, ORCHESTRATION_QUERY_MAX_ROWS) + } + return Math.min(ORCHESTRATION_QUERY_MAX_ROWS, Math.max(0, Math.floor(requested))) +} + +export function orchestrationRowRetainedUtf8Bytes(row: object): number { + let bytes = 0 + for (const value of Object.values(row)) { + if (typeof value === 'string') { + bytes += Buffer.byteLength(value, 'utf8') + } + } + return bytes +} + +export function retainOrchestrationQueryRows( + rows: Iterable, + requestedLimit?: number +): T[] { + const limit = clampOrchestrationQueryLimit(requestedLimit) + if (limit === 0) { + return [] + } + const retained: T[] = [] + let retainedBytes = 0 + + for (const row of rows) { + const rowBytes = orchestrationRowRetainedUtf8Bytes(row) + if (rowBytes > ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES) { + continue + } + if (retainedBytes + rowBytes > ORCHESTRATION_QUERY_MAX_RETAINED_UTF8_BYTES) { + break + } + retained.push(row) + retainedBytes += rowBytes + if (retained.length >= limit) { + break + } + } + return retained +} + +export function assertOrchestrationWriteFits(label: string, values: unknown[]): void { + let retainedBytes = 0 + for (const value of values) { + if (typeof value !== 'string') { + continue + } + retainedBytes += Buffer.byteLength(value, 'utf8') + if (retainedBytes > ORCHESTRATION_WRITE_MAX_UTF8_BYTES) { + throw new Error( + `${label} exceeds the ${ORCHESTRATION_WRITE_MAX_UTF8_BYTES}-byte orchestration limit` + ) + } + } +} + +export function assertOrchestrationWaitTypeFilterFits(value: string | undefined): void { + if ( + value && + measureUtf8ByteLength(value, { + stopAfterBytes: ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES + }).exceededLimit + ) { + throw new Error( + `Message type filter exceeds the ${ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES}-byte orchestration wait limit` + ) + } +} + +export function assertOrchestrationStringListFits(label: string, values: string[]): void { + if (values.length > ORCHESTRATION_WRITE_MAX_ITEMS) { + throw new Error( + `${label} exceeds the ${ORCHESTRATION_WRITE_MAX_ITEMS}-item orchestration limit` + ) + } + assertOrchestrationWriteFits(label, values) +} + +export function truncateOrchestrationDiagnostic(value: string): string { + if (Buffer.byteLength(value, 'utf8') <= ORCHESTRATION_WRITE_MAX_UTF8_BYTES) { + return value + } + + let bytes = 0 + let end = 0 + for (const codePoint of value) { + const codePointBytes = Buffer.byteLength(codePoint, 'utf8') + if (bytes + codePointBytes > ORCHESTRATION_WRITE_MAX_UTF8_BYTES) { + break + } + bytes += codePointBytes + end += codePoint.length + } + return value.slice(0, end) +} diff --git a/src/main/runtime/recent-pty-output-buffer.test.ts b/src/main/runtime/recent-pty-output-buffer.test.ts index 6c21284e163..5b0dac0742b 100644 --- a/src/main/runtime/recent-pty-output-buffer.test.ts +++ b/src/main/runtime/recent-pty-output-buffer.test.ts @@ -93,6 +93,34 @@ describe('RecentPtyOutputBuffer', () => { expectEquivalent(chunks) }) + it('retains 100,000 tiny writes in compact payload storage and typed boundary metadata', () => { + const buffer = new RecentPtyOutputBuffer() + let reference: string | undefined + for (let index = 0; index < 100_000; index += 1) { + const chunk = String.fromCharCode(97 + (index % 26)) + buffer.append(chunk) + reference = referenceAppend(reference, chunk) + } + + const internals = buffer as unknown as { + chunks: string[] + originalChunkLengths: { length: number; capacity: number } + } + expect(internals.chunks.length).toBeLessThanOrEqual(1024) + expect(internals.originalChunkLengths.length).toBe(RECENT_PTY_OUTPUT_LIMIT) + expect(internals.originalChunkLengths.capacity).toBe(RECENT_PTY_OUTPUT_LIMIT) + expect(buffer.read()).toBe(reference) + + let streamedLength = 0 + let streamedChunks = 0 + buffer.forEachRetainedChunk((chunk) => { + streamedLength += chunk.length + streamedChunks += 1 + }) + expect(streamedLength).toBe(RECENT_PTY_OUTPUT_LIMIT) + expect(streamedChunks).toBe(RECENT_PTY_OUTPUT_LIMIT) + }) + it('matches the old UTF-16 slice behavior for multi-byte content at the boundary', () => { // JS string .slice counts UTF-16 code units, so trimming can split a // surrogate pair; the buffer must reproduce that split exactly. diff --git a/src/main/runtime/recent-pty-output-buffer.ts b/src/main/runtime/recent-pty-output-buffer.ts index a10559b75c9..e18c46b96dd 100644 --- a/src/main/runtime/recent-pty-output-buffer.ts +++ b/src/main/runtime/recent-pty-output-buffer.ts @@ -1,31 +1,69 @@ export const RECENT_PTY_OUTPUT_LIMIT = 64 * 1024 -// Compact the backing array once this many fully-dropped head slots accumulate, -// so the array itself stays bounded under long chunk floods. -const DROPPED_HEAD_COMPACT_THRESHOLD = 1024 +const RETAINED_CONTENT_CHUNK_LIMIT = 1024 +const DROPPED_CONTENT_CHUNK_LIMIT = 1024 +const INITIAL_BOUNDARY_CAPACITY = 16 -/** - * Bounded deque of raw PTY output chunks retaining exactly the last - * RECENT_PTY_OUTPUT_LIMIT UTF-16 code units. - * - * Why: eagerly rebuilding a rolling 64KB string per PTY chunk flattened a - * ~128KB rope on every write; keep chunks and defer the join to rare readers. - */ +class RetainedChunkLengthQueue { + private values = new Uint32Array(INITIAL_BOUNDARY_CAPACITY) + private head = 0 + private count = 0 + + get length(): number { + return this.count + } + + get capacity(): number { + return this.values.length + } + + push(value: number): void { + if (this.count === this.values.length) { + const grown = new Uint32Array(this.values.length * 2) + for (let index = 0; index < this.count; index += 1) { + grown[index] = this.at(index) + } + this.values = grown + this.head = 0 + } + this.values[(this.head + this.count) % this.values.length] = value + this.count += 1 + } + + shift(): number { + if (this.count === 0) { + return 0 + } + const value = this.values[this.head]! + this.head = (this.head + 1) % this.values.length + this.count -= 1 + if (this.count === 0) { + this.head = 0 + } + return value + } + + at(index: number): number { + return this.values[(this.head + index) % this.values.length]! + } + + reset(): void { + this.values = new Uint32Array(INITIAL_BOUNDARY_CAPACITY) + this.head = 0 + this.count = 0 + } +} + +/** Bounded raw PTY tail that temporarily preserves source chunk boundaries for path backfill. */ export class RecentPtyOutputBuffer { private chunks: string[] = [] - private headIndex = 0 - // Code units already trimmed off the front of the head chunk. Deferred so - // repeated small trims never allocate a substring per append, and so the - // head chunk's original text stays available for candidate backfill. - private headOffset = 0 + private contentHeadIndex = 0 + private contentHeadOffset = 0 private totalLen = 0 - // True when the stored head chunk is not the full original PTY chunk (a - // single over-limit append is stored pre-sliced), so backfill replay knows - // the original line context of its leading text is gone. + private headOffset = 0 private headChunkIsPartial = false - // Original chunk boundaries are owed only to the one-time path-candidate - // backfill; compact() ends that obligation and lets read() collapse. private preserveChunkBoundaries: boolean + private readonly originalChunkLengths = new RetainedChunkLengthQueue() constructor(options?: { preserveChunkBoundaries?: boolean }) { this.preserveChunkBoundaries = options?.preserveChunkBoundaries ?? true @@ -36,88 +74,145 @@ export class RecentPtyOutputBuffer { return } if (data.length >= RECENT_PTY_OUTPUT_LIMIT) { - this.chunks = [data.slice(-RECENT_PTY_OUTPUT_LIMIT)] - this.headIndex = 0 - this.headOffset = 0 + this.replaceContent(data.slice(-RECENT_PTY_OUTPUT_LIMIT)) this.totalLen = RECENT_PTY_OUTPUT_LIMIT + this.headOffset = 0 this.headChunkIsPartial = data.length > RECENT_PTY_OUTPUT_LIMIT + this.originalChunkLengths.reset() + if (this.preserveChunkBoundaries) { + this.originalChunkLengths.push(RECENT_PTY_OUTPUT_LIMIT) + } return } - this.chunks.push(data) + + this.appendContent(data) this.totalLen += data.length + if (this.preserveChunkBoundaries) { + this.trimPreservingBoundaries() + this.originalChunkLengths.push(data.length) + } else { + this.trimContentToWindow() + } + } + + read(): string { + const stored = this.storedContent() + const value = this.headOffset > 0 ? stored.slice(this.headOffset) : stored + if (!this.preserveChunkBoundaries) { + this.replaceContent(value) + } + return value + } + + /** Original PTY chunks retained for the one-time path-candidate backfill. */ + retainedChunks(): { chunks: string[]; headChunkIsPartial: boolean } { + const chunks: string[] = [] + const state = this.forEachRetainedChunk((chunk) => chunks.push(chunk)) + return { chunks, headChunkIsPartial: state.headChunkIsPartial } + } + + /** Streams boundaries so activation does not allocate one string object per tiny write. */ + forEachRetainedChunk( + visit: (chunk: string, index: number, headChunkIsPartial: boolean) => void + ): { + headChunkIsPartial: boolean + } { + if (!this.preserveChunkBoundaries) { + const value = this.read() + if (value) { + visit(value, 0, false) + } + return { headChunkIsPartial: false } + } + const stored = this.storedContent() + let offset = 0 + for (let index = 0; index < this.originalChunkLengths.length; index += 1) { + const length = this.originalChunkLengths.at(index) + visit(stored.slice(offset, offset + length), index, this.headChunkIsPartial) + offset += length + } + return { headChunkIsPartial: this.headChunkIsPartial } + } + + /** Ends the boundary obligation and returns to compact steady-state storage. */ + compact(): void { + const value = this.read() + this.preserveChunkBoundaries = false + this.replaceContent(value) + this.headOffset = 0 + this.headChunkIsPartial = false + this.originalChunkLengths.reset() + } + + private appendContent(data: string): void { + this.chunks.push(data) + if (this.chunks.length - this.contentHeadIndex > RETAINED_CONTENT_CHUNK_LIMIT) { + this.replaceContent(this.storedContent()) + } + } + + private trimPreservingBoundaries(): void { while (this.totalLen > RECENT_PTY_OUTPUT_LIMIT) { - const headRemaining = this.chunks[this.headIndex].length - this.headOffset + const originalHeadLength = this.originalChunkLengths.at(0) + const headRemaining = originalHeadLength - this.headOffset const excess = this.totalLen - RECENT_PTY_OUTPUT_LIMIT if (headRemaining <= excess) { - // Release the dropped chunk's reference; the slot is reclaimed on compaction. - this.chunks[this.headIndex] = '' - this.headIndex += 1 + this.totalLen -= headRemaining this.headOffset = 0 this.headChunkIsPartial = false - this.totalLen -= headRemaining + this.originalChunkLengths.shift() + this.dropContentPrefix(originalHeadLength) } else { this.headOffset += excess this.totalLen -= excess } } - if (this.headIndex >= DROPPED_HEAD_COMPACT_THRESHOLD) { - this.chunks = this.chunks.slice(this.headIndex) - this.headIndex = 0 - } } - read(): string { - if (this.preserveChunkBoundaries) { - // Join without mutating: boundaries and the original head chunk are - // still owed to retainedChunks(); reads are rare before compact(). - if (this.chunks.length - this.headIndex > 1) { - const retained = this.chunks.slice(this.headIndex) - if (this.headOffset > 0) { - retained[0] = retained[0].slice(this.headOffset) - } - return retained.join('') + private trimContentToWindow(): void { + const excess = this.totalLen - RECENT_PTY_OUTPUT_LIMIT + if (excess <= 0) { + return + } + this.dropContentPrefix(excess) + this.totalLen -= excess + } + + private dropContentPrefix(length: number): void { + let remaining = length + while (remaining > 0) { + const chunk = this.chunks[this.contentHeadIndex] ?? '' + const available = chunk.length - this.contentHeadOffset + if (available <= remaining) { + this.chunks[this.contentHeadIndex] = '' + this.contentHeadIndex += 1 + this.contentHeadOffset = 0 + remaining -= available + } else { + this.contentHeadOffset += remaining + remaining = 0 } - const head = this.chunks[this.headIndex] ?? '' - return this.headOffset > 0 ? head.slice(this.headOffset) : head } - if (this.chunks.length - this.headIndex > 1) { - // Collapse to the joined tail so repeated reads stay O(1). - const retained = this.chunks.slice(this.headIndex) - if (this.headOffset > 0) { - retained[0] = retained[0].slice(this.headOffset) - this.headOffset = 0 - } - this.chunks = [retained.join('')] - this.headIndex = 0 - } else if (this.headOffset > 0) { - // Single retained chunk: apply the deferred head trim once, here. - this.chunks[this.headIndex] = this.chunks[this.headIndex].slice(this.headOffset) - this.headOffset = 0 - } - return this.chunks[this.headIndex] ?? '' - } - - /** - * Retained chunks with original PTY boundaries. The head chunk is its full - * original text (any window-trimmed prefix included) unless - * headChunkIsPartial. Why: path-candidate backfill must replay the eager - * per-chunk extraction exactly — trimming or joining chunks changes the - * candidate set. Only meaningful before compact(). - */ - retainedChunks(): { chunks: string[]; headChunkIsPartial: boolean } { - return { - chunks: this.chunks.slice(this.headIndex), - headChunkIsPartial: this.headChunkIsPartial + if (this.contentHeadIndex >= DROPPED_CONTENT_CHUNK_LIMIT) { + this.chunks = this.chunks.slice(this.contentHeadIndex) + this.contentHeadIndex = 0 } } - /** - * Ends the chunk-boundary obligation after the one-time backfill and - * collapses immediately, so the append/read hot path returns to the - * compact single-chunk steady state. - */ - compact(): void { - this.preserveChunkBoundaries = false - this.read() + private storedContent(): string { + const retained = this.chunks.slice(this.contentHeadIndex) + if (retained.length === 0) { + return '' + } + if (this.contentHeadOffset > 0) { + retained[0] = retained[0]!.slice(this.contentHeadOffset) + } + return retained.length === 1 ? retained[0]! : retained.join('') + } + + private replaceContent(value: string): void { + this.chunks = value ? [value] : [] + this.contentHeadIndex = 0 + this.contentHeadOffset = 0 } } diff --git a/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts b/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts new file mode 100644 index 00000000000..21e7f9aadec --- /dev/null +++ b/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts @@ -0,0 +1,237 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayAuthCoordinator, type RelayAuthContext } from './relay-auth-coordinator' +import { RelayHttpError } from './relay-http-client' + +const context: RelayAuthContext = { + identity: { userId: 'user-1', profileId: 'profile-1', organizationId: 'org-1' }, + accessToken: 'access-1', + relayEntitled: true +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('RelayAuthCoordinator transient recovery', () => { + it('retries a transient assignment failure and activates without an external event', async () => { + vi.useFakeTimers() + const broker = { closeNow: vi.fn() } + const openBroker = vi + .fn() + .mockRejectedValueOnce(new RelayHttpError('assignment', 500)) + .mockResolvedValueOnce(broker) + const statuses: string[] = [] + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: (status) => statuses.push(status), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + expect(statuses.at(-1)).toBe('offline') + + await vi.advanceTimersByTimeAsync(501) + expect(openBroker).toHaveBeenCalledTimes(2) + expect(coordinator.getActiveBroker()).toBe(broker) + expect(statuses.at(-1)).toBe('registered') + }) + + it('retries when cloud-session refresh fails before identity can be read', async () => { + vi.useFakeTimers() + const broker = { closeNow: vi.fn() } + const readContext = vi + .fn() + .mockRejectedValueOnce(new Error('temporary cloud session refresh failure')) + .mockResolvedValueOnce(context) + const openBroker = vi.fn().mockResolvedValue(broker) + const coordinator = new RelayAuthCoordinator({ + readContext, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(readContext).toHaveBeenCalledOnce() + expect(openBroker).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(501) + expect(readContext).toHaveBeenCalledTimes(2) + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBe(broker) + }) + + it('backs a sustained outage off to the five-minute jitter cap', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + for (const delayMs of [500, 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 64_000, 128_000]) { + await vi.advanceTimersByTimeAsync(delayMs) + } + expect(openBroker).toHaveBeenCalledTimes(10) + + await vi.advanceTimersByTimeAsync(149_999) + expect(openBroker).toHaveBeenCalledTimes(10) + await vi.advanceTimersByTimeAsync(1) + expect(openBroker).toHaveBeenCalledTimes(11) + + await vi.advanceTimersByTimeAsync(150_000) + expect(openBroker).toHaveBeenCalledTimes(12) + }) + + it('does not retry a permanent authorization response', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new RelayHttpError('token-exchange', 403)) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('cancels a pending retry as soon as demand disappears', async () => { + vi.useFakeTimers() + let demanded = true + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + hasDemand: () => demanded, + openBroker, + onStatus: vi.fn(), + random: () => 0.75 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + demanded = false + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('re-reads demand when the retry fires and stops without opening again', async () => { + vi.useFakeTimers() + let demanded = true + const statuses: string[] = [] + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + hasDemand: () => demanded, + openBroker, + onStatus: (status) => statuses.push(status), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + demanded = false + await vi.advanceTimersByTimeAsync(501) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(statuses.at(-1)).toBe('standby') + }) + + it('re-reads entitlement when the retry fires and stops after removal', async () => { + vi.useFakeTimers() + let current = context + const openBroker = vi.fn().mockRejectedValue(new RelayHttpError('assignment', 500)) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => current, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + current = { ...context, relayEntitled: false } + await vi.advanceTimersByTimeAsync(501) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('cancels a pending retry immediately when the coordinator is fenced', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + coordinator.fenceAndCloseNow() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('does not carry a pending retry across an identity switch', async () => { + vi.useFakeTimers() + let current = context + const broker = { closeNow: vi.fn() } + const openBroker = vi + .fn() + .mockRejectedValueOnce(new Error('temporary control open failure')) + .mockResolvedValueOnce(broker) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => current, + openBroker, + onStatus: vi.fn(), + random: () => 0.75 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + current = { + ...context, + identity: { ...context.identity, profileId: 'profile-2' } + } + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledTimes(2) + expect(coordinator.getActiveBroker()).toBe(broker) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/runtime/relay/relay-auth-coordinator.ts b/src/main/runtime/relay/relay-auth-coordinator.ts index f197edcc2b1..7f97e33dc41 100644 --- a/src/main/runtime/relay/relay-auth-coordinator.ts +++ b/src/main/runtime/relay/relay-auth-coordinator.ts @@ -1,4 +1,5 @@ import type { RelayBrokerStatus } from './relay-session-broker' +import { shouldRetryRelayConnectionError } from './relay-http-client' export type RelayAuthIdentity = { userId: string @@ -26,6 +27,7 @@ type RelayAuthCoordinatorOptions = { }) => Promise onStatus: (status: RelayBrokerStatus) => void lingerMs?: number + random?: () => number } type BrokerOwnership = { @@ -39,12 +41,17 @@ function identityKey(identity: RelayAuthIdentity): string { } export class RelayAuthCoordinator { + // Why: recover brief failures quickly without turning a sustained outage into auth/director load. + private static readonly RETRY_BASE_MS = 1_000 + private static readonly RETRY_MAX_MS = 5 * 60_000 private readonly options: RelayAuthCoordinatorOptions private authEpoch = 0 private ownership: BrokerOwnership | null = null private readonly pendingOwnerships = new Set() private latestReconcile: Promise = Promise.resolve() private lingerTimer: ReturnType | null = null + private retryTimer: ReturnType | null = null + private retryAttempt = 0 private stopped = false constructor(options: RelayAuthCoordinatorOptions) { @@ -52,12 +59,20 @@ export class RelayAuthCoordinator { } reconcile(): void { + this.beginReconcile(true) + } + + private beginReconcile(resetRetry: boolean, expectedIdentityKey?: string): void { if (this.stopped) { return } + this.cancelRetry() + if (resetRetry) { + this.retryAttempt = 0 + } const epoch = ++this.authEpoch this.invalidatePendingOwnerships() - const reconcile = this.reconcileEpoch(epoch) + const reconcile = this.reconcileEpoch(epoch, expectedIdentityKey) this.latestReconcile = reconcile void reconcile } @@ -65,6 +80,8 @@ export class RelayAuthCoordinator { fenceAndCloseNow(): void { ++this.authEpoch this.cancelLinger() + this.cancelRetry() + this.retryAttempt = 0 this.invalidatePendingOwnerships() this.invalidateOwnership() this.options.onStatus('offline') @@ -94,7 +111,8 @@ export class RelayAuthCoordinator { this.fenceAndCloseNow() } - private async reconcileEpoch(epoch: number): Promise { + private async reconcileEpoch(epoch: number, expectedIdentityKey?: string): Promise { + let retryIdentityKey: string | undefined try { const context = await this.options.readContext() if (!this.isEpochCurrent(epoch)) { @@ -102,12 +120,19 @@ export class RelayAuthCoordinator { } if (!context || !context.relayEntitled) { this.cancelLinger() + this.retryAttempt = 0 this.invalidateOwnership() this.options.onStatus('offline') return } const nextIdentityKey = identityKey(context.identity) + if (expectedIdentityKey && nextIdentityKey !== expectedIdentityKey) { + this.retryAttempt = 0 + this.options.onStatus('offline') + return + } if (!(this.options.hasDemand?.(context) ?? true)) { + this.retryAttempt = 0 if (this.ownership?.valid && this.ownership.identityKey !== nextIdentityKey) { this.cancelLinger() this.invalidateOwnership() @@ -119,9 +144,11 @@ export class RelayAuthCoordinator { } this.cancelLinger() if (this.ownership?.valid && this.ownership.identityKey === nextIdentityKey) { + this.retryAttempt = 0 this.options.onStatus('registered') return } + retryIdentityKey = nextIdentityKey this.invalidateOwnership() this.options.onStatus('connecting') const ownership: BrokerOwnership = { @@ -150,14 +177,42 @@ export class RelayAuthCoordinator { return } this.ownership = ownership + this.retryAttempt = 0 this.options.onStatus('registered') - } catch { + } catch (error) { if (this.isEpochCurrent(epoch)) { this.options.onStatus('offline') + if (shouldRetryRelayConnectionError(error)) { + this.scheduleRetry(epoch, retryIdentityKey) + } } } } + private scheduleRetry(epoch: number, expectedIdentityKey?: string): void { + if (this.retryTimer || !this.isEpochCurrent(epoch)) { + return + } + const exponent = Math.min( + this.retryAttempt, + Math.ceil(Math.log2(RelayAuthCoordinator.RETRY_MAX_MS / RelayAuthCoordinator.RETRY_BASE_MS)) + ) + const capMs = Math.min( + RelayAuthCoordinator.RETRY_MAX_MS, + RelayAuthCoordinator.RETRY_BASE_MS * 2 ** exponent + ) + this.retryAttempt++ + const random = this.options.random ?? Math.random + const delayMs = Math.floor(random() * (capMs + 1)) + this.retryTimer = setTimeout(() => { + this.retryTimer = null + if (this.isEpochCurrent(epoch)) { + // Retry still re-reads entitlement and demand; the timer grants no authority. + this.beginReconcile(false, expectedIdentityKey) + } + }, delayMs) + } + private async refreshAccessToken( ownership: { valid: boolean }, expectedIdentityKey: string @@ -212,6 +267,13 @@ export class RelayAuthCoordinator { } } + private cancelRetry(): void { + if (this.retryTimer) { + clearTimeout(this.retryTimer) + this.retryTimer = null + } + } + private invalidatePendingOwnerships(): void { for (const ownership of this.pendingOwnerships) { ownership.valid = false diff --git a/src/main/runtime/relay/relay-control-client.test.ts b/src/main/runtime/relay/relay-control-client.test.ts index 415b4583d46..e07fa5aa434 100644 --- a/src/main/runtime/relay/relay-control-client.test.ts +++ b/src/main/runtime/relay/relay-control-client.test.ts @@ -2,8 +2,13 @@ import { createHash, createHmac, randomBytes } from 'node:crypto' import { afterEach, describe, expect, it, vi } from 'vitest' import nacl from 'tweetnacl' import { WebSocketServer, type WebSocket } from 'ws' +import { JsonStringifyByteLimitError } from '../../../shared/node-bounded-json-stringify' import type { E2EEKeypair } from '../e2ee-keypair' -import { RelayControlClient } from './relay-control-client' +import { + RELAY_CONTROL_MAX_MESSAGE_BYTES, + RelayControlClient, + serializeRelayControlMessage +} from './relay-control-client' const encoder = new TextEncoder() const HOST_PROOF_DOMAIN = 'orca-relay-host-proof/v1' @@ -80,6 +85,21 @@ describe('RelayControlClient', () => { const servers: WebSocketServer[] = [] const clients: RelayControlClient[] = [] + it('preserves ordinary control message serialization byte-for-byte', () => { + const payload = { type: 'auth-refresh', relayJwt: 'token-😀' } + + expect(serializeRelayControlMessage(payload)).toBe(JSON.stringify(payload)) + }) + + it('rejects an oversized control message during bounded serialization', () => { + expect(() => + serializeRelayControlMessage({ + type: 'auth-refresh', + relayJwt: 'x'.repeat(RELAY_CONTROL_MAX_MESSAGE_BYTES) + }) + ).toThrow(JsonStringifyByteLimitError) + }) + afterEach(async () => { for (const client of clients.splice(0)) { client.closeNow() diff --git a/src/main/runtime/relay/relay-control-client.ts b/src/main/runtime/relay/relay-control-client.ts index 44f3d65798c..750e59359e3 100644 --- a/src/main/runtime/relay/relay-control-client.ts +++ b/src/main/runtime/relay/relay-control-client.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto' import WebSocket, { type RawData } from 'ws' import { MOBILE_RELAY_CLOSE_CODE } from '../../../shared/mobile-relay-close-codes' +import { stringifyJsonWithinByteLimit } from '../../../shared/node-bounded-json-stringify' import type { E2EEKeypair } from '../e2ee-keypair' import { RelayConnectionOpenMessageSchema, @@ -19,6 +20,7 @@ import type { DeviceCredentialInstallAuthorization } from './relay-control-reque import { answerRelayHostChallenge } from './relay-host-proof' type RelayControlState = 'idle' | 'opening' | 'proving' | 'active' | 'draining' | 'closed' +export const RELAY_CONTROL_MAX_MESSAGE_BYTES = 64 * 1024 type RelayControlClientOptions = { cellUrl: string @@ -74,7 +76,7 @@ export class RelayControlClient { new WebSocket(url, { headers: { authorization: `Bearer ${token}` }, perMessageDeflate: false, - maxPayload: 64 * 1024 + maxPayload: RELAY_CONTROL_MAX_MESSAGE_BYTES })) } @@ -165,7 +167,7 @@ export class RelayControlClient { } this.state = 'proving' this.socket.send( - JSON.stringify({ + serializeRelayControlMessage({ type: 'host-hello', v: 1, relayHostId: this.options.relayHostId, @@ -197,7 +199,7 @@ export class RelayControlClient { return } if (RelayPingMessageSchema.safeParse(message).success) { - this.socket?.send(JSON.stringify({ type: 'pong', t: message.t })) + this.socket?.send(serializeRelayControlMessage({ type: 'pong', t: message.t })) return } const connection = RelayConnectionOpenMessageSchema.safeParse(message) @@ -235,7 +237,7 @@ export class RelayControlClient { return } this.socket?.send( - JSON.stringify({ + serializeRelayControlMessage({ type: 'host-challenge-ack', challengeId: challenge.data.challengeId, proofB64 @@ -257,7 +259,7 @@ export class RelayControlClient { if (!this.socket || (this.state !== 'active' && this.state !== 'draining')) { throw new Error('relay_control_not_active') } - this.socket.send(JSON.stringify(payload)) + this.socket.send(serializeRelayControlMessage(payload)) } private failProtocol(reason: string): void { @@ -282,3 +284,7 @@ export class RelayControlClient { this.connectReject = null } } + +export function serializeRelayControlMessage(payload: unknown): string { + return stringifyJsonWithinByteLimit(payload, RELAY_CONTROL_MAX_MESSAGE_BYTES).serialized +} diff --git a/src/main/runtime/relay/relay-control-origin.ts b/src/main/runtime/relay/relay-control-origin.ts index ef607c9862b..a7bfc9809e9 100644 --- a/src/main/runtime/relay/relay-control-origin.ts +++ b/src/main/runtime/relay/relay-control-origin.ts @@ -31,6 +31,7 @@ export class RelayControlOrigin { readonly assignment: RelayAssignment readonly transport: CloudRelayTransport private readonly options: RelayControlOriginOptions + private readonly detachTransport: () => void private readonly controls = new Set() private readonly retiredControlTimers = new Map< RelayControlClient, @@ -53,7 +54,7 @@ export class RelayControlOrigin { createSocket: options.createDataSocket, onConnectionClosed: (connectionId) => options.onConnectionReleased(connectionId, this) }) - options.mobileSocketWiring.attachTransport(this.transport, (ws) => + this.detachTransport = options.mobileSocketWiring.attachTransport(this.transport, (ws) => this.transport.metadataFor(ws) ) } @@ -143,6 +144,7 @@ export class RelayControlOrigin { return } this.closed = true + this.detachTransport() for (const timer of this.retiredControlTimers.values()) { clearTimeout(timer) } diff --git a/src/main/runtime/relay/relay-control-requests.test.ts b/src/main/runtime/relay/relay-control-requests.test.ts new file mode 100644 index 00000000000..bafa0611d35 --- /dev/null +++ b/src/main/runtime/relay/relay-control-requests.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + RELAY_CONTROL_MAX_PENDING_REQUESTS, + RELAY_CONTROL_REQUEST_ID_MAX_UTF8_BYTES, + RelayControlRequests +} from './relay-control-requests' + +afterEach(() => { + vi.useRealTimers() +}) + +describe('RelayControlRequests admission', () => { + it('fails closed at the pending request cap', async () => { + vi.useFakeTimers() + const requests = new RelayControlRequests() + const send = vi.fn() + const admitted = Array.from({ length: RELAY_CONTROL_MAX_PENDING_REQUESTS }, (_, index) => + requests.createInvite(`request-${index}`, 'device-a', send) + ) + const settlements = Promise.allSettled(admitted) + + await expect(requests.createInvite('request-overflow', 'device-a', send)).rejects.toThrow( + 'relay_control_request_limit' + ) + expect(requests.size).toBe(RELAY_CONTROL_MAX_PENDING_REQUESTS) + expect(send).toHaveBeenCalledTimes(RELAY_CONTROL_MAX_PENDING_REQUESTS) + + requests.rejectAll(new Error('closed')) + await settlements + expect(requests.size).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('rejects an oversized request id before sending or retaining it', async () => { + const requests = new RelayControlRequests() + const send = vi.fn() + + await expect( + requests.createInvite('🙂'.repeat(RELAY_CONTROL_REQUEST_ID_MAX_UTF8_BYTES), 'device-a', send) + ).rejects.toThrow('relay_control_request_limit') + + expect(requests.size).toBe(0) + expect(send).not.toHaveBeenCalled() + }) + + it('releases admission when send throws', async () => { + vi.useFakeTimers() + const requests = new RelayControlRequests() + + await expect( + requests.createInvite('request-a', 'device-a', () => { + throw new Error('send failed') + }) + ).rejects.toThrow('send failed') + + expect(requests.size).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('releases admission when a request times out', async () => { + vi.useFakeTimers() + const requests = new RelayControlRequests() + const result = requests.createInvite('request-a', 'device-a', () => {}) + const rejection = expect(result).rejects.toThrow('relay_control_request_timeout') + + await vi.advanceTimersByTimeAsync(10_000) + + await rejection + expect(requests.size).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/src/main/runtime/relay/relay-control-requests.ts b/src/main/runtime/relay/relay-control-requests.ts index 2a96b94e3ec..36629a10239 100644 --- a/src/main/runtime/relay/relay-control-requests.ts +++ b/src/main/runtime/relay/relay-control-requests.ts @@ -16,14 +16,21 @@ type PendingRequest = { resolve: (value: unknown) => void reject: (error: Error) => void timer: ReturnType + reqIdBytes: number } +export const RELAY_CONTROL_MAX_PENDING_REQUESTS = 256 +export const RELAY_CONTROL_REQUEST_ID_MAX_UTF8_BYTES = 512 +export const RELAY_CONTROL_RETAINED_REQUEST_ID_MAX_UTF8_BYTES = + RELAY_CONTROL_MAX_PENDING_REQUESTS * RELAY_CONTROL_REQUEST_ID_MAX_UTF8_BYTES + export type DeviceCredentialInstallAuthorization = | { mode: 'relay-basis'; basisConnId: string } | { mode: 'authenticated-direct'; directAuthId: string } export class RelayControlRequests { private readonly pending = new Map() + private retainedRequestIdBytes = 0 get size(): number { return this.pending.size @@ -160,12 +167,22 @@ export class RelayControlRequests { if (this.pending.has(reqId)) { return Promise.reject(new Error('duplicate_relay_request_id')) } + const reqIdBytes = Buffer.byteLength(reqId, 'utf8') + if ( + reqId.length === 0 || + reqIdBytes > RELAY_CONTROL_REQUEST_ID_MAX_UTF8_BYTES || + this.pending.size >= RELAY_CONTROL_MAX_PENDING_REQUESTS || + reqIdBytes > RELAY_CONTROL_RETAINED_REQUEST_ID_MAX_UTF8_BYTES - this.retainedRequestIdBytes + ) { + return Promise.reject(new Error('relay_control_request_limit')) + } return new Promise((resolve, reject) => { const timer = setTimeout(() => { - this.pending.delete(reqId) + this.finish(reqId) reject(new Error('relay_control_request_timeout')) }, 10_000) - this.pending.set(reqId, { kind, resolve, reject, timer }) + this.pending.set(reqId, { kind, resolve, reject, timer, reqIdBytes }) + this.retainedRequestIdBytes += reqIdBytes try { send(payload) } catch (error) { @@ -180,6 +197,7 @@ export class RelayControlRequests { if (pending) { clearTimeout(pending.timer) this.pending.delete(reqId) + this.retainedRequestIdBytes -= pending.reqIdBytes } } } diff --git a/src/main/runtime/relay/relay-host-proof-bounds.test.ts b/src/main/runtime/relay/relay-host-proof-bounds.test.ts new file mode 100644 index 00000000000..602b1bee44b --- /dev/null +++ b/src/main/runtime/relay/relay-host-proof-bounds.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' +import nacl from 'tweetnacl' +import { + answerRelayHostChallenge, + RELAY_HOST_CHALLENGE_MAX_CIPHERTEXT_BASE64_CHARACTERS +} from './relay-host-proof' + +describe('relay host proof base64 admission', () => { + it('rejects oversized fixed-width fields before decoding', () => { + const decode = vi.spyOn(Buffer, 'from') + + expect( + answerRelayHostChallenge( + { + challengeId: 'challenge', + relayEphemeralPublicKeyB64: 'A'.repeat(45), + nonceB64: 'A'.repeat(33), + ciphertextB64: 'AAAA', + expiresAt: Date.now() + 1_000 + }, + context() + ) + ).toBeNull() + expect(decode).not.toHaveBeenCalled() + decode.mockRestore() + }) + + it('rejects oversized ciphertext before decoding that field', () => { + const relayKey = Buffer.from(new Uint8Array(32)).toString('base64') + const nonce = Buffer.from(new Uint8Array(24)).toString('base64') + const decode = vi.spyOn(Buffer, 'from') + + expect( + answerRelayHostChallenge( + { + challengeId: 'challenge', + relayEphemeralPublicKeyB64: relayKey, + nonceB64: nonce, + ciphertextB64: 'A'.repeat(RELAY_HOST_CHALLENGE_MAX_CIPHERTEXT_BASE64_CHARACTERS + 1), + expiresAt: Date.now() + 1_000 + }, + context() + ) + ).toBeNull() + expect(decode).toHaveBeenCalledTimes(2) + decode.mockRestore() + }) +}) + +function context() { + const host = nacl.box.keyPair() + return { + relayOrigin: 'https://relay.example', + userId: 'user', + profileId: 'profile', + organizationId: 'organization', + relayHostId: 'host', + hostPublicKey: host.publicKey, + hostSecretKey: host.secretKey, + assignmentEpoch: 1, + resumeRequested: false + } +} diff --git a/src/main/runtime/relay/relay-host-proof.ts b/src/main/runtime/relay/relay-host-proof.ts index 94db5b2f1b2..9fcc29ed9d9 100644 --- a/src/main/runtime/relay/relay-host-proof.ts +++ b/src/main/runtime/relay/relay-host-proof.ts @@ -5,6 +5,7 @@ const HOST_PROOF_TRANSCRIPT_DOMAIN = 'orca-relay-host-proof/v1' const HOST_CHALLENGE_PLAINTEXT_DOMAIN = 'orca-relay-host-challenge/v1' const textEncoder = new TextEncoder() const textDecoder = new TextDecoder() +export const RELAY_HOST_CHALLENGE_MAX_CIPHERTEXT_BASE64_CHARACTERS = 16 * 1024 export type RelayHostChallenge = { challengeId: string @@ -29,7 +30,10 @@ export type RelayHostProofContext = { } function decodeCanonicalBase64(value: string, expectedBytes: number): Uint8Array | null { - if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) { + if ( + value.length !== Math.ceil(expectedBytes / 3) * 4 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) + ) { return null } const decoded = Buffer.from(value, 'base64') @@ -129,8 +133,18 @@ export function answerRelayHostChallenge( ): string | null { const relayKey = decodeCanonicalBase64(challenge.relayEphemeralPublicKeyB64, 32) const nonce = decodeCanonicalBase64(challenge.nonceB64, 24) + if ( + !relayKey || + !nonce || + challenge.ciphertextB64.length > RELAY_HOST_CHALLENGE_MAX_CIPHERTEXT_BASE64_CHARACTERS || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( + challenge.ciphertextB64 + ) + ) { + return null + } const ciphertext = Buffer.from(challenge.ciphertextB64, 'base64') - if (!relayKey || !nonce || ciphertext.toString('base64') !== challenge.ciphertextB64) { + if (ciphertext.toString('base64') !== challenge.ciphertextB64) { return null } const plaintext = nacl.box.open(ciphertext, nonce, relayKey, context.hostSecretKey) diff --git a/src/main/runtime/relay/relay-http-client.test.ts b/src/main/runtime/relay/relay-http-client.test.ts index 57beb527497..68e27b157ac 100644 --- a/src/main/runtime/relay/relay-http-client.test.ts +++ b/src/main/runtime/relay/relay-http-client.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from 'vitest' import nacl from 'tweetnacl' +import { + API_RESPONSE_MAX_BYTES, + FetchResponseBodyTooLargeError +} from '../../lib/fetch-response-body' import { cancelTrackingResponse } from '../../lib/unread-response-body.test-fixtures' import { exchangeRelayAuthorization, requestRelayAssignment } from './relay-http-client' @@ -104,4 +108,30 @@ describe('relay HTTP client', () => { ).rejects.toThrow() expect(cancelledBodies).toBe(2) }) + + it('rejects oversized successful responses from injected standard fetch clients', async () => { + let cancelled = false + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([123])) + }, + cancel() { + cancelled = true + } + }), + { headers: { 'content-length': String(API_RESPONSE_MAX_BYTES + 1) } } + ) + const fetch = vi.fn(async () => response) + + await expect( + requestRelayAssignment({ + directorUrl: 'https://relay.example', + relayToken: 'scoped-token', + relayHostId: 'AbCdEf0123_-xyZ9', + fetch + }) + ).rejects.toBeInstanceOf(FetchResponseBodyTooLargeError) + expect(cancelled).toBe(true) + }) }) diff --git a/src/main/runtime/relay/relay-http-client.ts b/src/main/runtime/relay/relay-http-client.ts index 47118b88cfc..6862f26ce6c 100644 --- a/src/main/runtime/relay/relay-http-client.ts +++ b/src/main/runtime/relay/relay-http-client.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto' import { z } from 'zod' import type { E2EEKeypair } from '../e2ee-keypair' import { cancelUnreadResponseBody } from '../../lib/unread-response-body' +import { readFetchResponseJsonWithinLimit } from '../../lib/fetch-response-body' const RelayTokenResponseSchema = z .object({ @@ -37,6 +38,18 @@ export class RelayHttpError extends Error { } } +export function shouldRetryRelayConnectionError(error: unknown): boolean { + if (!(error instanceof RelayHttpError)) { + return true + } + return ( + error.statusCode >= 500 || + error.statusCode === 408 || + error.statusCode === 425 || + error.statusCode === 429 + ) +} + export function deriveRelayHostId(publicKey: Uint8Array): string { return createHash('sha256').update(publicKey).digest('base64url').slice(0, 16) } @@ -73,7 +86,9 @@ export async function exchangeRelayAuthorization(input: { await cancelUnreadResponseBody(response) throw new RelayHttpError('token-exchange', response.status) } - const parsed = RelayTokenResponseSchema.safeParse(await response.json()) + const parsed = RelayTokenResponseSchema.safeParse( + await readFetchResponseJsonWithinLimit(response) + ) if (!parsed.success) { throw new RelayHttpError('token-exchange', 502) } @@ -101,7 +116,9 @@ export async function requestRelayAssignment(input: { await cancelUnreadResponseBody(response) throw new RelayHttpError('assignment', response.status) } - const parsed = AssignmentResponseSchema.safeParse(await response.json()) + const parsed = AssignmentResponseSchema.safeParse( + await readFetchResponseJsonWithinLimit(response) + ) if (!parsed.success || !isAllowedRelayOrigin(parsed.data.cellUrl)) { throw new RelayHttpError('assignment', 502) } diff --git a/src/main/runtime/relay/relay-revoke-outbox.test.ts b/src/main/runtime/relay/relay-revoke-outbox.test.ts index 54ab0473049..a4a4724164f 100644 --- a/src/main/runtime/relay/relay-revoke-outbox.test.ts +++ b/src/main/runtime/relay/relay-revoke-outbox.test.ts @@ -1,8 +1,14 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { RelayRevokeOutbox } from './relay-revoke-outbox' +import { + MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES, + MAX_RELAY_REVOKE_OUTBOX_ITEMS, + RelayRevokeOutbox, + RelayRevokeOutboxCapacityError, + type RelayRevokeOutboxItem +} from './relay-revoke-outbox' describe('RelayRevokeOutbox', () => { const paths: string[] = [] @@ -29,4 +35,70 @@ describe('RelayRevokeOutbox', () => { new RelayRevokeOutbox(path).pendingFor(binding.ownerIdentityKey, binding.relayHostId) ).toEqual([]) }) + + it('treats an oversized sparse outbox file as unavailable', () => { + const path = mkdtempSync(join(tmpdir(), 'orca-relay-revoke-bound-')) + paths.push(path) + const outboxPath = join(path, 'mobile-relay-revoke-outbox.json') + writeFileSync(outboxPath, '[]') + truncateSync(outboxPath, MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES + 1) + + expect(new RelayRevokeOutbox(path).pendingFor('owner', 'host')).toEqual([]) + }) + + it('fails closed instead of dropping revocations beyond the retained-item bound', () => { + const path = mkdtempSync(join(tmpdir(), 'orca-relay-revoke-count-')) + paths.push(path) + const items: RelayRevokeOutboxItem[] = Array.from( + { length: MAX_RELAY_REVOKE_OUTBOX_ITEMS + 1 }, + (_, index) => ({ + reqId: `request-${index}`, + relayHostId: 'host', + relayDeviceId: `device-${index}`, + ownerIdentityKey: 'owner', + createdAt: index + }) + ) + writeFileSync(join(path, 'mobile-relay-revoke-outbox.json'), JSON.stringify(items)) + + expect(() => new RelayRevokeOutbox(path)).toThrow(RelayRevokeOutboxCapacityError) + }) + + it('keeps idempotent revokes usable at capacity and rejects only a new revoke', () => { + const path = mkdtempSync(join(tmpdir(), 'orca-relay-revoke-capacity-')) + paths.push(path) + const items: RelayRevokeOutboxItem[] = Array.from( + { length: MAX_RELAY_REVOKE_OUTBOX_ITEMS }, + (_, index) => ({ + reqId: `request-${index}`, + relayHostId: 'host', + relayDeviceId: `device-${index}`, + ownerIdentityKey: 'owner', + createdAt: index + }) + ) + writeFileSync(join(path, 'mobile-relay-revoke-outbox.json'), JSON.stringify(items)) + const outbox = new RelayRevokeOutbox(path) + + expect(outbox.enqueue(items[0]!).reqId).toBe(items[0]!.reqId) + expect(() => + outbox.enqueue({ relayHostId: 'host', relayDeviceId: 'new', ownerIdentityKey: 'owner' }) + ).toThrow(RelayRevokeOutboxCapacityError) + }) + + it('rejects a byte-oversized revoke without publishing partial in-memory state', () => { + const path = mkdtempSync(join(tmpdir(), 'orca-relay-revoke-byte-capacity-')) + paths.push(path) + const outbox = new RelayRevokeOutbox(path) + + expect(() => + outbox.enqueue({ + relayHostId: 'host', + relayDeviceId: 'device', + ownerIdentityKey: 'x'.repeat(MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES) + }) + ).toThrow(RelayRevokeOutboxCapacityError) + expect(outbox.pendingFor('owner', 'host')).toEqual([]) + expect(new RelayRevokeOutbox(path).pendingFor('owner', 'host')).toEqual([]) + }) }) diff --git a/src/main/runtime/relay/relay-revoke-outbox.ts b/src/main/runtime/relay/relay-revoke-outbox.ts index 8a7e8cc19bb..5fd866ad008 100644 --- a/src/main/runtime/relay/relay-revoke-outbox.ts +++ b/src/main/runtime/relay/relay-revoke-outbox.ts @@ -1,7 +1,10 @@ import { randomUUID } from 'node:crypto' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { join } from 'node:path' -import { hardenExistingSecureFile, writeSecureJsonFile } from '../../../shared/secure-file' +import { readNodeFileSyncWithinLimit } from '../../../shared/node-bounded-file-reader' +import { JsonStringifyByteLimitError } from '../../../shared/node-bounded-json-stringify' +import { writeSecureJsonFileWithinLimit } from '../../../shared/bounded-secure-json-file' +import { hardenExistingSecureFile } from '../../../shared/secure-file' export type RelayDeviceBinding = { relayHostId: string @@ -16,6 +19,15 @@ export type RelayRevokeOutboxItem = RelayDeviceBinding & { } const OUTBOX_FILENAME = 'mobile-relay-revoke-outbox.json' +export const MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES = 1024 * 1024 +export const MAX_RELAY_REVOKE_OUTBOX_ITEMS = 4096 + +export class RelayRevokeOutboxCapacityError extends Error { + constructor() { + super(`Relay revoke outbox exceeds ${MAX_RELAY_REVOKE_OUTBOX_ITEMS} items`) + this.name = 'RelayRevokeOutboxCapacityError' + } +} function isItem(value: unknown): value is RelayRevokeOutboxItem { if (!value || typeof value !== 'object') { @@ -53,9 +65,13 @@ export class RelayRevokeOutbox { if (existing) { return existing } + if (this.items.length >= MAX_RELAY_REVOKE_OUTBOX_ITEMS) { + throw new RelayRevokeOutboxCapacityError() + } const item = { ...binding, reqId: randomUUID(), createdAt: Date.now() } - this.items.push(item) - this.save() + const next = [...this.items, item] + this.save(next) + this.items = next return item } @@ -70,24 +86,44 @@ export class RelayRevokeOutbox { if (next.length === this.items.length) { return } + this.save(next) this.items = next - this.save() } private load(): RelayRevokeOutboxItem[] { if (!existsSync(this.path)) { return [] } + let parsed: unknown try { hardenExistingSecureFile(this.path) - const parsed: unknown = JSON.parse(readFileSync(this.path, 'utf-8')) - return Array.isArray(parsed) ? parsed.filter(isItem) : [] + parsed = JSON.parse( + readNodeFileSyncWithinLimit(this.path, MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) } catch { return [] } + if (!Array.isArray(parsed)) { + return [] + } + const items = parsed.filter(isItem) + if (items.length > MAX_RELAY_REVOKE_OUTBOX_ITEMS) { + // Why: silently dropping durable revocations could leave remote credentials active. + throw new RelayRevokeOutboxCapacityError() + } + return items } - private save(): void { - writeSecureJsonFile(this.path, this.items) + private save(items: RelayRevokeOutboxItem[]): void { + try { + writeSecureJsonFileWithinLimit(this.path, items, MAX_RELAY_REVOKE_OUTBOX_FILE_BYTES) + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new RelayRevokeOutboxCapacityError() + } + throw error + } } } diff --git a/src/main/runtime/relay/relay-session-broker.test.ts b/src/main/runtime/relay/relay-session-broker.test.ts index d2e0ff0021d..32eb8c896d7 100644 --- a/src/main/runtime/relay/relay-session-broker.test.ts +++ b/src/main/runtime/relay/relay-session-broker.test.ts @@ -115,6 +115,7 @@ describe('RelaySessionBroker lifecycle ownership', () => { let current = true const statuses: string[] = [] const keypair = nacl.box.keyPair() + const detachTransport = vi.fn() const connecting = RelaySessionBroker.connect({ authConfig: { relayTokenEndpoint: 'https://auth.example.test/v1/relay-token', @@ -127,7 +128,7 @@ describe('RelaySessionBroker lifecycle ownership', () => { publicKeyB64: Buffer.from(keypair.publicKey).toString('base64') }, appVersion: '1.0.0', - mobileSocketWiring: { attachTransport: vi.fn() } as never, + mobileSocketWiring: { attachTransport: vi.fn(() => detachTransport) } as never, isCurrent: () => current, refreshAccessToken: async () => null, onStatus: (status) => statuses.push(status) @@ -147,6 +148,7 @@ describe('RelaySessionBroker lifecycle ownership', () => { await expect(connecting).rejects.toBeInstanceOf(StaleRelayBrokerError) expect(fakes.controls[0]!.closeNow).toHaveBeenCalledOnce() expect(fakes.transports[0]!.stop).toHaveBeenCalledOnce() + expect(detachTransport).toHaveBeenCalledOnce() expect(statuses).toEqual(['connecting']) }) @@ -316,7 +318,7 @@ function brokerOptions( publicKeyB64: Buffer.from(keypair.publicKey).toString('base64') }, appVersion: '1.0.0', - mobileSocketWiring: { attachTransport: vi.fn() } as never, + mobileSocketWiring: { attachTransport: vi.fn(() => vi.fn()) } as never, isCurrent: () => true, refreshAccessToken: async () => null, onStatus: vi.fn(), diff --git a/src/main/runtime/rpc/dispatcher-output-bounds.test.ts b/src/main/runtime/rpc/dispatcher-output-bounds.test.ts new file mode 100644 index 00000000000..2bab56bbd08 --- /dev/null +++ b/src/main/runtime/rpc/dispatcher-output-bounds.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from 'vitest' +import { REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES } from '../../../shared/remote-runtime-memory-limits' +import type { OrcaRuntimeService } from '../orca-runtime' +import { defineMethod, defineStreamingMethod, type RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' + +const request = (method: string): RpcRequest => ({ + id: 'request-1', + authToken: 'unused', + method +}) + +function runtime(): OrcaRuntimeService { + return { + getRuntimeId: () => 'runtime-1', + recordFeatureInteraction: vi.fn() + } as unknown as OrcaRuntimeService +} + +describe('RpcDispatcher outbound response bounds', () => { + it('preserves ordinary streaming response serialization byte-for-byte', async () => { + const dispatcher = new RpcDispatcher({ + runtime: runtime(), + methods: [ + defineMethod({ + name: 'test.normal', + params: null, + handler: () => ({ text: 'hello 😀' }) + }) + ] + }) + const reply = vi.fn() + + await dispatcher.dispatchStreaming(request('test.normal'), reply) + + expect(reply).toHaveBeenCalledWith( + JSON.stringify({ + id: 'request-1', + ok: true, + result: { text: 'hello 😀' }, + _meta: { runtimeId: 'runtime-1' } + }) + ) + }) + + it('replaces an oversized one-shot result with a small correlated error', async () => { + const dispatcher = new RpcDispatcher({ + runtime: runtime(), + methods: [ + defineMethod({ + name: 'test.oversized', + params: null, + handler: () => ({ text: 'x'.repeat(REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES) }) + }) + ] + }) + + const response = await dispatcher.dispatch(request('test.oversized')) + + expect(response).toMatchObject({ + id: 'request-1', + ok: false, + error: { code: 'response_too_large' }, + _meta: { runtimeId: 'runtime-1' } + }) + expect(Buffer.byteLength(JSON.stringify(response), 'utf8')).toBeLessThan( + REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES + ) + }) + + it('bounds an oversized streaming emission before calling the reply transport', async () => { + const dispatcher = new RpcDispatcher({ + runtime: runtime(), + methods: [ + defineStreamingMethod({ + name: 'test.stream-oversized', + params: null, + handler: async (_params, _context, emit) => { + emit({ text: 'x'.repeat(REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES) }) + } + }) + ] + }) + const reply = vi.fn() + + await dispatcher.dispatchStreaming(request('test.stream-oversized'), reply) + + expect(reply).toHaveBeenCalledOnce() + const response = JSON.parse(reply.mock.calls[0]![0] as string) as { + id: string + ok: boolean + error: { code: string } + } + expect(response).toMatchObject({ + id: 'request-1', + ok: false, + error: { code: 'response_too_large' } + }) + }) +}) diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 5b677088074..da6c9ab2237 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -29,6 +29,10 @@ import { import { ALL_RPC_METHODS } from './methods' import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe' import type { OrcaRuntimeService } from '../orca-runtime' +import { + boundRuntimeRpcResponse, + serializeRuntimeRpcResponse +} from './runtime-rpc-response-serialization' export type DispatcherOptions = { runtime: OrcaRuntimeService @@ -48,28 +52,27 @@ export class RpcDispatcher { const meta = this.meta() const method = this.registry.get(request.method) if (!method) { - return errorResponse( - request.id, - meta, - 'method_not_found', - `Unknown method: ${request.method}` + return boundRuntimeRpcResponse( + errorResponse(request.id, meta, 'method_not_found', `Unknown method: ${request.method}`) ) } const parsedParams = this.parseParams(request, method, meta) if (parsedParams.error) { - return parsedParams.error + return boundRuntimeRpcResponse(parsedParams.error) } // Why: streaming methods are not supported over one-shot transports like // Unix sockets. They require a reply function that can be called multiple // times, which is only available via dispatchStreaming. if (isStreamingMethod(method)) { - return errorResponse( - request.id, - meta, - 'method_not_supported', - `Method ${request.method} requires a streaming transport` + return boundRuntimeRpcResponse( + errorResponse( + request.id, + meta, + 'method_not_supported', + `Method ${request.method} requires a streaming transport` + ) ) } @@ -83,12 +86,12 @@ export class RpcDispatcher { signal: options?.signal }) this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) - return successResponse(request.id, meta, result) + return boundRuntimeRpcResponse(successResponse(request.id, meta, result)) } catch (error) { if (isEmulator) { emulatorProbeError(`rpc ${request.method}`, error, { params: request.params }) } - return this.mapError(request, meta, error) + return boundRuntimeRpcResponse(this.mapError(request, meta, error)) } } @@ -116,7 +119,7 @@ export class RpcDispatcher { const method = this.registry.get(request.method) if (!method) { reply( - JSON.stringify( + serializeRuntimeRpcResponse( errorResponse(request.id, meta, 'method_not_found', `Unknown method: ${request.method}`) ) ) @@ -125,7 +128,7 @@ export class RpcDispatcher { const parsedParams = this.parseParams(request, method, meta) if (parsedParams.error) { - reply(JSON.stringify(parsedParams.error)) + reply(serializeRuntimeRpcResponse(parsedParams.error)) return } @@ -144,9 +147,9 @@ export class RpcDispatcher { registerBinaryStreamHandler: options?.registerBinaryStreamHandler }) this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) - reply(JSON.stringify(successResponse(request.id, meta, result))) + reply(serializeRuntimeRpcResponse(successResponse(request.id, meta, result))) } catch (error) { - reply(JSON.stringify(this.mapError(request, meta, error))) + reply(serializeRuntimeRpcResponse(this.mapError(request, meta, error))) } return } @@ -161,7 +164,7 @@ export class RpcDispatcher { ) const response = successResponse(request.id, meta, result) response.streaming = true - reply(JSON.stringify(response)) + reply(serializeRuntimeRpcResponse(response)) } try { @@ -188,7 +191,7 @@ export class RpcDispatcher { request.params ) } catch (error) { - reply(JSON.stringify(this.mapError(request, meta, error))) + reply(serializeRuntimeRpcResponse(this.mapError(request, meta, error))) } } diff --git a/src/main/runtime/rpc/e2ee-channel-text-backpressure.test.ts b/src/main/runtime/rpc/e2ee-channel-text-backpressure.test.ts index 4c45e97ea10..4e4b5f52ecb 100644 --- a/src/main/runtime/rpc/e2ee-channel-text-backpressure.test.ts +++ b/src/main/runtime/rpc/e2ee-channel-text-backpressure.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { WebSocket } from 'ws' import { E2EEChannel, type E2EEChannelOptions } from './e2ee-channel' import { deriveSharedKey, decrypt, encrypt, generateKeyPair } from './e2ee-crypto' +import { createMobileE2EEOutboundMemoryBudget } from './mobile-e2ee-outbound-memory-budget' // Repro for gap (a): the streaming JSON reply path (encryptedReply) had no // bufferedAmount gate, so a fast producer over a slow link (legacy @@ -98,4 +99,24 @@ describe('E2EE text reply backpressure', () => { expect(ctx.ws.sent.length).toBe(baseline + 1) expect(decrypt(ctx.ws.sent[baseline]!, ctx.sharedKey)).toBe('{"ok":true}') }) + + it('rejects aggregate queue growth across independently backpressured sockets', () => { + const outboundMemoryBudget = createMobileE2EEOutboundMemoryBudget({ + maxBufferedBytes: 1_000, + maxQueuedBytes: 150, + maxQueuedFrames: 10 + }) + const first = setup({ outboundMemoryBudget }) + const second = setup({ outboundMemoryBudget }) + first.ws.bufferedAmount = 1_001 + second.ws.bufferedAmount = 1_001 + + emitReply(first, 'x'.repeat(40)) + emitReply(second, 'x'.repeat(40)) + + expect(first.onError).not.toHaveBeenCalled() + expect(second.onError).toHaveBeenCalledWith(1013, 'Outbound reply buffer overflow') + first.channel.destroy() + expect(outboundMemoryBudget.evidence().queuedBytes).toBe(0) + }) }) diff --git a/src/main/runtime/rpc/e2ee-channel.test.ts b/src/main/runtime/rpc/e2ee-channel.test.ts index 400a14de86d..8ca3c2062cf 100644 --- a/src/main/runtime/rpc/e2ee-channel.test.ts +++ b/src/main/runtime/rpc/e2ee-channel.test.ts @@ -2,6 +2,11 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' import type { WebSocket } from 'ws' import { E2EEChannel, type E2EEChannelOptions } from './e2ee-channel' import { generateKeyPair, deriveSharedKey, encrypt, decrypt, encryptBytes } from './e2ee-crypto' +import { + REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES, + REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES +} from '../../../shared/remote-runtime-memory-limits' +import { REMOTE_RUNTIME_JSON_STRUCTURE_LIMITS } from '../../../shared/remote-runtime-request-frames' function publicKeyToBase64(key: Uint8Array): string { return Buffer.from(key).toString('base64') @@ -140,6 +145,39 @@ describe('E2EEChannel', () => { expect(ctx.onError).toHaveBeenCalledWith(4001, 'Invalid handshake message') }) + it('rejects structurally amplified hello JSON before parsing', () => { + const ctx = setup() + const amplified = `{"type":"e2ee_hello","padding":[${'0,'.repeat(REMOTE_RUNTIME_JSON_STRUCTURE_LIMITS.structuralTokens)}0]}` + const parse = vi.spyOn(JSON, 'parse') + + ctx.channel.handleRawMessage(amplified) + + expect(ctx.onError).toHaveBeenCalledWith(4001, 'Invalid handshake message') + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + ctx.channel.destroy() + }) + + it('rejects structurally amplified auth JSON before parsing', () => { + const ctx = setup() + ctx.channel.handleRawMessage( + JSON.stringify({ + type: 'e2ee_hello', + publicKeyB64: publicKeyToBase64(ctx.clientKeys.publicKey) + }) + ) + const sharedKey = deriveSharedKey(ctx.clientKeys.secretKey, ctx.serverKeys.publicKey) + const amplified = `{"type":"e2ee_auth","deviceToken":"valid-token","padding":[${'0,'.repeat(REMOTE_RUNTIME_JSON_STRUCTURE_LIMITS.structuralTokens)}0]}` + const parse = vi.spyOn(JSON, 'parse') + + ctx.channel.handleRawMessage(encrypt(amplified, sharedKey)) + + expect(ctx.onError).toHaveBeenCalledWith(4001, 'Invalid e2ee_auth') + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + ctx.channel.destroy() + }) + it('rejects missing fields', () => { const ctx = setup() ctx.channel.handleRawMessage(JSON.stringify({ type: 'e2ee_hello' })) @@ -159,6 +197,19 @@ describe('E2EEChannel', () => { expect(ctx.onError).toHaveBeenCalledWith(4001, 'Invalid public key') }) + it('rejects oversized public key text before base64 decoding', () => { + const ctx = setup() + const decode = vi.spyOn(Buffer, 'from') + + ctx.channel.handleRawMessage( + JSON.stringify({ type: 'e2ee_hello', publicKeyB64: 'A'.repeat(45) }) + ) + + expect(ctx.onError).toHaveBeenCalledWith(4001, 'Invalid public key') + expect(decode).not.toHaveBeenCalled() + decode.mockRestore() + }) + it('times out if no hello received', () => { const ctx = setup() @@ -209,6 +260,38 @@ describe('E2EEChannel', () => { expect(replyPlain).toBe('{"id":"rpc-1","ok":true}') }) + it('rejects an oversized text reply before encryption', () => { + const ctx = setup() + const sharedKey = doHandshake(ctx) + const sentBefore = ctx.ws.sent.length + + ctx.channel.onMessage((_plaintext, encryptedReply) => { + encryptedReply('x'.repeat(REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES + 1)) + }) + ctx.channel.handleRawMessage(encrypt('request', sharedKey)) + + expect(ctx.onError).toHaveBeenCalledWith(1013, 'Outbound reply buffer overflow') + expect(ctx.ws.sent).toHaveLength(sentBefore) + }) + + it('rejects an oversized binary reply before encryption', () => { + const ctx = setup() + const sharedKey = doHandshake(ctx) + const sentBefore = ctx.ws.sent.length + let accepted: boolean | void = undefined + + ctx.channel.onMessage((_plaintext, _encryptedReply, encryptedBinaryReply) => { + accepted = encryptedBinaryReply( + new Uint8Array(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1) + ) + }) + ctx.channel.handleRawMessage(encrypt('request', sharedKey)) + + expect(accepted).toBe(false) + expect(ctx.onError).toHaveBeenCalledWith(1013, 'Outbound reply buffer overflow') + expect(ctx.ws.sent).toHaveLength(sentBefore) + }) + it('decrypts and forwards binary messages after authentication', () => { const ctx = setup() const sharedKey = doHandshake(ctx) diff --git a/src/main/runtime/rpc/e2ee-channel.ts b/src/main/runtime/rpc/e2ee-channel.ts index eeb6097b4e3..95ae6f8d8cc 100644 --- a/src/main/runtime/rpc/e2ee-channel.ts +++ b/src/main/runtime/rpc/e2ee-channel.ts @@ -1,26 +1,24 @@ // Why: this channel keeps E2EE framing out of RPC handlers, which consume plaintext across transports. import type { WebSocket } from 'ws' import { deriveSharedKey, encrypt, decrypt, encryptBytes, decryptBytes } from './e2ee-crypto' -import { - createWsOutboundBackpressureQueue, - type WsOutboundBackpressureQueue -} from '../../../shared/ws-outbound-backpressure-queue' import { DesktopMobileE2EEV2Session, type DesktopMobileE2EEV2Context } from './mobile-e2ee-v2-desktop-session' -import { - createDesktopMobileE2EEV2OutboundQueue, - type DesktopMobileE2EEV2OutboundItem as V2OutboundItem -} from './mobile-e2ee-v2-desktop-outbound' +import type { DesktopMobileE2EEV2OutboundItem as V2OutboundItem } from './mobile-e2ee-v2-desktop-outbound' import { handleDesktopMobileE2EEV2Inbound } from './mobile-e2ee-v2-desktop-inbound' -import { isValidMobileE2EEAuthVersion, type MobileE2EEAuth } from './mobile-e2ee-auth-validation' - -type ChannelState = 'awaiting_hello' | 'awaiting_auth' | 'ready' +import { authenticateMobileE2EE, decodeMobileE2EEPublicKey } from './mobile-e2ee-auth-validation' +import { + isMobileE2EEBinaryPayloadWithinLimit, + isMobileE2EEOutboundItemWithinLimit, + isMobileE2EETextPayloadWithinLimit +} from './mobile-e2ee-outbound-admission' +import { parseRemoteRuntimeJsonText } from '../../../shared/remote-runtime-request-frames' +import type { MobileE2EEOutboundMemoryBudget } from './mobile-e2ee-outbound-memory-budget' +import { MobileE2EEDesktopOutboundOwner } from './mobile-e2ee-desktop-outbound-owner' const HANDSHAKE_TIMEOUT_MS = 10_000 const MAX_CONSECUTIVE_DECRYPT_FAILURES = 5 -const MAX_BINARY_BUFFERED_AMOUNT = 8 * 1024 * 1024 export type E2EEChannelOptions = { serverSecretKey: Uint8Array @@ -29,6 +27,7 @@ export type E2EEChannelOptions = { onError: (code: number, reason: string) => void transportContext?: DesktopMobileE2EEV2Context requireV2?: boolean + outboundMemoryBudget?: MobileE2EEOutboundMemoryBudget } export type E2EEAuthenticatedDevice = { @@ -38,7 +37,7 @@ export type E2EEAuthenticatedDevice = { } export class E2EEChannel { - private state: ChannelState = 'awaiting_hello' + private state: 'awaiting_hello' | 'awaiting_auth' | 'ready' = 'awaiting_hello' private sharedKey: Uint8Array | null = null private consecutiveFailures = 0 private handshakeTimer: ReturnType | null = null @@ -49,8 +48,8 @@ export class E2EEChannel { private readonly onError: (code: number, reason: string) => void private readonly transportContext: DesktopMobileE2EEV2Context private readonly requireV2: boolean + private readonly outbound: MobileE2EEDesktopOutboundOwner private v2Session: DesktopMobileE2EEV2Session | null = null - private v2OutboundQueue: WsOutboundBackpressureQueue | null = null // Why: the handler is set after readiness because its reply closure needs this channel's encryption state. private messageHandler: | (( @@ -60,11 +59,6 @@ export class E2EEChannel { ) => void) | null = null private binaryMessageHandler: ((plaintext: Uint8Array) => void) | null = null - // Why: the streaming JSON reply path (e.g. legacy terminal.subscribe) has no - // seq/resync, so it must never drop frames under backpressure. Hold text - // replies in order while bufferedAmount is over the cap and drain as it - // clears; only a wedged link (hard cap) closes the socket for a clean resync. - private textReplyQueue: WsOutboundBackpressureQueue | null = null deviceToken: string | null = null authenticatedDevice: E2EEAuthenticatedDevice | null = null @@ -77,6 +71,7 @@ export class E2EEChannel { this.onError = options.onError this.transportContext = options.transportContext ?? { transport: 'direct' } this.requireV2 = options.requireV2 ?? false + this.outbound = new MobileE2EEDesktopOutboundOwner(ws, options.outboundMemoryBudget) this.handshakeTimer = setTimeout(() => { this.onError(4002, 'E2EE handshake timeout') @@ -143,23 +138,30 @@ export class E2EEChannel { return } - // Why: streaming RPC handlers (e.g. terminal.subscribe) retain this - // closure and may fire emits long after the inbound message handled - // here. If destroy() runs in between (mobile disconnect, handshake - // failure) sharedKey becomes null and tweetnacl throws "unexpected - // type, use Uint8Array" from inside nacl.box.after. Guard both the - // socket state AND the key so late emits become silent no-ops. + // Why: streaming emits can outlive destroy(), so late replies must not encrypt with a cleared key. const encryptedReply = (response: string) => { if (!this.sharedKey || this.ws.readyState !== this.ws.OPEN) { return } - this.ensureTextReplyQueue().enqueue(encrypt(response, this.sharedKey)) + if (!isMobileE2EETextPayloadWithinLimit(response)) { + this.onError(1013, 'Outbound reply buffer overflow') + return + } + this.outbound.enqueueLegacyText( + encrypt(response, this.sharedKey), + () => Boolean(this.sharedKey), + () => this.onError(1013, 'Outbound reply buffer overflow') + ) } const encryptedBinaryReply = (response: Uint8Array): boolean => { if (!this.sharedKey || this.ws.readyState !== this.ws.OPEN) { return false } - if (this.ws.bufferedAmount > MAX_BINARY_BUFFERED_AMOUNT) { + if (!isMobileE2EEBinaryPayloadWithinLimit(response)) { + this.onError(1013, 'Outbound reply buffer overflow') + return false + } + if (!this.outbound.canSend(response.byteLength + 40)) { return false } this.ws.send(Buffer.from(encryptBytes(response, this.sharedKey)), { binary: true }) @@ -180,7 +182,7 @@ export class E2EEChannel { private handleHello(raw: string): void { let hello: Record try { - hello = JSON.parse(raw) as Record + hello = parseRemoteRuntimeJsonText(raw) as Record } catch { this.onError(4001, 'Invalid handshake message') return @@ -215,8 +217,8 @@ export class E2EEChannel { // Why: derive the shared key from our secret + client's public key. // Both sides compute the same shared secret via ECDH. - const clientPublicKey = Uint8Array.from(Buffer.from(hello.publicKeyB64, 'base64')) - if (clientPublicKey.length !== 32) { + const clientPublicKey = decodeMobileE2EEPublicKey(hello.publicKeyB64) + if (!clientPublicKey) { this.onError(4001, 'Invalid public key') return } @@ -232,32 +234,19 @@ export class E2EEChannel { } private handleAuth(plaintext: string): void { - let auth: MobileE2EEAuth - try { - auth = JSON.parse(plaintext) as MobileE2EEAuth - } catch { - this.sendEncryptedControl({ type: 'e2ee_error', error: { code: 'bad_auth' } }) - this.onError(4001, 'Invalid e2ee_auth') + const authentication = authenticateMobileE2EE({ + plaintext, + v2Session: this.v2Session, + resolveDevice: this.resolveAuthenticatedDevice + }) + if (!authentication.ok) { + this.sendEncryptedControl({ type: 'e2ee_error', error: { code: authentication.code } }) + this.onError(4001, authentication.code === 'bad_auth' ? 'Invalid e2ee_auth' : 'Unauthorized') return } + const authenticatedDevice = authentication.device - if ( - auth.type !== 'e2ee_auth' || - !auth.deviceToken || - !isValidMobileE2EEAuthVersion(auth, this.v2Session) - ) { - this.sendEncryptedControl({ type: 'e2ee_error', error: { code: 'bad_auth' } }) - this.onError(4001, 'Invalid e2ee_auth') - return - } - const authenticatedDevice = this.resolveAuthenticatedDevice(auth.deviceToken) - if (!authenticatedDevice || authenticatedDevice.deviceToken !== auth.deviceToken) { - this.sendEncryptedControl({ type: 'e2ee_error', error: { code: 'unauthorized' } }) - this.onError(4001, 'Unauthorized') - return - } - - this.deviceToken = auth.deviceToken + this.deviceToken = authenticatedDevice.deviceToken this.authenticatedDevice = authenticatedDevice this.state = 'ready' @@ -294,47 +283,33 @@ export class E2EEChannel { this.messageHandler?.( plaintext, (response) => this.enqueueV2({ kind: 'text', plaintext: response }), - (response) => (this.enqueueV2({ kind: 'binary', plaintext: response }), true) + (response) => this.enqueueV2({ kind: 'binary', plaintext: response }) ), onProtocolError: () => this.onError(4001, 'Invalid binary message before authentication') }) } - private enqueueV2(item: V2OutboundItem): void { + private enqueueV2(item: V2OutboundItem): boolean { if (!this.v2Session || this.ws.readyState !== this.ws.OPEN) { - return + return false } - if (!this.v2OutboundQueue) { - this.v2OutboundQueue = createDesktopMobileE2EEV2OutboundQueue({ - ws: this.ws, - session: this.v2Session, - onOverflow: () => this.onError(1013, 'Outbound reply buffer overflow') - }) + if (!isMobileE2EEOutboundItemWithinLimit(item)) { + this.onError(1013, 'Outbound reply buffer overflow') + return false } - this.v2OutboundQueue.enqueue(item) - } - - private ensureTextReplyQueue(): WsOutboundBackpressureQueue { - if (!this.textReplyQueue) { - this.textReplyQueue = createWsOutboundBackpressureQueue({ - send: (frame) => this.ws.send(frame), - // Encrypted replies are base64 ASCII strings, so length === byte count. - byteLengthOf: (frame) => frame.length, - getBufferedAmount: () => this.ws.bufferedAmount, - isWritable: () => Boolean(this.sharedKey) && this.ws.readyState === this.ws.OPEN, - // 1013 (Try Again Later): the link is wedged; drop the channel so the - // client reconnects and replays a full snapshot instead of unbounded RSS. - onOverflow: () => this.onError(1013, 'Outbound reply buffer overflow') - }) - } - return this.textReplyQueue + return this.outbound.enqueueV2(item, this.v2Session, () => + this.onError(1013, 'Outbound reply buffer overflow') + ) } private sendEncryptedControl(message: unknown): void { if (this.v2Session) { this.enqueueV2({ kind: 'text', plaintext: JSON.stringify(message) }) } else if (this.ws.readyState === this.ws.OPEN && this.sharedKey) { - this.ws.send(encrypt(JSON.stringify(message), this.sharedKey)) + const frame = encrypt(JSON.stringify(message), this.sharedKey) + this.outbound.sendLegacyFrame(frame, () => + this.onError(1013, 'Outbound reply buffer overflow') + ) } } @@ -348,9 +323,6 @@ export class E2EEChannel { this.v2Session = null this.messageHandler = null this.binaryMessageHandler = null - this.textReplyQueue?.dispose() - this.textReplyQueue = null - this.v2OutboundQueue?.dispose() - this.v2OutboundQueue = null + this.outbound.dispose() } } diff --git a/src/main/runtime/rpc/e2ee-crypto.test.ts b/src/main/runtime/rpc/e2ee-crypto.test.ts index 93fc12ff063..bdc61e4c29c 100644 --- a/src/main/runtime/rpc/e2ee-crypto.test.ts +++ b/src/main/runtime/rpc/e2ee-crypto.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import nacl from 'tweetnacl' import { generateKeyPair, @@ -6,7 +6,9 @@ import { encrypt, decrypt, encryptBytes, - decryptBytes + decryptBytes, + MAX_E2EE_ENCRYPTED_BASE64_CHARACTERS, + publicKeyFromBase64 } from './e2ee-crypto' import { MOBILE_E2EE_LEGACY_FIXTURE } from '../../../shared/mobile-e2ee-legacy-fixtures' @@ -88,6 +90,16 @@ describe('e2ee-crypto', () => { expect(decrypt('dG9vc2hvcnQ=', shared)).toBeNull() }) + it('rejects oversized encoded inputs before base64 decoding', () => { + const shared = deriveSharedKey(generateKeyPair().secretKey, generateKeyPair().publicKey) + const decode = vi.spyOn(Buffer, 'from') + + expect(() => publicKeyFromBase64('A'.repeat(45))).toThrow('encoded value is too large') + expect(decrypt('A'.repeat(MAX_E2EE_ENCRYPTED_BASE64_CHARACTERS + 1), shared)).toBeNull() + expect(decode).not.toHaveBeenCalled() + decode.mockRestore() + }) + it('decrypt returns null for tampered data', () => { const server = generateKeyPair() const client = generateKeyPair() diff --git a/src/main/runtime/rpc/e2ee-crypto.ts b/src/main/runtime/rpc/e2ee-crypto.ts index f0f4f460d83..81e8c0f9755 100644 --- a/src/main/runtime/rpc/e2ee-crypto.ts +++ b/src/main/runtime/rpc/e2ee-crypto.ts @@ -4,5 +4,7 @@ export { deriveSharedKey, encrypt, encryptBytes, - generateKeyPair + generateKeyPair, + MAX_E2EE_ENCRYPTED_BASE64_CHARACTERS, + publicKeyFromBase64 } from '../../../shared/e2ee-crypto' diff --git a/src/main/runtime/rpc/errors.test.ts b/src/main/runtime/rpc/errors.test.ts index 0e4519656cb..babc255d5ed 100644 --- a/src/main/runtime/rpc/errors.test.ts +++ b/src/main/runtime/rpc/errors.test.ts @@ -1,4 +1,8 @@ import { describe, expect, it } from 'vitest' +import { + MARKDOWN_DOCUMENT_LISTING_ERROR_CODE, + MarkdownDocumentListingCapacityError +} from '../../../shared/markdown-document-listing-limits' import { mapRuntimeError } from './errors' class LineageError extends Error { @@ -145,4 +149,19 @@ describe('mapRuntimeError', () => { _meta: { runtimeId: 'runtime-1' } }) }) + + it('preserves the Markdown listing capacity code across runtime RPC', () => { + expect( + mapRuntimeError( + 'req_1', + { runtimeId: 'runtime-1' }, + new MarkdownDocumentListingCapacityError() + ) + ).toMatchObject({ + ok: false, + error: { + code: MARKDOWN_DOCUMENT_LISTING_ERROR_CODE + } + }) + }) }) diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts index 39190c6dfcb..fdd4f143d83 100644 --- a/src/main/runtime/rpc/errors.ts +++ b/src/main/runtime/rpc/errors.ts @@ -7,6 +7,7 @@ import { computerUseErrorRecoveryData } from '../../../shared/computer-use-error import { COMPUTER_ERROR_CODES } from '../../../shared/runtime-types' import { LINEAR_ERROR_CODES } from '../../../shared/linear-agent-access' import { AGENT_SESSION_RPC_ERROR_CODES } from '../../../shared/agent-session-host-authority' +import { MARKDOWN_DOCUMENT_LISTING_ERROR_CODE } from '../../../shared/markdown-document-listing-limits' export function successResponse(id: string, meta: RpcEnvelopeMeta, result: unknown): RpcSuccess { return { @@ -60,7 +61,8 @@ const RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ const COMPUTER_PASSTHROUGH_CODES: ReadonlySet = new Set(Object.values(COMPUTER_ERROR_CODES)) const LINEAR_PASSTHROUGH_CODES: ReadonlySet = new Set(LINEAR_ERROR_CODES) const STRUCTURED_RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ - 'worktree_id_requires_full_path' + 'worktree_id_requires_full_path', + MARKDOWN_DOCUMENT_LISTING_ERROR_CODE ]) export function mapRuntimeError(id: string, meta: RpcEnvelopeMeta, error: unknown): RpcFailure { diff --git a/src/main/runtime/rpc/methods/ai-vault.test.ts b/src/main/runtime/rpc/methods/ai-vault.test.ts index a622ea4f835..1ecc168dc78 100644 --- a/src/main/runtime/rpc/methods/ai-vault.test.ts +++ b/src/main/runtime/rpc/methods/ai-vault.test.ts @@ -4,6 +4,7 @@ import type { RpcRequest } from '../core' import { OrcaRuntimeService } from '../../orca-runtime' import type { AiVaultListResult, AiVaultSession } from '../../../../shared/ai-vault-types' import type { AiVaultScanOptions } from '../../../ai-vault/session-scanner-types' +import { AI_VAULT_SESSION_ID_MAX_UTF8_BYTES } from '../../../ai-vault/session-list-retention' const { scanAiVaultSessions } = vi.hoisted(() => ({ scanAiVaultSessions: vi.fn() @@ -181,6 +182,24 @@ describe('aiVault.listSessions handler + shared cache', () => { expect(scanAiVaultSessions).toHaveBeenCalledTimes(1) }) + it('bounds a pathological result before retaining it in the shared cache', async () => { + const oversized = makeSession() + oversized.sessionId = 'x'.repeat(AI_VAULT_SESSION_ID_MAX_UTF8_BYTES + 1) + scanAiVaultSessions.mockResolvedValue({ + sessions: [oversized], + issues: [], + scannedAt: SCANNED_AT + }) + + const first = await listAiVaultSessions({ limit: 500 }) + const second = await listAiVaultSessions({ limit: 500 }) + + expect(first.sessions).toEqual([]) + expect(first.issues.at(-1)?.message).toContain('AI Vault omitted') + expect(second).toBe(first) + expect(scanAiVaultSessions).toHaveBeenCalledTimes(1) + }) + it('keeps a newer different-key scan dedupable after an older scan resolves', async () => { // Why: the resolving scan's cleanup must not clear tracking a concurrent // different-key scan replaced, or re-requests start a duplicate rescan. diff --git a/src/main/runtime/rpc/methods/clipboard-image-upload-buffer.ts b/src/main/runtime/rpc/methods/clipboard-image-upload-buffer.ts new file mode 100644 index 00000000000..eac0c809a33 --- /dev/null +++ b/src/main/runtime/rpc/methods/clipboard-image-upload-buffer.ts @@ -0,0 +1,121 @@ +const BASE64_PADDING_BYTE = '='.charCodeAt(0) + +function isBase64DataByte(byte: number): boolean { + return ( + (byte >= 65 && byte <= 90) || + (byte >= 97 && byte <= 122) || + (byte >= 48 && byte <= 57) || + byte === 43 || + byte === 47 + ) +} + +export class ClipboardImageUploadBuffer { + private readonly segments: Buffer[] = [] + private retainedLength = 0 + + constructor( + private readonly expectedLength: number, + private readonly segmentLength: number + ) {} + + get length(): number { + return this.retainedLength + } + + append(contentBase64: string): void { + if (contentBase64.length > this.expectedLength - this.retainedLength) { + throw new Error('Clipboard image upload exceeded expected size') + } + const source = Buffer.from(contentBase64, 'ascii') + const originalLength = this.retainedLength + const originalSegmentCount = this.segments.length + let sourceOffset = 0 + try { + while (sourceOffset < source.length) { + const segmentIndex = Math.floor(this.retainedLength / this.segmentLength) + const segmentOffset = this.retainedLength % this.segmentLength + let segment = this.segments[segmentIndex] + if (!segment) { + const segmentStart = segmentIndex * this.segmentLength + segment = Buffer.allocUnsafe( + Math.min(this.segmentLength, this.expectedLength - segmentStart) + ) + this.segments.push(segment) + } + const copyLength = Math.min(source.length - sourceOffset, segment.length - segmentOffset) + source.copy(segment, segmentOffset, sourceOffset, sourceOffset + copyLength) + sourceOffset += copyLength + this.retainedLength += copyLength + } + } catch (error) { + this.retainedLength = originalLength + this.segments.length = originalSegmentCount + throw error + } + } + + clear(): void { + this.retainedLength = 0 + this.segments.length = 0 + } + + decode(): Buffer { + const dataLength = this.validateAndGetDataLength() + const decoded = Buffer.allocUnsafe(Math.floor((dataLength * 3) / 4)) + let carry = '' + let remainingData = dataLength + let written = 0 + for (const segment of this.segments) { + if (remainingData === 0) { + break + } + const segmentDataLength = Math.min(segment.length, remainingData) + const data = segment.subarray(0, segmentDataLength).toString('ascii') + const combined = carry.length > 0 ? carry + data : data + const completeLength = combined.length - (combined.length % 4) + if (completeLength > 0) { + const complete = + completeLength === combined.length ? combined : combined.slice(0, completeLength) + written += decoded.write(complete, written, 'base64') + } + carry = completeLength === combined.length ? '' : combined.slice(completeLength) + remainingData -= segmentDataLength + } + if (carry.length > 0) { + written += decoded.write(carry, written, 'base64') + } + if (written !== decoded.length) { + throw new Error('Clipboard image content must be base64') + } + return decoded + } + + private validateAndGetDataLength(): number { + if (this.retainedLength % 4 === 1) { + throw new Error('Clipboard image content must be base64') + } + let dataLength = 0 + let paddingLength = 0 + let remaining = this.retainedLength + for (const segment of this.segments) { + const usedLength = Math.min(segment.length, remaining) + for (let index = 0; index < usedLength; index++) { + const byte = segment[index] + if (byte === BASE64_PADDING_BYTE) { + paddingLength += 1 + continue + } + if (paddingLength > 0 || !isBase64DataByte(byte)) { + throw new Error('Clipboard image content must be base64') + } + dataLength += 1 + } + remaining -= usedLength + } + if (paddingLength > 2) { + throw new Error('Clipboard image content must be base64') + } + return dataLength + } +} diff --git a/src/main/runtime/rpc/methods/clipboard.test.ts b/src/main/runtime/rpc/methods/clipboard.test.ts index 118b21c766a..a528239f000 100644 --- a/src/main/runtime/rpc/methods/clipboard.test.ts +++ b/src/main/runtime/rpc/methods/clipboard.test.ts @@ -4,6 +4,7 @@ import type { RpcRequest } from '../core' import type { OrcaRuntimeService } from '../../orca-runtime' import { CLIPBOARD_IMAGE_MAX_BASE64_CHARS, + CLIPBOARD_IMAGE_MAX_SOURCE_BYTES, CLIPBOARD_IMAGE_TOO_LARGE_ERROR } from '../../../../shared/clipboard-image' @@ -18,7 +19,9 @@ vi.mock('../../../window/clipboard-image-temp-file', () => ({ import { CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS, CLIPBOARD_IMAGE_UPLOAD_MAX_CONCURRENT, + CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS, CLIPBOARD_METHODS, + getRetainedClipboardImageUploadBase64CharsForTest, resetClipboardImageUploadsForTest } from './clipboard' @@ -100,7 +103,7 @@ describe('clipboard RPC methods', () => { it('accepts chunked uploads and forwards the recorded connectionId on commit', async () => { saveClipboardImageBufferAsTempFile.mockResolvedValue('/tmp/orca-paste-image.png') const dispatcher = makeDispatcher() - const contentBase64 = Buffer.from('png-bytes').toString('base64') + const contentBase64 = Buffer.from('png-byte').toString('base64') const start = await dispatcher.dispatch( makeRequest('clipboard.startImageUpload', { @@ -111,8 +114,8 @@ describe('clipboard RPC methods', () => { expect(start.ok).toBe(true) const uploadId = (start.ok ? start.result : null) as { uploadId: string } - const firstChunk = contentBase64.slice(0, 4) - const secondChunk = contentBase64.slice(4) + const firstChunk = contentBase64.slice(0, 2) + const secondChunk = contentBase64.slice(2) await expect( dispatcher.dispatch( makeRequest('clipboard.appendImageUploadChunk', { @@ -121,7 +124,7 @@ describe('clipboard RPC methods', () => { contentBase64: firstChunk }) ) - ).resolves.toMatchObject({ ok: true, result: { receivedBase64Length: 4 } }) + ).resolves.toMatchObject({ ok: true, result: { receivedBase64Length: 2 } }) await expect( dispatcher.dispatch( makeRequest('clipboard.appendImageUploadChunk', { @@ -131,13 +134,15 @@ describe('clipboard RPC methods', () => { }) ) ).resolves.toMatchObject({ ok: true, result: { receivedBase64Length: contentBase64.length } }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(contentBase64.length) await expect( dispatcher.dispatch( makeRequest('clipboard.commitImageUpload', { uploadId: uploadId.uploadId }) ) ).resolves.toMatchObject({ ok: true, result: '/tmp/orca-paste-image.png' }) - expect(saveClipboardImageBufferAsTempFile).toHaveBeenCalledWith(Buffer.from('png-bytes'), { + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) + expect(saveClipboardImageBufferAsTempFile).toHaveBeenCalledWith(Buffer.from('png-byte'), { connectionId: 'ssh-1' }) }) @@ -161,6 +166,7 @@ describe('clipboard RPC methods', () => { ) expect(response.ok).toBe(false) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) expect(saveClipboardImageBufferAsTempFile).not.toHaveBeenCalled() }) @@ -183,6 +189,7 @@ describe('clipboard RPC methods', () => { }) ) ).resolves.toMatchObject({ ok: false }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) await expect( dispatcher.dispatch( makeRequest('clipboard.appendImageUploadChunk', { @@ -258,6 +265,7 @@ describe('clipboard RPC methods', () => { ) expect(response.ok).toBe(false) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) expect(saveClipboardImageBufferAsTempFile).not.toHaveBeenCalled() }) @@ -290,6 +298,7 @@ describe('clipboard RPC methods', () => { ) expect(response.ok).toBe(false) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) expect(saveClipboardImageBufferAsTempFile).not.toHaveBeenCalled() }) @@ -302,12 +311,21 @@ describe('clipboard RPC methods', () => { }) ) const uploadId = (start.ok ? start.result : null) as { uploadId: string } + await dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: uploadId.uploadId, + offset: 0, + contentBase64: 'AAAA' + }) + ) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(4) await expect( dispatcher.dispatch( makeRequest('clipboard.abortImageUpload', { uploadId: uploadId.uploadId }) ) ).resolves.toMatchObject({ ok: true, result: { aborted: true } }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) await expect( dispatcher.dispatch( makeRequest('clipboard.abortImageUpload', { uploadId: uploadId.uploadId }) @@ -349,22 +367,137 @@ describe('clipboard RPC methods', () => { makeRequest('clipboard.commitImageUpload', { uploadId: uploadId.uploadId }) ) ).resolves.toMatchObject({ ok: false }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) expect(saveClipboardImageBufferAsTempFile).toHaveBeenCalledTimes(1) }) + it('caps retained chunks process-wide and keeps commit memory charged until save completes', async () => { + let releaseSave!: () => void + const saveGate = new Promise((resolve) => { + releaseSave = resolve + }) + saveClipboardImageBufferAsTempFile.mockImplementation(async (content: Buffer) => { + expect(content).toHaveLength(CLIPBOARD_IMAGE_MAX_SOURCE_BYTES) + await saveGate + return '/tmp/orca-paste-image.png' + }) + const dispatcher = makeDispatcher() + const firstStart = await dispatcher.dispatch( + makeRequest('clipboard.startImageUpload', { + expectedBase64Length: CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS, + connectionId: 'ssh-1' + }) + ) + const secondStart = await dispatcher.dispatch( + makeRequest('clipboard.startImageUpload', { + expectedBase64Length: 4, + connectionId: 'ssh-2' + }) + ) + const firstUpload = (firstStart.ok ? firstStart.result : null) as { uploadId: string } + const secondUpload = (secondStart.ok ? secondStart.result : null) as { uploadId: string } + const fullChunk = 'A'.repeat(CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS) + + for ( + let offset = 0; + offset < CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS; + offset += fullChunk.length + ) { + await expect( + dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: firstUpload.uploadId, + offset, + contentBase64: fullChunk + }) + ) + ).resolves.toMatchObject({ ok: true }) + } + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe( + CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS + ) + + await expect( + dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: secondUpload.uploadId, + offset: 0, + contentBase64: 'AAAA' + }) + ) + ).resolves.toMatchObject({ ok: false }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe( + CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS + ) + + const commit = dispatcher.dispatch( + makeRequest('clipboard.commitImageUpload', { uploadId: firstUpload.uploadId }) + ) + await vi.waitFor(() => expect(saveClipboardImageBufferAsTempFile).toHaveBeenCalledTimes(1)) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe( + CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS + ) + releaseSave() + await expect(commit).resolves.toMatchObject({ ok: true }) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) + + await expect( + dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: secondUpload.uploadId, + offset: 0, + contentBase64: 'AAAA' + }) + ) + ).resolves.toMatchObject({ ok: true, result: { receivedBase64Length: 4 } }) + }) + + it('releases retained chunks when test state is reset', async () => { + const dispatcher = makeDispatcher() + const start = await dispatcher.dispatch( + makeRequest('clipboard.startImageUpload', { + expectedBase64Length: 4, + connectionId: null + }) + ) + const uploadId = (start.ok ? start.result : null) as { uploadId: string } + await dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: uploadId.uploadId, + offset: 0, + contentBase64: 'AAAA' + }) + ) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(4) + + resetClipboardImageUploadsForTest() + + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) + }) + it('bounds concurrent uploads and releases slots through TTL cleanup', async () => { vi.useFakeTimers() const dispatcher = makeDispatcher() for (let index = 0; index < CLIPBOARD_IMAGE_UPLOAD_MAX_CONCURRENT; index++) { - await expect( - dispatcher.dispatch( - makeRequest('clipboard.startImageUpload', { - expectedBase64Length: 4, - connectionId: null - }) - ) - ).resolves.toMatchObject({ ok: true }) + const start = await dispatcher.dispatch( + makeRequest('clipboard.startImageUpload', { + expectedBase64Length: 4, + connectionId: null + }) + ) + expect(start).toMatchObject({ ok: true }) + const uploadId = (start.ok ? start.result : null) as { uploadId: string } + await dispatcher.dispatch( + makeRequest('clipboard.appendImageUploadChunk', { + uploadId: uploadId.uploadId, + offset: 0, + contentBase64: 'AAAA' + }) + ) } + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe( + CLIPBOARD_IMAGE_UPLOAD_MAX_CONCURRENT * 4 + ) await expect( dispatcher.dispatch( makeRequest('clipboard.startImageUpload', { @@ -375,6 +508,7 @@ describe('clipboard RPC methods', () => { ).resolves.toMatchObject({ ok: false }) vi.advanceTimersByTime(5 * 60 * 1000 + 1) + expect(getRetainedClipboardImageUploadBase64CharsForTest()).toBe(0) await expect( dispatcher.dispatch( diff --git a/src/main/runtime/rpc/methods/clipboard.ts b/src/main/runtime/rpc/methods/clipboard.ts index 3d5212c7a52..72476f677e6 100644 --- a/src/main/runtime/rpc/methods/clipboard.ts +++ b/src/main/runtime/rpc/methods/clipboard.ts @@ -6,23 +6,28 @@ import { CLIPBOARD_IMAGE_MAX_BASE64_CHARS, CLIPBOARD_IMAGE_TOO_LARGE_ERROR } from '../../../../shared/clipboard-image' +import { ClipboardImageUploadBuffer } from './clipboard-image-upload-buffer' const MAX_CLIPBOARD_IMAGE_BASE64_CHARS = CLIPBOARD_IMAGE_MAX_BASE64_CHARS export const CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS = 512 * 1024 export const CLIPBOARD_IMAGE_UPLOAD_MAX_CONCURRENT = 8 +export const CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS = CLIPBOARD_IMAGE_MAX_BASE64_CHARS const CLIPBOARD_IMAGE_UPLOAD_TTL_MS = 5 * 60 * 1000 +const CLIPBOARD_IMAGE_UPLOAD_MEMORY_ERROR = 'Too much clipboard image upload data is in progress' const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/ type ClipboardImageUpload = { expectedBase64Length: number connectionId?: string | null - chunks: string[] - receivedBase64Length: number + content: ClipboardImageUploadBuffer + reservedBase64Length: number expiresAt: number ttlTimer: ReturnType + committing: boolean } const clipboardImageUploads = new Map() +let retainedClipboardImageUploadBase64Chars = 0 function isValidBase64(value: string): boolean { return value.length % 4 !== 1 && BASE64_PATTERN.test(value) @@ -38,7 +43,7 @@ function pruneExpiredUploads(now = Date.now()): void { function scheduleUploadExpiry(uploadId: string): ReturnType { const timer = setTimeout(() => { - clipboardImageUploads.delete(uploadId) + deleteUpload(uploadId) }, CLIPBOARD_IMAGE_UPLOAD_TTL_MS) if (typeof timer === 'object' && 'unref' in timer) { timer.unref() @@ -52,12 +57,26 @@ function refreshUploadExpiry(uploadId: string, upload: ClipboardImageUpload): vo upload.ttlTimer = scheduleUploadExpiry(uploadId) } +function releaseUploadRetention(upload: ClipboardImageUpload): void { + retainedClipboardImageUploadBase64Chars -= upload.reservedBase64Length + upload.reservedBase64Length = 0 + upload.content.clear() +} + +function finishUpload(uploadId: string, upload: ClipboardImageUpload): void { + clearTimeout(upload.ttlTimer) + if (clipboardImageUploads.get(uploadId) === upload) { + clipboardImageUploads.delete(uploadId) + } + releaseUploadRetention(upload) +} + function deleteUpload(uploadId: string): void { const upload = clipboardImageUploads.get(uploadId) - if (upload) { - clearTimeout(upload.ttlTimer) + if (!upload || upload.committing) { + return } - clipboardImageUploads.delete(uploadId) + finishUpload(uploadId, upload) } function getUpload(uploadId: string): ClipboardImageUpload { @@ -66,13 +85,21 @@ function getUpload(uploadId: string): ClipboardImageUpload { if (!upload) { throw new Error('Clipboard image upload was not found') } + if (upload.committing) { + throw new Error('Clipboard image upload is already committing') + } return upload } -function assertValidBase64Content(value: string): void { - if (!isValidBase64(value)) { - throw new Error('Clipboard image content must be base64') +function reserveUploadBase64(chars: number): boolean { + if ( + chars > + CLIPBOARD_IMAGE_UPLOAD_MAX_RETAINED_BASE64_CHARS - retainedClipboardImageUploadBase64Chars + ) { + return false } + retainedClipboardImageUploadBase64Chars += chars + return true } function clipboardImageBase64Payload(maxChars: number, tooLargeMessage: string) { @@ -148,10 +175,14 @@ export const CLIPBOARD_METHODS: RpcMethod[] = [ clipboardImageUploads.set(uploadId, { expectedBase64Length: params.expectedBase64Length, connectionId: params.connectionId, - chunks: [], - receivedBase64Length: 0, + content: new ClipboardImageUploadBuffer( + params.expectedBase64Length, + CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS + ), + reservedBase64Length: 0, expiresAt: Date.now() + CLIPBOARD_IMAGE_UPLOAD_TTL_MS, - ttlTimer: scheduleUploadExpiry(uploadId) + ttlTimer: scheduleUploadExpiry(uploadId), + committing: false }) return { uploadId } } @@ -161,17 +192,25 @@ export const CLIPBOARD_METHODS: RpcMethod[] = [ params: AppendImageUploadChunk, handler: (params) => { const upload = getUpload(params.uploadId) - if (params.offset !== upload.receivedBase64Length) { + if (params.offset !== upload.content.length) { throw new Error('Clipboard image chunk offset is out of order') } - const nextLength = upload.receivedBase64Length + params.contentBase64.length + const nextLength = upload.content.length + params.contentBase64.length if (nextLength > upload.expectedBase64Length) { throw new Error('Clipboard image upload exceeded expected size') } - upload.chunks.push(params.contentBase64) - upload.receivedBase64Length = nextLength + if (!reserveUploadBase64(params.contentBase64.length)) { + throw new Error(CLIPBOARD_IMAGE_UPLOAD_MEMORY_ERROR) + } + try { + upload.content.append(params.contentBase64) + } catch (error) { + retainedClipboardImageUploadBase64Chars -= params.contentBase64.length + throw error + } + upload.reservedBase64Length += params.contentBase64.length refreshUploadExpiry(params.uploadId, upload) - return { receivedBase64Length: upload.receivedBase64Length } + return { receivedBase64Length: upload.content.length } } }), defineMethod({ @@ -179,19 +218,19 @@ export const CLIPBOARD_METHODS: RpcMethod[] = [ params: CommitImageUpload, handler: async (params) => { const upload = getUpload(params.uploadId) + upload.committing = true + clearTimeout(upload.ttlTimer) try { - if (upload.receivedBase64Length !== upload.expectedBase64Length) { + if (upload.content.length !== upload.expectedBase64Length) { throw new Error('Clipboard image upload is incomplete') } - const contentBase64 = upload.chunks.join('') - assertValidBase64Content(contentBase64) - return await saveClipboardImageBufferAsTempFile(Buffer.from(contentBase64, 'base64'), { + const content = upload.content.decode() + upload.content.clear() + return await saveClipboardImageBufferAsTempFile(content, { connectionId: upload.connectionId }) } finally { - // Why: failed SSH or filesystem commits must not leave bounded upload - // memory pinned until TTL cleanup. - deleteUpload(params.uploadId) + finishUpload(params.uploadId, upload) } } }), @@ -206,7 +245,12 @@ export const CLIPBOARD_METHODS: RpcMethod[] = [ ] export function resetClipboardImageUploadsForTest(): void { - for (const uploadId of clipboardImageUploads.keys()) { - deleteUpload(uploadId) + for (const [uploadId, upload] of clipboardImageUploads) { + finishUpload(uploadId, upload) } + retainedClipboardImageUploadBase64Chars = 0 +} + +export function getRetainedClipboardImageUploadBase64CharsForTest(): number { + return retainedClipboardImageUploadBase64Chars } diff --git a/src/main/runtime/rpc/methods/index.ts b/src/main/runtime/rpc/methods/index.ts index 7212729348b..537e7ce97f5 100644 --- a/src/main/runtime/rpc/methods/index.ts +++ b/src/main/runtime/rpc/methods/index.ts @@ -5,6 +5,7 @@ import { AUTOMATION_METHODS } from './automations' import { REPO_METHODS } from './repo' import { WORKTREE_METHODS } from './worktree' import { TERMINAL_METHODS } from './terminal' +import { TERMINAL_ORPHAN_METHODS } from './terminal-orphan' import { BROWSER_CORE_METHODS } from './browser-core' import { BROWSER_EXTRA_METHODS } from './browser-extras' import { BROWSER_SCREENCAST_METHODS } from './browser-screencast' @@ -49,6 +50,7 @@ export const ALL_RPC_METHODS: readonly RpcAnyMethod[] = [ ...WORKTREE_METHODS, ...AGENT_SESSION_METHODS, ...TERMINAL_METHODS, + ...TERMINAL_ORPHAN_METHODS, ...BROWSER_CORE_METHODS, ...BROWSER_SCREENCAST_METHODS, ...BROWSER_EXTRA_METHODS, diff --git a/src/main/runtime/rpc/methods/orchestration-gates.ts b/src/main/runtime/rpc/methods/orchestration-gates.ts index ca16b9719f0..0b6853fc746 100644 --- a/src/main/runtime/rpc/methods/orchestration-gates.ts +++ b/src/main/runtime/rpc/methods/orchestration-gates.ts @@ -3,6 +3,10 @@ import { defineMethod, type RpcMethod } from '../core' import { OptionalFiniteNumber, OptionalString, requiredString } from '../schemas' import type { GateStatus } from '../../orchestration/db' import { Coordinator } from '../../orchestration/coordinator' +import { + assertOrchestrationStringListFits, + assertOrchestrationWriteFits +} from '../../orchestration/query-retention' // Why: the coordinator instance is stored at module scope so orchestration.runStop // can signal it to halt. Only one coordinator can run at a time (enforced by @@ -108,10 +112,12 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ let options: string[] | undefined if (params.options) { try { + assertOrchestrationWriteFits('Decision gate options', [params.options]) const parsed = JSON.parse(params.options) if (!Array.isArray(parsed) || !parsed.every((option) => typeof option === 'string')) { throw new Error('not an array of strings') } + assertOrchestrationStringListFits('Decision gate options', parsed) options = parsed } catch { throw new Error('Invalid --options: must be a JSON array of strings') @@ -144,11 +150,17 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ params: GateListParams, handler: (params, { runtime }) => { const db = runtime.getOrchestrationDb() - const gates = db.listGates({ + const filter = { taskId: params.task, status: params.status as GateStatus - }) - return { gates, count: gates.length } + } + const gates = db.listGates(filter) + const total = db.countGates(filter) + return { + gates, + count: gates.length, + ...(total > gates.length ? { total, truncated: true as const } : {}) + } } }) ] diff --git a/src/main/runtime/rpc/methods/orchestration-query-retention.test.ts b/src/main/runtime/rpc/methods/orchestration-query-retention.test.ts new file mode 100644 index 00000000000..56f6c4fc9c7 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-query-retention.test.ts @@ -0,0 +1,185 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type Database from '../../../sqlite/sync-database' +import { OrchestrationDb } from '../../orchestration/db' +import { + ORCHESTRATION_QUERY_MAX_ROWS, + ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES, + ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES +} from '../../orchestration/query-retention' +import { OrcaRuntimeService } from '../../orca-runtime' +import type { RpcContext } from '../core' +import { ORCHESTRATION_METHODS } from './orchestration' + +function sqliteFor(db: OrchestrationDb): Database.Database { + return (db as unknown as { db: Database.Database }).db +} + +describe('orchestration RPC query retention', () => { + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + }) + + async function call(name: string, params: Record): Promise { + db ??= new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + return callWithRuntime(runtime, name, params) + } + + async function callWithRuntime( + runtime: OrcaRuntimeService, + name: string, + params: Record + ): Promise { + const ctx: RpcContext = { runtime } + const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name) + if (!method) { + throw new Error(`Method not found: ${name}`) + } + const parsed = method.params ? method.params.parse(params) : undefined + return method.handler(parsed, ctx) + } + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + it('reports remaining unread messages while successive checks drain the queue', async () => { + const d = createDb() + for (let index = 0; index < ORCHESTRATION_QUERY_MAX_ROWS + 1; index += 1) { + d.insertMessage({ from: 'sender', to: 'worker', subject: `message-${index}` }) + } + + const first = (await call('orchestration.check', { terminal: 'worker' })) as { + count: number + truncated?: boolean + remaining?: number + } + expect(first).toMatchObject({ + count: ORCHESTRATION_QUERY_MAX_ROWS, + truncated: true, + remaining: 1 + }) + + const second = (await call('orchestration.check', { terminal: 'worker' })) as { + count: number + truncated?: boolean + } + expect(second).toMatchObject({ count: 1 }) + expect(second.truncated).toBeUndefined() + }) + + it('returns truncation instead of waiting forever on an oversized legacy row', async () => { + const d = createDb() + sqliteFor(d) + .prepare( + `INSERT INTO messages (id, from_handle, to_handle, subject, body) + VALUES ('msg_oversized', 'sender', 'worker', 'oversized', ?)` + ) + .run('x'.repeat(ORCHESTRATION_QUERY_MAX_ROW_UTF8_BYTES + 1)) + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(d) + const waitForMessage = vi.spyOn(runtime, 'waitForMessage') + + const result = await callWithRuntime(runtime, 'orchestration.check', { + terminal: 'worker', + wait: true, + timeoutMs: 60_000 + }) + + expect(result).toMatchObject({ + count: 0, + truncated: true, + remaining: 1 + }) + expect(waitForMessage).not.toHaveBeenCalled() + }) + + it('rejects oversized waiting type filters before retaining the request', async () => { + const d = createDb() + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(d) + const waitForMessage = vi.spyOn(runtime, 'waitForMessage') + const types = 'status,'.repeat( + Math.ceil(ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES / 'status,'.length) + 1 + ) + + await expect( + callWithRuntime(runtime, 'orchestration.check', { + terminal: 'worker', + wait: true, + types + }) + ).rejects.toThrow( + `${ORCHESTRATION_WAIT_TYPE_FILTER_MAX_UTF8_BYTES}-byte orchestration wait limit` + ) + expect(waitForMessage).not.toHaveBeenCalled() + }) + + it('preserves normal waiting type filters while removing duplicate retention', async () => { + const d = createDb() + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(d) + const waitForMessage = vi.spyOn(runtime, 'waitForMessage').mockResolvedValue() + + await expect( + callWithRuntime(runtime, 'orchestration.check', { + terminal: 'worker', + wait: true, + timeoutMs: 100, + types: 'status,status,worker_done,escalation,decision_gate' + }) + ).resolves.toMatchObject({ count: 0, messages: [] }) + expect(waitForMessage).toHaveBeenCalledWith('worker', { + typeFilter: ['status', 'worker_done', 'escalation', 'decision_gate'], + timeoutMs: 100, + signal: undefined + }) + }) + + it('reports exact totals when task and gate lists are capped', async () => { + const d = createDb() + const task = d.createTask({ spec: 'gated' }) + for (let index = 0; index < ORCHESTRATION_QUERY_MAX_ROWS + 1; index += 1) { + d.createTask({ spec: `task-${index}` }) + d.createGate({ taskId: task.id, question: `question-${index}` }) + } + + const tasks = (await call('orchestration.taskList', {})) as { + count: number + total?: number + truncated?: boolean + } + expect(tasks).toMatchObject({ + count: ORCHESTRATION_QUERY_MAX_ROWS, + total: ORCHESTRATION_QUERY_MAX_ROWS + 2, + truncated: true + }) + + const gates = (await call('orchestration.gateList', {})) as { + count: number + total?: number + truncated?: boolean + } + expect(gates).toMatchObject({ + count: ORCHESTRATION_QUERY_MAX_ROWS, + total: ORCHESTRATION_QUERY_MAX_ROWS + 1, + truncated: true + }) + }) + + it('reports the exact inbox total when a requested page omits rows', async () => { + const d = createDb() + d.insertMessage({ from: 'a', to: 'b', subject: 'one' }) + d.insertMessage({ from: 'a', to: 'b', subject: 'two' }) + + await expect(call('orchestration.inbox', { limit: 1 })).resolves.toMatchObject({ + count: 1, + total: 2, + truncated: true + }) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index 7a731f91774..00b47658863 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -7,6 +7,11 @@ import { buildDispatchPreamble } from '../../orchestration/preamble' import { formatMessageBanner } from '../../orchestration/formatter' import { isGroupAddress, resolveGroupAddress } from '../../orchestration/groups' import { reconcileLifecycleMessage } from '../../orchestration/lifecycle-reconciliation' +import { + assertOrchestrationStringListFits, + assertOrchestrationWaitTypeFilterFits, + assertOrchestrationWriteFits +} from '../../orchestration/query-retention' import { abbreviateOrchestrationTasks } from '../../../../shared/orchestration-task-summary' import { ORCHESTRATION_GATE_METHODS } from './orchestration-gates' @@ -21,6 +26,21 @@ const MESSAGE_TYPES: MessageType[] = [ 'heartbeat' ] +function parseMessageTypeFilter(types: string | undefined): MessageType[] | undefined { + if (!types) { + return undefined + } + const parsed = types + .split(',') + .map((type) => type.trim()) + .filter(Boolean) as MessageType[] + const invalidTypes = parsed.filter((type) => !MESSAGE_TYPES.includes(type)) + if (invalidTypes.length > 0) { + throw new Error(`Invalid --types: ${invalidTypes.join(',')}`) + } + return Array.from(new Set(parsed)) +} + const TASK_STATUSES: TaskStatus[] = [ 'pending', 'ready', @@ -272,22 +292,20 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ handler: async (params, { runtime, signal }) => { const db = runtime.getOrchestrationDb() const handle = params.terminal ?? 'unknown' - const typeFilter = params.types - ? (params.types - .split(',') - .map((t) => t.trim()) - .filter(Boolean) as MessageType[]) - : undefined - const invalidTypes = typeFilter?.filter((t) => !MESSAGE_TYPES.includes(t)) - if (invalidTypes && invalidTypes.length > 0) { - throw new Error(`Invalid --types: ${invalidTypes.join(',')}`) + if (params.wait) { + assertOrchestrationWaitTypeFilterFits(params.types) } + assertOrchestrationWriteFits('Message type filter', [params.types]) + const typeFilter = parseMessageTypeFilter(params.types) // Why: unread:false is honored for one release as a compat shim so in-flight callers don't break (design doc §5). const showAll = params.all === true || (params.unread === false && params.peek !== true) const consumeUnread = !showAll && params.peek !== true const readAndReturn = () => { + const totalBeforeRead = showAll + ? db.countAllMessagesForHandle(handle, typeFilter) + : db.countUnreadMessages(handle, typeFilter) const messages = showAll ? db.getAllMessagesForHandle(handle, undefined, typeFilter) : db.getUnreadMessages(handle, typeFilter) @@ -304,19 +322,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ db.markAsRead(messages.map((m) => m.id)) } + const remaining = consumeUnread + ? db.countUnreadMessages(handle, typeFilter) + : Math.max(0, totalBeforeRead - messages.length) + const saturation = remaining > 0 ? { truncated: true as const, remaining } : {} if (params.inject) { const formatted = visibleMessages.map(formatMessageBanner).join('\n\n') - return { messages: visibleMessages, formatted, count: visibleMessages.length } + return { + messages: visibleMessages, + formatted, + count: visibleMessages.length, + ...saturation + } } - return { messages: visibleMessages, count: visibleMessages.length } + return { messages: visibleMessages, count: visibleMessages.length, ...saturation } } if (signal?.aborted) { return { messages: [], count: 0 } } const result = readAndReturn() - if (result.count > 0 || !params.wait) { + if (result.count > 0 || result.truncated || !params.wait) { return result } @@ -367,7 +394,14 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const messages = params.terminal ? db.getAllMessagesForHandle(params.terminal, params.limit) : db.getInbox(params.limit) - return { messages, count: messages.length } + const total = params.terminal + ? db.countAllMessagesForHandle(params.terminal) + : db.countInbox() + return { + messages, + count: messages.length, + ...(total > messages.length ? { total, truncated: true as const } : {}) + } } }), @@ -379,10 +413,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ let deps: string[] | undefined if (params.deps) { try { + assertOrchestrationWriteFits('Task dependencies', [params.deps]) const parsed = JSON.parse(params.deps) if (!Array.isArray(parsed) || !parsed.every((d) => typeof d === 'string')) { throw new Error('not an array of strings') } + assertOrchestrationStringListFits('Task dependencies', parsed) deps = parsed } catch { throw new Error('Invalid --deps: must be a JSON array of task IDs') @@ -405,11 +441,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ params: TaskListParams, handler: (params, { runtime }) => { const db = runtime.getOrchestrationDb() - // Why: listTasksWithDispatch adds assignee_handle + dispatch_id (NULL for non-dispatched), so legacy-shape consumers are unaffected. - const joined = db.listTasksWithDispatch({ + const filter = { status: params.status as TaskStatus, ready: params.ready - }) + } + // Why: listTasksWithDispatch adds assignee_handle + dispatch_id (NULL for non-dispatched), so legacy-shape consumers are unaffected. + const joined = db.listTasksWithDispatch(filter) const tasks = joined.map((row) => { const { assignee_handle, dispatch_id, ...base } = row if (base.status === 'dispatched') { @@ -417,9 +454,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } return base }) + const total = db.countTasks(filter) return { tasks: params.brief ? abbreviateOrchestrationTasks(tasks) : tasks, - count: tasks.length + count: tasks.length, + ...(total > tasks.length ? { total, truncated: true as const } : {}) } } }), @@ -568,11 +607,13 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' const timeoutMs = params.timeoutMs ?? 600_000 + assertOrchestrationWriteFits('Decision gate options', [params.options]) const options = params.options ?.split(',') .map((s) => s.trim()) .filter(Boolean) ?? [] + assertOrchestrationStringListFits('Decision gate options', options) const payload = JSON.stringify({ question: params.question, options }) const outbound = db.insertMessage({ diff --git a/src/main/runtime/rpc/methods/session-tabs-schemas.ts b/src/main/runtime/rpc/methods/session-tabs-schemas.ts index ac600e661ed..c035099d083 100644 --- a/src/main/runtime/rpc/methods/session-tabs-schemas.ts +++ b/src/main/runtime/rpc/methods/session-tabs-schemas.ts @@ -81,7 +81,10 @@ function parseTerminalPaneLayoutNode(value: unknown): TerminalPaneLayoutNodeInpu } if ( node.ratio !== undefined && - (typeof node.ratio !== 'number' || node.ratio < 0 || node.ratio > 1) + (typeof node.ratio !== 'number' || + !Number.isFinite(node.ratio) || + node.ratio < 0 || + node.ratio > 1) ) { return null } @@ -93,7 +96,7 @@ function parseTerminalPaneLayoutNode(value: unknown): TerminalPaneLayoutNodeInpu return value as TerminalPaneLayoutNodeInput } -const TerminalPaneLayoutNodeSchema = z +export const TerminalPaneLayoutNodeSchema = z .unknown() .transform((value) => parseTerminalPaneLayoutNode(value)) .pipe( diff --git a/src/main/runtime/rpc/methods/terminal-orphan.ts b/src/main/runtime/rpc/methods/terminal-orphan.ts new file mode 100644 index 00000000000..979c6aa747c --- /dev/null +++ b/src/main/runtime/rpc/methods/terminal-orphan.ts @@ -0,0 +1,113 @@ +import { z } from 'zod' +import type { TabGroupLayoutNode } from '../../../../shared/types' +import { isPtyIncarnationId, type PtyIncarnationId } from '../../../../shared/pty-incarnation' +import { defineMethod, type RpcAnyMethod } from '../core' +import { OptionalString, requiredString } from '../schemas' +import { TerminalPaneLayoutNodeSchema } from './session-tabs-schemas' + +function parseOrphanGroupLayout(value: unknown): TabGroupLayoutNode | null { + const stack: { value: unknown; depth: number }[] = [{ value, depth: 0 }] + let count = 0 + while (stack.length > 0) { + const current = stack.pop()! + if ( + current.depth > 64 || + ++count > 1_024 || + !current.value || + typeof current.value !== 'object' + ) { + return null + } + const node = current.value as Record + if (node.type === 'leaf') { + if ( + typeof node.groupId !== 'string' || + node.groupId.length < 1 || + node.groupId.length > 256 + ) { + return null + } + continue + } + if ( + node.type !== 'split' || + (node.direction !== 'horizontal' && node.direction !== 'vertical') || + (node.ratio !== undefined && + (typeof node.ratio !== 'number' || + !Number.isFinite(node.ratio) || + node.ratio < 0 || + node.ratio > 1)) + ) { + return null + } + stack.push( + { value: node.first, depth: current.depth + 1 }, + { value: node.second, depth: current.depth + 1 } + ) + } + return value as TabGroupLayoutNode +} + +const TerminalOrphanGroupLayout = z + .unknown() + .transform(parseOrphanGroupLayout) + .pipe(z.custom((value) => value !== null, 'Invalid orphan group layout')) + +const TerminalOrphanTopology = z.object({ + tabs: z + .array( + z.object({ + tabId: requiredString('Missing topology tab id').pipe(z.string().max(256)), + root: TerminalPaneLayoutNodeSchema, + activeLeafId: requiredString('Missing active leaf id').pipe(z.string().max(128)), + expandedLeafId: z.string().max(128).nullable() + }) + ) + .min(1) + .max(64), + groups: z + .array( + z.object({ + id: z.string().min(1).max(256), + activeTabId: z.string().min(1).max(256), + tabOrder: z.array(z.string().min(1).max(256)).min(1).max(64), + recentTabIds: z.array(z.string().min(1).max(256)).max(64).optional() + }) + ) + .min(1) + .max(64), + groupLayout: TerminalOrphanGroupLayout.optional() +}) + +const TerminalOrphanIncarnationId = z.custom( + isPtyIncarnationId, + 'Invalid PTY incarnation' +) + +const TerminalAdoptOrphans = z.object({ + worktree: requiredString('Missing worktree selector').pipe(z.string().max(32_768)), + expectedTopologyRevision: z.number().int().nonnegative(), + claims: z + .array( + z.object({ + terminal: requiredString('Missing terminal handle').pipe(z.string().max(256)), + ptyId: requiredString('Missing PTY id').pipe(z.string().max(8_192)), + incarnationId: TerminalOrphanIncarnationId, + tabId: requiredString('Missing tab id').pipe(z.string().max(256)), + leafId: requiredString('Missing leaf id').pipe(z.string().max(128)) + }) + ) + .min(1) + .max(64), + activeTabId: OptionalString.pipe(z.string().max(256).optional()), + activeGroupId: OptionalString.pipe(z.string().max(256).optional()), + topology: TerminalOrphanTopology.optional() +}) + +export const TERMINAL_ORPHAN_METHODS: RpcAnyMethod[] = [ + defineMethod({ + name: 'terminal.adoptOrphans', + params: TerminalAdoptOrphans, + handler: async (params, { runtime }) => runtime.adoptTerminalOrphans(params) + }) +] diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index 8f952fcc15f..d1847e690c9 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -42,17 +42,21 @@ import { navigationTargetsHost, resolveRuntimeNavigationTarget } from '../../../../shared/runtime-navigation' +import { + TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_STREAM_MAX_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES, + TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION, + TERMINAL_MULTIPLEX_PENDING_MAX_BYTES, + TERMINAL_OUTPUT_BATCH_MAX_BYTES, + TERMINAL_STREAM_CHUNK_BYTES +} from '../../../../shared/terminal-multiplex-flow-control' +import { drainTerminalMultiplexRoundRobin } from '../terminal-multiplex-round-robin' +import { appendCompactedStringChunk } from '../../../../shared/string-chunk-compaction' const REQUESTED_SNAPSHOT_BYTE_BUDGET = 2 * 1024 * 1024 -const TERMINAL_STREAM_CHUNK_BYTES = 48 * 1024 const TERMINAL_OUTPUT_FLUSH_MS = 5 -// Why: output batches become binary stream payloads; byte size is the transport cost. -const TERMINAL_OUTPUT_BATCH_MAX_BYTES = 64 * 1024 -// Why: remote clients can apply output pressure without pausing runtime PTY ingestion. -const TERMINAL_MULTIPLEX_ACK_STREAM_HIGH_WATER_BYTES = 512 * 1024 -const TERMINAL_MULTIPLEX_ACK_TOTAL_HIGH_WATER_BYTES = 2 * 1024 * 1024 -// Why: pending output becomes binary frames, so cap encoded payload bytes, not UTF-16 code units. -const TERMINAL_MULTIPLEX_PENDING_MAX_BYTES = 256 * 1024 const TERMINAL_QUERY_REPLAY_MAX_CHARS = 16 * 1024 // Why: bound initial subscribe latency; readiness after this deadline triggers an in-stream recovery snapshot. const MOBILE_RENDERER_MOUNT_READY_TIMEOUT_MS = 3_000 @@ -102,6 +106,7 @@ type TerminalMultiplexStream = { isMobile: boolean ackOutput: boolean ackInFlightBytes: number + ackWindowBytes: number supportsDesktopViewportClaims: boolean desktopClaimTail: Promise // Whether THIS stream registered the width driver, so detach won't release a peer stream's floor. @@ -204,7 +209,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp flush() pendingCwd = meta.cwd } - chunks.push(data) + appendCompactedStringChunk(chunks, data) pendingRawLength += rawLength const remainingBudget = Math.max(1, TERMINAL_OUTPUT_BATCH_MAX_BYTES - bytes) const measurement = measureTerminalStreamByteLength(data, { @@ -831,6 +836,10 @@ const TerminalFocus = TerminalHandle.extend({ const TerminalListParams = z.object({ worktree: OptionalString, limit: OptionalFiniteNumber, + handles: z + .array(requiredString('Missing terminal handle').pipe(z.string().max(256))) + .max(64) + .optional(), requireFreshPtyLiveness: z.boolean().optional() }) @@ -1103,6 +1112,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ params: TerminalListParams, handler: async (params, { runtime }) => runtime.listTerminals(params.worktree, params.limit, { + handles: params.handles, requireFreshPtyLiveness: params.requireFreshPtyLiveness }) }), @@ -1598,6 +1608,8 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const streams = new Map() const pendingPtyWaitControllers = new Map>() let ackTotalInFlightBytes = 0 + let ackTotalWindowBytes = TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES + let ackFlushCursorStreamId: number | null = null let resolveMultiplex = (): void => {} const multiplexClosed = new Promise((resolve) => { resolveMultiplex = resolve @@ -1614,10 +1626,21 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ // Why: a seq-less Output chunk must carry sentinel 0, not the control-frame cursor, or it poisons the client's frame-drop tracker. const resolvedSeq = typeof seq === 'number' ? seq : opcode === TerminalStreamOpcode.Output ? 0 : cursor++ - const sent = sendBinary( - encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload }) - ) - return sent !== false + let sent: boolean | void + try { + sent = sendBinary( + encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload }) + ) + } catch { + closeMultiplex() + return false + } + if (sent === false) { + // Why: false means the transport discarded this frame; reconnect is the only available retry boundary with an authoritative snapshot. + closeMultiplex() + return false + } + return true } const sendStreamError = (streamId: number, message: string): void => { sendFrame(streamId, TerminalStreamOpcode.Error, encodeTerminalStreamText(message)) @@ -1645,24 +1668,28 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return true } return ( - stream.ackInFlightBytes + bytes <= TERMINAL_MULTIPLEX_ACK_STREAM_HIGH_WATER_BYTES && - ackTotalInFlightBytes + bytes <= TERMINAL_MULTIPLEX_ACK_TOTAL_HIGH_WATER_BYTES + stream.ackInFlightBytes + bytes <= stream.ackWindowBytes && + ackTotalInFlightBytes + bytes <= ackTotalWindowBytes ) } const sendAckGatedOutput = ( stream: TerminalMultiplexStream, chunk: TerminalOutputFrameChunk - ): void => { - sendFrame( + ): boolean => { + const sent = sendFrame( stream.streamId, chunk.opcode ?? TerminalStreamOpcode.Output, chunk.bytes, chunk.seq ) + if (!sent) { + return false + } if (stream.ackOutput) { stream.ackInFlightBytes += chunk.bytes.byteLength ackTotalInFlightBytes += chunk.bytes.byteLength } + return true } const queueOrSendOutput = ( stream: TerminalMultiplexStream, @@ -1695,23 +1722,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ if (closed || streams.get(stream.streamId) !== stream) { return } - const size = runtime.getTerminalSize(stream.ptyId) + if (!serialized) { + throw new Error('Remote terminal recovery snapshot unavailable.') + } const displayMode = runtime.getMobileDisplayMode(stream.ptyId) // Why: dropped ACK-pending output breaks live replay; send a fresh snapshot before resuming output. - // Why: clients discard truncated snapshots, so mark truncated only when serialization actually failed. sendSnapshotFrames((opcode, payload) => sendFrame(stream.streamId, opcode, payload), { kind: 'scrollback', - cols: serialized?.cols ?? size?.cols ?? 80, - rows: serialized?.rows ?? size?.rows ?? 24, + cols: serialized.cols, + rows: serialized.rows, displayMode, reason: 'ack-pending-overflow', - seq: serialized?.seq, - source: serialized?.source, - truncated: !serialized, - truncatedByByteBudget: serialized?.truncatedByByteBudget, - data: serialized?.data ?? '' + seq: serialized.seq, + source: serialized.source, + truncatedByByteBudget: serialized.truncatedByByteBudget, + data: serialized.data }) - if (serialized && typeof serialized.seq === 'number') { + if (typeof serialized.seq === 'number') { // Why: chunks queued before the snapshot serialized are already in it; replaying them would duplicate output. const snapshotSeq = serialized.seq const retained = stream.ackPendingOutput.filter( @@ -1729,24 +1756,32 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ stream.streamId, error instanceof Error ? error.message : 'Remote terminal recovery snapshot failed.' ) + // Why: retrying the same failed recovery from finally creates an unbounded error loop. + detachStream(stream.streamId, true) } finally { if (streams.get(stream.streamId) === stream) { stream.ackRecoverySnapshotInFlight = false - flushAckPendingOutput(stream) + flushAllAckPendingOutput() } } } - const flushAckPendingOutput = (stream: TerminalMultiplexStream): void => { + const flushAckPendingOutput = ( + stream: TerminalMultiplexStream, + maxChunks = Number.POSITIVE_INFINITY + ): number => { if (stream.ackPendingOutputOverflowed) { void sendAckRecoverySnapshot(stream) - return + return 0 } let flushed = 0 while ( flushed < stream.ackPendingOutput.length && + flushed < maxChunks && canSendAckGatedOutput(stream, stream.ackPendingOutput[flushed]!.bytes.byteLength) ) { - sendAckGatedOutput(stream, stream.ackPendingOutput[flushed]!) + if (!sendAckGatedOutput(stream, stream.ackPendingOutput[flushed]!)) { + return flushed + } flushed += 1 } if (flushed > 0) { @@ -1756,17 +1791,38 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ 0 ) } + return flushed } const flushAllAckPendingOutput = (): void => { - for (const stream of streams.values()) { - flushAckPendingOutput(stream) - } + const ordered = Array.from(streams.values()) + ackFlushCursorStreamId = drainTerminalMultiplexRoundRobin({ + streams: ordered, + cursorStreamId: ackFlushCursorStreamId, + canContinue: () => !closed, + drainOne: (stream) => { + if (streams.get(stream.streamId) !== stream) { + return false + } + if (flushAckPendingOutput(stream, 1) > 0) { + return true + } + return false + } + }) } const acknowledgeOutput = (stream: TerminalMultiplexStream, bytes: number): void => { if (!stream.ackOutput || bytes <= 0) { return } const acknowledged = Math.min(stream.ackInFlightBytes, bytes) + stream.ackWindowBytes = Math.min( + TERMINAL_MULTIPLEX_ACK_STREAM_MAX_WINDOW_BYTES, + stream.ackWindowBytes + acknowledged + ) + ackTotalWindowBytes = Math.min( + TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES, + ackTotalWindowBytes + acknowledged + ) stream.ackInFlightBytes -= acknowledged ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - acknowledged) flushAllAckPendingOutput() @@ -1986,6 +2042,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ stream.pendingOutputOverflowed = false stream.buffering = true const requestId = request.requestId + let sentSnapshotOutputSeq: number | undefined try { const scrollbackRows = normalizeMultiplexSnapshotScrollbackRows(request.scrollbackRows) let serialized = await serializeBudgetedRequestedSnapshot( @@ -2027,6 +2084,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return } } + sentSnapshotOutputSeq = serialized?.seq sendSnapshotFrames((opcode, payload) => sendFrame(stream.streamId, opcode, payload), { kind: 'scrollback', cols: serialized?.cols ?? size?.cols ?? 80, @@ -2054,7 +2112,17 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const pendingOutput = stream.pendingOutput.splice(0) if (shouldFlushPendingOutput) { for (const chunk of pendingOutput) { - stream.outputBatcher.push(chunk.data, chunk.meta) + // Why: an untagged reply resets the client to the snapshot's + // high-water, so covered bytes would render twice; tagged + // snapshots feed a side consumer and the live view still + // needs every buffered chunk. + const uncoveredData = + typeof requestId === 'number' + ? chunk.data + : getOutputAfterSnapshotSeq(chunk, sentSnapshotOutputSeq) + if (uncoveredData) { + stream.outputBatcher.push(uncoveredData, chunk.meta) + } } } stream.pendingOutputBytes = 0 @@ -2091,6 +2159,15 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ } const request = parsed.data detachStream(request.streamId, false) + cancelPendingPtyWaits(request.streamId) + if ( + streams.size + pendingPtyWaitControllers.size >= + TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION + ) { + sendStreamError(request.streamId, 'terminal_stream_limit_exceeded') + emit({ type: 'end', streamId: request.streamId }) + return + } const isMobile = request.client?.type === 'mobile' let leaf: { ptyId: string | null } | null @@ -2163,6 +2240,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ isMobile, ackOutput: request.capabilities?.ackOutput === 1, ackInFlightBytes: 0, + ackWindowBytes: TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, supportsDesktopViewportClaims: request.capabilities?.desktopViewportClaims === 1, desktopClaimTail: Promise.resolve(true), registeredRemoteDesktopDriver: false, diff --git a/src/main/runtime/rpc/mobile-e2ee-auth-validation.ts b/src/main/runtime/rpc/mobile-e2ee-auth-validation.ts index e5039c28794..faeb9b9afb3 100644 --- a/src/main/runtime/rpc/mobile-e2ee-auth-validation.ts +++ b/src/main/runtime/rpc/mobile-e2ee-auth-validation.ts @@ -1,4 +1,6 @@ import type { DesktopMobileE2EEV2Session } from './mobile-e2ee-v2-desktop-session' +import { publicKeyFromBase64 } from './e2ee-crypto' +import { parseRemoteRuntimeJsonText } from '../../../shared/remote-runtime-request-frames' export type MobileE2EEAuth = { type: 'e2ee_auth' @@ -20,3 +22,35 @@ export function isValidMobileE2EEAuthVersion( auth.transcriptHashB64 === v2Session.transcriptHashB64 ) } + +export function authenticateMobileE2EE(args: { + plaintext: string + v2Session: DesktopMobileE2EEV2Session | null + resolveDevice: (token: string) => TDevice | null +}): { ok: true; device: TDevice } | { ok: false; code: 'bad_auth' | 'unauthorized' } { + let auth: MobileE2EEAuth + try { + auth = parseRemoteRuntimeJsonText(args.plaintext) as MobileE2EEAuth + } catch { + return { ok: false, code: 'bad_auth' } + } + if ( + auth.type !== 'e2ee_auth' || + !auth.deviceToken || + !isValidMobileE2EEAuthVersion(auth, args.v2Session) + ) { + return { ok: false, code: 'bad_auth' } + } + const device = args.resolveDevice(auth.deviceToken) + return device?.deviceToken === auth.deviceToken + ? { ok: true, device } + : { ok: false, code: 'unauthorized' } +} + +export function decodeMobileE2EEPublicKey(value: string): Uint8Array | null { + try { + return publicKeyFromBase64(value) + } catch { + return null + } +} diff --git a/src/main/runtime/rpc/mobile-e2ee-desktop-outbound-owner.ts b/src/main/runtime/rpc/mobile-e2ee-desktop-outbound-owner.ts new file mode 100644 index 00000000000..4eb6157bbd9 --- /dev/null +++ b/src/main/runtime/rpc/mobile-e2ee-desktop-outbound-owner.ts @@ -0,0 +1,83 @@ +import type { WebSocket } from 'ws' +import type { WsOutboundBackpressureQueue } from '../../../shared/ws-outbound-backpressure-queue' +import { createLegacyMobileE2EETextReplyQueue } from './mobile-e2ee-outbound-admission' +import { + createDesktopMobileE2EEV2OutboundQueue, + type DesktopMobileE2EEV2OutboundItem +} from './mobile-e2ee-v2-desktop-outbound' +import type { DesktopMobileE2EEV2Session } from './mobile-e2ee-v2-desktop-session' +import { + createMobileE2EEOutboundMemoryBudget, + type MobileE2EEOutboundMemoryBudget, + type MobileE2EEOutboundSocketMemory +} from './mobile-e2ee-outbound-memory-budget' + +export class MobileE2EEDesktopOutboundOwner { + private readonly memoryBudget: MobileE2EEOutboundMemoryBudget + private readonly socketMemory: MobileE2EEOutboundSocketMemory | null + private legacyQueue: WsOutboundBackpressureQueue | null = null + private v2Queue: WsOutboundBackpressureQueue | null = null + + constructor( + private readonly ws: WebSocket, + memoryBudget: MobileE2EEOutboundMemoryBudget = createMobileE2EEOutboundMemoryBudget() + ) { + this.memoryBudget = memoryBudget + this.socketMemory = memoryBudget.registerBufferedAmount(() => ws.bufferedAmount) + } + + canSend(bytes: number): boolean { + return this.socketMemory?.canSend(bytes) === true + } + + sendLegacyFrame(frame: string, onOverflow: () => void): boolean { + if (!this.canSend(frame.length) || this.ws.readyState !== this.ws.OPEN) { + onOverflow() + return false + } + this.ws.send(frame) + return true + } + + enqueueLegacyText(frame: string, isKeyed: () => boolean, onOverflow: () => void): boolean { + if (!this.socketMemory) { + onOverflow() + return false + } + this.legacyQueue ??= createLegacyMobileE2EETextReplyQueue({ + ws: this.ws, + isKeyed, + memoryBudget: this.memoryBudget, + socketMemory: this.socketMemory, + onOverflow + }) + return this.legacyQueue.enqueue(frame) + } + + enqueueV2( + item: DesktopMobileE2EEV2OutboundItem, + session: DesktopMobileE2EEV2Session, + onOverflow: () => void + ): boolean { + if (!this.socketMemory) { + onOverflow() + return false + } + this.v2Queue ??= createDesktopMobileE2EEV2OutboundQueue({ + ws: this.ws, + session, + memoryBudget: this.memoryBudget, + socketMemory: this.socketMemory, + onOverflow + }) + return this.v2Queue.enqueue(item) + } + + dispose(): void { + this.legacyQueue?.dispose() + this.legacyQueue = null + this.v2Queue?.dispose() + this.v2Queue = null + this.socketMemory?.release() + } +} diff --git a/src/main/runtime/rpc/mobile-e2ee-outbound-admission.ts b/src/main/runtime/rpc/mobile-e2ee-outbound-admission.ts new file mode 100644 index 00000000000..f759f6f5c47 --- /dev/null +++ b/src/main/runtime/rpc/mobile-e2ee-outbound-admission.ts @@ -0,0 +1,61 @@ +import type { WebSocket } from 'ws' +import { + createWsOutboundBackpressureQueue, + type WsOutboundBackpressureQueue +} from '../../../shared/ws-outbound-backpressure-queue' +import { + REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES, + REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES +} from '../../../shared/remote-runtime-memory-limits' +import type { + MobileE2EEOutboundMemoryBudget, + MobileE2EEOutboundSocketMemory +} from './mobile-e2ee-outbound-memory-budget' + +export function mobileE2EETextPayloadAdmissionBytes(value: string): number { + const bytes = Buffer.byteLength(value, 'utf8') + return bytes <= REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES ? bytes : Number.POSITIVE_INFINITY +} + +export function isMobileE2EETextPayloadWithinLimit(value: string): boolean { + return Number.isFinite(mobileE2EETextPayloadAdmissionBytes(value)) +} + +export function mobileE2EEBinaryPayloadAdmissionBytes(value: Uint8Array): number { + return value.byteLength <= REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + ? value.byteLength + : Number.POSITIVE_INFINITY +} + +export function isMobileE2EEBinaryPayloadWithinLimit(value: Uint8Array): boolean { + return Number.isFinite(mobileE2EEBinaryPayloadAdmissionBytes(value)) +} + +export function isMobileE2EEOutboundItemWithinLimit( + item: + | { kind: 'text'; plaintext: string } + | { kind: 'binary'; plaintext: Uint8Array } +): boolean { + return item.kind === 'text' + ? isMobileE2EETextPayloadWithinLimit(item.plaintext) + : isMobileE2EEBinaryPayloadWithinLimit(item.plaintext) +} + +export function createLegacyMobileE2EETextReplyQueue(args: { + ws: WebSocket + isKeyed: () => boolean + onOverflow: () => void + memoryBudget: MobileE2EEOutboundMemoryBudget + socketMemory: MobileE2EEOutboundSocketMemory +}): WsOutboundBackpressureQueue { + return createWsOutboundBackpressureQueue({ + send: (frame) => args.ws.send(frame), + // Encrypted replies are base64 ASCII strings, so length === byte count. + byteLengthOf: (frame) => frame.length, + getBufferedAmount: () => args.ws.bufferedAmount, + isWritable: () => args.isKeyed() && args.ws.readyState === args.ws.OPEN, + canSend: (bytes) => args.socketMemory.canSend(bytes), + claimQueuedBytes: (bytes) => args.memoryBudget.claimQueuedBytes(bytes), + onOverflow: args.onOverflow + }) +} diff --git a/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.test.ts b/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.test.ts new file mode 100644 index 00000000000..147b850619f --- /dev/null +++ b/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { createMobileE2EEOutboundMemoryBudget } from './mobile-e2ee-outbound-memory-budget' + +describe('mobile E2EE outbound memory budget', () => { + it('bounds aggregate queued frames and releases claims exactly once', () => { + const budget = createMobileE2EEOutboundMemoryBudget({ + maxQueuedBytes: 5, + maxQueuedFrames: 2 + }) + const first = budget.claimQueuedBytes(3) + const second = budget.claimQueuedBytes(2) + + expect(first).not.toBeNull() + expect(second).not.toBeNull() + expect(budget.claimQueuedBytes(0)).toBeNull() + expect(budget.evidence()).toMatchObject({ queuedBytes: 5, queuedFrames: 2 }) + + first?.() + first?.() + expect(budget.claimQueuedBytes(3)).not.toBeNull() + }) + + it('bounds prospective native buffering across registered sockets', () => { + let firstBuffered = 3 + let secondBuffered = 2 + const budget = createMobileE2EEOutboundMemoryBudget({ + maxBufferedBytes: 8, + maxSocketSources: 2 + }) + const first = budget.registerBufferedAmount(() => firstBuffered)! + const second = budget.registerBufferedAmount(() => secondBuffered)! + + expect(first.canSend(3)).toBe(true) + expect(second.canSend(4)).toBe(false) + expect(budget.registerBufferedAmount(() => 0)).toBeNull() + + first.release() + firstBuffered = 100 + secondBuffered = 0 + expect(second.canSend(8)).toBe(true) + expect(budget.evidence()).toMatchObject({ bufferedBytes: 0, sockets: 1 }) + }) +}) diff --git a/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.ts b/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.ts new file mode 100644 index 00000000000..4defe917f57 --- /dev/null +++ b/src/main/runtime/rpc/mobile-e2ee-outbound-memory-budget.ts @@ -0,0 +1,106 @@ +export const MOBILE_E2EE_PROCESS_MAX_BUFFERED_BYTES = 32 * 1024 * 1024 +export const MOBILE_E2EE_PROCESS_MAX_QUEUED_BYTES = 128 * 1024 * 1024 +export const MOBILE_E2EE_PROCESS_MAX_QUEUED_FRAMES = 16_384 +export const MOBILE_E2EE_PROCESS_MAX_SOCKET_SOURCES = 256 + +export type MobileE2EEOutboundSocketMemory = { + canSend: (bytes: number) => boolean + release: () => void +} + +export type MobileE2EEOutboundMemoryBudget = { + claimQueuedBytes: (bytes: number) => (() => void) | null + registerBufferedAmount: ( + readBufferedAmount: () => number + ) => MobileE2EEOutboundSocketMemory | null + evidence: () => { + bufferedBytes: number + queuedBytes: number + queuedFrames: number + sockets: number + } +} + +export function createMobileE2EEOutboundMemoryBudget(options?: { + maxBufferedBytes?: number + maxQueuedBytes?: number + maxQueuedFrames?: number + maxSocketSources?: number +}): MobileE2EEOutboundMemoryBudget { + const maxBufferedBytes = options?.maxBufferedBytes ?? MOBILE_E2EE_PROCESS_MAX_BUFFERED_BYTES + const maxQueuedBytes = options?.maxQueuedBytes ?? MOBILE_E2EE_PROCESS_MAX_QUEUED_BYTES + const maxQueuedFrames = options?.maxQueuedFrames ?? MOBILE_E2EE_PROCESS_MAX_QUEUED_FRAMES + const maxSocketSources = options?.maxSocketSources ?? MOBILE_E2EE_PROCESS_MAX_SOCKET_SOURCES + const bufferedSources = new Set<() => number>() + let queuedBytes = 0 + let queuedFrames = 0 + + const bufferedBytes = (): number => { + let total = 0 + for (const read of bufferedSources) { + try { + const value = read() + if (Number.isFinite(value) && value > 0) { + total += value + } + } catch { + // Closed sockets can reject a late read before channel teardown releases the source. + } + } + return total + } + + return { + claimQueuedBytes(bytes): (() => void) | null { + if ( + !Number.isFinite(bytes) || + bytes < 0 || + queuedFrames >= maxQueuedFrames || + queuedBytes + bytes > maxQueuedBytes + ) { + return null + } + queuedBytes += bytes + queuedFrames += 1 + return createRelease(() => { + queuedBytes -= bytes + queuedFrames -= 1 + }) + }, + registerBufferedAmount(readBufferedAmount): MobileE2EEOutboundSocketMemory | null { + if (bufferedSources.size >= maxSocketSources) { + return null + } + bufferedSources.add(readBufferedAmount) + let registered = true + return { + canSend: (bytes) => + registered && + Number.isFinite(bytes) && + bytes >= 0 && + bytes <= maxBufferedBytes - bufferedBytes(), + release: createRelease(() => { + registered = false + bufferedSources.delete(readBufferedAmount) + }) + } + }, + evidence: () => ({ + bufferedBytes: bufferedBytes(), + queuedBytes, + queuedFrames, + sockets: bufferedSources.size + }) + } +} + +function createRelease(release: () => void): () => void { + let released = false + return () => { + if (released) { + return + } + released = true + release() + } +} diff --git a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-outbound.ts b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-outbound.ts index 4dc6d1fc121..9bf581037f2 100644 --- a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-outbound.ts +++ b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-outbound.ts @@ -4,6 +4,14 @@ import { type WsOutboundBackpressureQueue } from '../../../shared/ws-outbound-backpressure-queue' import type { DesktopMobileE2EEV2Session } from './mobile-e2ee-v2-desktop-session' +import { + mobileE2EEBinaryPayloadAdmissionBytes, + mobileE2EETextPayloadAdmissionBytes +} from './mobile-e2ee-outbound-admission' +import type { + MobileE2EEOutboundMemoryBudget, + MobileE2EEOutboundSocketMemory +} from './mobile-e2ee-outbound-memory-budget' export type DesktopMobileE2EEV2OutboundItem = | { kind: 'text'; plaintext: string } @@ -13,6 +21,8 @@ export function createDesktopMobileE2EEV2OutboundQueue(args: { ws: WebSocket session: DesktopMobileE2EEV2Session onOverflow: () => void + memoryBudget: MobileE2EEOutboundMemoryBudget + socketMemory: MobileE2EEOutboundSocketMemory }): WsOutboundBackpressureQueue { return createWsOutboundBackpressureQueue({ // Why: sealing happens only after queue admission, so counters cannot be @@ -24,12 +34,17 @@ export function createDesktopMobileE2EEV2OutboundQueue(args: { args.ws.send(Buffer.from(args.session.sealBinary(item.plaintext)), { binary: true }) } }, - byteLengthOf: (item) => - (item.kind === 'text' - ? new TextEncoder().encode(item.plaintext).length - : item.plaintext.length) + 82, + byteLengthOf: (item) => { + const bytes = + item.kind === 'text' + ? mobileE2EETextPayloadAdmissionBytes(item.plaintext) + : mobileE2EEBinaryPayloadAdmissionBytes(item.plaintext) + return Number.isFinite(bytes) ? bytes + 82 : bytes + }, getBufferedAmount: () => args.ws.bufferedAmount, isWritable: () => args.ws.readyState === args.ws.OPEN, + canSend: (bytes) => args.socketMemory.canSend(bytes), + claimQueuedBytes: (bytes) => args.memoryBudget.claimQueuedBytes(bytes), onOverflow: args.onOverflow }) } diff --git a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.test.ts b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.test.ts index 70123a9a683..0329bfb702d 100644 --- a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.test.ts +++ b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import nacl from 'tweetnacl' import { deriveSharedKey } from './e2ee-crypto' import { deriveMobileE2EEV2KeySchedule } from './mobile-e2ee-v2-key-schedule' @@ -8,7 +8,10 @@ import { type MobileE2EEV2Hello } from '../../../shared/mobile-e2ee-v2-contract' import { sealMobileE2EEV2Frame } from '../../../shared/mobile-e2ee-v2-framing' -import { DesktopMobileE2EEV2Session } from './mobile-e2ee-v2-desktop-session' +import { + DesktopMobileE2EEV2Session, + MAX_MOBILE_E2EE_V2_TEXT_FRAME_BASE64_CHARACTERS +} from './mobile-e2ee-v2-desktop-session' const server = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(1)) const client = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(2)) @@ -31,6 +34,22 @@ function hello(): MobileE2EEV2Hello { } describe('desktop mobile E2EE v2 session', () => { + it('rejects oversized text frames before base64 decoding', () => { + const session = DesktopMobileE2EEV2Session.create({ + hello: hello(), + serverSecretKey: server.secretKey, + expectedContext: { transport: 'relay', relayHostId: 'AbCdEf0123_-xyZ9' }, + randomBytes: () => new Uint8Array(32).fill(4) + })! + const decode = vi.spyOn(Buffer, 'from') + + expect( + session.openText('A'.repeat(MAX_MOBILE_E2EE_V2_TEXT_FRAME_BASE64_CHARACTERS + 1)) + ).toBeNull() + expect(decode).not.toHaveBeenCalled() + decode.mockRestore() + }) + it('creates a fresh ready message and opens exact-next auth counter zero', () => { const clientHello = hello() const session = DesktopMobileE2EEV2Session.create({ diff --git a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.ts b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.ts index e656d30ed01..2fa54548cf1 100644 --- a/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.ts +++ b/src/main/runtime/rpc/mobile-e2ee-v2-desktop-session.ts @@ -12,6 +12,11 @@ import { } from '../../../shared/mobile-e2ee-v2-framing' import { deriveSharedKey } from './e2ee-crypto' import { deriveMobileE2EEV2KeySchedule } from './mobile-e2ee-v2-key-schedule' +import { REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES } from '../../../shared/remote-runtime-memory-limits' + +const MOBILE_E2EE_V2_FRAME_OVERHEAD_BYTES = 82 +export const MAX_MOBILE_E2EE_V2_TEXT_FRAME_BASE64_CHARACTERS = + Math.ceil((REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES + MOBILE_E2EE_V2_FRAME_OVERHEAD_BYTES) / 3) * 4 export type DesktopMobileE2EEV2Context = { transport: MobileE2EETransport @@ -72,7 +77,7 @@ export class DesktopMobileE2EEV2Session { } openText(frameB64: string): string | null { - const frame = decodeCanonicalBase64(frameB64) + const frame = decodeCanonicalBase64(frameB64, MAX_MOBILE_E2EE_V2_TEXT_FRAME_BASE64_CHARACTERS) if (!frame) { return null } @@ -138,8 +143,11 @@ function hasExpectedContext( ) } -function decodeCanonicalBase64(value: string): Uint8Array | null { - if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) { +function decodeCanonicalBase64(value: string, maxEncodedCharacters: number): Uint8Array | null { + if ( + value.length > maxEncodedCharacters || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) + ) { return null } const bytes = Buffer.from(value, 'base64') diff --git a/src/main/runtime/rpc/mobile-socket-wiring.test.ts b/src/main/runtime/rpc/mobile-socket-wiring.test.ts index b53f20e572e..de492191563 100644 --- a/src/main/runtime/rpc/mobile-socket-wiring.test.ts +++ b/src/main/runtime/rpc/mobile-socket-wiring.test.ts @@ -92,6 +92,37 @@ describe('MobileSocketWiring', () => { expect(relay.terminateClientConnections).toHaveBeenCalledWith('valid-token') }) + it('releases detached transports from revocation fanout under origin churn', () => { + const desktop = generateKeyPair() + const wiring = new MobileSocketWiring({ + deviceRegistry: registryFor('device-1', 'valid-token'), + e2eeKeypair: { + publicKey: desktop.publicKey, + secretKey: desktop.secretKey, + publicKeyB64: Buffer.from(desktop.publicKey).toString('base64') + }, + onText: vi.fn(), + onBinary: vi.fn(), + onClose: vi.fn() + }) + const live = new FakeTransport() + wiring.attachTransport(live) + const retired = Array.from({ length: 1_000 }, () => new FakeTransport()) + + for (const transport of retired) { + const detach = wiring.attachTransport(transport) + detach() + detach() + } + + expect(wiring['transports'].size).toBe(1) + expect(wiring.terminateDeviceConnections('valid-token')).toBe(0) + expect(live.terminateClientConnections).toHaveBeenCalledOnce() + expect( + retired.every((transport) => transport.terminateClientConnections.mock.calls.length === 0) + ).toBe(true) + }) + it('preserves the legacy direct handshake, identity, and close cleanup', () => { const desktop = generateKeyPair() const phone = generateKeyPair() diff --git a/src/main/runtime/rpc/mobile-socket-wiring.ts b/src/main/runtime/rpc/mobile-socket-wiring.ts index 637b5185866..3235bb8f94a 100644 --- a/src/main/runtime/rpc/mobile-socket-wiring.ts +++ b/src/main/runtime/rpc/mobile-socket-wiring.ts @@ -3,6 +3,7 @@ import type { WebSocket } from 'ws' import type { DeviceEntry, DeviceRegistry } from '../device-registry' import type { E2EEKeypair } from '../e2ee-keypair' import { E2EEChannel, type E2EEAuthenticatedDevice } from './e2ee-channel' +import { createMobileE2EEOutboundMemoryBudget } from './mobile-e2ee-outbound-memory-budget' type MobileSocketPayload = string | Uint8Array @@ -74,6 +75,7 @@ export class MobileSocketWiring { private readonly connectionIds = new Map() private readonly authenticatedSockets = new Map() private readonly transports = new Set() + private readonly outboundMemoryBudget = createMobileE2EEOutboundMemoryBudget() constructor(options: MobileSocketWiringOptions) { this.deviceRegistry = options.deviceRegistry @@ -90,12 +92,20 @@ export class MobileSocketWiring { getMetadata: (ws: WebSocket) => MobileSocketTransportMetadata = () => ({ transport: 'direct' }) - ): void { + ): () => void { this.transports.add(transport) transport.onMessage((message, _reply, ws) => { this.handleRawMessage(transport, ws, message, getMetadata(ws)) }) transport.onConnectionClose((_clientId, ws) => this.handleClose(ws)) + let attached = true + return () => { + if (!attached) { + return + } + attached = false + this.transports.delete(transport) + } } getConnectionId(ws: WebSocket): string | undefined { @@ -135,6 +145,7 @@ export class MobileSocketWiring { ? { transport: 'relay', relayHostId: metadata.relayHostId } : { transport: 'direct' }, requireV2: metadata.transport === 'relay', + outboundMemoryBudget: this.outboundMemoryBudget, resolveAuthenticatedDevice: (token) => { const device = this.deviceRegistry.validateToken(token) if (!device) { diff --git a/src/main/runtime/rpc/relay-transport.ts b/src/main/runtime/rpc/relay-transport.ts index 1a34afdf849..1eb9bd07a82 100644 --- a/src/main/runtime/rpc/relay-transport.ts +++ b/src/main/runtime/rpc/relay-transport.ts @@ -1,8 +1,10 @@ import WebSocket from 'ws' +import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' import type { RpcTransport } from './transport' import type { MobileSocketTransport, MobileSocketTransportMetadata } from './mobile-socket-wiring' const MAX_RELAY_MESSAGE_BYTES = 1024 * 1024 +const RELAY_SOCKET_CLOSE_WAIT_CONCURRENCY = 32 type RelayMessagePayload = string | Uint8Array @@ -117,7 +119,9 @@ export class CloudRelayTransport implements RpcTransport, MobileSocketTransport for (const socket of sockets) { socket.terminate() } - await Promise.all(sockets.map((socket) => this.waitForClose(socket))) + await forEachWithConcurrency(sockets, RELAY_SOCKET_CLOSE_WAIT_CONCURRENCY, (socket) => + this.waitForClose(socket) + ) } async openConnection(connection: RelayConnectionOpen): Promise { diff --git a/src/main/runtime/rpc/runtime-rpc-response-serialization.ts b/src/main/runtime/rpc/runtime-rpc-response-serialization.ts new file mode 100644 index 00000000000..2a888e42f82 --- /dev/null +++ b/src/main/runtime/rpc/runtime-rpc-response-serialization.ts @@ -0,0 +1,45 @@ +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../../../shared/node-bounded-json-stringify' +import { REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES } from '../../../shared/remote-runtime-memory-limits' +import type { RpcResponse } from './core' +import { errorResponse } from './errors' + +const RESPONSE_TOO_LARGE_MESSAGE = `RPC response exceeds ${REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES} bytes` + +export function boundRuntimeRpcResponse(response: RpcResponse): RpcResponse { + try { + serializeWithinLimit(response) + return response + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + return responseTooLargeError(response) + } + throw error + } +} + +export function serializeRuntimeRpcResponse(response: RpcResponse): string { + try { + return serializeWithinLimit(response) + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + return serializeWithinLimit(responseTooLargeError(response)) + } + throw error + } +} + +function serializeWithinLimit(response: RpcResponse): string { + return stringifyJsonWithinByteLimit(response, REMOTE_RUNTIME_MAX_OUTBOUND_JSON_BYTES).serialized +} + +function responseTooLargeError(response: RpcResponse): RpcResponse { + return errorResponse( + response.id, + response._meta, + 'response_too_large', + RESPONSE_TOO_LARGE_MESSAGE + ) +} diff --git a/src/main/runtime/rpc/schemas.test.ts b/src/main/runtime/rpc/schemas.test.ts index 010ef2bea11..8ba47496545 100644 --- a/src/main/runtime/rpc/schemas.test.ts +++ b/src/main/runtime/rpc/schemas.test.ts @@ -17,6 +17,7 @@ import { Wait } from './methods/browser-schemas' import { TERMINAL_METHODS } from './methods/terminal' +import { TERMINAL_ORPHAN_METHODS } from './methods/terminal-orphan' import { WORKTREE_METHODS } from './methods/worktree' function expectParses(schema: ZodType, value: unknown): void { @@ -85,6 +86,90 @@ describe('RPC optional pipe schemas', () => { expectParses(methodParams(WORKTREE_METHODS, 'worktree.prefetchCreateBase'), { repo: 'repo-1' }) }) + it('requires complete, bounded orphan adoption claims and a topology revision', () => { + const adopt = methodParams(TERMINAL_ORPHAN_METHODS, 'terminal.adoptOrphans') + const claim = { + terminal: 'term-live', + ptyId: 'pty-live', + incarnationId: 'inc-live', + tabId: 'tab-live', + leafId: 'leaf-live' + } + + expectParses(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [claim], + topology: { + tabs: [ + { + tabId: 'tab-live', + root: { type: 'leaf', leafId: 'leaf-live' }, + activeLeafId: 'leaf-live', + expandedLeafId: null + } + ], + groups: [{ id: 'group-live', activeTabId: 'tab-live', tabOrder: ['tab-live'] }], + groupLayout: { type: 'leaf', groupId: 'group-live' } + } + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: -1, + claims: [claim] + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [{ ...claim, incarnationId: '' }] + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [{ ...claim, incarnationId: 'i'.repeat(129) }] + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [] + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [{ ...claim, terminal: 't'.repeat(257) }] + }) + expectRejects(adopt, { + worktree: 'id:repo::/worktree', + expectedTopologyRevision: 4, + claims: [claim], + topology: { + tabs: [ + { + tabId: 'tab-live', + root: { + type: 'split', + direction: 'horizontal', + ratio: Number.NaN, + first: { type: 'leaf', leafId: 'leaf-live' }, + second: { type: 'leaf', leafId: 'leaf-other' } + }, + activeLeafId: 'leaf-live', + expandedLeafId: null + } + ], + groups: [{ id: 'group-live', activeTabId: 'tab-live', tabOrder: ['tab-live'] }] + } + }) + }) + + it('bounds targeted terminal listing used by orphan recovery', () => { + const list = methodParams(TERMINAL_METHODS, 'terminal.list') + + expectParses(list, { worktree: 'id:repo::/worktree', handles: ['term-live'] }) + expectRejects(list, { handles: [''] }) + expectRejects(list, { handles: Array.from({ length: 65 }, (_, index) => `term-${index}`) }) + }) + it('accepts worktree.create payloads sent by the previous mobile protocol', () => { const create = methodParams(WORKTREE_METHODS, 'worktree.create') diff --git a/src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts b/src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts new file mode 100644 index 00000000000..cfa660ce707 --- /dev/null +++ b/src/main/runtime/rpc/terminal-multiplex-flow-control.bench.test.ts @@ -0,0 +1,208 @@ +import { performance } from 'node:perf_hooks' +import { describe, expect, it } from 'vitest' +import { + TERMINAL_MULTIPLEX_ACK_BATCH_BYTES, + TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_STREAM_MAX_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES, + TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES, + TERMINAL_STREAM_CHUNK_BYTES +} from '../../../shared/terminal-multiplex-flow-control' +import { drainTerminalMultiplexRoundRobin } from './terminal-multiplex-round-robin' + +const MIB = 1024 * 1024 +const PAYLOAD_BYTES_PER_STREAM = 64 * MIB +const PARSER_PAYLOAD_BYTES_PER_VIEWER = 4 * MIB +const benchEnabled = process.env.ORCA_TERMINAL_PERF_BENCH === '1' + +type SimulationResult = { + throughputMiBPerSecond: number + perStreamCompletionMs: number[] + maxInFlightBytes: number + outputFrames: number + ackFrames: number + loopIterations: number +} + +type ParserMeasurement = { + aggregateMiBPerSecond: number + cpuMs: number + retainedHeapKiB: number + xtermWrites: number +} + +async function measureHeadlessXtermParsing(viewers: number): Promise { + const { Terminal } = await import('@xterm/headless') + const sample = '\x1b[?25l\x1b[38;5;45mremote output | build | status | 0123456789\x1b[0m\r\n' + const chunk = sample + .repeat(Math.ceil(TERMINAL_STREAM_CHUNK_BYTES / sample.length)) + .slice(0, TERMINAL_STREAM_CHUNK_BYTES) + const terminals = Array.from( + { length: viewers }, + () => new Terminal({ cols: 120, rows: 40, scrollback: 5_000 }) + ) + const heapBefore = process.memoryUsage().heapUsed + const cpuBefore = process.cpuUsage() + const startedAt = performance.now() + let xtermWrites = 0 + await Promise.all( + terminals.map(async (terminal) => { + let remaining = PARSER_PAYLOAD_BYTES_PER_VIEWER + while (remaining > 0) { + const data = remaining >= chunk.length ? chunk : chunk.slice(0, remaining) + xtermWrites += 1 + await new Promise((resolve) => terminal.write(data, resolve)) + remaining -= data.length + } + }) + ) + const elapsedMs = performance.now() - startedAt + const cpu = process.cpuUsage(cpuBefore) + const heapAfter = process.memoryUsage().heapUsed + for (const terminal of terminals) { + terminal.dispose() + } + return { + aggregateMiBPerSecond: (PARSER_PAYLOAD_BYTES_PER_VIEWER * viewers) / MIB / (elapsedMs / 1_000), + cpuMs: (cpu.user + cpu.system) / 1_000, + retainedHeapKiB: Math.max(0, heapAfter - heapBefore) / 1_024, + xtermWrites + } +} + +function simulateParsedCredit(streamCount: number, rttMs: number): SimulationResult { + const remaining = Array.from({ length: streamCount }, () => PAYLOAD_BYTES_PER_STREAM) + const inFlight = Array.from({ length: streamCount }, () => 0) + const windows = Array.from( + { length: streamCount }, + () => TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES + ) + const streams = Array.from({ length: streamCount }, (_, streamIndex) => ({ + streamId: streamIndex + 1, + streamIndex + })) + const perStreamCompletionMs = Array.from({ length: streamCount }, () => 0) + const acknowledgements = new Map() + let totalInFlight = 0 + let totalWindow = TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES + let maxInFlightBytes = 0 + let outputFrames = 0 + let ackFrames = 0 + let nowMs = 0 + let loopIterations = 0 + let sendCursorStreamId: number | null = null + while (remaining.some((bytes) => bytes > 0) || totalInFlight > 0) { + for (const acknowledgement of acknowledgements.get(nowMs) ?? []) { + inFlight[acknowledgement.streamIndex] -= acknowledgement.bytes + totalInFlight -= acknowledgement.bytes + if ( + remaining[acknowledgement.streamIndex] === 0 && + inFlight[acknowledgement.streamIndex] === 0 + ) { + perStreamCompletionMs[acknowledgement.streamIndex] = nowMs + } + windows[acknowledgement.streamIndex] = Math.min( + TERMINAL_MULTIPLEX_ACK_STREAM_MAX_WINDOW_BYTES, + windows[acknowledgement.streamIndex]! + acknowledgement.bytes + ) + totalWindow = Math.min( + TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES, + totalWindow + acknowledgement.bytes + ) + ackFrames += Math.ceil(acknowledgement.bytes / TERMINAL_MULTIPLEX_ACK_BATCH_BYTES) + } + acknowledgements.delete(nowMs) + sendCursorStreamId = drainTerminalMultiplexRoundRobin({ + streams, + cursorStreamId: sendCursorStreamId, + drainOne: ({ streamIndex }) => { + if ( + remaining[streamIndex]! <= 0 || + inFlight[streamIndex]! >= windows[streamIndex]! || + totalInFlight >= totalWindow + ) { + return false + } + const bytes = Math.min( + TERMINAL_STREAM_CHUNK_BYTES, + remaining[streamIndex]!, + windows[streamIndex]! - inFlight[streamIndex]!, + totalWindow - totalInFlight + ) + remaining[streamIndex] -= bytes + inFlight[streamIndex] += bytes + totalInFlight += bytes + outputFrames += 1 + const due = nowMs + rttMs + const dueAcks = acknowledgements.get(due) ?? [] + const existingAck = dueAcks.find((ack) => ack.streamIndex === streamIndex) + if (existingAck) { + existingAck.bytes += bytes + } else { + dueAcks.push({ streamIndex, bytes }) + } + acknowledgements.set(due, dueAcks) + return true + } + }) + maxInFlightBytes = Math.max(maxInFlightBytes, totalInFlight) + nowMs += 1 + loopIterations += 1 + } + return { + throughputMiBPerSecond: (PAYLOAD_BYTES_PER_STREAM * streamCount) / MIB / (nowMs / 1000), + perStreamCompletionMs, + maxInFlightBytes, + outputFrames, + ackFrames, + loopIterations + } +} + +describe('terminal multiplex parsed-credit bounds', () => { + it('keeps aggregate memory bounded and streams fair at 100 ms RTT', () => { + const result = simulateParsedCredit(8, 100) + expect(result.maxInFlightBytes).toBeLessThanOrEqual( + TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES + ) + expect(result.throughputMiBPerSecond / 8).toBeGreaterThan(7) + expect(result.ackFrames).toBeLessThan(result.outputFrames / 3) + expect( + Math.max(...result.perStreamCompletionMs) - Math.min(...result.perStreamCompletionMs) + ).toBeLessThan(200) + }) +}) + +describe.skipIf(!benchEnabled)('terminal multiplex parsed-credit benchmark', () => { + it('reports RTT, fairness, protocol allocations, and measured xterm parser cost', async () => { + const parserMeasurements = new Map() + for (const viewers of [1, 4, 8]) { + parserMeasurements.set(viewers, await measureHeadlessXtermParsing(viewers)) + } + const rows = [1, 20, 100].flatMap((rttMs) => + [1, 4, 8].map((viewers) => { + const startedAt = performance.now() + const result = simulateParsedCredit(viewers, rttMs) + const parser = parserMeasurements.get(viewers)! + return { + rttMs, + viewers, + aggregateMiBps: Number(result.throughputMiBPerSecond.toFixed(1)), + perViewerMiBps: Number((result.throughputMiBPerSecond / viewers).toFixed(1)), + schedulerCpuMs: Number((performance.now() - startedAt).toFixed(2)), + protocolFrameAllocations: result.outputFrames + result.ackFrames, + loopIterations: result.loopIterations, + maxInFlightKiB: result.maxInFlightBytes / 1024, + completionSpreadMs: + Math.max(...result.perStreamCompletionMs) - Math.min(...result.perStreamCompletionMs), + measuredParserMiBps: Number(parser.aggregateMiBPerSecond.toFixed(1)), + parserCpuMs: Number(parser.cpuMs.toFixed(1)), + parserRetainedHeapKiB: Number(parser.retainedHeapKiB.toFixed(0)), + xtermWriteAllocations: parser.xtermWrites + } + }) + ) + // eslint-disable-next-line no-console -- opt-in benchmark evidence + console.table(rows) + }) +}) diff --git a/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts b/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts new file mode 100644 index 00000000000..99c386c8492 --- /dev/null +++ b/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it, vi } from 'vitest' +import { RpcDispatcher } from './dispatcher' +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import { TERMINAL_METHODS } from './methods/terminal' +import type { RuntimeTerminalWait } from '../../../shared/runtime-types' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + decodeTerminalStreamText, + encodeTerminalStreamFrame, + encodeTerminalStreamJson +} from '../../../shared/terminal-stream-protocol' + +// An untagged SnapshotRequest is the client's frame-drop resync: its reply +// resets the client terminal to the snapshot's output high-water. Output +// buffered while the snapshot serialized is already inside that snapshot up to +// its seq, so replaying the covered bytes afterward renders the recovered tail +// twice. A tagged (requestId) snapshot feeds a side consumer instead — the +// live view still needs every buffered chunk, so that replay must stay whole. + +type OutputMeta = { seq?: number; rawLength?: number } + +async function setupMultiplexStream(): Promise<{ + binaryFrames: Uint8Array[] + sendClientFrame: (opcode: TerminalStreamOpcode, payload: Uint8Array) => void + emitOutput: (data: string, meta?: OutputMeta) => void + setSnapshot: (snapshot: { data: string; seq?: number }) => void + deferNextSerialize: () => void + releaseSerialize: () => Promise + finish: () => Promise +}> { + const messages: string[] = [] + const binaryFrames: Uint8Array[] = [] + const handlers = new Map< + number, + (frame: NonNullable>) => void + >() + const cleanups = new Map void>() + let emitOutput: ((data: string, meta?: OutputMeta) => void) | null = null + let snapshot: { data: string; seq?: number } = { data: 'INITIAL', seq: 0 } + let deferSerialize = false + let releaseDeferredSerialize: (() => void) | null = null + + const runtime = { + getRuntimeId: () => 'test-runtime', + resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + serializeTerminalBuffer: vi.fn(async () => { + if (deferSerialize) { + deferSerialize = false + await new Promise((resolve) => { + releaseDeferredSerialize = resolve + }) + } + return { data: snapshot.data, cols: 80, rows: 24, seq: snapshot.seq } + }), + getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + registerRemoteTerminalViewSubscriber: vi.fn(() => () => {}), + subscribeToTerminalData: vi.fn( + (_ptyId: string, cb: (data: string, meta?: OutputMeta) => void) => { + emitOutput = cb + return vi.fn() + } + ), + subscribeToTerminalResize: vi.fn().mockReturnValue(vi.fn()), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + subscribeToDriverChanges: vi.fn().mockReturnValue(vi.fn()), + getTerminalFitOverride: vi.fn().mockReturnValue(null), + getDriver: vi.fn().mockReturnValue({ kind: 'idle' }), + registerSubscriptionCleanup: vi.fn((id: string, cleanup: () => void) => { + cleanups.set(id, cleanup) + }), + cleanupSubscription: vi.fn((id: string) => { + const cleanup = cleanups.get(id) + cleanups.delete(id) + cleanup?.() + }), + waitForTerminal: vi.fn(() => new Promise(() => {})), + updateDesktopViewport: vi.fn().mockResolvedValue(true) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + + const request: RpcRequest = { + id: 'req-1', + authToken: 'tok', + method: 'terminal.multiplex', + params: {} + } + const dispatchPromise = dispatcher.dispatchStreaming(request, (msg) => messages.push(msg), { + connectionId: 'conn-1', + sendBinary: (bytes) => { + binaryFrames.push(bytes) + }, + registerBinaryStreamHandler: (streamId, handler) => { + handlers.set(streamId, handler) + return () => handlers.delete(streamId) + } + }) + + await vi.runOnlyPendingTimersAsync() + expect(messages.some((msg) => JSON.parse(msg).result?.type === 'ready')).toBe(true) + + handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 5, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' } + }) + }) + )! + ) + for (let i = 0; i < 5; i += 1) { + await vi.runOnlyPendingTimersAsync() + } + expect(emitOutput).not.toBeNull() + + return { + binaryFrames, + sendClientFrame: (opcode, payload) => { + handlers.get(5)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ opcode, streamId: 5, seq: 1, payload }) + )! + ) + }, + emitOutput: (data, meta) => emitOutput!(data, meta), + setSnapshot: (next) => { + snapshot = next + }, + deferNextSerialize: () => { + deferSerialize = true + }, + releaseSerialize: async () => { + releaseDeferredSerialize?.() + releaseDeferredSerialize = null + for (let i = 0; i < 5; i += 1) { + await vi.runOnlyPendingTimersAsync() + } + }, + finish: async () => { + runtime.cleanupSubscription('terminal-multiplex:conn-1') + await dispatchPromise + } + } +} + +function outputTextsAfterLastSnapshotEnd(frames: Uint8Array[]): string[] { + const decoded = frames.map((frame) => decodeTerminalStreamFrame(frame)) + const lastEnd = decoded.reduce( + (last, frame, index) => (frame?.opcode === TerminalStreamOpcode.SnapshotEnd ? index : last), + -1 + ) + return decoded.slice(lastEnd + 1).flatMap((frame) => { + if (frame?.opcode === TerminalStreamOpcode.Output) { + return [decodeTerminalStreamText(frame.payload)] + } + if (frame?.opcode === TerminalStreamOpcode.OutputSpan) { + return [decodeTerminalStreamJson<{ data?: string }>(frame.payload)?.data ?? ''] + } + return [] + }) +} + +describe('terminal.multiplex requested-snapshot replay trim', () => { + it('drops snapshot-covered buffered output after an untagged resync reply', async () => { + vi.useFakeTimers() + try { + const harness = await setupMultiplexStream() + + harness.setSnapshot({ data: 'RECOVERED', seq: 12 }) + harness.deferNextSerialize() + harness.sendClientFrame(TerminalStreamOpcode.SnapshotRequest, encodeTerminalStreamJson({})) + // Buffered while the snapshot serialized: fully covered by seq 12, and a + // partial chunk straddling the boundary whose tail the client still needs. + harness.emitOutput('xxx', { seq: 9, rawLength: 3 }) + harness.emitOutput('bbbccc', { seq: 15, rawLength: 6 }) + await harness.releaseSerialize() + + const snapshotStart = harness.binaryFrames + .map((frame) => decodeTerminalStreamFrame(frame)) + .findLast((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart)! + expect(decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({ seq: 12 }) + expect(outputTextsAfterLastSnapshotEnd(harness.binaryFrames).join('')).toBe('ccc') + + await harness.finish() + } finally { + vi.useRealTimers() + } + }) + + it('replays all buffered output untouched after a tagged snapshot reply', async () => { + vi.useFakeTimers() + try { + const harness = await setupMultiplexStream() + + harness.setSnapshot({ data: 'MANUAL', seq: 12 }) + harness.deferNextSerialize() + harness.sendClientFrame( + TerminalStreamOpcode.SnapshotRequest, + encodeTerminalStreamJson({ requestId: 7 }) + ) + harness.emitOutput('xxx', { seq: 9, rawLength: 3 }) + harness.emitOutput('bbbccc', { seq: 15, rawLength: 6 }) + await harness.releaseSerialize() + + const snapshotStart = harness.binaryFrames + .map((frame) => decodeTerminalStreamFrame(frame)) + .findLast((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart)! + expect(decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({ requestId: 7 }) + expect(outputTextsAfterLastSnapshotEnd(harness.binaryFrames).join('')).toBe('xxxbbbccc') + + await harness.finish() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/runtime/rpc/terminal-multiplex-round-robin.test.ts b/src/main/runtime/rpc/terminal-multiplex-round-robin.test.ts new file mode 100644 index 00000000000..96e04718c91 --- /dev/null +++ b/src/main/runtime/rpc/terminal-multiplex-round-robin.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { drainTerminalMultiplexRoundRobin } from './terminal-multiplex-round-robin' + +describe('terminal multiplex round-robin drain', () => { + it('admits a later interactive stream before older bulk queues refill the window', () => { + const streams = Array.from({ length: 8 }, (_, index) => ({ + streamId: index + 1, + pendingChunks: index === 7 ? 1 : 8 + })) + const order: number[] = [] + let remainingSlots = 8 + + const cursor = drainTerminalMultiplexRoundRobin({ + streams, + cursorStreamId: null, + canContinue: () => remainingSlots > 0, + drainOne: (stream) => { + if (stream.pendingChunks === 0) { + return false + } + stream.pendingChunks -= 1 + remainingSlots -= 1 + order.push(stream.streamId) + return true + } + }) + + expect(order).toEqual([1, 2, 3, 4, 5, 6, 7, 8]) + expect(cursor).toBe(8) + }) + + it('resumes after the previous sender on the next release', () => { + const streams = [1, 2, 3].map((streamId) => ({ streamId, pendingChunks: 2 })) + let slots = 2 + const firstOrder: number[] = [] + const cursor = drainTerminalMultiplexRoundRobin({ + streams, + cursorStreamId: null, + canContinue: () => slots > 0, + drainOne: (stream) => { + stream.pendingChunks -= 1 + slots -= 1 + firstOrder.push(stream.streamId) + return true + } + }) + slots = 2 + const secondOrder: number[] = [] + drainTerminalMultiplexRoundRobin({ + streams, + cursorStreamId: cursor, + canContinue: () => slots > 0, + drainOne: (stream) => { + if (stream.pendingChunks === 0) { + return false + } + stream.pendingChunks -= 1 + slots -= 1 + secondOrder.push(stream.streamId) + return true + } + }) + + expect(firstOrder).toEqual([1, 2]) + expect(secondOrder).toEqual([3, 1]) + }) +}) diff --git a/src/main/runtime/rpc/terminal-multiplex-round-robin.ts b/src/main/runtime/rpc/terminal-multiplex-round-robin.ts new file mode 100644 index 00000000000..d5c8f57d83b --- /dev/null +++ b/src/main/runtime/rpc/terminal-multiplex-round-robin.ts @@ -0,0 +1,41 @@ +type TerminalMultiplexDrainStream = { streamId: number } + +export function drainTerminalMultiplexRoundRobin(args: { + streams: readonly T[] + cursorStreamId: number | null + drainOne: (stream: T) => boolean + canContinue?: () => boolean +}): number | null { + const { streams, drainOne } = args + if (streams.length === 0) { + return null + } + const canContinue = args.canContinue ?? (() => true) + let cursorStreamId = args.cursorStreamId + let startIndex = getStartIndex(streams, cursorStreamId) + while (canContinue()) { + let progressed = false + for (let offset = 0; offset < streams.length && canContinue(); offset += 1) { + const stream = streams[(startIndex + offset) % streams.length]! + if (drainOne(stream)) { + cursorStreamId = stream.streamId + progressed = true + } + } + if (!progressed) { + break + } + startIndex = getStartIndex(streams, cursorStreamId) + } + return cursorStreamId +} + +function getStartIndex( + streams: readonly T[], + cursorStreamId: number | null +): number { + if (cursorStreamId === null) { + return 0 + } + return (streams.findIndex((stream) => stream.streamId === cursorStreamId) + 1) % streams.length +} diff --git a/src/main/runtime/rpc/terminal-multiplex.test.ts b/src/main/runtime/rpc/terminal-multiplex.test.ts index f281d72e48e..d86161d0321 100644 --- a/src/main/runtime/rpc/terminal-multiplex.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex.test.ts @@ -43,7 +43,8 @@ function makeRequest(method: string, params?: unknown): RpcRequest { function startDesktopMultiplexSubscribe( overrides: Partial = {}, - trace?: string[] + trace?: string[], + sendBinaryOverride?: (bytes: Uint8Array) => boolean | void ) { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] @@ -87,6 +88,10 @@ function startDesktopMultiplexSubscribe( { connectionId: 'conn-desktop-first-paint', sendBinary: (bytes) => { + const sent = sendBinaryOverride?.(bytes) + if (sent === false) { + return false + } binaryFrames.push(bytes) const opcode = decodeTerminalStreamFrame(bytes)?.opcode if ( @@ -96,10 +101,15 @@ function startDesktopMultiplexSubscribe( ) { trace?.push('snapshot') } + return sent }, registerBinaryStreamHandler: (streamId, handler) => { handlers.set(streamId, handler) - return () => handlers.delete(streamId) + return () => { + if (handlers.get(streamId) === handler) { + handlers.delete(streamId) + } + } } } ) @@ -128,6 +138,65 @@ function sendDesktopMultiplexSubscribe( } describe('terminal multiplex RPC', () => { + it.each(['refuses', 'throws'] as const)( + 'closes without reserving ACK debt when the transport %s an output frame', + async (failureMode) => { + let dataListener: + | ((data: string, meta?: { seq?: number; rawLength?: number }) => void) + | null = null + let rejectOutput = false + const unsubscribeData = vi.fn() + const harness = startDesktopMultiplexSubscribe( + { + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return unsubscribeData + }) + }, + undefined, + (bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + if (!rejectOutput || frame?.opcode !== TerminalStreamOpcode.Output) { + return true + } + if (failureMode === 'throws') { + throw new Error('socket closed') + } + return false + } + ) + + await vi.waitFor(() => + expect(harness.messages.some((msg) => JSON.parse(msg).result?.type === 'ready')).toBe(true) + ) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect(harness.messages.some((msg) => JSON.parse(msg).result?.type === 'subscribed')).toBe( + true + ) + ) + await vi.waitFor(() => expect(dataListener).not.toBeNull()) + harness.binaryFrames.splice(0) + rejectOutput = true + + const output = 'x'.repeat(64 * 1024) + const deliverData = dataListener as unknown as ( + data: string, + meta?: { seq?: number; rawLength?: number } + ) => void + deliverData(output, { seq: output.length, rawLength: output.length }) + + await vi.waitFor(() => expect(unsubscribeData).toHaveBeenCalledOnce()) + await harness.dispatchPromise + expect( + harness.binaryFrames + .map((bytes) => decodeTerminalStreamFrame(bytes)) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.Output) + ).toEqual([]) + expect(harness.handlers.size).toBe(0) + } + ) + it('multiplexes terminal streams and routes desktop resize to the source PTY', async () => { vi.useFakeTimers() try { @@ -1197,7 +1266,7 @@ describe('terminal multiplex RPC', () => { await dispatchPromise }) - it('releases shared ACK budget to other stalled multiplex streams', async () => { + it('round-robins released ACK budget to a later interactive stream', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] const handlers = new Map< @@ -1272,7 +1341,7 @@ describe('terminal multiplex RPC', () => { expect(messages.some((msg) => JSON.parse(msg).result?.type === 'ready')).toBe(true) ) - const streamIds = [21, 22, 23, 24, 25, 26] + const streamIds = [21, 22, 23, 24, 25, 26, 27, 28] for (const streamId of streamIds) { handlers.get(0)?.( decodeTerminalStreamFrame( @@ -1302,22 +1371,33 @@ describe('terminal multiplex RPC', () => { await vi.waitFor(() => expect(dataListeners.size).toBe(streamIds.length)) binaryFrames.splice(0) - const fillerOutput = 'f'.repeat(480 * 1024) + const fillerOutput = 'f'.repeat(512 * 1024) for (let index = 1; index <= 4; index += 1) { dataListeners.get(`pty-${index}`)?.(fillerOutput, { seq: fillerOutput.length, rawLength: fillerOutput.length }) } - const stalledOutput = 's'.repeat(700 * 1024) - dataListeners.get('pty-5')?.(stalledOutput, { - seq: stalledOutput.length, - rawLength: stalledOutput.length - }) - dataListeners.get('pty-6')?.(stalledOutput, { - seq: stalledOutput.length, - rawLength: stalledOutput.length + const queuedFillerOutput = 'q'.repeat(256 * 1024) + for (let index = 1; index <= 4; index += 1) { + dataListeners.get(`pty-${index}`)?.(queuedFillerOutput, { + seq: fillerOutput.length + queuedFillerOutput.length, + rawLength: queuedFillerOutput.length + }) + } + const stalledOutput = 's'.repeat(256 * 1024) + for (let index = 5; index <= 7; index += 1) { + dataListeners.get(`pty-${index}`)?.(stalledOutput, { + seq: stalledOutput.length, + rawLength: stalledOutput.length + }) + } + const interactiveOutput = 'interactive-output\r\n' + dataListeners.get('pty-8')?.(interactiveOutput, { + seq: interactiveOutput.length, + rawLength: interactiveOutput.length }) + await new Promise((resolve) => setTimeout(resolve, 10)) const initialOutputFrames = binaryFrames .map((frame) => decodeTerminalStreamFrame(frame)) @@ -1337,25 +1417,27 @@ describe('terminal multiplex RPC', () => { 0 ) expect(initialBytes).toBeLessThanOrEqual(2 * 1024 * 1024) - expect(initialBytesByStream.get(21)).toBe(480 * 1024) - expect(initialBytesByStream.get(22)).toBe(480 * 1024) - expect(initialBytesByStream.get(23)).toBe(480 * 1024) - expect(initialBytesByStream.get(24)).toBe(480 * 1024) - expect(initialBytesByStream.get(25)).toBeGreaterThan(0) + expect(initialBytesByStream.get(21)).toBe(512 * 1024) + expect(initialBytesByStream.get(22)).toBe(512 * 1024) + expect(initialBytesByStream.get(23)).toBe(512 * 1024) + expect(initialBytesByStream.get(24)).toBe(512 * 1024) + expect(initialBytesByStream.get(25) ?? 0).toBe(0) expect(initialBytesByStream.get(26) ?? 0).toBe(0) + expect(initialBytesByStream.get(27) ?? 0).toBe(0) + expect(initialBytesByStream.get(28) ?? 0).toBe(0) - handlers.get(26)?.( + handlers.get(28)?.( decodeTerminalStreamFrame( encodeTerminalStreamFrame({ opcode: TerminalStreamOpcode.Input, - streamId: 26, + streamId: 28, seq: 200, payload: encodeTerminalStreamText('remote-still-interactive\r') }) )! ) await vi.waitFor(() => - expect(runtime.sendTerminal).toHaveBeenCalledWith('terminal-6', { + expect(runtime.sendTerminal).toHaveBeenCalledWith('terminal-8', { text: 'remote-still-interactive\r', enter: false, interrupt: false @@ -1379,25 +1461,17 @@ describe('terminal multiplex RPC', () => { binaryFrames .slice(frameCountBeforeAck) .map((frame) => decodeTerminalStreamFrame(frame)) - .some((frame) => { - if (frame?.streamId !== 25 || frame.opcode !== TerminalStreamOpcode.SnapshotStart) { - return false - } - const payload = decodeTerminalStreamJson<{ reason?: string }>(frame.payload) - return payload?.reason === 'ack-pending-overflow' - }) + .some( + (frame) => + frame?.streamId === 28 && + frame.opcode === TerminalStreamOpcode.Output && + decodeTerminalStreamText(frame.payload) === interactiveOutput + ) ).toBe(true) ) const framesAfterAck = binaryFrames .slice(frameCountBeforeAck) .map((frame) => decodeTerminalStreamFrame(frame)) - const snapshotStartIndex = framesAfterAck.findIndex((frame) => { - if (frame?.streamId !== 25 || frame.opcode !== TerminalStreamOpcode.SnapshotStart) { - return false - } - const payload = decodeTerminalStreamJson<{ reason?: string }>(frame.payload) - return payload?.reason === 'ack-pending-overflow' - }) const outputFramesAfterAck = framesAfterAck.filter( (frame) => frame?.opcode === TerminalStreamOpcode.Output ) @@ -1411,17 +1485,14 @@ describe('terminal multiplex RPC', () => { (bytesAfterAckByStream.get(frame.streamId) ?? 0) + frame.payload.byteLength ) } - expect(snapshotStartIndex).toBeGreaterThanOrEqual(0) - expect( - framesAfterAck - .filter((frame) => frame?.streamId === 25 && frame.opcode === TerminalStreamOpcode.Output) - .every((frame) => framesAfterAck.indexOf(frame) > snapshotStartIndex) - ).toBe(true) expect(bytesAfterAckByStream.get(25) ?? 0).toBeGreaterThan(0) - expect(bytesAfterAckByStream.get(21) ?? 0).toBe(0) + expect(bytesAfterAckByStream.get(26) ?? 0).toBeGreaterThan(0) + expect(bytesAfterAckByStream.get(27) ?? 0).toBeGreaterThan(0) + expect(bytesAfterAckByStream.get(28) ?? 0).toBe(interactiveOutput.length) + expect(bytesAfterAckByStream.get(21) ?? 0).toBeGreaterThan(0) expect( outputFramesAfterAck.reduce((total, frame) => total + (frame?.payload.byteLength ?? 0), 0) - ).toBeLessThanOrEqual(initialBytesByStream.get(21) ?? 0) + ).toBeLessThanOrEqual((initialBytesByStream.get(21) ?? 0) * 2) runtime.cleanupSubscription('terminal-multiplex:conn-ack-shared-budget') await dispatchPromise @@ -1605,6 +1676,81 @@ describe('terminal multiplex RPC', () => { await dispatchPromise }) + it.each([ + { + failure: 'throws', + recover: () => Promise.reject(new Error('snapshot unavailable')) + }, + { + failure: 'returns no snapshot', + recover: () => Promise.resolve(null) + } + ])('ends a stream when ACK overflow recovery serialization $failure', async ({ recover }) => { + const dataListenerRef: { + current?: (data: string, meta?: { seq?: number; rawLength?: number }) => void + } = {} + const serializeTerminalBuffer = vi + .fn() + .mockResolvedValueOnce({ data: 'initial snapshot', cols: 120, rows: 40 }) + .mockImplementation(recover) + const harness = startDesktopMultiplexSubscribe({ + serializeTerminalBuffer, + subscribeToTerminalData: vi.fn( + ( + _: string, + listener: (data: string, meta?: { seq?: number; rawLength?: number }) => void + ) => { + dataListenerRef.current = listener + return vi.fn() + } + ) + }) + + await vi.waitFor(() => + expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( + true + ) + ) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + harness.binaryFrames.splice(0) + + const output = 'x'.repeat(3 * 1024 * 1024) + dataListenerRef.current?.(output, { seq: output.length, rawLength: output.length }) + const inFlightBytes = harness.binaryFrames + .map((bytes) => decodeTerminalStreamFrame(bytes)) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.Output) + .reduce((total, frame) => total + (frame?.payload.byteLength ?? 0), 0) + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 2, + payload: encodeTerminalStreamJson({ bytes: inFlightBytes }) + }) + )! + ) + + await vi.waitFor(() => { + const eventTypes = harness.messages.map((message) => JSON.parse(message).result?.type) + expect(eventTypes).toContain('error') + expect(eventTypes).toContain('end') + }) + expect(harness.handlers.has(7)).toBe(false) + expect(serializeTerminalBuffer).toHaveBeenCalledTimes(2) + await Promise.resolve() + await Promise.resolve() + expect(serializeTerminalBuffer).toHaveBeenCalledTimes(2) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it('trims recovery-covered ACK pending output instead of replaying it', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] @@ -2881,6 +3027,81 @@ describe('terminal multiplex RPC', () => { await harness.dispatchPromise }) + it('cancels an older pending PTY wait when the same multiplex slot resubscribes', async () => { + const waitSignals: AbortSignal[] = [] + const waitForLeafPtyId = vi.fn( + (_handle: string, _timeoutMs?: number, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + if (signal) { + waitSignals.push(signal) + } + signal?.addEventListener('abort', () => reject(new Error('request_aborted')), { + once: true + }) + }) + ) + const harness = startDesktopMultiplexSubscribe({ + resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: null }), + waitForLeafPtyId + }) + await vi.waitFor(() => + expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( + true + ) + ) + + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => expect(waitSignals).toHaveLength(1)) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => expect(waitSignals).toHaveLength(2)) + + expect(waitSignals[0]?.aborted).toBe(true) + expect(waitSignals[1]?.aborted).toBe(false) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await vi.waitFor(() => expect(waitSignals[1]?.aborted).toBe(true)) + await harness.dispatchPromise + }) + + it('caps multiplex stream slots so aggregate pending output stays bounded', async () => { + const harness = startDesktopMultiplexSubscribe() + await vi.waitFor(() => + expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( + true + ) + ) + for (let streamId = 1; streamId <= 33; streamId += 1) { + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: streamId, + payload: encodeTerminalStreamJson({ + streamId, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1 } + }) + }) + )! + ) + } + + await vi.waitFor(() => { + const results = harness.messages.map((message) => JSON.parse(message).result) + expect(results.filter((result) => result?.type === 'subscribed')).toHaveLength(32) + expect(results).toContainEqual({ + type: 'error', + streamId: 33, + message: 'terminal_stream_limit_exceeded' + }) + expect(results).toContainEqual({ type: 'end', streamId: 33 }) + }) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it("still reports no_connected_pty when a desktop multiplex subscriber's PTY never appears", async () => { const runtime = stubRuntime({ resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: null }), diff --git a/src/main/runtime/rpc/unix-socket-transport.test.ts b/src/main/runtime/rpc/unix-socket-transport.test.ts index e321c2c59e9..8ed4d518075 100644 --- a/src/main/runtime/rpc/unix-socket-transport.test.ts +++ b/src/main/runtime/rpc/unix-socket-transport.test.ts @@ -63,7 +63,7 @@ describe('UnixSocketTransport', () => { ;(transport as unknown as UnixSocketTransportInternals).handleConnection( socket as unknown as Socket ) - socket.emit('data', '{"id":"pending","method":"wait"}\n') + socket.emit('data', Buffer.from('{"id":"pending","method":"wait"}\n')) vi.advanceTimersByTime(100) expect(socket.writes).toHaveLength(1) @@ -74,4 +74,26 @@ describe('UnixSocketTransport', () => { vi.advanceTimersByTime(500) expect(socket.writes).toHaveLength(1) }) + + it('parses a request delivered as 100,000 one-byte fragments', () => { + const transport = new UnixSocketTransport({ + endpoint: '/tmp/orca-runtime-rpc-test.sock', + kind: 'unix' + }) + const socket = new FakeSocket() + let received = '' + transport.onMessage((message) => { + received = message + }) + ;(transport as unknown as UnixSocketTransportInternals).handleConnection( + socket as unknown as Socket + ) + + const request = Buffer.from(`${' '.repeat(99_960)}{"id":"tiny","method":"status"}\n`) + for (let index = 0; index < request.byteLength; index += 1) { + socket.emit('data', request.subarray(index, index + 1)) + } + + expect(received).toBe('{"id":"tiny","method":"status"}') + }) }) diff --git a/src/main/runtime/rpc/unix-socket-transport.ts b/src/main/runtime/rpc/unix-socket-transport.ts index ed66f94d907..24b53350bc6 100644 --- a/src/main/runtime/rpc/unix-socket-transport.ts +++ b/src/main/runtime/rpc/unix-socket-transport.ts @@ -6,6 +6,7 @@ // away. See design doc §3.1. import { createServer, type Server, type Socket } from 'node:net' import { chmodSync, existsSync, rmSync } from 'node:fs' +import { GrowingByteBuffer } from '../../../shared/growing-byte-buffer' import type { RpcMessageContext, RpcTransport } from './transport' const MAX_RUNTIME_RPC_MESSAGE_BYTES = 1024 * 1024 @@ -104,7 +105,7 @@ export class UnixSocketTransport implements RpcTransport { private handleConnection(socket: Socket): void { this.activeSockets.add(socket) - let buffer = '' + const buffer = new GrowingByteBuffer() let oversized = false // Why: each in-flight dispatch registers its own AbortController here so // `socket.on('close')` can abort them all at once. Keeping the set scoped @@ -114,7 +115,6 @@ export class UnixSocketTransport implements RpcTransport { // multiplexes sequential requests. const inflight = new Set<() => void>() - socket.setEncoding('utf8') socket.setNoDelay(true) socket.setTimeout(RUNTIME_RPC_SOCKET_IDLE_TIMEOUT_MS, () => { socket.destroy() @@ -129,30 +129,31 @@ export class UnixSocketTransport implements RpcTransport { inflight.clear() this.activeSockets.delete(socket) }) - socket.on('data', (chunk: string) => { + socket.on('data', (chunk: Buffer) => { if (oversized) { return } - buffer += chunk + buffer.append(chunk) // Why: the Orca runtime lives in Electron main, so it must reject // oversized local RPC frames instead of letting a local client grow an // unbounded buffer and stall the app. - if (Buffer.byteLength(buffer, 'utf8') > MAX_RUNTIME_RPC_MESSAGE_BYTES) { + if (buffer.byteLength > MAX_RUNTIME_RPC_MESSAGE_BYTES) { oversized = true + buffer.clear() this.messageHandler?.('', (response) => { socket.write(`${response}\n`) socket.end() }) return } - let newlineIndex = buffer.indexOf('\n') + let newlineIndex = buffer.indexOfByte(0x0a) while (newlineIndex !== -1) { - const rawMessage = buffer.slice(0, newlineIndex).trim() - buffer = buffer.slice(newlineIndex + 1) + const rawMessage = buffer.takePrefixString(newlineIndex).trim() + buffer.discardPrefix(1) if (rawMessage) { this.dispatchMessage(socket, rawMessage, inflight) } - newlineIndex = buffer.indexOf('\n') + newlineIndex = buffer.indexOfByte(0x0a) } }) } diff --git a/src/main/runtime/rpc/ws-fallback-port-store.test.ts b/src/main/runtime/rpc/ws-fallback-port-store.test.ts index 2edd314c212..2c3ae73381a 100644 --- a/src/main/runtime/rpc/ws-fallback-port-store.test.ts +++ b/src/main/runtime/rpc/ws-fallback-port-store.test.ts @@ -1,8 +1,12 @@ -import { mkdtempSync, writeFileSync } from 'node:fs' +import { mkdtempSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' -import { readWsFallbackPort, writeWsFallbackPort } from './ws-fallback-port-store' +import { + MAX_FALLBACK_PORT_FILE_BYTES, + readWsFallbackPort, + writeWsFallbackPort +} from './ws-fallback-port-store' function makeUserDataPath(): string { return mkdtempSync(join(tmpdir(), 'ws-fallback-port-test-')) @@ -32,4 +36,13 @@ describe('ws-fallback-port-store', () => { writeWsFallbackPort(userDataPath, 70000) expect(readWsFallbackPort(userDataPath)).toBeUndefined() }) + + it('ignores an oversized sparse fallback-port file', () => { + const userDataPath = makeUserDataPath() + const path = join(userDataPath, 'mobile-ws-fallback-port.json') + writeFileSync(path, '{"port":54321}', 'utf8') + truncateSync(path, MAX_FALLBACK_PORT_FILE_BYTES + 1) + + expect(readWsFallbackPort(userDataPath)).toBeUndefined() + }) }) diff --git a/src/main/runtime/rpc/ws-fallback-port-store.ts b/src/main/runtime/rpc/ws-fallback-port-store.ts index beedcea8cf2..3b12a15f18c 100644 --- a/src/main/runtime/rpc/ws-fallback-port-store.ts +++ b/src/main/runtime/rpc/ws-fallback-port-store.ts @@ -1,5 +1,6 @@ -import { readFileSync, writeFileSync } from 'node:fs' +import { writeFileSync } from 'node:fs' import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../../shared/node-bounded-file-reader' // Why: when the preferred WS port is taken (second Orca instance), the OS // assigns a random port. Paired mobile devices store ws://ip:port endpoints, @@ -10,6 +11,7 @@ import { join } from 'node:path' // again. const FALLBACK_PORT_FILE = 'mobile-ws-fallback-port.json' +export const MAX_FALLBACK_PORT_FILE_BYTES = 1024 function isValidPort(value: unknown): value is number { return typeof value === 'number' && Number.isInteger(value) && value > 0 && value <= 65535 @@ -17,7 +19,10 @@ function isValidPort(value: unknown): value is number { export function readWsFallbackPort(userDataPath: string): number | undefined { try { - const raw = readFileSync(join(userDataPath, FALLBACK_PORT_FILE), 'utf8') + const raw = readNodeFileSyncWithinLimit( + join(userDataPath, FALLBACK_PORT_FILE), + MAX_FALLBACK_PORT_FILE_BYTES + ).buffer.toString('utf8') const parsed: unknown = JSON.parse(raw) if ( typeof parsed === 'object' && diff --git a/src/main/runtime/runtime-control-file-bounds.test.ts b/src/main/runtime/runtime-control-file-bounds.test.ts new file mode 100644 index 00000000000..2d08c9c5bc9 --- /dev/null +++ b/src/main/runtime/runtime-control-file-bounds.test.ts @@ -0,0 +1,121 @@ +import { mkdtempSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { + getRuntimeMetadataPath, + MAX_RUNTIME_METADATA_FILE_BYTES, + MAX_RUNTIME_METADATA_JSON_STRUCTURAL_TOKENS +} from '../../shared/runtime-bootstrap' +import { + DeviceRegistry, + DeviceRegistryCapacityError, + MAX_DEVICE_REGISTRY_FILE_BYTES +} from './device-registry' +import { + loadOrCreateE2EEKeypair, + MAX_KEYPAIR_FILE_BYTES, + MAX_KEYPAIR_JSON_STRUCTURAL_TOKENS +} from './e2ee-keypair' +import { readRuntimeMetadata, writeRuntimeMetadata } from './runtime-metadata' + +describe('runtime control-file bounds', () => { + const paths: string[] = [] + + afterEach(() => { + for (const path of paths.splice(0)) { + rmSync(path, { recursive: true, force: true }) + } + }) + + function makeUserDataPath(prefix: string): string { + const path = mkdtempSync(join(tmpdir(), prefix)) + paths.push(path) + return path + } + + it('rejects oversized runtime metadata before parsing it', () => { + const userDataPath = makeUserDataPath('orca-runtime-metadata-bound-') + const metadataPath = getRuntimeMetadataPath(userDataPath) + writeFileSync(metadataPath, '{"runtimeId":"runtime-1"}') + truncateSync(metadataPath, MAX_RUNTIME_METADATA_FILE_BYTES + 1) + + expect(() => readRuntimeMetadata(userDataPath)).toThrow(NodeFileReadTooLargeError) + }) + + it('rejects structurally amplified runtime metadata before parsing it', () => { + const userDataPath = makeUserDataPath('orca-runtime-metadata-structure-') + const metadataPath = getRuntimeMetadataPath(userDataPath) + writeFileSync( + metadataPath, + `{"transports":[${'0,'.repeat(MAX_RUNTIME_METADATA_JSON_STRUCTURAL_TOKENS)}0]}` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + + expect(() => readRuntimeMetadata(userDataPath)).toThrow('JSON structure exceeds') + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('preserves prior runtime metadata when serialization exceeds its read ceiling', () => { + const userDataPath = makeUserDataPath('orca-runtime-metadata-write-') + const metadataPath = getRuntimeMetadataPath(userDataPath) + const initial = { + runtimeId: 'runtime-1', + pid: 42, + transports: [], + authToken: 'token', + startedAt: 1 + } + writeRuntimeMetadata(userDataPath, initial) + const before = readFileSync(metadataPath, 'utf8') + + expect(() => + writeRuntimeMetadata(userDataPath, { + ...initial, + transports: [{ kind: 'unix', endpoint: 'x'.repeat(MAX_RUNTIME_METADATA_FILE_BYTES) }] + }) + ).toThrow('JSON output exceeds') + expect(readFileSync(metadataPath, 'utf8')).toBe(before) + }) + + it('treats an oversized device registry as unavailable', () => { + const userDataPath = makeUserDataPath('orca-device-registry-bound-') + const registryPath = join(userDataPath, 'orca-devices.json') + writeFileSync(registryPath, '[]') + truncateSync(registryPath, MAX_DEVICE_REGISTRY_FILE_BYTES + 1) + + expect(new DeviceRegistry(userDataPath).listDevices()).toEqual([]) + }) + + it('rejects a byte-oversized device without publishing a partial credential', () => { + const userDataPath = makeUserDataPath('orca-device-registry-write-bound-') + const registry = new DeviceRegistry(userDataPath) + + expect(() => registry.addDevice('x'.repeat(MAX_DEVICE_REGISTRY_FILE_BYTES), 'mobile')).toThrow( + DeviceRegistryCapacityError + ) + expect(registry.listDevices()).toEqual([]) + expect(new DeviceRegistry(userDataPath).listDevices()).toEqual([]) + }) + + it('regenerates an oversized E2EE keypair without loading it', () => { + const userDataPath = makeUserDataPath('orca-keypair-bound-') + const keypairPath = join(userDataPath, 'orca-e2ee-keypair.json') + writeFileSync(keypairPath, '{"v":1}') + truncateSync(keypairPath, MAX_KEYPAIR_FILE_BYTES + 1) + + expect(loadOrCreateE2EEKeypair(userDataPath).publicKey).toHaveLength(32) + expect(statSync(keypairPath).size).toBeLessThan(MAX_KEYPAIR_FILE_BYTES) + }) + + it('regenerates a structurally amplified E2EE keypair before parsing it', () => { + const userDataPath = makeUserDataPath('orca-keypair-structure-') + const keypairPath = join(userDataPath, 'orca-e2ee-keypair.json') + writeFileSync(keypairPath, `{"padding":[${'0,'.repeat(MAX_KEYPAIR_JSON_STRUCTURAL_TOKENS)}0]}`) + const parseSpy = vi.spyOn(JSON, 'parse') + + expect(loadOrCreateE2EEKeypair(userDataPath).publicKey).toHaveLength(32) + expect(parseSpy).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/runtime-file-watcher-admission.test.ts b/src/main/runtime/runtime-file-watcher-admission.test.ts new file mode 100644 index 00000000000..ecb5752c84d --- /dev/null +++ b/src/main/runtime/runtime-file-watcher-admission.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { + RUNTIME_FILE_WATCHER_MAX_LEASES, + RUNTIME_FILE_WATCHER_MAX_PATH_BYTES, + RuntimeFileWatcherAdmission +} from './runtime-file-watcher-admission' + +describe('runtime file watcher admission', () => { + it('caps pending and active leases, then recovers after release', () => { + const admission = new RuntimeFileWatcherAdmission() + const releases = Array.from({ length: RUNTIME_FILE_WATCHER_MAX_LEASES }, (_, index) => + admission.claim('runtime-a', index % 2 === 0 ? 'ssh-a' : undefined, `/repo-${index}`) + ) + + expect(admission.evidence().leases).toBe(RUNTIME_FILE_WATCHER_MAX_LEASES) + expect(() => admission.claim('runtime-a', undefined, '/overflow')).toThrow('capacity reached') + + releases[0]?.() + releases[0]?.() + expect(() => admission.claim('runtime-a', undefined, '/recovered')).not.toThrow() + }) + + it('bounds multibyte roots before retaining a lease', () => { + const admission = new RuntimeFileWatcherAdmission() + const exact = 'é'.repeat(RUNTIME_FILE_WATCHER_MAX_PATH_BYTES / 2) + + expect(() => admission.claim('runtime-a', 'ssh-a', exact)).not.toThrow() + expect(() => admission.claim('runtime-a', 'ssh-a', `${exact}x`)).toThrow( + `exceeds ${RUNTIME_FILE_WATCHER_MAX_PATH_BYTES} UTF-8 bytes` + ) + expect(admission.evidence().leases).toBe(1) + }) +}) diff --git a/src/main/runtime/runtime-file-watcher-admission.ts b/src/main/runtime/runtime-file-watcher-admission.ts new file mode 100644 index 00000000000..34a04647adb --- /dev/null +++ b/src/main/runtime/runtime-file-watcher-admission.ts @@ -0,0 +1,53 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const RUNTIME_FILE_WATCHER_MAX_LEASES = 256 +export const RUNTIME_FILE_WATCHER_MAX_RETAINED_IDENTITY_BYTES = 16 * 1024 * 1024 +export const RUNTIME_FILE_WATCHER_MAX_PATH_BYTES = 64 * 1024 +const RUNTIME_FILE_WATCHER_MAX_OWNER_ID_BYTES = 8 * 1024 + +export class RuntimeFileWatcherAdmission { + private leases = 0 + private retainedBytes = 0 + + claim(runtimeId: string, connectionId: string | undefined, rootPath: string): () => void { + const identityBytes = + boundedIdentityBytes(runtimeId, RUNTIME_FILE_WATCHER_MAX_OWNER_ID_BYTES, 'runtime id') + + boundedIdentityBytes(rootPath, RUNTIME_FILE_WATCHER_MAX_PATH_BYTES, 'root path') + + (connectionId + ? boundedIdentityBytes( + connectionId, + RUNTIME_FILE_WATCHER_MAX_OWNER_ID_BYTES, + 'connection id' + ) + : 0) + if ( + this.leases >= RUNTIME_FILE_WATCHER_MAX_LEASES || + this.retainedBytes + identityBytes > RUNTIME_FILE_WATCHER_MAX_RETAINED_IDENTITY_BYTES + ) { + throw new Error('Runtime file watcher capacity reached; close an existing watch and retry.') + } + this.leases += 1 + this.retainedBytes += identityBytes + let claimed = true + return () => { + if (!claimed) { + return + } + claimed = false + this.leases -= 1 + this.retainedBytes -= identityBytes + } + } + + evidence(): { leases: number; retainedBytes: number } { + return { leases: this.leases, retainedBytes: this.retainedBytes } + } +} + +function boundedIdentityBytes(value: string, limit: number, field: string): number { + const measured = measureUtf8ByteLength(value, { stopAfterBytes: limit }) + if (measured.exceededLimit) { + throw new Error(`Runtime file watcher ${field} exceeds ${limit} UTF-8 bytes.`) + } + return measured.byteLength +} diff --git a/src/main/runtime/runtime-metadata.ts b/src/main/runtime/runtime-metadata.ts index bd43203252d..18218ee8242 100644 --- a/src/main/runtime/runtime-metadata.ts +++ b/src/main/runtime/runtime-metadata.ts @@ -1,6 +1,13 @@ -import { existsSync, readFileSync, rmSync } from 'node:fs' -import { getRuntimeMetadataPath, type RuntimeMetadata } from '../../shared/runtime-bootstrap' -import { writeSecureJsonFile } from '../../shared/secure-file' +import { existsSync, rmSync } from 'node:fs' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { + getRuntimeMetadataPath, + MAX_RUNTIME_METADATA_FILE_BYTES, + parseRuntimeMetadataJson, + type RuntimeMetadata +} from '../../shared/runtime-bootstrap' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { writeSecureFile } from '../../shared/secure-file' export function writeRuntimeMetadata(userDataPath: string, metadata: RuntimeMetadata): void { const metadataPath = getRuntimeMetadataPath(userDataPath) @@ -12,7 +19,11 @@ export function readRuntimeMetadata(userDataPath: string): RuntimeMetadata | nul if (!existsSync(metadataPath)) { return null } - return JSON.parse(readFileSync(metadataPath, 'utf-8')) as RuntimeMetadata + return parseRuntimeMetadataJson( + readNodeFileSyncWithinLimit(metadataPath, MAX_RUNTIME_METADATA_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) } export function clearRuntimeMetadata(userDataPath: string): void { @@ -52,5 +63,6 @@ export function clearRuntimeMetadataIfOwned( } function writeMetadataFile(path: string, metadata: RuntimeMetadata): void { - writeSecureJsonFile(path, metadata) + const { serialized } = stringifyJsonWithinByteLimit(metadata, MAX_RUNTIME_METADATA_FILE_BYTES, 2) + writeSecureFile(path, serialized) } diff --git a/src/main/runtime/runtime-mobile-file-path-search.test.ts b/src/main/runtime/runtime-mobile-file-path-search.test.ts index 495a7933842..177fd993e25 100644 --- a/src/main/runtime/runtime-mobile-file-path-search.test.ts +++ b/src/main/runtime/runtime-mobile-file-path-search.test.ts @@ -95,4 +95,21 @@ describe('RuntimeMobileFilePathSearchCache', () => { vi.useRealTimers() } }) + + it('rejects distinct scans beyond the in-flight cache bound', async () => { + const cache = new RuntimeMobileFilePathSearchCache(2, 100) + const loads: ((value: RuntimeMobileFilePathInventory) => void)[] = [] + const load = () => + new Promise((resolve) => { + loads.push(resolve) + }) + const first = cache.get('a', load) + const second = cache.get('b', load) + + await expect(cache.get('c', load)).rejects.toThrow('search is busy') + + const inventory = { paths: [], totalCount: 0, truncated: false } + loads.forEach((resolve) => resolve(inventory)) + await expect(Promise.all([first, second])).resolves.toEqual([inventory, inventory]) + }) }) diff --git a/src/main/runtime/runtime-mobile-file-path-search.ts b/src/main/runtime/runtime-mobile-file-path-search.ts index 13213d6dfcf..61001b04bec 100644 --- a/src/main/runtime/runtime-mobile-file-path-search.ts +++ b/src/main/runtime/runtime-mobile-file-path-search.ts @@ -34,6 +34,9 @@ export class RuntimeMobileFilePathSearchCache { if (pending) { return pending } + if (this.inFlight.size >= this.maxEntries) { + throw new Error('Mobile file path search is busy; retry after current searches finish.') + } const next = load() .then((loaded) => { // Why: a slow SSH scan should receive a full TTL after it becomes usable, diff --git a/src/main/runtime/runtime-operation-generations.test.ts b/src/main/runtime/runtime-operation-generations.test.ts new file mode 100644 index 00000000000..310fd79add2 --- /dev/null +++ b/src/main/runtime/runtime-operation-generations.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_RUNTIME_OPERATION_GENERATIONS, + MAX_RUNTIME_OPERATION_GENERATION_KEY_BYTES, + MAX_RUNTIME_OPERATION_GENERATION_RETAINED_KEY_BYTES, + RuntimeOperationGenerations +} from './runtime-operation-generations' + +const BOUNDS = { + maxEntries: 2, + maxKeyBytes: 4, + maxRetainedKeyBytes: 6 +} + +describe('RuntimeOperationGenerations', () => { + it('keeps a stable generation until the operation advances', () => { + const generations = new RuntimeOperationGenerations(BOUNDS) + const initial = generations.current('a') + + expect(generations.current('a')).toBe(initial) + expect(generations.advance('a')).not.toBe(initial) + expect(generations.isCurrent('a', initial)).toBe(false) + }) + + it('evicts least-recently-used keys and fails closed for stale captures', () => { + const generations = new RuntimeOperationGenerations(BOUNDS) + const first = generations.current('a') + const evicted = generations.current('b') + generations.current('a') + const latest = generations.current('cc') + + expect(generations.isCurrent('a', first)).toBe(true) + expect(generations.isCurrent('b', evicted)).toBe(false) + expect(generations.isCurrent('cc', latest)).toBe(true) + expect(generations.evidence()).toEqual({ entries: 2, retainedKeyBytes: 3 }) + }) + + it('bounds aggregate retained key bytes', () => { + const generations = new RuntimeOperationGenerations({ + maxEntries: 3, + maxKeyBytes: 4, + maxRetainedKeyBytes: 3 + }) + const evicted = generations.current('aa') + generations.current('b') + generations.current('cc') + + expect(generations.isCurrent('aa', evicted)).toBe(false) + expect(generations.evidence()).toEqual({ entries: 2, retainedKeyBytes: 3 }) + }) + + it('does not retain oversized keys', () => { + const generations = new RuntimeOperationGenerations(BOUNDS) + const generation = generations.current('oversized') + + expect(generations.isCurrent('oversized', generation)).toBe(false) + expect(generations.evidence()).toEqual({ entries: 0, retainedKeyBytes: 0 }) + }) + + it('invalidates forgotten owners without retaining tombstones', () => { + const generations = new RuntimeOperationGenerations(BOUNDS) + const stale = generations.current('a') + generations.forget('a') + + expect(generations.isCurrent('a', stale)).toBe(false) + expect(generations.current('a')).not.toBe(stale) + expect(generations.evidence()).toEqual({ entries: 1, retainedKeyBytes: 1 }) + }) + + it('publishes explicit production bounds', () => { + expect(MAX_RUNTIME_OPERATION_GENERATIONS).toBe(8_192) + expect(MAX_RUNTIME_OPERATION_GENERATION_KEY_BYTES).toBe(64 * 1024) + expect(MAX_RUNTIME_OPERATION_GENERATION_RETAINED_KEY_BYTES).toBe(4 * 1024 * 1024) + }) +}) diff --git a/src/main/runtime/runtime-operation-generations.ts b/src/main/runtime/runtime-operation-generations.ts new file mode 100644 index 00000000000..a35d1aee5db --- /dev/null +++ b/src/main/runtime/runtime-operation-generations.ts @@ -0,0 +1,122 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MAX_RUNTIME_OPERATION_GENERATIONS = 8_192 +export const MAX_RUNTIME_OPERATION_GENERATION_KEY_BYTES = 64 * 1024 +export const MAX_RUNTIME_OPERATION_GENERATION_RETAINED_KEY_BYTES = 4 * 1024 * 1024 + +export type RuntimeOperationGenerationBounds = { + maxEntries: number + maxKeyBytes: number + maxRetainedKeyBytes: number +} + +const DEFAULT_BOUNDS: RuntimeOperationGenerationBounds = { + maxEntries: MAX_RUNTIME_OPERATION_GENERATIONS, + maxKeyBytes: MAX_RUNTIME_OPERATION_GENERATION_KEY_BYTES, + maxRetainedKeyBytes: MAX_RUNTIME_OPERATION_GENERATION_RETAINED_KEY_BYTES +} + +type RetainedGeneration = { + generation: number + keyBytes: number +} + +export class RuntimeOperationGenerations { + private readonly generations = new Map() + private nextGeneration = 1 + private retainedKeyBytes = 0 + + constructor(private readonly bounds: RuntimeOperationGenerationBounds = DEFAULT_BOUNDS) { + if ( + !Number.isSafeInteger(bounds.maxEntries) || + bounds.maxEntries < 1 || + !Number.isSafeInteger(bounds.maxKeyBytes) || + bounds.maxKeyBytes < 1 || + !Number.isSafeInteger(bounds.maxRetainedKeyBytes) || + bounds.maxRetainedKeyBytes < 1 + ) { + throw new RangeError('Runtime operation generation bounds must be positive integers') + } + } + + current(key: string): number { + const retained = this.generations.get(key) + if (!retained) { + // Why: a missing key may have been evicted, so it must never reuse a stale token. + return this.replace(key) + } + this.generations.delete(key) + this.generations.set(key, retained) + return retained.generation + } + + advance(key: string): number { + return this.replace(key) + } + + isCurrent(key: string, generation: number): boolean { + const retained = this.generations.get(key) + if (retained?.generation !== generation) { + return false + } + this.generations.delete(key) + this.generations.set(key, retained) + return true + } + + forget(key: string): void { + this.delete(key) + } + + evidence(): { entries: number; retainedKeyBytes: number } { + return { + entries: this.generations.size, + retainedKeyBytes: this.retainedKeyBytes + } + } + + private replace(key: string): number { + this.delete(key) + const generation = this.takeGeneration() + const measurement = measureUtf8ByteLength(key, { + stopAfterBytes: this.bounds.maxKeyBytes + }) + if (measurement.exceededLimit || measurement.byteLength > this.bounds.maxRetainedKeyBytes) { + return generation + } + while ( + this.generations.size >= this.bounds.maxEntries || + this.retainedKeyBytes + measurement.byteLength > this.bounds.maxRetainedKeyBytes + ) { + const oldest = this.generations.keys().next().value + if (oldest === undefined) { + return generation + } + this.delete(oldest) + } + this.generations.set(key, { + generation, + keyBytes: measurement.byteLength + }) + this.retainedKeyBytes += measurement.byteLength + return generation + } + + private takeGeneration(): number { + if (!Number.isSafeInteger(this.nextGeneration)) { + throw new Error('Runtime operation generation exhausted') + } + const generation = this.nextGeneration + this.nextGeneration += 1 + return generation + } + + private delete(key: string): void { + const retained = this.generations.get(key) + if (!retained) { + return + } + this.generations.delete(key) + this.retainedKeyBytes -= retained.keyBytes + } +} diff --git a/src/main/runtime/runtime-rpc-request-admission.test.ts b/src/main/runtime/runtime-rpc-request-admission.test.ts new file mode 100644 index 00000000000..4253cfc14fa --- /dev/null +++ b/src/main/runtime/runtime-rpc-request-admission.test.ts @@ -0,0 +1,107 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import type { WebSocket } from 'ws' +import { REMOTE_RUNTIME_JSON_STRUCTURE_LIMITS } from '../../shared/remote-runtime-request-frames' +import { DeviceRegistry } from './device-registry' +import type { OrcaRuntimeService } from './orca-runtime' +import { OrcaRuntimeRpcServer } from './runtime-rpc' +import type { RpcResponse } from './rpc/core' + +function createServer(shortRequestCap = 64): OrcaRuntimeRpcServer { + const runtime = { getRuntimeId: () => 'runtime-test' } as OrcaRuntimeService + return new OrcaRuntimeRpcServer({ + runtime, + userDataPath: mkdtempSync(join(tmpdir(), 'orca-runtime-admission-')), + shortRequestCap + }) +} + +function localRequest(server: OrcaRuntimeRpcServer, id: string): string { + return JSON.stringify({ + id, + authToken: server['authToken'], + method: 'status.get' + }) +} + +describe('runtime RPC request admission', () => { + it('rejects structural amplification before JSON.parse', async () => { + const server = createServer() + const amplified = `{"id":"amplified","authToken":"${server['authToken']}","method":"status.get","params":[${'0,'.repeat(REMOTE_RUNTIME_JSON_STRUCTURE_LIMITS.structuralTokens)}0]}` + const parse = vi.spyOn(JSON, 'parse') + + const response = await server['handleMessage'](amplified) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'bad_request' } + }) + expect(parse).not.toHaveBeenCalled() + parse.mockRestore() + }) + + it('caps concurrent short local requests and releases capacity on settle', async () => { + const server = createServer(1) + let releaseFirst: ((response: RpcResponse) => void) | undefined + const firstResponse = new Promise((resolve) => { + releaseFirst = resolve + }) + const dispatch = vi + .spyOn(server['dispatcher'], 'dispatch') + .mockImplementationOnce(() => firstResponse) + .mockResolvedValue({ + id: 'after', + ok: true, + result: null, + _meta: { runtimeId: 'runtime-test' } + }) + + const first = server['handleMessage'](localRequest(server, 'first')) + await vi.waitFor(() => expect(server['activeShortRequests']).toBe(1)) + + await expect(server['handleMessage'](localRequest(server, 'overflow'))).resolves.toMatchObject({ + id: 'overflow', + ok: false, + error: { code: 'runtime_busy' } + }) + expect(dispatch).toHaveBeenCalledTimes(1) + + releaseFirst?.({ + id: 'first', + ok: true, + result: null, + _meta: { runtimeId: 'runtime-test' } + }) + await first + expect(server['activeShortRequests']).toBe(0) + await server['handleMessage'](localRequest(server, 'after')) + expect(dispatch).toHaveBeenCalledTimes(2) + }) + + it('applies the same short-request cap to WebSocket dispatch', async () => { + const server = createServer(1) + const registry = new DeviceRegistry(mkdtempSync(join(tmpdir(), 'orca-runtime-device-'))) + const device = registry.addDevice('runtime-test', 'runtime') + server['deviceRegistry'] = registry + server['activeShortRequests'] = 1 + const replies: RpcResponse[] = [] + + await server['handleWebSocketMessage']( + JSON.stringify({ id: 'overflow', method: 'status.get', deviceToken: device.token }), + (response) => replies.push(JSON.parse(response) as RpcResponse), + () => {}, + undefined, + undefined as WebSocket | undefined + ) + + expect(replies).toEqual([ + expect.objectContaining({ + id: 'overflow', + ok: false, + error: expect.objectContaining({ code: 'runtime_busy' }) + }) + ]) + }) +}) diff --git a/src/main/runtime/runtime-rpc.ts b/src/main/runtime/runtime-rpc.ts index 57029a95762..7cf30eabdc9 100644 --- a/src/main/runtime/runtime-rpc.ts +++ b/src/main/runtime/runtime-rpc.ts @@ -1,7 +1,7 @@ /* eslint-disable max-lines -- Why: this file is the single security boundary for the bundled CLI — transport setup, auth-token enforcement, admission control, keepalive framing, and orphan-socket sweeping all co-locate deliberately so a reviewer can audit the boundary in one sitting. Splitting this across files would scatter the invariants without reducing complexity. */ // Why: the single security boundary for the bundled CLI — auth-token enforcement, metadata publication, transport orchestration. import { randomBytes } from 'node:crypto' -import { readdirSync, rmSync } from 'node:fs' +import { opendirSync, rmSync } from 'node:fs' import { join } from 'node:path' import type { RuntimeMetadata, RuntimeTransportMetadata } from '../../shared/runtime-bootstrap' import type { OrcaRuntimeService } from './orca-runtime' @@ -41,6 +41,7 @@ import { decodeTerminalStreamFrame, type TerminalStreamFrame } from '../../shared/terminal-stream-protocol' +import { parseRemoteRuntimeJsonText } from '../../shared/remote-runtime-request-frames' const DEFAULT_WS_PORT = 6768 @@ -54,9 +55,10 @@ type OrcaRuntimeRpcServerOptions = { // Why: true when the caller pinned a port (`orca serve --port`) so bind order prefers it over a stale STA-1511 fallback (#8535). preferPinnedWsPort?: boolean webClientRoot?: string - // Why: test-only overrides for the two constants below; production must not pass these (defaults set by §3.1). + // Why: test-only overrides for the admission constants below; production uses the defaults. keepaliveIntervalMs?: number longPollCap?: number + shortRequestCap?: number } export type PairingOfferUnavailableReason = @@ -114,6 +116,8 @@ const KEEPALIVE_INTERVAL_MS = 10_000 // Why: cap long-polls at half the 32-slot connection budget so they can't starve short RPCs; overflow → runtime_busy. See §7 risk #2. const LONG_POLL_CAP = 16 +// Why: multiplexed sockets otherwise let one peer retain an unbounded number of active request graphs. +const SHORT_REQUEST_CAP = 64 function createWebClientUrl(endpoint: string, pairingUrl: string): string { const url = new URL(endpoint) @@ -362,6 +366,7 @@ const MOBILE_RPC_METHOD_ALLOWLIST = new Set([ 'terminal.ensureAgentSession', 'terminal.focus', 'terminal.agentStatus', + 'terminal.adoptOrphans', 'terminal.getAutoRestoreFit', 'terminal.isRunningAgent', 'terminal.list', @@ -430,6 +435,7 @@ export class OrcaRuntimeRpcServer { private readonly authToken = randomBytes(24).toString('hex') private readonly keepaliveIntervalMs: number private readonly longPollCap: number + private readonly shortRequestCap: number private readonly relayRevokeOutbox: RelayRevokeOutbox private deviceRegistry: DeviceRegistry | null = null private e2eeKeypair: E2EEKeypair | null = null @@ -451,6 +457,7 @@ export class OrcaRuntimeRpcServer { >() // Why: separate from server.maxConnections — count only long-running dispatches, not short RPCs. See §3.1 + §7 risk #2. private activeLongPolls = 0 + private activeShortRequests = 0 constructor({ runtime, @@ -462,7 +469,8 @@ export class OrcaRuntimeRpcServer { preferPinnedWsPort = false, webClientRoot, keepaliveIntervalMs = KEEPALIVE_INTERVAL_MS, - longPollCap = LONG_POLL_CAP + longPollCap = LONG_POLL_CAP, + shortRequestCap = SHORT_REQUEST_CAP }: OrcaRuntimeRpcServerOptions) { this.runtime = runtime this.dispatcher = new RpcDispatcher({ runtime }) @@ -475,6 +483,7 @@ export class OrcaRuntimeRpcServer { this.webClientRoot = webClientRoot this.keepaliveIntervalMs = keepaliveIntervalMs this.longPollCap = longPollCap + this.shortRequestCap = shortRequestCap this.relayRevokeOutbox = new RelayRevokeOutbox(userDataPath) } @@ -861,7 +870,7 @@ export class OrcaRuntimeRpcServer { // Why: best-effort id recovery so the client can correlate the error frame to its pending request. let id = 'unknown' try { - const parsed = JSON.parse(msg) as { id?: unknown } + const parsed = parseRemoteRuntimeJsonText(msg) as { id?: unknown } if (typeof parsed.id === 'string' && parsed.id.length > 0) { id = parsed.id } @@ -1008,7 +1017,7 @@ export class OrcaRuntimeRpcServer { } const request = parsed.request - // Why: long-poll admission fence; short RPCs bypass the counter. See §7 risk #2. + // Why: long-polls and short work have separate budgets so waits cannot starve ordinary RPCs. const longPoll = isLongPollRequest(request) if (longPoll && this.activeLongPolls >= this.longPollCap) { return this.buildError( @@ -1017,10 +1026,19 @@ export class OrcaRuntimeRpcServer { 'long-poll capacity reached; retry with backoff' ) } + if (!longPoll && this.activeShortRequests >= this.shortRequestCap) { + return this.buildError( + request.id, + 'runtime_busy', + 'short-request capacity reached; retry with backoff' + ) + } if (longPoll) { this.activeLongPolls += 1 // Why: arm keepalive only for long-polls; short RPCs never create the setInterval. See §3.1. context?.startKeepalive() + } else { + this.activeShortRequests += 1 } try { @@ -1030,6 +1048,8 @@ export class OrcaRuntimeRpcServer { } finally { if (longPoll) { this.activeLongPolls = Math.max(0, this.activeLongPolls - 1) + } else { + this.activeShortRequests = Math.max(0, this.activeShortRequests - 1) } } } @@ -1037,7 +1057,7 @@ export class OrcaRuntimeRpcServer { private parseAndAuth(rawMessage: string): { request: RpcRequest } | { error: RpcResponse } { let request: RpcRequest try { - request = JSON.parse(rawMessage) as RpcRequest + request = parseRemoteRuntimeJsonText(rawMessage) as RpcRequest } catch { return { error: this.buildError('unknown', 'bad_request', 'Invalid JSON request') } } @@ -1070,7 +1090,7 @@ export class OrcaRuntimeRpcServer { ): Promise { let request: RpcRequest try { - request = JSON.parse(rawMessage) as RpcRequest + request = parseRemoteRuntimeJsonText(rawMessage) as RpcRequest } catch { reply(JSON.stringify(this.buildError('unknown', 'bad_request', 'Invalid JSON request'))) return @@ -1135,10 +1155,24 @@ export class OrcaRuntimeRpcServer { ) return } + if (!longPoll && this.activeShortRequests >= this.shortRequestCap) { + reply( + JSON.stringify( + this.buildError( + request.id, + 'runtime_busy', + 'short-request capacity reached; retry with backoff' + ) + ) + ) + return + } const abortRegistration = ws ? this.registerWebSocketDispatchAbort(ws) : null if (longPoll) { this.activeLongPolls += 1 + } else { + this.activeShortRequests += 1 } // Why: older pairings may lack scope metadata, so stamp the authenticated scope onto status.get. @@ -1189,6 +1223,8 @@ export class OrcaRuntimeRpcServer { abortRegistration?.dispose() if (longPoll) { this.activeLongPolls = Math.max(0, this.activeLongPolls - 1) + } else { + this.activeShortRequests = Math.max(0, this.activeShortRequests - 1) } } } @@ -1213,38 +1249,50 @@ export class OrcaRuntimeRpcServer { export const RUNTIME_SOCKET_NAME_REGEX = /^o-(\d+)-[A-Za-z0-9_-]+\.sock$/ export function sweepOrphanedRuntimeSockets(userDataPath: string, ownPid: number): void { - let entries: string[] + let directory: ReturnType try { - entries = readdirSync(userDataPath) + directory = opendirSync(userDataPath) } catch { // Why: first-launch userData may not exist yet; nothing to sweep. return } - for (const entry of entries) { - const match = RUNTIME_SOCKET_NAME_REGEX.exec(entry) - if (!match) { - continue - } - const pid = Number(match[1]) - if (!Number.isFinite(pid)) { - continue - } - // Why: never delete our own socket — a bug here would rmSync one we're about to bind. - if (pid === ownPid) { - continue - } - try { - // Why: signal 0 is the POSIX liveness probe (sends nothing); ESRCH = dead pid, EPERM = foreign owner (left alone). - process.kill(pid, 0) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') { - try { - rmSync(join(userDataPath, entry), { force: true }) - } catch { - // Why: best-effort sweep; a later start() or OS reboot cleans any socket we can't unlink. + try { + while (true) { + const entry = directory.readSync() + if (!entry) { + break + } + const match = RUNTIME_SOCKET_NAME_REGEX.exec(entry.name) + if (!match) { + continue + } + const pid = Number(match[1]) + if (!Number.isFinite(pid)) { + continue + } + // Why: never delete our own socket — a bug here would rmSync one we're about to bind. + if (pid === ownPid) { + continue + } + try { + // Why: signal 0 is the POSIX liveness probe (sends nothing); ESRCH = dead pid, EPERM = foreign owner (left alone). + process.kill(pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') { + try { + rmSync(join(userDataPath, entry.name), { force: true }) + } catch { + // Why: best-effort sweep; a later start() or OS reboot cleans any socket we can't unlink. + } } } } + } finally { + try { + directory.closeSync() + } catch { + // Best-effort sweep cleanup. + } } } diff --git a/src/main/runtime/runtime-ssh-relay-recovery-generations.test.ts b/src/main/runtime/runtime-ssh-relay-recovery-generations.test.ts new file mode 100644 index 00000000000..bca69548e8e --- /dev/null +++ b/src/main/runtime/runtime-ssh-relay-recovery-generations.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_RUNTIME_SSH_RELAY_RECOVERIES, + MAX_RUNTIME_SSH_RELAY_RECOVERY_RETAINED_TARGET_ID_BYTES, + MAX_RUNTIME_SSH_RELAY_RECOVERY_TARGET_ID_BYTES, + RuntimeSshRelayRecoveryGenerations +} from './runtime-ssh-relay-recovery-generations' + +describe('RuntimeSshRelayRecoveryGenerations', () => { + it('releases completed recoveries under sequential unique-target churn', () => { + const generations = new RuntimeSshRelayRecoveryGenerations() + + for (let index = 0; index < 10_000; index += 1) { + const targetId = `ssh-${index}` + const lease = generations.begin(targetId) + expect(lease).not.toBeNull() + lease?.release() + } + + expect(generations.evidence()).toEqual({ recoveries: 0, retainedTargetIdBytes: 0 }) + }) + + it('keeps a replacement current when an older recovery finishes late', () => { + const generations = new RuntimeSshRelayRecoveryGenerations() + const stale = generations.begin('ssh-1')! + const replacement = generations.begin('ssh-1')! + + stale.release() + + expect(stale.isCurrent()).toBe(false) + expect(replacement.isCurrent()).toBe(true) + expect(generations.evidence()).toEqual({ + recoveries: 1, + retainedTargetIdBytes: 5 + }) + }) + + it('caps same-target hung replacements by outstanding lease count', () => { + const generations = new RuntimeSshRelayRecoveryGenerations() + const leases = Array.from({ length: 10_000 }, () => generations.begin('ssh-1')).filter( + (lease) => lease !== null + ) + + expect(leases).toHaveLength(MAX_RUNTIME_SSH_RELAY_RECOVERIES) + expect(leases.slice(0, -1).every((lease) => !lease.isCurrent())).toBe(true) + expect(leases.at(-1)?.isCurrent()).toBe(true) + expect(generations.evidence()).toEqual({ + recoveries: MAX_RUNTIME_SSH_RELAY_RECOVERIES, + retainedTargetIdBytes: MAX_RUNTIME_SSH_RELAY_RECOVERIES * 5 + }) + + for (const lease of leases) { + lease.release() + } + expect(generations.evidence()).toEqual({ recoveries: 0, retainedTargetIdBytes: 0 }) + }) + + it('keeps invalidated attempts admitted until their work releases', () => { + const generations = new RuntimeSshRelayRecoveryGenerations({ + maxRecoveries: 1, + maxTargetIdBytes: 8, + maxRetainedTargetIdBytes: 8 + }) + const stale = generations.begin('ssh-1')! + + generations.invalidate('ssh-1') + + expect(stale.isCurrent()).toBe(false) + expect(generations.begin('ssh-2')).toBeNull() + stale.release() + expect(generations.begin('ssh-2')?.isCurrent()).toBe(true) + }) + + it('rejects unique hung recoveries at the concurrent-entry cap and admits after release', () => { + const generations = new RuntimeSshRelayRecoveryGenerations() + const leases = Array.from({ length: MAX_RUNTIME_SSH_RELAY_RECOVERIES }, (_, index) => + generations.begin(`ssh-${index}`) + ) + + expect(leases.every(Boolean)).toBe(true) + expect(generations.begin('ssh-overflow')).toBeNull() + leases[0]?.release() + + const replacement = generations.begin('ssh-overflow') + expect(replacement?.isCurrent()).toBe(true) + for (const lease of leases) { + lease?.release() + } + replacement?.release() + expect(generations.evidence()).toEqual({ recoveries: 0, retainedTargetIdBytes: 0 }) + }) + + it('bounds individual and aggregate retained target-id bytes', () => { + const generations = new RuntimeSshRelayRecoveryGenerations({ + maxRecoveries: 3, + maxTargetIdBytes: 8, + maxRetainedTargetIdBytes: 10 + }) + const first = generations.begin('123456') + const second = generations.begin('abcd') + + expect(first?.isCurrent()).toBe(true) + expect(second?.isCurrent()).toBe(true) + expect(generations.begin('x')).toBeNull() + expect(generations.begin('🌊🌊🌊')).toBeNull() + expect(generations.evidence()).toEqual({ recoveries: 2, retainedTargetIdBytes: 10 }) + }) + + it('publishes explicit production bounds', () => { + expect(MAX_RUNTIME_SSH_RELAY_RECOVERIES).toBe(256) + expect(MAX_RUNTIME_SSH_RELAY_RECOVERY_TARGET_ID_BYTES).toBe(64 * 1024) + expect(MAX_RUNTIME_SSH_RELAY_RECOVERY_RETAINED_TARGET_ID_BYTES).toBe(4 * 1024 * 1024) + }) +}) diff --git a/src/main/runtime/runtime-ssh-relay-recovery-generations.ts b/src/main/runtime/runtime-ssh-relay-recovery-generations.ts new file mode 100644 index 00000000000..658521eb988 --- /dev/null +++ b/src/main/runtime/runtime-ssh-relay-recovery-generations.ts @@ -0,0 +1,108 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const MAX_RUNTIME_SSH_RELAY_RECOVERIES = 256 +export const MAX_RUNTIME_SSH_RELAY_RECOVERY_TARGET_ID_BYTES = 64 * 1024 +export const MAX_RUNTIME_SSH_RELAY_RECOVERY_RETAINED_TARGET_ID_BYTES = 4 * 1024 * 1024 + +export type RuntimeSshRelayRecoveryGenerationBounds = { + maxRecoveries: number + maxTargetIdBytes: number + maxRetainedTargetIdBytes: number +} + +export type RuntimeSshRelayRecoveryGenerationLease = { + isCurrent: () => boolean + release: () => void +} + +const DEFAULT_BOUNDS: RuntimeSshRelayRecoveryGenerationBounds = { + maxRecoveries: MAX_RUNTIME_SSH_RELAY_RECOVERIES, + maxTargetIdBytes: MAX_RUNTIME_SSH_RELAY_RECOVERY_TARGET_ID_BYTES, + maxRetainedTargetIdBytes: MAX_RUNTIME_SSH_RELAY_RECOVERY_RETAINED_TARGET_ID_BYTES +} + +type RetainedRecovery = { + generation: number + targetIdBytes: number +} + +export class RuntimeSshRelayRecoveryGenerations { + private readonly generationByTargetId = new Map() + private activeRecoveries = 0 + private retainedTargetIdBytes = 0 + private nextGeneration = 1 + + constructor(private readonly bounds: RuntimeSshRelayRecoveryGenerationBounds = DEFAULT_BOUNDS) { + if ( + !Number.isSafeInteger(bounds.maxRecoveries) || + bounds.maxRecoveries < 1 || + !Number.isSafeInteger(bounds.maxTargetIdBytes) || + bounds.maxTargetIdBytes < 1 || + !Number.isSafeInteger(bounds.maxRetainedTargetIdBytes) || + bounds.maxRetainedTargetIdBytes < 1 + ) { + throw new RangeError('SSH relay recovery generation bounds must be positive integers') + } + } + + begin(targetId: string): RuntimeSshRelayRecoveryGenerationLease | null { + const targetIdMeasurement = measureUtf8ByteLength(targetId, { + stopAfterBytes: this.bounds.maxTargetIdBytes + }) + if (targetIdMeasurement.exceededLimit) { + return null + } + if ( + this.activeRecoveries >= this.bounds.maxRecoveries || + this.retainedTargetIdBytes + targetIdMeasurement.byteLength > + this.bounds.maxRetainedTargetIdBytes + ) { + return null + } + if (!Number.isSafeInteger(this.nextGeneration)) { + throw new Error('SSH relay recovery generation exhausted') + } + const generation = this.nextGeneration + this.nextGeneration += 1 + const retained = { + generation, + targetIdBytes: targetIdMeasurement.byteLength + } + this.generationByTargetId.set(targetId, retained) + this.activeRecoveries += 1 + this.retainedTargetIdBytes += retained.targetIdBytes + let active = true + return { + isCurrent: () => + active && + this.generationByTargetId.get(targetId) === retained && + retained.generation === generation, + release: () => { + if (!active) { + return + } + active = false + this.releaseLease(targetId, retained) + } + } + } + + invalidate(targetId: string): void { + this.generationByTargetId.delete(targetId) + } + + evidence(): { recoveries: number; retainedTargetIdBytes: number } { + return { + recoveries: this.activeRecoveries, + retainedTargetIdBytes: this.retainedTargetIdBytes + } + } + + private releaseLease(targetId: string, retained: RetainedRecovery): void { + this.activeRecoveries -= 1 + this.retainedTargetIdBytes -= retained.targetIdBytes + if (this.generationByTargetId.get(targetId) === retained) { + this.generationByTargetId.delete(targetId) + } + } +} diff --git a/src/main/runtime/runtime-terminal-file-grant-retention.test.ts b/src/main/runtime/runtime-terminal-file-grant-retention.test.ts new file mode 100644 index 00000000000..7634f0e1bec --- /dev/null +++ b/src/main/runtime/runtime-terminal-file-grant-retention.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from 'vitest' +import { retainRuntimeTerminalFileGrant } from './runtime-terminal-file-grant-retention' + +describe('retainRuntimeTerminalFileGrant', () => { + it('preserves grants below the cap and evicts the oldest at admission', () => { + const grants = new Map() + const release = vi.fn((id: string) => grants.delete(id)) + + retainRuntimeTerminalFileGrant(grants, { id: 'a' }, release, 2) + retainRuntimeTerminalFileGrant(grants, { id: 'b' }, release, 2) + expect([...grants.keys()]).toEqual(['a', 'b']) + expect(release).not.toHaveBeenCalled() + + retainRuntimeTerminalFileGrant(grants, { id: 'c' }, release, 2) + expect([...grants.keys()]).toEqual(['b', 'c']) + expect(release).toHaveBeenCalledWith('a', { id: 'a' }) + }) +}) diff --git a/src/main/runtime/runtime-terminal-file-grant-retention.ts b/src/main/runtime/runtime-terminal-file-grant-retention.ts new file mode 100644 index 00000000000..24ef7efea76 --- /dev/null +++ b/src/main/runtime/runtime-terminal-file-grant-retention.ts @@ -0,0 +1,22 @@ +export const RUNTIME_TERMINAL_FILE_GRANT_MAX_ENTRIES = 1024 + +export function retainRuntimeTerminalFileGrant( + grants: Map, + grant: T, + release: (id: string, retained: T) => void, + maxEntries = RUNTIME_TERMINAL_FILE_GRANT_MAX_ENTRIES +): void { + while (grants.size >= Math.max(1, maxEntries)) { + const oldestId = grants.keys().next().value as string | undefined + if (oldestId === undefined) { + break + } + const oldest = grants.get(oldestId) + if (oldest) { + release(oldestId, oldest) + } else { + grants.delete(oldestId) + } + } + grants.set(grant.id, grant) +} diff --git a/src/main/runtime/runtime-text-search-admission.test.ts b/src/main/runtime/runtime-text-search-admission.test.ts new file mode 100644 index 00000000000..e245160aba6 --- /dev/null +++ b/src/main/runtime/runtime-text-search-admission.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'vitest' +import { assertRuntimeTextSearchAdmission } from './runtime-text-search-admission' + +describe('assertRuntimeTextSearchAdmission', () => { + it('allows ordinary and replacement searches but rejects excess distinct roots', () => { + const active = new Map([ + ['repo-a', {}], + ['repo-b', {}] + ]) + + expect(() => assertRuntimeTextSearchAdmission(active, 'repo-a', 2)).not.toThrow() + expect(() => assertRuntimeTextSearchAdmission(active, 'repo-c', 2)).toThrow('search is busy') + }) +}) diff --git a/src/main/runtime/runtime-text-search-admission.ts b/src/main/runtime/runtime-text-search-admission.ts new file mode 100644 index 00000000000..78517515a32 --- /dev/null +++ b/src/main/runtime/runtime-text-search-admission.ts @@ -0,0 +1,11 @@ +export const RUNTIME_TEXT_SEARCH_MAX_ACTIVE = 16 + +export function assertRuntimeTextSearchAdmission( + activeSearchKeys: ReadonlyMap, + requestedKey: string, + maxActive = RUNTIME_TEXT_SEARCH_MAX_ACTIVE +): void { + if (!activeSearchKeys.has(requestedKey) && activeSearchKeys.size >= maxActive) { + throw new Error('Runtime text search is busy; retry after current searches finish.') + } +} diff --git a/src/main/runtime/terminal-orphan-owner.test.ts b/src/main/runtime/terminal-orphan-owner.test.ts new file mode 100644 index 00000000000..37e24e63b14 --- /dev/null +++ b/src/main/runtime/terminal-orphan-owner.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { terminalOrphanExecutionOwnersEqual } from './terminal-orphan-owner' + +describe('terminal orphan execution owner', () => { + it('requires exact SSH host ownership', () => { + expect( + terminalOrphanExecutionOwnersEqual( + { connectionId: 'ssh-a', wslDistro: null }, + { connectionId: 'ssh-b', wslDistro: null } + ) + ).toBe(false) + expect( + terminalOrphanExecutionOwnersEqual( + { connectionId: 'ssh-a', wslDistro: null }, + { connectionId: 'ssh-a' } + ) + ).toBe(true) + }) + + it('matches WSL distro case-insensitively but never crosses native or another distro', () => { + const expected = { connectionId: null, wslDistro: 'Ubuntu' } + expect( + terminalOrphanExecutionOwnersEqual(expected, { + connectionId: null, + wslDistro: 'ubuntu' + }) + ).toBe(true) + expect( + terminalOrphanExecutionOwnersEqual(expected, { connectionId: null, wslDistro: null }) + ).toBe(false) + expect( + terminalOrphanExecutionOwnersEqual(expected, { + connectionId: null, + wslDistro: 'Debian' + }) + ).toBe(false) + expect(terminalOrphanExecutionOwnersEqual(expected, { connectionId: null })).toBe(false) + }) +}) diff --git a/src/main/runtime/terminal-orphan-owner.ts b/src/main/runtime/terminal-orphan-owner.ts new file mode 100644 index 00000000000..2b098b288ff --- /dev/null +++ b/src/main/runtime/terminal-orphan-owner.ts @@ -0,0 +1,25 @@ +export type TerminalOrphanExecutionOwner = { + connectionId: string | null + wslDistro?: string | null +} + +function normalizeWslDistro(distro: string | null): string | null { + const normalized = distro?.trim().toLowerCase() ?? '' + return normalized || null +} + +export function terminalOrphanExecutionOwnersEqual( + expected: TerminalOrphanExecutionOwner, + actual: TerminalOrphanExecutionOwner +): boolean { + if (expected.connectionId !== actual.connectionId) { + return false + } + if (expected.connectionId !== null) { + return true + } + if (expected.wslDistro === undefined || actual.wslDistro === undefined) { + return false + } + return normalizeWslDistro(expected.wslDistro) === normalizeWslDistro(actual.wslDistro) +} diff --git a/src/main/runtime/terminal-orphan-topology.test.ts b/src/main/runtime/terminal-orphan-topology.test.ts new file mode 100644 index 00000000000..aea55932462 --- /dev/null +++ b/src/main/runtime/terminal-orphan-topology.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from 'vitest' +import { + hasExactTerminalOrphanGroupLayout, + mergeTerminalOrphanGroupLayout +} from './terminal-orphan-topology' + +describe('terminal orphan topology', () => { + it('rejects duplicate and missing group leaves', () => { + expect( + hasExactTerminalOrphanGroupLayout( + { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', groupId: 'group-a' }, + second: { type: 'leaf', groupId: 'group-a' } + }, + new Set(['group-a', 'group-b']) + ) + ).toBe(false) + }) + + it('keeps current host layout while preserving an unrelated proposed subtree', () => { + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout: { type: 'leaf', groupId: 'group-live' }, + existingGroupIds: ['group-live'], + proposedLayout: { + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-old-left' }, + second: { type: 'leaf', groupId: 'group-old-right' } + }, + proposedGroupIds: ['group-old-left', 'group-old-right'], + mergedGroupIds: ['group-live', 'group-old-left', 'group-old-right'] + }) + ).toEqual({ + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-live' }, + second: { + type: 'split', + direction: 'vertical', + ratio: 0.6, + first: { type: 'leaf', groupId: 'group-old-left' }, + second: { type: 'leaf', groupId: 'group-old-right' } + } + }) + }) + + it('grafts proposed groups beside their one current anchor without duplicating it', () => { + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout: { + type: 'split', + direction: 'vertical', + first: { type: 'leaf', groupId: 'group-live' }, + second: { type: 'leaf', groupId: 'group-other' } + }, + existingGroupIds: ['group-live', 'group-other'], + proposedLayout: { + type: 'split', + direction: 'horizontal', + ratio: 0.7, + first: { type: 'leaf', groupId: 'group-live' }, + second: { type: 'leaf', groupId: 'group-recovered' } + }, + proposedGroupIds: ['group-live', 'group-recovered'], + mergedGroupIds: ['group-live', 'group-other', 'group-recovered'] + }) + ).toEqual({ + type: 'split', + direction: 'vertical', + first: { + type: 'split', + direction: 'horizontal', + ratio: 0.7, + first: { type: 'leaf', groupId: 'group-live' }, + second: { type: 'leaf', groupId: 'group-recovered' } + }, + second: { type: 'leaf', groupId: 'group-other' } + }) + }) + + it('uses the proposed layout when the host has no groups', () => { + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout: null, + existingGroupIds: [], + proposedLayout: { type: 'leaf', groupId: 'group-recovered' }, + proposedGroupIds: ['group-recovered'], + mergedGroupIds: ['group-recovered'] + }) + ).toEqual({ type: 'leaf', groupId: 'group-recovered' }) + }) + + it('keeps the host layout when the proposal has no groups', () => { + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout: { type: 'leaf', groupId: 'group-live' }, + existingGroupIds: ['group-live'], + proposedLayout: null, + proposedGroupIds: [], + mergedGroupIds: ['group-live'] + }) + ).toEqual({ type: 'leaf', groupId: 'group-live' }) + }) + + it('does not rewrite host layout when the proposal adds no groups', () => { + const existingLayout = { + type: 'split' as const, + direction: 'horizontal' as const, + ratio: 0.4, + first: { type: 'leaf' as const, groupId: 'group-a' }, + second: { type: 'leaf' as const, groupId: 'group-b' } + } + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout, + existingGroupIds: ['group-a', 'group-b'], + proposedLayout: existingLayout, + proposedGroupIds: ['group-a', 'group-b'], + mergedGroupIds: ['group-a', 'group-b'] + }) + ).toEqual(existingLayout) + }) + + it('appends new groups when multiple shared anchors make placement ambiguous', () => { + expect( + mergeTerminalOrphanGroupLayout({ + existingLayout: { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', groupId: 'group-a' }, + second: { type: 'leaf', groupId: 'group-b' } + }, + existingGroupIds: ['group-a', 'group-b'], + proposedLayout: { + type: 'split', + direction: 'vertical', + ratio: 0.7, + first: { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', groupId: 'group-a' }, + second: { type: 'leaf', groupId: 'group-b' } + }, + second: { type: 'leaf', groupId: 'group-recovered' } + }, + proposedGroupIds: ['group-a', 'group-b', 'group-recovered'], + mergedGroupIds: ['group-a', 'group-b', 'group-recovered'] + }) + ).toEqual({ + type: 'split', + direction: 'vertical', + ratio: 0.7, + first: { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', groupId: 'group-a' }, + second: { type: 'leaf', groupId: 'group-b' } + }, + second: { type: 'leaf', groupId: 'group-recovered' } + }) + }) +}) diff --git a/src/main/runtime/terminal-orphan-topology.ts b/src/main/runtime/terminal-orphan-topology.ts new file mode 100644 index 00000000000..bcb0b6e60ca --- /dev/null +++ b/src/main/runtime/terminal-orphan-topology.ts @@ -0,0 +1,139 @@ +import type { TabGroupLayoutNode } from '../../shared/types' + +function collectLayoutGroupIds(node: TabGroupLayoutNode | null | undefined, ids: string[]): void { + if (!node) { + return + } + if (node.type === 'leaf') { + ids.push(node.groupId) + return + } + collectLayoutGroupIds(node.first, ids) + collectLayoutGroupIds(node.second, ids) +} + +function pruneLayout( + node: TabGroupLayoutNode | null | undefined, + retainedGroupIds: ReadonlySet +): TabGroupLayoutNode | null { + if (!node) { + return null + } + if (node.type === 'leaf') { + return retainedGroupIds.has(node.groupId) ? node : null + } + const first = pruneLayout(node.first, retainedGroupIds) + const second = pruneLayout(node.second, retainedGroupIds) + if (!first) { + return second + } + if (!second) { + return first + } + return { ...node, first, second } +} + +function appendMissingGroups( + layout: TabGroupLayoutNode | null, + orderedGroupIds: readonly string[] +): TabGroupLayoutNode | null { + const present: string[] = [] + collectLayoutGroupIds(layout, present) + const presentSet = new Set(present) + let next = layout + for (const groupId of orderedGroupIds) { + if (presentSet.has(groupId)) { + continue + } + const leaf = { type: 'leaf' as const, groupId } + next = next + ? { type: 'split', direction: 'horizontal', first: next, second: leaf, ratio: 0.5 } + : leaf + presentSet.add(groupId) + } + return next +} + +function replaceLeaf( + node: TabGroupLayoutNode, + groupId: string, + replacement: TabGroupLayoutNode +): TabGroupLayoutNode { + if (node.type === 'leaf') { + return node.groupId === groupId ? replacement : node + } + return { + ...node, + first: replaceLeaf(node.first, groupId, replacement), + second: replaceLeaf(node.second, groupId, replacement) + } +} + +export function hasExactTerminalOrphanGroupLayout( + layout: TabGroupLayoutNode, + expectedGroupIds: ReadonlySet +): boolean { + const groupIds: string[] = [] + collectLayoutGroupIds(layout, groupIds) + return ( + groupIds.length === expectedGroupIds.size && + new Set(groupIds).size === groupIds.length && + groupIds.every((groupId) => expectedGroupIds.has(groupId)) + ) +} + +export function mergeTerminalOrphanGroupLayout(args: { + existingLayout: TabGroupLayoutNode | null | undefined + existingGroupIds: readonly string[] + proposedLayout: TabGroupLayoutNode | null | undefined + proposedGroupIds: readonly string[] + mergedGroupIds: readonly string[] +}): TabGroupLayoutNode | undefined { + const mergedGroupIdSet = new Set(args.mergedGroupIds) + const existingGroupIdSet = new Set(args.existingGroupIds) + const proposedGroupIdSet = new Set(args.proposedGroupIds) + let existing = appendMissingGroups( + pruneLayout(args.existingLayout, existingGroupIdSet), + args.existingGroupIds + ) + const proposed = appendMissingGroups( + pruneLayout(args.proposedLayout, proposedGroupIdSet), + args.proposedGroupIds + ) + + if (!existing) { + return appendMissingGroups(proposed, args.mergedGroupIds) ?? undefined + } + if (!proposed) { + return appendMissingGroups(existing, args.mergedGroupIds) ?? undefined + } + + const sharedGroupIds = args.proposedGroupIds.filter((groupId) => existingGroupIdSet.has(groupId)) + const newGroupIds = new Set( + args.proposedGroupIds.filter((groupId) => !existingGroupIdSet.has(groupId)) + ) + if (newGroupIds.size > 0 && sharedGroupIds.length === 1) { + // One shared group identifies where the recovered subtree belonged without disturbing unrelated host layout. + const anchorGroupId = sharedGroupIds[0]! + const proposalAtAnchor = pruneLayout(proposed, new Set([anchorGroupId, ...newGroupIds])) + if (proposalAtAnchor) { + existing = replaceLeaf(existing, anchorGroupId, proposalAtAnchor) + } + } else if (newGroupIds.size > 0) { + // With no unique anchor, append the intact recovered subtree so ambiguous client metadata cannot rewrite host groups. + const newSubtree = pruneLayout(proposed, newGroupIds) + if (newSubtree) { + existing = { + type: 'split', + direction: proposed.type === 'split' ? proposed.direction : 'horizontal', + first: existing, + second: newSubtree, + ratio: proposed.type === 'split' ? proposed.ratio : 0.5 + } + } + } + + return ( + appendMissingGroups(pruneLayout(existing, mergedGroupIdSet), args.mergedGroupIds) ?? undefined + ) +} diff --git a/src/main/runtime/tls-certificate-bounds.test.ts b/src/main/runtime/tls-certificate-bounds.test.ts new file mode 100644 index 00000000000..019738e250f --- /dev/null +++ b/src/main/runtime/tls-certificate-bounds.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const execFileSyncMock = vi.hoisted(() => vi.fn()) + +vi.mock('node:child_process', () => ({ execFileSync: execFileSyncMock })) + +import { loadOrCreateTlsCertificate } from './tls-certificate' + +const CERTIFICATE = '-----BEGIN CERTIFICATE-----\nYQ==\n-----END CERTIFICATE-----\n' +const PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nYg==\n-----END PRIVATE KEY-----\n' +const roots: string[] = [] + +beforeEach(() => { + execFileSyncMock.mockImplementation((_command: string, args: string[]) => { + const keyIndex = args.indexOf('-keyout') + const certIndex = args.indexOf('-out') + writeFileSync(args[keyIndex + 1], PRIVATE_KEY) + writeFileSync(args[certIndex + 1], CERTIFICATE) + }) +}) + +afterEach(() => { + execFileSyncMock.mockReset() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function makeUserDataPath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-tls-bounds-')) + roots.push(root) + return root +} + +describe('TLS certificate file bounds', () => { + it('preserves existing ordinary certificate material', () => { + const userDataPath = makeUserDataPath() + writeFileSync(join(userDataPath, 'orca-tls-cert.pem'), CERTIFICATE) + writeFileSync(join(userDataPath, 'orca-tls-key.pem'), PRIVATE_KEY) + + expect(loadOrCreateTlsCertificate(userDataPath)).toMatchObject({ + cert: CERTIFICATE, + key: PRIVATE_KEY + }) + expect(execFileSyncMock).not.toHaveBeenCalled() + }) + + it('regenerates instead of retaining an oversized sparse PEM file', () => { + const userDataPath = makeUserDataPath() + const certPath = join(userDataPath, 'orca-tls-cert.pem') + const descriptor = openSync(certPath, 'w') + ftruncateSync(descriptor, 1024 * 1024 + 1) + closeSync(descriptor) + writeFileSync(join(userDataPath, 'orca-tls-key.pem'), PRIVATE_KEY) + + expect(loadOrCreateTlsCertificate(userDataPath)).toMatchObject({ + cert: CERTIFICATE, + key: PRIVATE_KEY + }) + expect(execFileSyncMock).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/runtime/tls-certificate.ts b/src/main/runtime/tls-certificate.ts index 6c502c31442..b3e37d42572 100644 --- a/src/main/runtime/tls-certificate.ts +++ b/src/main/runtime/tls-certificate.ts @@ -4,11 +4,16 @@ // app pins the certificate fingerprint received during QR pairing. import { createHash } from 'node:crypto' import { execFileSync } from 'node:child_process' -import { existsSync, readFileSync, chmodSync } from 'node:fs' +import { existsSync, chmodSync } from 'node:fs' import { join } from 'node:path' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit +} from '../../shared/node-bounded-file-reader' const TLS_CERT_FILENAME = 'orca-tls-cert.pem' const TLS_KEY_FILENAME = 'orca-tls-key.pem' +const MAX_TLS_PEM_FILE_BYTES = 1024 * 1024 export type TlsCertificate = { cert: string @@ -21,11 +26,17 @@ export function loadOrCreateTlsCertificate(userDataPath: string): TlsCertificate const keyPath = join(userDataPath, TLS_KEY_FILENAME) if (existsSync(certPath) && existsSync(keyPath)) { - const cert = readFileSync(certPath, 'utf-8') - const key = readFileSync(keyPath, 'utf-8') - const fingerprint = computeFingerprint(cert) - if (fingerprint) { - return { cert, key, fingerprint } + try { + const cert = readTlsPemFile(certPath) + const key = readTlsPemFile(keyPath) + const fingerprint = computeFingerprint(cert) + if (fingerprint) { + return { cert, key, fingerprint } + } + } catch (error) { + if (!(error instanceof NodeFileReadTooLargeError)) { + throw error + } } // Why: if the existing cert is malformed (e.g., from a buggy earlier // generation), regenerate rather than failing the WebSocket transport. @@ -66,11 +77,15 @@ export function loadOrCreateTlsCertificate(userDataPath: string): TlsCertificate chmodSync(keyPath_, 0o600) chmodSync(certPath_, 0o600) - const cert = readFileSync(certPath_, 'utf-8') - const key = readFileSync(keyPath_, 'utf-8') + const cert = readTlsPemFile(certPath_) + const key = readTlsPemFile(keyPath_) return { cert, key, fingerprint: computeFingerprint(cert)! } } +function readTlsPemFile(filePath: string): string { + return readNodeFileSyncWithinLimit(filePath, MAX_TLS_PEM_FILE_BYTES).buffer.toString('utf-8') +} + function resolveOpenSslExecutable(): string { if (process.platform !== 'win32') { return 'openssl' diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 1025c6be020..606b62d064e 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -375,6 +375,53 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.shutdown).toHaveBeenCalledTimes(1) }) + it('accepts a failed Windows stop when a fresh inventory proves the PTY exited', async () => { + const worktreeId = 'repo-1::C:/Users/User/orca/workspaces/repo/feature' + const ptyId = `${worktreeId}@@windows-pty` + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => boolean + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty(ptyId, () => false)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + let inventoryCount = 0 + const localProvider = createProviderStub(async () => { + inventoryCount += 1 + return inventoryCount === 1 + ? [{ id: ptyId, cwd: 'C:/Users/User/orca/workspaces/repo/feature', title: 'shell' }] + : [] + }) + ;(localProvider.shutdown as unknown as ReturnType).mockRejectedValue( + new Error(`Session not found: ${ptyId}`) + ) + listRegisteredPtysMock.mockReturnValue([ + { ptyId, worktreeId, sessionId: null, paneKey: null, pid: 100 } + ]) + + await expect( + killAllProcessesForWorktree(worktreeId, { + runtime, + localProvider, + requirePhysicalStop: true + }) + ).resolves.toEqual({ + runtimeStopped: 0, + providerStopped: 0, + registryStopped: 0 + }) + expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) + }) + it('keeps duplicate sweeps behind the runtime physical-stop promise', async () => { let releasePhysicalStop: () => void = () => undefined const physicalStop = new Promise((resolve) => { diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 8dfe63a9f7e..db6c0f5d6e1 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -26,9 +26,8 @@ export type WorktreeTeardownResult = { export const WORKTREE_PROCESS_SWEEP_TIMEOUT_MS = 10_000 -// Why: margin so a bounded daemon RPC rejects BEFORE the sweep deadline and its -// rejection can propagate — otherwise the outer deadline wins with a confusing -// "Timed out waiting for physical PTY teardown" instead of the accurate stop failure. +// Why: reserve time after bounded stop RPCs to recheck whether a reported +// failure actually left a live PTY before the outer sweep deadline. export const WORKTREE_TEARDOWN_RPC_MARGIN_MS = 500 // Absolute deadline (epoch ms) threaded into provider RPCs on the destructive @@ -150,15 +149,41 @@ export async function killAllProcessesForWorktree( result.providerStopped = providerStopped result.registryStopped = registryStopped if (deps.requirePhysicalStop) { - const stops = await Promise.all(stopAttempts.values()) - if (stops.some((stopped) => !stopped)) { + const stopResults = await Promise.all( + [...stopAttempts].map(async ([ptyId, stopped]) => [ptyId, await stopped] as const) + ) + const failedPtyIds = stopResults.filter(([, stopped]) => !stopped).map(([ptyId]) => ptyId) + const failedPtysExited = + failedPtyIds.length === 0 || + (await verifyFailedPtysExited(failedPtyIds, deps.localProvider, deadline)) + if (!failedPtysExited) { throw new Error(`Failed to physically stop every PTY for worktree: ${worktreeId}`) } + for (const ptyId of failedPtyIds) { + clearStoppedPtyState(ptyId, deps.onPtyStopped) + } } return result } +async function verifyFailedPtysExited( + failedPtyIds: readonly string[], + provider: IPtyProvider, + deadline: number +): Promise { + const sessions = await settleBeforeDeadline( + () => provider.listProcesses({ deadlineMs: deadline }), + null, + deadline + ).catch(() => null) + if (!sessions) { + return false + } + const livePtyIds = new Set(sessions.map((session) => session.id)) + return failedPtyIds.every((ptyId) => !livePtyIds.has(ptyId)) +} + async function settleBeforeDeadline( run: () => Promise, fallback: T, diff --git a/src/main/serve-update-handoff.test.ts b/src/main/serve-update-handoff.test.ts index d649243a8f5..140ea03da33 100644 --- a/src/main/serve-update-handoff.test.ts +++ b/src/main/serve-update-handoff.test.ts @@ -1,10 +1,12 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { EventEmitter } from 'node:events' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SERVE_UPDATE_HANDOFF_PATH_ENV, + MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES, + MAX_SERVE_UPDATE_HANDOFF_JSON_STRUCTURAL_TOKENS, getServeUpdateHandoffPath, parseServeUpdateHandoffState } from '../shared/serve-update-handoff' @@ -81,6 +83,38 @@ describe('serve update handoff', () => { expect(requestServeUpdateHandoff('1.0.61')).toBe(false) }) + it('ignores oversized sparse handoff state', async () => { + const handoffPath = getServeUpdateHandoffPath(root) + writeFileSync(handoffPath, '{"schemaVersion":1}') + truncateSync(handoffPath, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES + 1) + const { getServeUpdateHandoffFailure } = await import('./serve-update-handoff') + + expect(getServeUpdateHandoffFailure()).toBeNull() + }) + + it('rejects structurally amplified handoff state before parsing', async () => { + const handoffPath = getServeUpdateHandoffPath(root) + writeFileSync( + handoffPath, + `{"padding":[${'0,'.repeat(MAX_SERVE_UPDATE_HANDOFF_JSON_STRUCTURAL_TOKENS)}0]}` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + const { getServeUpdateHandoffFailure } = await import('./serve-update-handoff') + + expect(getServeUpdateHandoffFailure()).toBeNull() + expect(parseSpy).not.toHaveBeenCalled() + }) + + it('preserves prior handoff state when the replacement exceeds its read ceiling', async () => { + const { requestServeUpdateHandoff } = await import('./serve-update-handoff') + expect(requestServeUpdateHandoff('1.0.61')).toBe(true) + const handoffPath = getServeUpdateHandoffPath(root) + const before = readFileSync(handoffPath, 'utf8') + + expect(requestServeUpdateHandoff('x'.repeat(MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES))).toBe(false) + expect(readFileSync(handoffPath, 'utf8')).toBe(before) + }) + it.runIf(process.platform === 'darwin')( 'quits a supervised serve child when its CLI parent is lost', async () => { diff --git a/src/main/serve-update-handoff.ts b/src/main/serve-update-handoff.ts index ab831a90d49..e0ee6b681ce 100644 --- a/src/main/serve-update-handoff.ts +++ b/src/main/serve-update-handoff.ts @@ -1,10 +1,13 @@ -import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' import { dirname, resolve } from 'node:path' import { app } from 'electron' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../shared/node-bounded-json-stringify' import { + MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES, SERVE_UPDATE_HANDOFF_PATH_ENV, getServeUpdateHandoffPath, - parseServeUpdateHandoffState, + parseServeUpdateHandoffJson, type ServeSupervisorMessage, type ServeUpdateHandoffState } from '../shared/serve-update-handoff' @@ -43,7 +46,11 @@ export function failServeUpdateHandoff(reason: string): void { return } try { - const state = parseServeUpdateHandoffState(JSON.parse(readFileSync(handoffPath, 'utf8'))) + const state = parseServeUpdateHandoffJson( + readNodeFileSyncWithinLimit(handoffPath, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) if (state?.phase !== 'install-requested' || state.servingPid !== process.pid) { return } @@ -90,7 +97,11 @@ export function getServeUpdateHandoffFailure(): string | null { return null } try { - const state = parseServeUpdateHandoffState(JSON.parse(readFileSync(handoffPath, 'utf8'))) + const state = parseServeUpdateHandoffJson( + readNodeFileSyncWithinLimit(handoffPath, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES).buffer.toString( + 'utf8' + ) + ) if (state?.phase !== 'failed') { return null } @@ -107,11 +118,17 @@ export function getServeUpdateHandoffFailure(): string | null { function writeHandoffState(path: string, state: ServeUpdateHandoffState): boolean { const temporaryPath = `${path}.${process.pid}.tmp` try { + const { serialized } = stringifyJsonWithinByteLimit(state, MAX_SERVE_UPDATE_HANDOFF_FILE_BYTES) mkdirSync(dirname(path), { recursive: true }) - writeFileSync(temporaryPath, JSON.stringify(state), { mode: 0o600 }) + writeFileSync(temporaryPath, serialized, { mode: 0o600 }) renameSync(temporaryPath, path) return true } catch { + try { + unlinkSync(temporaryPath) + } catch { + // The temporary handoff is best-effort state and may not have been created. + } return false } } diff --git a/src/main/setup-script-import-file.test.ts b/src/main/setup-script-import-file.test.ts new file mode 100644 index 00000000000..9d0bfbe4f36 --- /dev/null +++ b/src/main/setup-script-import-file.test.ts @@ -0,0 +1,56 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { NodeFileReadTooLargeError } from '../shared/node-bounded-file-reader' +import { + isSetupScriptImportTextWithinLimit, + readSetupScriptImportFile, + SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES +} from './setup-script-import-file' + +describe('setup script import file bounds', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + async function makeFile(contents: string): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-setup-script-import-')) + roots.push(root) + const filePath = join(root, 'config.json') + await writeFile(filePath, contents) + return filePath + } + + it('preserves ordinary configuration text', async () => { + const filePath = await makeFile('{"setup":"pnpm install"}') + + await expect(readSetupScriptImportFile(filePath)).resolves.toBe('{"setup":"pnpm install"}') + }) + + it('rejects an oversized sparse local configuration', async () => { + const filePath = await makeFile('x') + await truncate(filePath, SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES + 1) + + await expect(readSetupScriptImportFile(filePath)).rejects.toBeInstanceOf( + NodeFileReadTooLargeError + ) + }) + + it('measures remote configuration strings by UTF-8 bytes', () => { + expect(isSetupScriptImportTextWithinLimit('a'.repeat(SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES))).toBe( + true + ) + expect( + isSetupScriptImportTextWithinLimit('a'.repeat(SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES + 1)) + ).toBe(false) + expect( + isSetupScriptImportTextWithinLimit('é'.repeat(SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES / 2)) + ).toBe(true) + expect( + isSetupScriptImportTextWithinLimit('é'.repeat(SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES / 2 + 1)) + ).toBe(false) + }) +}) diff --git a/src/main/setup-script-import-file.ts b/src/main/setup-script-import-file.ts new file mode 100644 index 00000000000..66b9dd537da --- /dev/null +++ b/src/main/setup-script-import-file.ts @@ -0,0 +1,18 @@ +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' +import { + isSetupScriptImportTextWithinLimit, + SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS +} from '../shared/setup-script-import-limits' + +export { + isSetupScriptImportTextWithinLimit, + SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES, + SETUP_SCRIPT_IMPORT_MAX_CODE_UNITS +} + +export async function readSetupScriptImportFile(filePath: string): Promise { + return ( + await readNodeFileWithinLimit(filePath, SETUP_SCRIPT_IMPORT_FILE_MAX_BYTES) + ).buffer.toString('utf-8') +} diff --git a/src/main/skills/claude-plugin-skill-sources-wsl.test.ts b/src/main/skills/claude-plugin-skill-sources-wsl.test.ts index 4d4f1add82c..216d8385699 100644 --- a/src/main/skills/claude-plugin-skill-sources-wsl.test.ts +++ b/src/main/skills/claude-plugin-skill-sources-wsl.test.ts @@ -35,4 +35,8 @@ describe('WSL Claude plugin metadata', () => { 'invalid response' ) }) + + it('parses delimiter-heavy output without materializing a field array', () => { + expect(parseWslClaudePluginMetadataOutput('\0'.repeat(1_000_000), 2)).toEqual([null, null]) + }) }) diff --git a/src/main/skills/claude-plugin-skill-sources-wsl.ts b/src/main/skills/claude-plugin-skill-sources-wsl.ts index d3adddfde55..bf22d9aba0f 100644 --- a/src/main/skills/claude-plugin-skill-sources-wsl.ts +++ b/src/main/skills/claude-plugin-skill-sources-wsl.ts @@ -7,6 +7,10 @@ import { type ClaudePluginMetadata } from './claude-plugin-skill-sources' import type { SkillScanRoot } from './skill-discovery-sources' +import { + readWslSkillProtocolField, + type WslSkillProtocolFieldCursor +} from './wsl-skill-protocol-fields' const MAX_PLUGIN_METADATA_BYTES = 4 * 1024 * 1024 const WSL_METADATA_TIMEOUT_MS = 5_000 @@ -39,17 +43,26 @@ export function parseWslClaudePluginMetadataOutput( fileCount: number ): (string | null)[] { const contents = Array(fileCount).fill(null) - const fields = output.split('\0') - let index = 0 - while (index < fields.length && fields[index]) { - const kind = fields[index++] - const fileIndex = Number.parseInt(fields[index++] ?? '', 10) - const exists = fields[index++] === '1' - const encoded = fields[index++] + const cursor: WslSkillProtocolFieldCursor = { offset: 0 } + const readField = (): string => + readWslSkillProtocolField( + output, + cursor, + 'WSL Claude plugin metadata returned an incomplete response.' + ) + while (cursor.offset < output.length) { + // Why: cursor reads keep delimiter-heavy subprocess output from amplifying into a huge array. + const kind = readField() + if (!kind) { + break + } + const fileIndex = Number.parseInt(readField(), 10) + const exists = readField() === '1' + const encoded = readField() if (kind !== 'F' || !Number.isInteger(fileIndex) || fileIndex < 0 || fileIndex >= fileCount) { throw new Error('WSL Claude plugin metadata returned an invalid response.') } - if (exists && encoded !== undefined) { + if (exists) { contents[fileIndex] = Buffer.from(encoded, 'base64').toString('utf8') } } diff --git a/src/main/skills/discovery.test.ts b/src/main/skills/discovery.test.ts index a5d113aebe9..89fed1befe2 100644 --- a/src/main/skills/discovery.test.ts +++ b/src/main/skills/discovery.test.ts @@ -328,4 +328,25 @@ describe('skill discovery', () => { expect(result.skills.map((skill) => skill.name)).not.toContain('Too Deep') }) + + it('rejects pathological retained skill metadata instead of growing the result unboundedly', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) + const home = join(root, 'home') + const skillsRoot = join(home, '.agents', 'skills') + const description = 'x'.repeat(150_000) + await Promise.all( + Array.from({ length: 16 }, async (_, index) => { + const skillDir = join(skillsRoot, `large-${index}`) + await mkdir(skillDir, { recursive: true }) + await writeFile( + join(skillDir, 'SKILL.md'), + `---\nname: large-${index}\ndescription: ${description}\n---\n` + ) + }) + ) + + await expect( + discoverSkills({ homeDir: home, cwd: join(root, 'missing-cwd'), repos: [] }) + ).rejects.toThrow(/result is too large/) + }) }) diff --git a/src/main/skills/discovery.ts b/src/main/skills/discovery.ts index 00edfc8e705..e8965d6e1b2 100644 --- a/src/main/skills/discovery.ts +++ b/src/main/skills/discovery.ts @@ -1,5 +1,4 @@ -import type { Dirent } from 'node:fs' -import { open, readdir, realpath, stat } from 'node:fs/promises' +import { open, opendir, realpath, stat } from 'node:fs/promises' import { homedir } from 'node:os' import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path' import { summarizeSkillMarkdown } from '../../shared/skill-metadata' @@ -18,6 +17,15 @@ import { type SkillScanRoot } from './skill-discovery-sources' import { discoverClaudePluginSkillSources } from './claude-plugin-skill-sources' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' +import { + MAX_CONCURRENT_SKILL_DISCOVERY_CANDIDATES, + MAX_CONCURRENT_SKILL_DISCOVERY_ROOTS, + MAX_SKILL_PACKAGE_DIRECTORIES, + MAX_SKILL_PACKAGE_ENTRIES, + SkillDiscoveryBudget, + SkillDiscoveryLimitError +} from './skill-discovery-limits' export { buildSkillDiscoverySources } from './skill-discovery-sources' @@ -46,13 +54,18 @@ function isWithinDepth(rootPath: string, childPath: string, maxDepth: number): b return rel.split(sep).length <= maxDepth } -async function findSkillFiles(rootPath: string, maxDepth: number): Promise { +async function findSkillFiles( + rootPath: string, + maxDepth: number, + budget: SkillDiscoveryBudget +): Promise { const out: string[] = [] const visitedDirectoryPaths = new Set() async function visit(dirPath: string): Promise { if (!isWithinDepth(rootPath, dirPath, maxDepth)) { return } + budget.visitDirectory() let resolvedDirPath: string try { resolvedDirPath = await realpath(dirPath) @@ -64,45 +77,46 @@ async function findSkillFiles(rootPath: string, maxDepth: number): Promise null) + if (!directory) { return } - for (const entry of entries) { - const entryPath = join(dirPath, entry.name) - if (entry.name === SKILL_FILE_NAME) { - if (entry.isFile()) { - out.push(entryPath) + try { + for await (const entry of directory) { + budget.visitEntry() + const entryPath = join(dirPath, entry.name) + if (entry.name === SKILL_FILE_NAME) { + if (entry.isFile()) { + budget.admitCandidate() + out.push(entryPath) + continue + } + if (entry.isSymbolicLink()) { + const linkedStat = await stat(entryPath).catch(() => null) + if (linkedStat?.isFile()) { + budget.admitCandidate() + out.push(entryPath) + } + } + continue + } + if (entry.isDirectory()) { + await visit(entryPath) continue } if (entry.isSymbolicLink()) { - try { - if ((await stat(entryPath)).isFile()) { - out.push(entryPath) - } - } catch { - // Broken links are not valid skill files. - } - } - continue - } - if (entry.isDirectory()) { - await visit(entryPath) - continue - } - if (entry.isSymbolicLink()) { - // Why: users commonly symlink agent skill dirs across providers; follow - // directory links but guard by realpath so recursive links cannot loop. - try { - if ((await stat(entryPath)).isDirectory()) { + // Why: users commonly symlink agent skill dirs across providers; follow + // directory links but guard by realpath so recursive links cannot loop. + const linkedStat = await stat(entryPath).catch(() => null) + if (linkedStat?.isDirectory()) { await visit(entryPath) } - } catch { - // Broken links are not valid skill directories. } } + } catch (error) { + if (error instanceof SkillDiscoveryLimitError) { + throw error + } } } await visit(rootPath) @@ -111,9 +125,16 @@ async function findSkillFiles(rootPath: string, maxDepth: number): Promise { let count = 0 + let entriesVisited = 0 + let stoppedEarly = false const visitedDirectoryPaths = new Set() async function visit(currentPath: string): Promise { - if (count >= MAX_SKILL_FILES) { + if ( + stoppedEarly || + count >= MAX_SKILL_FILES || + visitedDirectoryPaths.size >= MAX_SKILL_PACKAGE_DIRECTORIES + ) { + stoppedEarly = true return } let resolvedPath: string @@ -127,30 +148,34 @@ async function countFiles(dirPath: string): Promise { } visitedDirectoryPaths.add(resolvedPath) - let entries: Dirent[] - try { - entries = await readdir(currentPath, { withFileTypes: true }) - } catch { + const directory = await opendir(currentPath).catch(() => null) + if (!directory) { return } - for (const entry of entries) { - if (count >= MAX_SKILL_FILES) { - return - } - const entryPath = join(currentPath, entry.name) - if (entry.isFile()) { - count += 1 - } else if (entry.isDirectory()) { - await visit(entryPath) - } else if (entry.isSymbolicLink()) { - try { - if ((await stat(entryPath)).isFile()) { - count += 1 + try { + for await (const entry of directory) { + entriesVisited += 1 + if (count >= MAX_SKILL_FILES || entriesVisited > MAX_SKILL_PACKAGE_ENTRIES) { + stoppedEarly = true + return + } + const entryPath = join(currentPath, entry.name) + if (entry.isFile()) { + count += 1 + } else if (entry.isDirectory()) { + await visit(entryPath) + } else if (entry.isSymbolicLink()) { + try { + if ((await stat(entryPath)).isFile()) { + count += 1 + } + } catch { + // Broken links do not contribute to the skill package file count. } - } catch { - // Broken links do not contribute to the skill package file count. } } + } catch { + // Preserve the partial count when a directory changes during enumeration. } } await visit(dirPath) @@ -184,11 +209,16 @@ async function readSkillSummary(skillFilePath: string): Promise<{ type ScannedSkill = DiscoveredSkill & { canonicalSkillFilePath: string } -async function scanRoot(root: SkillScanRoot): Promise { +async function scanRoot( + root: SkillScanRoot, + budget: SkillDiscoveryBudget +): Promise { const maxDepth = root.sourceKind === 'plugin' ? 9 : 4 - const skillFiles = await findSkillFiles(root.path, maxDepth) - const skills = await Promise.all( - skillFiles.map(async (skillFilePath): Promise => { + const skillFiles = await findSkillFiles(root.path, maxDepth, budget) + const skills = await mapWithConcurrency( + skillFiles, + MAX_CONCURRENT_SKILL_DISCOVERY_CANDIDATES, + async (skillFilePath): Promise => { // Why: path identity belongs to the scanning host; canonicalizing before // returning prevents symlinked roots from becoming duplicate picker rows. const canonicalSkillFilePath = await realpath(skillFilePath).catch(() => skillFilePath) @@ -198,7 +228,7 @@ async function scanRoot(root: SkillScanRoot): Promise { return null } const sourceKind = sourceKindForSkill(root, skillFilePath, { relative, sep }) - return { + const skill = { id: stablePathId(canonicalSkillFilePath), name: summary.name ?? basename(directoryPath), description: summary.description, @@ -215,7 +245,9 @@ async function scanRoot(root: SkillScanRoot): Promise { updatedAt: summary.updatedAt, canonicalSkillFilePath } satisfies ScannedSkill - }) + budget.retainSkill(skill) + return skill + } ) return skills.filter((skill): skill is ScannedSkill => skill !== null) } @@ -235,54 +267,58 @@ export async function discoverSkills(args: { ? await discoverClaudePluginSkillSources({ homeDir, cwd: args.cwd }) : []) ] - const sources: SkillDiscoverySource[] = [] - const skillGroups = await Promise.all( - roots.map(async (root) => { + const budget = new SkillDiscoveryBudget(roots) + const scannedRoots = await mapWithConcurrency( + roots, + MAX_CONCURRENT_SKILL_DISCOVERY_ROOTS, + async (root) => { const exists = await pathExists(root.path) - sources.push({ + const source: SkillDiscoverySource = { ...root, providers: [...root.providers], exists, skippedReason: exists ? undefined : 'missing' - }) - if (!exists) { - return [] } - return scanRoot(root) - }) + if (!exists) { + return { source, skills: [] } + } + return { source, skills: await scanRoot(root, budget) } + } ) const seen = new Map() - for (const skill of skillGroups.flat()) { - // Why: overlapping repo/cwd roots and symlinked provider homes can reach - // the same file. Keep the first source's higher-level scope identity, but - // record every contributing root so per-agent visibility survives dedup. - const existing = seen.get(skill.canonicalSkillFilePath) - if (existing) { - if (existing.rootPaths && !existing.rootPaths.includes(skill.rootPath)) { - existing.rootPaths.push(skill.rootPath) - } - // Why: providers is per-agent visibility just like rootPaths; keeping only - // the first root's tags makes a shared/symlinked skill under-report which - // agents can see it on the Settings provider badges/filter. Reassign a - // fresh array — `providers` aliases the scan root's array, so pushing in - // place would mutate the root and every sibling skill/source sharing it. - const mergedProviders = [...existing.providers] - for (const provider of skill.providers) { - if (!mergedProviders.includes(provider)) { - mergedProviders.push(provider) + for (const group of scannedRoots) { + for (const skill of group.skills) { + // Why: overlapping repo/cwd roots and symlinked provider homes can reach + // the same file. Keep the first source's higher-level scope identity, but + // record every contributing root so per-agent visibility survives dedup. + const existing = seen.get(skill.canonicalSkillFilePath) + if (existing) { + if (existing.rootPaths && !existing.rootPaths.includes(skill.rootPath)) { + existing.rootPaths.push(skill.rootPath) } + // Why: providers is per-agent visibility just like rootPaths; keeping only + // the first root's tags makes a shared/symlinked skill under-report which + // agents can see it on the Settings provider badges/filter. Reassign a + // fresh array — `providers` aliases the scan root's array, so pushing in + // place would mutate the root and every sibling skill/source sharing it. + const mergedProviders = [...existing.providers] + for (const provider of skill.providers) { + if (!mergedProviders.includes(provider)) { + mergedProviders.push(provider) + } + } + existing.providers = mergedProviders + continue } - existing.providers = mergedProviders - continue + const { canonicalSkillFilePath, ...publicSkill } = skill + seen.set(canonicalSkillFilePath, { ...publicSkill, rootPaths: [skill.rootPath] }) } - const { canonicalSkillFilePath, ...publicSkill } = skill - seen.set(canonicalSkillFilePath, { ...publicSkill, rootPaths: [skill.rootPath] }) } return { skills: Array.from(seen.values()).sort(compareSkills), - sources: sources.sort((a, b) => - a.label.localeCompare(b.label, undefined, { sensitivity: 'base' }) - ), + sources: scannedRoots + .map(({ source }) => source) + .sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: 'base' })), scannedAt: Date.now() } } diff --git a/src/main/skills/skill-bundle-artifacts.test.ts b/src/main/skills/skill-bundle-artifacts.test.ts index cf58caebe09..459c093df8b 100644 --- a/src/main/skills/skill-bundle-artifacts.test.ts +++ b/src/main/skills/skill-bundle-artifacts.test.ts @@ -1,8 +1,15 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { loadSkillBundleArtifacts } from './skill-bundle-artifacts' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' +import { + loadSkillBundleArtifacts, + readSkillBundleArtifactJson, + SKILL_BUNDLE_CURRENT_MANIFEST_MAX_BYTES, + SKILL_BUNDLE_JSON_MAX_STRUCTURAL_TOKENS, + SKILL_BUNDLE_SNAPSHOT_REGISTRY_MAX_BYTES +} from './skill-bundle-artifacts' const temporaryDirectories: string[] = [] @@ -76,4 +83,45 @@ describe('skill bundle artifacts', () => { 'Invalid skill release mapping' ) }) + + it('rejects a sparse oversized artifact without poisoning a later bounded load', async () => { + const resourceRoot = await mkdtemp(join(tmpdir(), 'orca-skill-artifacts-')) + temporaryDirectories.push(resourceRoot) + const target = join(resourceRoot, 'skills') + const source = resolve('resources', 'skills') + await mkdir(target, { recursive: true }) + const [manifest, registry, releaseMapping] = await Promise.all( + ['current-manifest.json', 'snapshot-registry.json', 'release-mapping.json'].map((name) => + readFile(join(source, name), 'utf8') + ) + ) + const manifestPath = join(target, 'current-manifest.json') + await Promise.all([ + writeFile(manifestPath, ''), + writeFile(join(target, 'snapshot-registry.json'), registry), + writeFile(join(target, 'release-mapping.json'), releaseMapping) + ]) + await truncate(manifestPath, SKILL_BUNDLE_CURRENT_MANIFEST_MAX_BYTES + 1) + + await expect(loadSkillBundleArtifacts(resourceRoot)).rejects.toThrow(NodeFileReadTooLargeError) + + await writeFile(manifestPath, manifest) + await expect(loadSkillBundleArtifacts(resourceRoot)).resolves.toMatchObject({ + manifest: { schemaVersion: 2 } + }) + }) + + it('rejects structural amplification before parsing a bounded artifact', async () => { + const resourceRoot = await mkdtemp(join(tmpdir(), 'orca-skill-artifacts-')) + temporaryDirectories.push(resourceRoot) + const path = join(resourceRoot, 'amplified.json') + await writeFile(path, `[${'0,'.repeat(SKILL_BUNDLE_JSON_MAX_STRUCTURAL_TOKENS)}0]`) + const parseSpy = vi.spyOn(JSON, 'parse') + + await expect( + readSkillBundleArtifactJson(path, SKILL_BUNDLE_SNAPSHOT_REGISTRY_MAX_BYTES) + ).rejects.toThrow('JSON structure exceeds') + expect(parseSpy).not.toHaveBeenCalled() + parseSpy.mockRestore() + }) }) diff --git a/src/main/skills/skill-bundle-artifacts.ts b/src/main/skills/skill-bundle-artifacts.ts index 08e33a0eeba..fe54454e0df 100644 --- a/src/main/skills/skill-bundle-artifacts.ts +++ b/src/main/skills/skill-bundle-artifacts.ts @@ -1,7 +1,8 @@ import { app } from 'electron' -import { readFile } from 'node:fs/promises' import { join, resolve } from 'node:path' import { z, type ZodType } from 'zod' +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' import type { SkillBundleManifest, SkillKnownSnapshot, @@ -9,6 +10,12 @@ import type { SkillSnapshotRegistry } from '../../shared/skill-freshness' +export const SKILL_BUNDLE_CURRENT_MANIFEST_MAX_BYTES = 2 * 1024 * 1024 +export const SKILL_BUNDLE_SNAPSHOT_REGISTRY_MAX_BYTES = 16 * 1024 * 1024 +export const SKILL_BUNDLE_RELEASE_MAPPING_MAX_BYTES = 2 * 1024 * 1024 +export const SKILL_BUNDLE_JSON_MAX_STRUCTURAL_TOKENS = 1_000_000 +export const SKILL_BUNDLE_JSON_MAX_NESTING_DEPTH = 128 + export type SkillBundleArtifacts = { manifest: SkillBundleManifest registry: SkillSnapshotRegistry @@ -103,9 +110,18 @@ export function loadSkillBundleArtifacts( async function readSkillBundleArtifacts(resourceRoot: string): Promise { const bundleRoot = join(resourceRoot, 'skills') const [manifestValue, registryValue, releaseMappingValue] = await Promise.all([ - readFile(join(bundleRoot, 'current-manifest.json'), 'utf8').then(JSON.parse), - readFile(join(bundleRoot, 'snapshot-registry.json'), 'utf8').then(JSON.parse), - readFile(join(bundleRoot, 'release-mapping.json'), 'utf8').then(JSON.parse) + readSkillBundleArtifactJson( + join(bundleRoot, 'current-manifest.json'), + SKILL_BUNDLE_CURRENT_MANIFEST_MAX_BYTES + ), + readSkillBundleArtifactJson( + join(bundleRoot, 'snapshot-registry.json'), + SKILL_BUNDLE_SNAPSHOT_REGISTRY_MAX_BYTES + ), + readSkillBundleArtifactJson( + join(bundleRoot, 'release-mapping.json'), + SKILL_BUNDLE_RELEASE_MAPPING_MAX_BYTES + ) ]) const manifest: SkillBundleManifest = parseArtifact( manifestSchema, @@ -157,3 +173,16 @@ async function readSkillBundleArtifacts(resourceRoot: string): Promise { + const { buffer } = await readNodeFileWithinLimit(path, maxBytes) + const content = buffer.toString('utf8') + assertJsonTextStructureWithinLimits(content, { + structuralTokens: SKILL_BUNDLE_JSON_MAX_STRUCTURAL_TOKENS, + nestingDepth: SKILL_BUNDLE_JSON_MAX_NESTING_DEPTH + }) + return JSON.parse(content) as unknown +} diff --git a/src/main/skills/skill-discovery-limits.test.ts b/src/main/skills/skill-discovery-limits.test.ts new file mode 100644 index 00000000000..659b0383db1 --- /dev/null +++ b/src/main/skills/skill-discovery-limits.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import type { DiscoveredSkill } from '../../shared/skills' +import type { SkillScanRoot } from './skill-discovery-sources' +import { + MAX_SKILL_DISCOVERY_CANDIDATES, + MAX_SKILL_DISCOVERY_RESULT_BYTES, + MAX_SKILL_DISCOVERY_ROOTS, + SkillDiscoveryBudget, + WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES, + assertSkillDiscoveryOutputWithinLimit +} from './skill-discovery-limits' + +function root(index: number): SkillScanRoot { + return { + id: `root-${index}`, + label: `Root ${index}`, + path: `/skills/${index}`, + sourceKind: 'home', + providers: ['agent-skills'], + owner: null + } +} + +function skill(description: string): DiscoveredSkill { + return { + id: 'skill-id', + name: 'Skill', + description, + providers: ['agent-skills'], + sourceKind: 'home', + sourceLabel: 'Agent skills home', + rootPath: '/skills', + directoryPath: '/skills/example', + skillFilePath: '/skills/example/SKILL.md', + installed: true, + fileCount: 1, + updatedAt: null + } +} + +describe('installed-skill discovery limits', () => { + it('rejects root and candidate counts beyond their fixed budgets', () => { + expect( + () => + new SkillDiscoveryBudget( + Array.from({ length: MAX_SKILL_DISCOVERY_ROOTS + 1 }, (_, index) => root(index)) + ) + ).toThrow(/too many roots/) + + const budget = new SkillDiscoveryBudget([root(0)]) + for (let index = 0; index < MAX_SKILL_DISCOVERY_CANDIDATES; index += 1) { + budget.admitCandidate() + } + expect(() => budget.admitCandidate()).toThrow(/too many skills/) + }) + + it('rejects retained result text and WSL output beyond their byte budgets', () => { + const budget = new SkillDiscoveryBudget([root(0)]) + expect(() => + budget.retainSkill(skill('x'.repeat(Math.floor(MAX_SKILL_DISCOVERY_RESULT_BYTES / 2)))) + ).toThrow(/result is too large/) + + expect(() => + assertSkillDiscoveryOutputWithinLimit('x'.repeat(WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES + 1)) + ).toThrow(/output is too large/) + }) +}) diff --git a/src/main/skills/skill-discovery-limits.ts b/src/main/skills/skill-discovery-limits.ts new file mode 100644 index 00000000000..038bf2f6ba7 --- /dev/null +++ b/src/main/skills/skill-discovery-limits.ts @@ -0,0 +1,119 @@ +import type { DiscoveredSkill } from '../../shared/skills' +import type { SkillScanRoot } from './skill-discovery-sources' + +export const MAX_SKILL_DISCOVERY_ROOTS = 512 +export const MAX_SKILL_DISCOVERY_CANDIDATES = 1_024 +export const MAX_SKILL_DISCOVERY_TRAVERSED_DIRECTORIES = 20_000 +export const MAX_SKILL_DISCOVERY_TRAVERSED_ENTRIES = 100_000 +export const MAX_SKILL_DISCOVERY_RESULT_BYTES = 4 * 1024 * 1024 +export const MAX_CONCURRENT_SKILL_DISCOVERY_ROOTS = 4 +export const MAX_CONCURRENT_SKILL_DISCOVERY_CANDIDATES = 4 +export const MAX_SKILL_PACKAGE_DIRECTORIES = 512 +export const MAX_SKILL_PACKAGE_ENTRIES = 10_000 +export const WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES = 8 * 1024 * 1024 + +export class SkillDiscoveryLimitError extends Error { + constructor(message: string) { + super(message) + this.name = 'SkillDiscoveryLimitError' + } +} + +export function assertSkillDiscoveryRootsWithinLimit(roots: readonly SkillScanRoot[]): void { + if (roots.length > MAX_SKILL_DISCOVERY_ROOTS) { + throw new SkillDiscoveryLimitError( + `Installed-skill discovery has too many roots (${roots.length}; max ${MAX_SKILL_DISCOVERY_ROOTS}).` + ) + } +} + +function stringBytes(value: string | null | undefined): number { + return (value?.length ?? 0) * 2 +} + +export function estimateDiscoveredSkillBytes(skill: DiscoveredSkill): number { + return ( + 256 + + stringBytes(skill.id) + + stringBytes(skill.name) + + stringBytes(skill.description) + + stringBytes(skill.sourceLabel) + + stringBytes(skill.rootPath) + + stringBytes(skill.directoryPath) + + stringBytes(skill.skillFilePath) + + skill.providers.reduce((bytes, provider) => bytes + stringBytes(provider), 0) + + (skill.rootPaths ?? []).reduce((bytes, rootPath) => bytes + stringBytes(rootPath), 0) + ) +} + +function estimateSkillSourceBytes(source: SkillScanRoot): number { + return ( + 192 + + stringBytes(source.id) + + stringBytes(source.label) + + stringBytes(source.path) + + source.providers.reduce((bytes, provider) => bytes + stringBytes(provider), 0) + ) +} + +export class SkillDiscoveryBudget { + private candidates = 0 + private directories = 0 + private entries = 0 + private retainedBytes = 0 + + constructor(roots: readonly SkillScanRoot[]) { + assertSkillDiscoveryRootsWithinLimit(roots) + for (const root of roots) { + this.retainBytes(estimateSkillSourceBytes(root)) + } + } + + visitDirectory(): void { + this.directories += 1 + if (this.directories > MAX_SKILL_DISCOVERY_TRAVERSED_DIRECTORIES) { + throw new SkillDiscoveryLimitError( + `Installed-skill discovery visited too many directories (max ${MAX_SKILL_DISCOVERY_TRAVERSED_DIRECTORIES}).` + ) + } + } + + visitEntry(): void { + this.entries += 1 + if (this.entries > MAX_SKILL_DISCOVERY_TRAVERSED_ENTRIES) { + throw new SkillDiscoveryLimitError( + `Installed-skill discovery visited too many entries (max ${MAX_SKILL_DISCOVERY_TRAVERSED_ENTRIES}).` + ) + } + } + + admitCandidate(): void { + this.candidates += 1 + if (this.candidates > MAX_SKILL_DISCOVERY_CANDIDATES) { + throw new SkillDiscoveryLimitError( + `Installed-skill discovery found too many skills (max ${MAX_SKILL_DISCOVERY_CANDIDATES}).` + ) + } + } + + retainSkill(skill: DiscoveredSkill): void { + this.retainBytes(estimateDiscoveredSkillBytes(skill)) + } + + private retainBytes(bytes: number): void { + this.retainedBytes += bytes + if (this.retainedBytes > MAX_SKILL_DISCOVERY_RESULT_BYTES) { + throw new SkillDiscoveryLimitError( + `Installed-skill discovery result is too large (max ${MAX_SKILL_DISCOVERY_RESULT_BYTES} bytes).` + ) + } + } +} + +export function assertSkillDiscoveryOutputWithinLimit(output: string): void { + if (Buffer.byteLength(output, 'utf8') > WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES) { + throw new SkillDiscoveryLimitError( + `WSL installed-skill discovery output is too large (max ${WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES} bytes).` + ) + } +} diff --git a/src/main/skills/skill-discovery-wsl.test.ts b/src/main/skills/skill-discovery-wsl.test.ts index 621b9f4c0a3..81e6c260316 100644 --- a/src/main/skills/skill-discovery-wsl.test.ts +++ b/src/main/skills/skill-discovery-wsl.test.ts @@ -85,12 +85,29 @@ describe('WSL skill discovery', () => { expect(script).toContain('find -L "$root_path"') expect(script).toContain('realpath -- "$skill_file"') expect(script).toContain('head -c 262144 -- "$skill_file"') + expect(script).toContain('skill_count=$((skill_count + 1))') + expect(script).toContain("printf '%s\\0%s\\0' E skill-limit") expect(script).toContain(`'/work/alice'\\''s project/.agents/skills'`) }) + it('rejects an explicit distro-side candidate-limit marker', () => { + expect(() => parseWslSkillDiscoveryOutput(record('E', 'skill-limit'), [homeRoot])).toThrow( + /too many skills/ + ) + }) + it('rejects malformed host responses instead of reporting an empty scan', () => { expect(() => parseWslSkillDiscoveryOutput(record('S', '9'), [homeRoot])).toThrow( 'unknown source' ) }) + + it('parses delimiter-heavy output without materializing a field array', () => { + const result = parseWslSkillDiscoveryOutput('\0'.repeat(1_000_000), [homeRoot], 42) + + expect(result.skills).toEqual([]) + expect(result.sources).toEqual([ + expect.objectContaining({ id: 'home-codex', exists: false, skippedReason: 'missing' }) + ]) + }) }) diff --git a/src/main/skills/skill-discovery-wsl.ts b/src/main/skills/skill-discovery-wsl.ts index c4f553225ae..189d4fa11af 100644 --- a/src/main/skills/skill-discovery-wsl.ts +++ b/src/main/skills/skill-discovery-wsl.ts @@ -16,16 +16,29 @@ import { type SkillScanRoot } from './skill-discovery-sources' import { discoverClaudePluginSkillSourcesInWsl } from './claude-plugin-skill-sources-wsl' +import { + MAX_SKILL_DISCOVERY_CANDIDATES, + SkillDiscoveryBudget, + SkillDiscoveryLimitError, + WSL_SKILL_DISCOVERY_MAX_OUTPUT_BYTES, + assertSkillDiscoveryOutputWithinLimit, + assertSkillDiscoveryRootsWithinLimit +} from './skill-discovery-limits' +import { + readWslSkillProtocolField, + type WslSkillProtocolFieldCursor +} from './wsl-skill-protocol-fields' const MAX_MARKDOWN_BYTES = 256 * 1024 const MAX_PACKAGE_FILES = 200 const WSL_SCAN_TIMEOUT_MS = 10_000 -const WSL_SCAN_MAX_BUFFER_BYTES = 128 * 1024 * 1024 export function buildWslSkillDiscoveryCommand(roots: readonly SkillScanRoot[]): string { + assertSkillDiscoveryRootsWithinLimit(roots) const lines = [ 'set -u', 'set -o pipefail', + 'skill_count=0', 'scan_root() {', ' root_index=$1', ' root_path=$2', @@ -40,6 +53,11 @@ export function buildWslSkillDiscoveryCommand(roots: readonly SkillScanRoot[]): ` directory_path=\${skill_file%/*}`, ` updated_at=$(stat -c '%Y' -- "$skill_file" 2>/dev/null || true)`, ` encoded_markdown=$(head -c ${MAX_MARKDOWN_BYTES} -- "$skill_file" 2>/dev/null | base64 | tr -d '\\n') || continue`, + ' skill_count=$((skill_count + 1))', + ` if [ "$skill_count" -gt ${MAX_SKILL_DISCOVERY_CANDIDATES} ]; then`, + ` printf '%s\\0%s\\0' E skill-limit`, + ' exit 0', + ' fi', ' file_count=0', ` while IFS= read -r -d '' package_file; do`, ' file_count=$((file_count + 1))', @@ -65,7 +83,7 @@ function executeWslSkillDiscovery(distro: string, command: string): Promise() const skillsByCanonicalPath = new Map() - let index = 0 - while (index < fields.length && fields[index]) { - const recordKind = fields[index++] - const rootIndex = Number.parseInt(readProtocolField(fields, index++), 10) + const cursor: WslSkillProtocolFieldCursor = { offset: 0 } + while (cursor.offset < output.length) { + // Why: cursor reads avoid turning delimiter-heavy bounded output into millions of string slots. + const recordKind = readProtocolField(output, cursor) + if (!recordKind) { + break + } + if (recordKind === 'E') { + const reason = readProtocolField(output, cursor) + throw new SkillDiscoveryLimitError( + reason === 'skill-limit' + ? `WSL installed-skill discovery found too many skills (max ${MAX_SKILL_DISCOVERY_CANDIDATES}).` + : 'WSL installed-skill discovery exceeded a safety limit.' + ) + } + const rootIndex = Number.parseInt(readProtocolField(output, cursor), 10) const root = roots[rootIndex] if (!root) { throw new Error('WSL skill discovery returned an unknown source.') } if (recordKind === 'R') { - rootExists.set(rootIndex, readProtocolField(fields, index++) === '1') + rootExists.set(rootIndex, readProtocolField(output, cursor) === '1') continue } if (recordKind !== 'S') { throw new Error('WSL skill discovery returned an invalid response.') } + budget.admitCandidate() - const skillFilePath = readProtocolField(fields, index++) - const canonicalSkillFilePath = readProtocolField(fields, index++) - const updatedAtSeconds = Number.parseInt(readProtocolField(fields, index++), 10) - const fileCount = Number.parseInt(readProtocolField(fields, index++), 10) - const markdown = Buffer.from(readProtocolField(fields, index++), 'base64').toString('utf8') + const skillFilePath = readProtocolField(output, cursor) + const canonicalSkillFilePath = readProtocolField(output, cursor) + const updatedAtSeconds = Number.parseInt(readProtocolField(output, cursor), 10) + const fileCount = Number.parseInt(readProtocolField(output, cursor), 10) + const markdown = Buffer.from(readProtocolField(output, cursor), 'base64').toString('utf8') const existing = skillsByCanonicalPath.get(canonicalSkillFilePath) if (existing) { // Why: dedup keeps one row, but every contributing root must survive so @@ -139,7 +171,7 @@ export function parseWslSkillDiscoveryOutput( const directoryPath = pathPosix.dirname(skillFilePath) const summary = summarizeSkillMarkdown(markdown) const sourceKind = sourceKindForSkill(root, skillFilePath, pathPosix) - skillsByCanonicalPath.set(canonicalSkillFilePath, { + const skill: DiscoveredSkill = { id: stablePathId(canonicalSkillFilePath), name: summary.name ?? pathPosix.basename(directoryPath), description: summary.description, @@ -155,7 +187,9 @@ export function parseWslSkillDiscoveryOutput( installed: true, fileCount: Number.isFinite(fileCount) ? fileCount : 0, updatedAt: Number.isFinite(updatedAtSeconds) ? updatedAtSeconds * 1000 : null - }) + } + budget.retainSkill(skill) + skillsByCanonicalPath.set(canonicalSkillFilePath, skill) } const sources: SkillDiscoverySource[] = roots.map((root, rootIndex) => { diff --git a/src/main/skills/wsl-skill-protocol-fields.ts b/src/main/skills/wsl-skill-protocol-fields.ts new file mode 100644 index 00000000000..4af7bde4aea --- /dev/null +++ b/src/main/skills/wsl-skill-protocol-fields.ts @@ -0,0 +1,20 @@ +export type WslSkillProtocolFieldCursor = { offset: number } + +export function readWslSkillProtocolField( + output: string, + cursor: WslSkillProtocolFieldCursor, + incompleteMessage: string +): string { + if (cursor.offset >= output.length) { + throw new Error(incompleteMessage) + } + const end = output.indexOf('\0', cursor.offset) + if (end === -1) { + const value = output.slice(cursor.offset) + cursor.offset = output.length + return value + } + const value = output.slice(cursor.offset, end) + cursor.offset = end + 1 + return value +} diff --git a/src/main/source-control/hosted-review-api-request.ts b/src/main/source-control/hosted-review-api-request.ts index dfdb0dbad70..3c1582bd616 100644 --- a/src/main/source-control/hosted-review-api-request.ts +++ b/src/main/source-control/hosted-review-api-request.ts @@ -1,3 +1,8 @@ +import { + readFetchResponseJsonWithinLimit, + readFetchResponseTextWithinLimit +} from '../lib/fetch-response-body' + export class HostedReviewApiRequestError extends Error { readonly status: number | null readonly timedOut: boolean @@ -12,7 +17,7 @@ export class HostedReviewApiRequestError extends Error { async function readResponseText(response: Response): Promise { try { - return await response.text() + return await readFetchResponseTextWithinLimit(response) } catch { return '' } @@ -31,7 +36,7 @@ export async function requestHostedReviewJson( status: response.status }) } - return (await response.json()) as T + return await readFetchResponseJsonWithinLimit(response) } catch (error) { if (error instanceof HostedReviewApiRequestError) { throw error diff --git a/src/main/source-control/pull-request-template.test.ts b/src/main/source-control/pull-request-template.test.ts new file mode 100644 index 00000000000..5b0bf33fcda --- /dev/null +++ b/src/main/source-control/pull-request-template.test.ts @@ -0,0 +1,48 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { MAX_HOSTED_REVIEW_TEMPLATE_BYTES, readHostedReviewTemplate } from './pull-request-template' + +describe('readHostedReviewTemplate', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + async function makeRepo(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-review-template-')) + roots.push(root) + await mkdir(join(root, '.github'), { recursive: true }) + return root + } + + it('preserves the first conventional template under the limit', async () => { + const repoPath = await makeRepo() + await writeFile(join(repoPath, '.github', 'pull_request_template.md'), 'normal template\n') + await writeFile(join(repoPath, 'PULL_REQUEST_TEMPLATE.md'), 'fallback\n') + + await expect(readHostedReviewTemplate(repoPath)).resolves.toBe('normal template\n') + }) + + it('skips an oversized sparse template and reads the next candidate', async () => { + const repoPath = await makeRepo() + const oversizedPath = join(repoPath, '.github', 'pull_request_template.md') + await writeFile(oversizedPath, 'x') + await truncate(oversizedPath, MAX_HOSTED_REVIEW_TEMPLATE_BYTES + 1) + await writeFile(join(repoPath, '.github', 'PULL_REQUEST_TEMPLATE.md'), 'bounded fallback\n') + + await expect(readHostedReviewTemplate(repoPath)).resolves.toBe('bounded fallback\n') + }) + + it('accepts a template exactly at the byte limit', async () => { + const repoPath = await makeRepo() + const body = 'a'.repeat(MAX_HOSTED_REVIEW_TEMPLATE_BYTES) + await writeFile(join(repoPath, '.github', 'pull_request_template.md'), body) + + await expect(readHostedReviewTemplate(repoPath)).resolves.toHaveLength( + MAX_HOSTED_REVIEW_TEMPLATE_BYTES + ) + }) +}) diff --git a/src/main/source-control/pull-request-template.ts b/src/main/source-control/pull-request-template.ts index e5f474a4958..f3ecaa323db 100644 --- a/src/main/source-control/pull-request-template.ts +++ b/src/main/source-control/pull-request-template.ts @@ -1,8 +1,10 @@ -import { readFile } from 'node:fs/promises' import { join } from 'node:path' import type { HostedReviewProvider } from '../../shared/hosted-review' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { joinWorktreeRelativePath } from '../runtime/runtime-relative-paths' +import { readNodeFileWithinLimit } from '../../shared/node-bounded-file-reader' + +export const MAX_HOSTED_REVIEW_TEMPLATE_BYTES = 1024 * 1024 const PULL_REQUEST_TEMPLATE_CANDIDATES = [ '.github/pull_request_template.md', @@ -28,6 +30,11 @@ function getTemplateCandidates(provider?: HostedReviewProvider | null): string[] if (provider === 'gitlab') { return [...MERGE_REQUEST_TEMPLATE_CANDIDATES, ...PULL_REQUEST_TEMPLATE_CANDIDATES] } + if (provider === 'github') { + return PULL_REQUEST_TEMPLATE_CANDIDATES.filter( + (candidate) => !candidate.startsWith('.azuredevops/') && !candidate.startsWith('.gitea/') + ) + } return PULL_REQUEST_TEMPLATE_CANDIDATES } @@ -38,27 +45,54 @@ export async function readHostedPullRequestTemplate( return readHostedReviewTemplate(repoPath, connectionId) } +export async function readGitLabMergeRequestTemplate( + repoPath: string, + connectionId?: string | null +): Promise { + return readHostedReviewTemplateCandidates( + repoPath, + connectionId, + MERGE_REQUEST_TEMPLATE_CANDIDATES + ) +} + export async function readHostedReviewTemplate( repoPath: string, connectionId?: string | null, provider?: HostedReviewProvider | null +): Promise { + return readHostedReviewTemplateCandidates(repoPath, connectionId, getTemplateCandidates(provider)) +} + +async function readHostedReviewTemplateCandidates( + repoPath: string, + connectionId: string | null | undefined, + candidates: readonly string[] ): Promise { const remoteProvider = connectionId ? getSshFilesystemProvider(connectionId) : undefined if (connectionId && !remoteProvider) { return '' } - for (const relativeCandidate of getTemplateCandidates(provider)) { + for (const relativeCandidate of candidates) { try { if (remoteProvider) { const result = await remoteProvider.readFile( joinWorktreeRelativePath(repoPath, relativeCandidate) ) - if (result.isBinary) { + if ( + result.isBinary || + Buffer.byteLength(result.content, 'utf8') > MAX_HOSTED_REVIEW_TEMPLATE_BYTES + ) { continue } return result.content } - return await readFile(join(repoPath, relativeCandidate), 'utf8') + return ( + await readNodeFileWithinLimit( + join(repoPath, relativeCandidate), + MAX_HOSTED_REVIEW_TEMPLATE_BYTES + ) + ).buffer.toString('utf8') } catch { // Try the next conventional hosted-review template path. } diff --git a/src/main/source-control/repository-ref-cache.test.ts b/src/main/source-control/repository-ref-cache.test.ts new file mode 100644 index 00000000000..41206fe10ad --- /dev/null +++ b/src/main/source-control/repository-ref-cache.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import { + buildRepositoryRefCacheKey, + REPOSITORY_REF_CACHE_KEY_MAX_BYTES, + REPOSITORY_REF_CACHE_MAX_ENTRIES, + REPOSITORY_REF_CACHE_VALUE_MAX_BYTES, + RepositoryRefCache +} from './repository-ref-cache' + +describe('repository ref cache bounds', () => { + it('admits an exact-boundary UTF-8 key and rejects one byte over', () => { + expect( + buildRepositoryRefCacheKey(['a'.repeat(REPOSITORY_REF_CACHE_KEY_MAX_BYTES - 1), '']) + ).not.toBeNull() + expect( + buildRepositoryRefCacheKey(['a'.repeat(REPOSITORY_REF_CACHE_KEY_MAX_BYTES), '']) + ).toBeNull() + }) + + it('measures multibyte keys by bytes', () => { + expect( + buildRepositoryRefCacheKey(['😀'.repeat(REPOSITORY_REF_CACHE_KEY_MAX_BYTES / 4)]) + ).not.toBeNull() + expect( + buildRepositoryRefCacheKey(['😀'.repeat(REPOSITORY_REF_CACHE_KEY_MAX_BYTES / 4 + 1)]) + ).toBeNull() + }) + + it('retains exact-boundary values and skips oversized values', () => { + const cache = new RepositoryRefCache<{ value: string }>() + const exactKey = buildRepositoryRefCacheKey(['exact']) + const oversizedKey = buildRepositoryRefCacheKey(['oversized']) + const exact = 'a'.repeat(REPOSITORY_REF_CACHE_VALUE_MAX_BYTES) + const oversized = 'a'.repeat(REPOSITORY_REF_CACHE_VALUE_MAX_BYTES + 1) + + cache.remember(exactKey, { value: exact }, [exact]) + cache.remember(oversizedKey, { value: oversized }, [oversized]) + + expect(cache.get(exactKey)).toEqual({ found: true, value: { value: exact } }) + expect(cache.get(oversizedKey)).toEqual({ found: false }) + }) + + it('bounds entry count and refreshes hits before LRU eviction', () => { + const cache = new RepositoryRefCache<{ value: number }>() + for (let index = 0; index < REPOSITORY_REF_CACHE_MAX_ENTRIES; index += 1) { + cache.remember(`key-${index}`, { value: index }, []) + } + expect(cache.get('key-0')).toEqual({ found: true, value: { value: 0 } }) + cache.remember('new-key', { value: -1 }, []) + + expect(cache.size).toBe(REPOSITORY_REF_CACHE_MAX_ENTRIES) + expect(cache.get('key-0').found).toBe(true) + expect(cache.get('key-1')).toEqual({ found: false }) + }) + + it('never retains an inadmissible key', () => { + const cache = new RepositoryRefCache<{ value: number }>() + cache.remember(null, { value: 1 }, []) + expect(cache.size).toBe(0) + }) +}) diff --git a/src/main/source-control/repository-ref-cache.ts b/src/main/source-control/repository-ref-cache.ts new file mode 100644 index 00000000000..3c12ef0d3a8 --- /dev/null +++ b/src/main/source-control/repository-ref-cache.ts @@ -0,0 +1,71 @@ +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' + +export const REPOSITORY_REF_CACHE_MAX_ENTRIES = 512 +export const REPOSITORY_REF_CACHE_KEY_MAX_BYTES = 4 * 1024 +export const REPOSITORY_REF_CACHE_VALUE_MAX_BYTES = 16 * 1024 + +export type RepositoryRefCacheLookup = { found: true; value: T | null } | { found: false } + +export function buildRepositoryRefCacheKey(parts: readonly string[]): string | null { + let remainingBytes = REPOSITORY_REF_CACHE_KEY_MAX_BYTES - Math.max(0, parts.length - 1) + if (remainingBytes < 0) { + return null + } + for (const part of parts) { + const measured = measureUtf8ByteLength(part, { stopAfterBytes: remainingBytes }) + if (measured.exceededLimit) { + return null + } + remainingBytes -= measured.byteLength + } + return parts.join('\0') +} + +export class RepositoryRefCache { + private readonly entries = new Map() + + clear(): void { + this.entries.clear() + } + + get size(): number { + return this.entries.size + } + + get(cacheKey: string | null): RepositoryRefCacheLookup { + if (cacheKey === null || !this.entries.has(cacheKey)) { + return { found: false } + } + const value = this.entries.get(cacheKey) ?? null + this.entries.delete(cacheKey) + this.entries.set(cacheKey, value) + return { found: true, value } + } + + remember(cacheKey: string | null, value: T | null, retainedStrings: readonly string[]): void { + if (cacheKey === null || !fitsValueBudget(retainedStrings)) { + return + } + this.entries.delete(cacheKey) + this.entries.set(cacheKey, value) + while (this.entries.size > REPOSITORY_REF_CACHE_MAX_ENTRIES) { + const oldestKey = this.entries.keys().next().value + if (oldestKey === undefined) { + return + } + this.entries.delete(oldestKey) + } + } +} + +function fitsValueBudget(values: readonly string[]): boolean { + let remainingBytes = REPOSITORY_REF_CACHE_VALUE_MAX_BYTES + for (const value of values) { + const measured = measureUtf8ByteLength(value, { stopAfterBytes: remainingBytes }) + if (measured.exceededLimit) { + return false + } + remainingBytes -= measured.byteLength + } + return true +} diff --git a/src/main/speech/model-cache-path-bounds.test.ts b/src/main/speech/model-cache-path-bounds.test.ts new file mode 100644 index 00000000000..a1e9bea9821 --- /dev/null +++ b/src/main/speech/model-cache-path-bounds.test.ts @@ -0,0 +1,108 @@ +import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { migrateSpeechModelCacheIfNeeded } from './model-cache-path' + +describe('speech model cache migration bounds', () => { + const cleanupPaths: string[] = [] + + afterEach(async () => { + vi.restoreAllMocks() + await Promise.all( + cleanupPaths.splice(0).map((path) => rm(path, { recursive: true, force: true })) + ) + }) + + async function createMigrationDirs(): Promise<{ sourceDir: string; targetDir: string }> { + const root = await mkdtemp(join(tmpdir(), 'orca-speech-migration-')) + cleanupPaths.push(root) + const sourceDir = join(root, 'source') + const targetDir = join(root, 'target') + await Promise.all([ + mkdir(sourceDir, { recursive: true }), + mkdir(targetDir, { recursive: true }) + ]) + return { sourceDir, targetDir } + } + + it('copies every entry at the exact count limit', async () => { + const { sourceDir, targetDir } = await createMigrationDirs() + await Promise.all([ + writeFile(join(sourceDir, 'encoder.onnx'), 'encoder'), + writeFile(join(sourceDir, 'tokens.txt'), 'tokens') + ]) + + await migrateSpeechModelCacheIfNeeded(sourceDir, targetDir, { maxEntries: 2 }) + + expect((await readdir(targetDir)).sort()).toEqual(['encoder.onnx', 'tokens.txt']) + }) + + it('stops before copying the first entry over the count limit', async () => { + const { sourceDir, targetDir } = await createMigrationDirs() + await Promise.all([ + writeFile(join(sourceDir, 'encoder.onnx'), 'encoder'), + writeFile(join(sourceDir, 'decoder.onnx'), 'decoder'), + writeFile(join(sourceDir, 'tokens.txt'), 'tokens') + ]) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await migrateSpeechModelCacheIfNeeded(sourceDir, targetDir, { maxEntries: 2 }) + + expect(await readdir(targetDir)).toHaveLength(2) + expect(warn).toHaveBeenCalledWith( + '[speech] Failed to migrate speech model cache to ASCII path:', + expect.objectContaining({ message: expect.stringContaining('exceeded 2 entries') }) + ) + }) + + it('stops traversal beyond the configured depth', async () => { + const { sourceDir, targetDir } = await createMigrationDirs() + await mkdir(join(sourceDir, 'model', 'nested'), { recursive: true }) + await writeFile(join(sourceDir, 'model', 'nested', 'encoder.onnx'), 'encoder') + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await migrateSpeechModelCacheIfNeeded(sourceDir, targetDir, { maxDepth: 2 }) + + await expect(readFile(join(targetDir, 'model', 'nested', 'encoder.onnx'))).rejects.toThrow() + expect(warn).toHaveBeenCalledWith( + '[speech] Failed to migrate speech model cache to ASCII path:', + expect.objectContaining({ message: expect.stringContaining('exceeded depth 2') }) + ) + }) + + it('accepts the exact visited-path byte budget and rejects one byte less', async () => { + const exact = await createMigrationDirs() + await writeFile(join(exact.sourceDir, 'tokens.txt'), 'tokens') + const exactBytes = + Buffer.byteLength(join(exact.sourceDir, 'tokens.txt'), 'utf8') + + Buffer.byteLength(join(exact.targetDir, 'tokens.txt'), 'utf8') + + await migrateSpeechModelCacheIfNeeded(exact.sourceDir, exact.targetDir, { + maxVisitedPathBytes: exactBytes + }) + await expect(readFile(join(exact.targetDir, 'tokens.txt'), 'utf8')).resolves.toBe('tokens') + + const overflow = await createMigrationDirs() + await writeFile(join(overflow.sourceDir, 'tokens.txt'), 'tokens') + const overflowBytes = + Buffer.byteLength(join(overflow.sourceDir, 'tokens.txt'), 'utf8') + + Buffer.byteLength(join(overflow.targetDir, 'tokens.txt'), 'utf8') + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await migrateSpeechModelCacheIfNeeded(overflow.sourceDir, overflow.targetDir, { + maxVisitedPathBytes: overflowBytes - 1 + }) + await expect(readFile(join(overflow.targetDir, 'tokens.txt'))).rejects.toThrow() + }) + + it('keeps an existing target file unchanged', async () => { + const { sourceDir, targetDir } = await createMigrationDirs() + await writeFile(join(sourceDir, 'tokens.txt'), 'source') + await writeFile(join(targetDir, 'tokens.txt'), 'existing') + + await migrateSpeechModelCacheIfNeeded(sourceDir, targetDir) + + await expect(readFile(join(targetDir, 'tokens.txt'), 'utf8')).resolves.toBe('existing') + }) +}) diff --git a/src/main/speech/model-cache-path.ts b/src/main/speech/model-cache-path.ts index 82dd286f1dd..3361fbca05b 100644 --- a/src/main/speech/model-cache-path.ts +++ b/src/main/speech/model-cache-path.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' import { existsSync } from 'node:fs' -import { cp, mkdir, readdir, rename, stat } from 'node:fs/promises' +import { cp, mkdir, opendir, rename, stat } from 'node:fs/promises' import { join, resolve } from 'node:path' export type SpeechModelCacheDir = { @@ -9,6 +9,72 @@ export type SpeechModelCacheDir = { } const WINDOWS_SAFE_CACHE_HASH_LENGTH = 16 +export const SPEECH_MODEL_CACHE_MIGRATION_MAX_ENTRIES = 16_384 +export const SPEECH_MODEL_CACHE_MIGRATION_MAX_DEPTH = 16 +export const SPEECH_MODEL_CACHE_MIGRATION_MAX_VISITED_PATH_BYTES = 16 * 1024 * 1024 + +export type SpeechModelCacheMigrationLimits = { + maxEntries: number + maxDepth: number + maxVisitedPathBytes: number +} + +class SpeechModelCacheMigrationBudget { + private entries = 0 + private visitedPathBytes = 0 + private readonly limits: SpeechModelCacheMigrationLimits + + constructor(requested: Partial) { + this.limits = { + maxEntries: resolveMigrationLimit( + requested.maxEntries, + SPEECH_MODEL_CACHE_MIGRATION_MAX_ENTRIES, + 'maxEntries' + ), + maxDepth: resolveMigrationLimit( + requested.maxDepth, + SPEECH_MODEL_CACHE_MIGRATION_MAX_DEPTH, + 'maxDepth' + ), + maxVisitedPathBytes: resolveMigrationLimit( + requested.maxVisitedPathBytes, + SPEECH_MODEL_CACHE_MIGRATION_MAX_VISITED_PATH_BYTES, + 'maxVisitedPathBytes' + ) + } + } + + claim(sourcePath: string, targetPath: string, depth: number): void { + if (depth > this.limits.maxDepth) { + throw new Error(`Speech model cache migration exceeded depth ${this.limits.maxDepth}`) + } + this.entries += 1 + if (this.entries > this.limits.maxEntries) { + throw new Error(`Speech model cache migration exceeded ${this.limits.maxEntries} entries`) + } + this.visitedPathBytes += + Buffer.byteLength(sourcePath, 'utf8') + Buffer.byteLength(targetPath, 'utf8') + if (this.visitedPathBytes > this.limits.maxVisitedPathBytes) { + throw new Error( + `Speech model cache migration exceeded ${this.limits.maxVisitedPathBytes} visited path bytes` + ) + } + } +} + +function resolveMigrationLimit( + requested: number | undefined, + maximum: number, + name: string +): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} function hasNonAsciiCharacters(value: string): boolean { for (const character of value) { @@ -65,12 +131,24 @@ export function getSpeechModelCacheDirCandidates( return [...candidates, { modelsDir: requestedModelsDir, migrationSourceDir: null }] } -async function copyMissingCacheEntry(sourcePath: string, targetPath: string): Promise { +async function copyMissingCacheEntry( + sourcePath: string, + targetPath: string, + depth: number, + budget: SpeechModelCacheMigrationBudget +): Promise { + budget.claim(sourcePath, targetPath, depth) const sourceStat = await stat(sourcePath) if (sourceStat.isDirectory()) { await mkdir(targetPath, { recursive: true }) - for (const entry of await readdir(sourcePath, { withFileTypes: true })) { - await copyMissingCacheEntry(join(sourcePath, entry.name), join(targetPath, entry.name)) + const directory = await opendir(sourcePath, { bufferSize: 32 }) + for await (const entry of directory) { + await copyMissingCacheEntry( + join(sourcePath, entry.name), + join(targetPath, entry.name), + depth + 1, + budget + ) } return } @@ -88,15 +166,23 @@ async function copyMissingCacheEntry(sourcePath: string, targetPath: string): Pr export async function migrateSpeechModelCacheIfNeeded( sourceDir: string | null, - targetDir: string + targetDir: string, + limits: Partial = {} ): Promise { if (!sourceDir || resolve(sourceDir) === resolve(targetDir) || !existsSync(sourceDir)) { return } try { - for (const entry of await readdir(sourceDir, { withFileTypes: true })) { - await copyMissingCacheEntry(join(sourceDir, entry.name), join(targetDir, entry.name)) + const budget = new SpeechModelCacheMigrationBudget(limits) + const directory = await opendir(sourceDir, { bufferSize: 32 }) + for await (const entry of directory) { + await copyMissingCacheEntry( + join(sourceDir, entry.name), + join(targetDir, entry.name), + 1, + budget + ) } } catch (error) { console.warn('[speech] Failed to migrate speech model cache to ASCII path:', error) diff --git a/src/main/speech/model-manager.test.ts b/src/main/speech/model-manager.test.ts index 07d78749b08..97a18e83fbb 100644 --- a/src/main/speech/model-manager.test.ts +++ b/src/main/speech/model-manager.test.ts @@ -5,6 +5,7 @@ import { dirname, join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' import { SPEECH_MODEL_CATALOG } from './model-catalog' import { ModelManager } from './model-manager' +import { SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES } from './speech-model-extraction-stderr' const { hasOpenAiSpeechApiKeyMock, netRequestMock, spawnMock } = vi.hoisted(() => ({ hasOpenAiSpeechApiKeyMock: vi.fn(), @@ -452,4 +453,59 @@ describe('ModelManager', () => { rmSync(dir, { recursive: true, force: true }) } }) + + it('surfaces bounded prefix and tail evidence when tar stderr is oversized', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) + try { + const handlers: Record void)[]> = { + close: [], + error: [] + } + const stderrHandlers: ((chunk: Buffer) => void)[] = [] + const child = { + stderr: { + on: vi.fn((_event: string, cb: (chunk: Buffer) => void) => { + stderrHandlers.push(cb) + return child.stderr + }), + off: vi.fn((_event: string, cb: (chunk: Buffer) => void) => { + const index = stderrHandlers.indexOf(cb) + if (index !== -1) { + stderrHandlers.splice(index, 1) + } + return child.stderr + }) + }, + kill: vi.fn(), + on: vi.fn((event: string, cb: (arg?: unknown) => void) => { + handlers[event]?.push(cb) + return child + }), + off: vi.fn((event: string, cb: (arg?: unknown) => void) => { + handlers[event] = handlers[event]?.filter((handler) => handler !== cb) ?? [] + return child + }) + } + spawnMock.mockReturnValue(child) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const extraction = manager.extractArchive(join(dir, 'model.tar.bz2'), dir, 'm', () => false) + stderrHandlers[0]( + Buffer.concat([ + Buffer.from('PREFIX'), + Buffer.alloc(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES * 2, 0x78), + Buffer.from('TAIL') + ]) + ) + handlers.close[0](2) + + await expect(extraction).rejects.toThrow( + /tar exited with code 2: PREFIX[\s\S]+stderr bytes omitted[\s\S]+TAIL$/ + ) + expect(stderrHandlers).toHaveLength(0) + expect(handlers.close).toHaveLength(0) + expect(handlers.error).toHaveLength(0) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) }) diff --git a/src/main/speech/model-manager.ts b/src/main/speech/model-manager.ts index 2b8bca8bbca..396fb146570 100644 --- a/src/main/speech/model-manager.ts +++ b/src/main/speech/model-manager.ts @@ -9,7 +9,7 @@ import { rmSync, statSync } from 'node:fs' -import { readdir, rm } from 'node:fs/promises' +import { rm } from 'node:fs/promises' import { createHash } from 'node:crypto' import { pipeline } from 'node:stream/promises' import { spawn } from 'node:child_process' @@ -26,6 +26,8 @@ import { migrateSpeechModelCacheIfNeeded, type SpeechModelCacheDir } from './model-cache-path' +import { findNestedSpeechModelDirectory } from './speech-model-directory-scanner' +import { SpeechModelExtractionStderr } from './speech-model-extraction-stderr' type DownloadHandle = { abort: () => void @@ -835,7 +837,7 @@ export class ModelManager { } ) - let stderr = '' + const stderr = new SpeechModelExtractionStderr() let settled = false let timeout: ReturnType | null = null let abortPoll: ReturnType | null = null @@ -864,7 +866,7 @@ export class ModelManager { reject(error) } const onStderrData = (chunk: Buffer): void => { - stderr += chunk.toString() + stderr.append(chunk) } const onClose = (code: number | null): void => { if (settled) { @@ -875,7 +877,7 @@ export class ModelManager { if (code === 0) { resolve() } else { - reject(new Error(`tar exited with code ${code}: ${stderr.slice(0, 500)}`)) + reject(new Error(`tar exited with code ${code}: ${stderr.errorEvidence()}`)) } } const onError = (err: Error): void => { @@ -902,22 +904,15 @@ export class ModelManager { if (!manifest.files) { return } - const entries = await readdir(modelDir, { withFileTypes: true }) - for (const entry of entries) { - if (entry.isDirectory()) { - const nestedDir = join(modelDir, entry.name) - const nestedFiles = await readdir(nestedDir) - const hasExpected = manifest.files.some((f) => nestedFiles.includes(f)) - if (hasExpected) { - const { rename: fsRename } = await import('node:fs/promises') - for (const file of nestedFiles) { - await fsRename(join(nestedDir, file), join(modelDir, file)) - } - await rm(nestedDir, { recursive: true, force: true }) - return - } - } + const nested = await findNestedSpeechModelDirectory(modelDir, manifest.files) + if (!nested) { + return } + const { rename: fsRename } = await import('node:fs/promises') + for (const file of nested.entryNames) { + await fsRename(join(nested.directoryPath, file), join(modelDir, file)) + } + await rm(nested.directoryPath, { recursive: true, force: true }) } private cleanup(modelId: string, archivePath: string): void { diff --git a/src/main/speech/openai-api-key-store.test.ts b/src/main/speech/openai-api-key-store.test.ts index 9475d5afa54..b783a976361 100644 --- a/src/main/speech/openai-api-key-store.test.ts +++ b/src/main/speech/openai-api-key-store.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import type * as Os from 'node:os' import { join } from 'node:path' @@ -85,4 +85,13 @@ describe('OpenAI speech API key store', () => { expect(existsSync(join(tempHome, '.orca'))).toBe(false) expect(safeStorageMock.decryptString).not.toHaveBeenCalled() }) + + it('rejects an oversized sparse key file before decrypting it', async () => { + writeStoredOpenAiKey('x') + truncateSync(join(tempHome, '.orca', 'openai-speech-token.enc'), 1024 * 1024 + 1) + const store = await loadStoreModule() + + expect(() => store.readOpenAiSpeechApiKey()).toThrow('could not be decrypted') + expect(safeStorageMock.decryptString).not.toHaveBeenCalled() + }) }) diff --git a/src/main/speech/openai-api-key-store.ts b/src/main/speech/openai-api-key-store.ts index dc7345f3037..b2a96cd0e95 100644 --- a/src/main/speech/openai-api-key-store.ts +++ b/src/main/speech/openai-api-key-store.ts @@ -1,7 +1,8 @@ import { safeStorage } from 'electron' -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' +import { readIntegrationCredentialFileSync } from '../integration-credential-file' type StoredOpenAiKey = { encryptedKeyBase64: string @@ -31,7 +32,9 @@ function readLegacyJsonStoredOpenAiKey(): StoredOpenAiKey | null { return null } try { - const parsed = JSON.parse(readFileSync(keyPath, 'utf8')) as Partial + const parsed = JSON.parse( + readIntegrationCredentialFileSync(keyPath).toString('utf8') + ) as Partial if (typeof parsed.encryptedKeyBase64 !== 'string' || parsed.encryptedKeyBase64 === '') { return null } @@ -76,7 +79,7 @@ export function readOpenAiSpeechApiKey(): string { throw new Error('OpenAI API key is not configured') } try { - const raw = readFileSync(keyPath) + const raw = readIntegrationCredentialFileSync(keyPath) const legacyJson = readLegacyJsonStoredOpenAiKey() if (legacyJson) { cachedOpenAiSpeechApiKey = safeStorage.decryptString( diff --git a/src/main/speech/openai-transcription-client.test.ts b/src/main/speech/openai-transcription-client.test.ts index 8e20f593b08..bbe4da6576b 100644 --- a/src/main/speech/openai-transcription-client.test.ts +++ b/src/main/speech/openai-transcription-client.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { sanitizeOpenAiTranscriptionErrorMessage } from './openai-transcription-client' +import { + OpenAiTranscriptionSession, + sanitizeOpenAiTranscriptionErrorMessage +} from './openai-transcription-client' describe('sanitizeOpenAiTranscriptionErrorMessage', () => { it('does not expose the invalid OpenAI API key echoed by the provider', () => { @@ -17,4 +20,16 @@ describe('sanitizeOpenAiTranscriptionErrorMessage', () => { ) ).toBe('Request failed for [redacted] with Authorization: Bearer [redacted]') }) + + it('retains one growable sample buffer for adversarial one-sample feeds', () => { + const session = new OpenAiTranscriptionSession('openai-gpt-4o-mini-transcribe', () => 'key') + const sample = new Float32Array([0.25]) + + for (let index = 0; index < 100_000; index += 1) { + session.feedAudio(sample, 16_000) + } + + expect(Reflect.get(session, 'sampleCount')).toBe(100_000) + expect(Reflect.get(session, 'samples')).toBeInstanceOf(Float32Array) + }) }) diff --git a/src/main/speech/openai-transcription-client.ts b/src/main/speech/openai-transcription-client.ts index c61dc940f10..f8b6e9b5336 100644 --- a/src/main/speech/openai-transcription-client.ts +++ b/src/main/speech/openai-transcription-client.ts @@ -1,4 +1,5 @@ import { resampleToRate } from './stt-audio-resample' +import { readFetchResponseJsonWithinLimit } from '../lib/fetch-response-body' export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record = { 'openai-gpt-4o-mini-transcribe': 'gpt-4o-mini-transcribe', @@ -8,6 +9,7 @@ export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record = { const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions' const CLOUD_TRANSCRIPTION_SAMPLE_RATE = 16000 const MAX_CLOUD_AUDIO_SECONDS = 10 * 60 +const MAX_CLOUD_AUDIO_SAMPLES = CLOUD_TRANSCRIPTION_SAMPLE_RATE * MAX_CLOUD_AUDIO_SECONDS type OpenAiTranscriptionResponse = { text?: unknown @@ -29,8 +31,8 @@ export function sanitizeOpenAiTranscriptionErrorMessage(message: string): string return sanitized || 'OpenAI transcription request failed' } -function encodePcm16Wav(samples: Float32Array, sampleRate: number): Buffer { - const dataBytes = samples.length * 2 +function encodePcm16Wav(samples: Float32Array, sampleCount: number, sampleRate: number): Buffer { + const dataBytes = sampleCount * 2 const buffer = Buffer.alloc(44 + dataBytes) buffer.write('RIFF', 0) @@ -47,7 +49,7 @@ function encodePcm16Wav(samples: Float32Array, sampleRate: number): Buffer { buffer.write('data', 36) buffer.writeUInt32LE(dataBytes, 40) - for (let i = 0; i < samples.length; i += 1) { + for (let i = 0; i < sampleCount; i += 1) { const clamped = Math.max(-1, Math.min(1, samples[i])) const value = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff buffer.writeInt16LE(Math.round(value), 44 + i * 2) @@ -56,17 +58,6 @@ function encodePcm16Wav(samples: Float32Array, sampleRate: number): Buffer { return buffer } -function combineChunks(chunks: Float32Array[]): Float32Array { - const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0) - const combined = new Float32Array(totalLength) - let offset = 0 - for (const chunk of chunks) { - combined.set(chunk, offset) - offset += chunk.length - } - return combined -} - function parseOpenAiTranscriptionResponse(data: OpenAiTranscriptionResponse): string { if (typeof data.text === 'string') { return data.text.trim() @@ -78,8 +69,8 @@ function parseOpenAiTranscriptionResponse(data: OpenAiTranscriptionResponse): st } export class OpenAiTranscriptionSession { - private chunks: Float32Array[] = [] - private audioSeconds = 0 + private samples = new Float32Array(0) + private sampleCount = 0 constructor( private readonly modelId: string, @@ -88,15 +79,25 @@ export class OpenAiTranscriptionSession { feedAudio(samples: Float32Array, sampleRate: number): void { const normalized = resampleToRate(samples, sampleRate, CLOUD_TRANSCRIPTION_SAMPLE_RATE) - this.audioSeconds += normalized.length / CLOUD_TRANSCRIPTION_SAMPLE_RATE - if (this.audioSeconds > MAX_CLOUD_AUDIO_SECONDS) { + const nextSampleCount = this.sampleCount + normalized.length + if (nextSampleCount > MAX_CLOUD_AUDIO_SAMPLES) { throw new Error('Cloud transcription is limited to 10 minutes per dictation') } - this.chunks.push(new Float32Array(normalized)) + if (this.samples.length < nextSampleCount) { + const nextCapacity = Math.min( + MAX_CLOUD_AUDIO_SAMPLES, + Math.max(CLOUD_TRANSCRIPTION_SAMPLE_RATE, this.samples.length * 2, nextSampleCount) + ) + const next = new Float32Array(nextCapacity) + next.set(this.samples.subarray(0, this.sampleCount)) + this.samples = next + } + this.samples.set(normalized, this.sampleCount) + this.sampleCount = nextSampleCount } async finish(): Promise { - if (this.chunks.length === 0) { + if (this.sampleCount === 0) { return '' } @@ -105,9 +106,9 @@ export class OpenAiTranscriptionSession { throw new Error(`Unknown OpenAI transcription model: ${this.modelId}`) } - const audio = combineChunks(this.chunks) - this.chunks = [] - const wav = encodePcm16Wav(audio, CLOUD_TRANSCRIPTION_SAMPLE_RATE) + const wav = encodePcm16Wav(this.samples, this.sampleCount, CLOUD_TRANSCRIPTION_SAMPLE_RATE) + this.samples = new Float32Array(0) + this.sampleCount = 0 const form = new FormData() form.append('model', apiModel) form.append('response_format', 'json') @@ -123,7 +124,9 @@ export class OpenAiTranscriptionSession { body: form }) - const data = (await response.json().catch(() => ({}))) as OpenAiTranscriptionResponse + const data = await readFetchResponseJsonWithinLimit( + response + ).catch((): OpenAiTranscriptionResponse => ({})) if (!response.ok) { const message = typeof data.error?.message === 'string' diff --git a/src/main/speech/speech-model-directory-scanner.test.ts b/src/main/speech/speech-model-directory-scanner.test.ts new file mode 100644 index 00000000000..831b1e2dafb --- /dev/null +++ b/src/main/speech/speech-model-directory-scanner.test.ts @@ -0,0 +1,172 @@ +import type * as NodeFs from 'node:fs' +import type * as NodeFsPromises from 'node:fs/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { opendirMock, opendirSyncMock } = vi.hoisted(() => ({ + opendirMock: vi.fn(), + opendirSyncMock: vi.fn() +})) + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), + opendirSync: opendirSyncMock +})) + +vi.mock('node:fs/promises', async (importOriginal) => ({ + ...(await importOriginal()), + opendir: opendirMock +})) + +import { + findNestedSpeechModelDirectory, + findSpeechModelBpeVocabFile, + SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES, + SpeechModelDirectoryCapacityError +} from './speech-model-directory-scanner' +import { buildHotwordsConfig } from './stt-worker-model-config' + +type FakeEntry = { name: string; directory?: boolean } + +function fakeAsyncDirectory(entries: FakeEntry[], onClose?: () => void): object { + return { + async *[Symbol.asyncIterator]() { + try { + for (const entry of entries) { + yield { + name: entry.name, + isDirectory: () => entry.directory === true + } + } + } finally { + onClose?.() + } + } + } +} + +function useSyncEntries(entries: FakeEntry[]): ReturnType { + let index = 0 + const closeSync = vi.fn() + opendirSyncMock.mockReturnValue({ + closeSync, + readSync: vi.fn(() => { + const entry = entries[index] + index += 1 + return entry ? { name: entry.name } : null + }) + }) + return closeSync +} + +describe('speech model directory scanner', () => { + beforeEach(() => { + opendirMock.mockReset() + opendirSyncMock.mockReset() + }) + + it('preserves the first matching nested directory and its entry order', async () => { + opendirMock.mockImplementation(async (path: string) => { + if (path === '/models') { + return fakeAsyncDirectory([ + { name: 'unrelated', directory: true }, + { name: 'archive', directory: true } + ]) + } + if (path === '/models/unrelated') { + return fakeAsyncDirectory([{ name: 'README.md' }]) + } + return fakeAsyncDirectory([{ name: 'encoder.onnx' }, { name: 'tokens.txt' }]) + }) + + await expect(findNestedSpeechModelDirectory('/models', ['encoder.onnx'])).resolves.toEqual({ + directoryPath: '/models/archive', + entryNames: ['encoder.onnx', 'tokens.txt'] + }) + }) + + it('accepts the exact archive entry limit', async () => { + opendirMock.mockImplementation(async (path: string) => + path === '/models' + ? fakeAsyncDirectory([{ name: 'archive', directory: true }]) + : fakeAsyncDirectory([{ name: 'encoder.onnx' }, { name: 'tokens.txt' }]) + ) + + await expect( + findNestedSpeechModelDirectory('/models', ['encoder.onnx'], { + maxEntries: 3 + }) + ).resolves.toMatchObject({ entryNames: ['encoder.onnx', 'tokens.txt'] }) + }) + + it('closes both streams and rejects the first entry over the archive limit', async () => { + const closed: string[] = [] + opendirMock.mockImplementation(async (path: string) => + path === '/models' + ? fakeAsyncDirectory([{ name: 'archive', directory: true }], () => closed.push('root')) + : fakeAsyncDirectory([{ name: 'one' }, { name: 'two' }, { name: 'three' }], () => + closed.push('nested') + ) + ) + + await expect( + findNestedSpeechModelDirectory('/models', ['missing'], { + maxEntries: 3 + }) + ).rejects.toBeInstanceOf(SpeechModelDirectoryCapacityError) + expect(closed).toEqual(['nested', 'root']) + }) + + it('rejects a nested listing above its retained-name budget', async () => { + opendirMock.mockImplementation(async (path: string) => + path === '/models' + ? fakeAsyncDirectory([{ name: 'archive', directory: true }]) + : fakeAsyncDirectory([{ name: 'x' }]) + ) + + await expect( + findNestedSpeechModelDirectory('/models', ['x'], { + maxRetainedNameBytes: 65 + }) + ).rejects.toThrow('retained name bytes') + }) + + it('finds a vocab at the exact production scan boundary', () => { + const entries = Array.from({ length: SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES - 1 }, (_, index) => ({ + name: `file-${index}` + })) + entries.push({ name: 'tokens.vocab' }) + const closeSync = useSyncEntries(entries) + + expect( + buildHotwordsConfig({ + modelDir: '/models', + modelType: 'transducer', + hotwordsFilePath: '/hotwords.txt', + modelingUnit: 'bpe' + }) + ).toMatchObject({ + decodingMethod: 'modified_beam_search', + bpeVocab: '/models/tokens.vocab' + }) + expect(closeSync).toHaveBeenCalledOnce() + }) + + it('ignores a vocab beyond the production scan boundary and closes the stream', () => { + const entries = Array.from({ length: SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES }, (_, index) => ({ + name: `file-${index}` + })) + entries.push({ name: 'too-late.vocab' }) + const closeSync = useSyncEntries(entries) + + expect( + buildHotwordsConfig({ + modelDir: '/models', + modelType: 'transducer', + hotwordsFilePath: '/hotwords.txt', + modelingUnit: 'bpe' + }) + ).toEqual({ decodingMethod: 'greedy_search' }) + expect(findSpeechModelBpeVocabFile('/models', 0)).toBeUndefined() + expect(closeSync).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/speech/speech-model-directory-scanner.ts b/src/main/speech/speech-model-directory-scanner.ts new file mode 100644 index 00000000000..5ca7ef01b14 --- /dev/null +++ b/src/main/speech/speech-model-directory-scanner.ts @@ -0,0 +1,130 @@ +import { opendirSync } from 'node:fs' +import { opendir } from 'node:fs/promises' +import { join } from 'node:path' + +export const SPEECH_MODEL_ARCHIVE_SCAN_MAX_ENTRIES = 32_768 +export const SPEECH_MODEL_ARCHIVE_MAX_RETAINED_NAME_BYTES = 4 * 1024 * 1024 +export const SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES = 4_096 + +export type SpeechModelArchiveScanLimits = { + maxEntries: number + maxRetainedNameBytes: number +} + +export type NestedSpeechModelDirectory = { + directoryPath: string + entryNames: string[] +} + +export class SpeechModelDirectoryCapacityError extends Error { + constructor(resource: string, limit: number) { + super(`Speech model directory exceeded its ${resource} limit (${limit})`) + this.name = 'SpeechModelDirectoryCapacityError' + } +} + +function resolveLimit(requested: number | undefined, maximum: number, name: string): number { + if (requested === undefined) { + return maximum + } + if (!Number.isSafeInteger(requested) || requested < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + return Math.min(requested, maximum) +} + +function estimateRetainedNameBytes(name: string): number { + return name.length * 2 + 64 +} + +export async function findNestedSpeechModelDirectory( + modelDir: string, + expectedFiles: readonly string[], + requestedLimits: Partial = {} +): Promise { + const maxEntries = resolveLimit( + requestedLimits.maxEntries, + SPEECH_MODEL_ARCHIVE_SCAN_MAX_ENTRIES, + 'maxEntries' + ) + const maxRetainedNameBytes = resolveLimit( + requestedLimits.maxRetainedNameBytes, + SPEECH_MODEL_ARCHIVE_MAX_RETAINED_NAME_BYTES, + 'maxRetainedNameBytes' + ) + const expectedFileNames = new Set(expectedFiles) + let scannedEntries = 0 + + const rootDirectory = await opendir(modelDir, { bufferSize: 32 }) + for await (const entry of rootDirectory) { + scannedEntries += 1 + if (scannedEntries > maxEntries) { + throw new SpeechModelDirectoryCapacityError('entry count', maxEntries) + } + if (!entry.isDirectory()) { + continue + } + + const nestedDirectoryPath = join(modelDir, entry.name) + const entryNames: string[] = [] + let retainedNameBytes = 0 + const nestedDirectory = await opendir(nestedDirectoryPath, { bufferSize: 32 }) + for await (const nestedEntry of nestedDirectory) { + scannedEntries += 1 + if (scannedEntries > maxEntries) { + throw new SpeechModelDirectoryCapacityError('entry count', maxEntries) + } + retainedNameBytes += estimateRetainedNameBytes(nestedEntry.name) + if (retainedNameBytes > maxRetainedNameBytes) { + throw new SpeechModelDirectoryCapacityError('retained name bytes', maxRetainedNameBytes) + } + entryNames.push(nestedEntry.name) + } + if (entryNames.some((name) => expectedFileNames.has(name))) { + return { directoryPath: nestedDirectoryPath, entryNames } + } + } + return null +} + +function closeSpeechModelDirectory(directory: ReturnType): void { + try { + directory.closeSync() + } catch { + // The OS may already have closed a fully consumed directory stream. + } +} + +export function findSpeechModelBpeVocabFile( + modelDir: string, + requestedMaxEntries = SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES +): string | undefined { + const maxEntries = resolveLimit( + requestedMaxEntries, + SPEECH_MODEL_VOCAB_SCAN_MAX_ENTRIES, + 'maxEntries' + ) + let directory: ReturnType + try { + directory = opendirSync(modelDir, { bufferSize: 32 }) + } catch { + return undefined + } + + let scannedEntries = 0 + try { + while (scannedEntries < maxEntries) { + const entry = directory.readSync() + if (entry === null) { + return undefined + } + scannedEntries += 1 + if (entry.name.endsWith('.vocab')) { + return join(modelDir, entry.name) + } + } + return undefined + } finally { + closeSpeechModelDirectory(directory) + } +} diff --git a/src/main/speech/speech-model-extraction-stderr.test.ts b/src/main/speech/speech-model-extraction-stderr.test.ts new file mode 100644 index 00000000000..9a851ccc517 --- /dev/null +++ b/src/main/speech/speech-model-extraction-stderr.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { + SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES, + SpeechModelExtractionStderr +} from './speech-model-extraction-stderr' + +describe('speech model extraction stderr', () => { + it('preserves ordinary error evidence exactly', () => { + const stderr = new SpeechModelExtractionStderr() + stderr.append(Buffer.from('tar: archive is corrupt\n')) + stderr.append(Buffer.from('tar: exiting with failure')) + + expect(stderr.errorEvidence()).toBe('tar: archive is corrupt\ntar: exiting with failure') + expect(stderr.retainedByteLength()).toBe(49) + expect(stderr.wasTruncated()).toBe(false) + }) + + it('retains the exact byte limit without truncation', () => { + const stderr = new SpeechModelExtractionStderr() + stderr.append(Buffer.alloc(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES, 0x61)) + + expect(stderr.retainedByteLength()).toBe(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES) + expect(stderr.wasTruncated()).toBe(false) + expect(stderr.errorEvidence()).toBe('a'.repeat(500)) + }) + + it('caps one byte over the limit while preserving prefix and tail evidence', () => { + const stderr = new SpeechModelExtractionStderr() + stderr.append(Buffer.alloc(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES, 0x61)) + stderr.append(Buffer.from('Z')) + + expect(stderr.retainedByteLength()).toBe(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES) + expect(stderr.wasTruncated()).toBe(true) + expect(stderr.errorEvidence()).toMatch( + /^a{500}\n\[\.\.\. 1 stderr bytes omitted \.\.\.\]\na{499}Z$/ + ) + }) + + it('bounds a single oversized chunk without converting it to an oversized string', () => { + const stderr = new SpeechModelExtractionStderr() + const chunk = Buffer.concat([ + Buffer.from('PREFIX'), + Buffer.alloc(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES * 2, 0x78), + Buffer.from('TAIL') + ]) + + stderr.append(chunk) + + expect(stderr.retainedByteLength()).toBe(SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES) + expect(stderr.errorEvidence()).toContain('PREFIX') + expect(stderr.errorEvidence()).toMatch(/TAIL$/) + }) +}) diff --git a/src/main/speech/speech-model-extraction-stderr.ts b/src/main/speech/speech-model-extraction-stderr.ts new file mode 100644 index 00000000000..146d97dda37 --- /dev/null +++ b/src/main/speech/speech-model-extraction-stderr.ts @@ -0,0 +1,74 @@ +export const SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES = 64 * 1024 + +const STDERR_PREFIX_BYTES = 4 * 1024 +const STDERR_TAIL_BYTES = SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES - STDERR_PREFIX_BYTES +const STDERR_EVIDENCE_CHARS_PER_END = 500 + +export class SpeechModelExtractionStderr { + private readonly prefix = Buffer.alloc(STDERR_PREFIX_BYTES) + private readonly tail = Buffer.alloc(STDERR_TAIL_BYTES) + private prefixLength = 0 + private tailLength = 0 + private tailWriteOffset = 0 + private observedBytes = 0 + + append(chunk: Buffer): void { + if (chunk.length === 0) { + return + } + if (this.prefixLength < this.prefix.length) { + const prefixBytes = Math.min(chunk.length, this.prefix.length - this.prefixLength) + chunk.copy(this.prefix, this.prefixLength, 0, prefixBytes) + this.prefixLength += prefixBytes + } + this.appendTail(chunk) + this.observedBytes = Math.min(Number.MAX_SAFE_INTEGER, this.observedBytes + chunk.length) + } + + retainedByteLength(): number { + return Math.min(this.observedBytes, SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES) + } + + wasTruncated(): boolean { + return this.observedBytes > SPEECH_MODEL_EXTRACTION_STDERR_MAX_RETAINED_BYTES + } + + errorEvidence(): string { + const prefix = this.prefix + .subarray(0, this.prefixLength) + .toString('utf8') + .slice(0, STDERR_EVIDENCE_CHARS_PER_END) + if (!this.wasTruncated()) { + return prefix + } + const omittedBytes = this.observedBytes - this.retainedByteLength() + const tail = this.orderedTail().toString('utf8').slice(-STDERR_EVIDENCE_CHARS_PER_END) + return `${prefix}\n[... ${omittedBytes} stderr bytes omitted ...]\n${tail}` + } + + private appendTail(chunk: Buffer): void { + if (chunk.length >= this.tail.length) { + chunk.copy(this.tail, 0, chunk.length - this.tail.length) + this.tailLength = this.tail.length + this.tailWriteOffset = 0 + return + } + const firstBytes = Math.min(chunk.length, this.tail.length - this.tailWriteOffset) + chunk.copy(this.tail, this.tailWriteOffset, 0, firstBytes) + if (firstBytes < chunk.length) { + chunk.copy(this.tail, 0, firstBytes) + } + this.tailWriteOffset = (this.tailWriteOffset + chunk.length) % this.tail.length + this.tailLength = Math.min(this.tail.length, this.tailLength + chunk.length) + } + + private orderedTail(): Buffer { + if (this.tailLength < this.tail.length || this.tailWriteOffset === 0) { + return this.tail.subarray(0, this.tailLength) + } + return Buffer.concat([ + this.tail.subarray(this.tailWriteOffset), + this.tail.subarray(0, this.tailWriteOffset) + ]) + } +} diff --git a/src/main/speech/stt-offline-audio-chunker.test.ts b/src/main/speech/stt-offline-audio-chunker.test.ts index 4b90976b2c2..21a552a75c2 100644 --- a/src/main/speech/stt-offline-audio-chunker.test.ts +++ b/src/main/speech/stt-offline-audio-chunker.test.ts @@ -97,4 +97,17 @@ describe('OfflineAudioChunker', () => { expect(chunker.push(new Float32Array(0))).toEqual([]) expect(chunker.flush()).toBeNull() }) + + it('preserves 100,000 one-sample fragments without retaining one object per push', () => { + const chunker = new OfflineAudioChunker(10_000) + const expected = new Float32Array(100_000) + + for (let index = 0; index < expected.length; index += 1) { + const value = (index % 251) / 251 + expected[index] = value + expect(chunker.push(Float32Array.of(value))).toEqual([]) + } + + expect(chunker.flush()).toEqual(expected) + }) }) diff --git a/src/main/speech/stt-offline-audio-chunker.ts b/src/main/speech/stt-offline-audio-chunker.ts index 291454bf083..15df3b79f21 100644 --- a/src/main/speech/stt-offline-audio-chunker.ts +++ b/src/main/speech/stt-offline-audio-chunker.ts @@ -13,7 +13,7 @@ const SPLIT_SEARCH_SECONDS = 5 const SPLIT_ENERGY_WINDOW_SECONDS = 0.1 export class OfflineAudioChunker { - private buffered: Float32Array[] = [] + private buffered = new Float32Array(0) private bufferedSamples = 0 private readonly chunkSampleLimit: number private readonly splitSearchSamples: number @@ -30,18 +30,16 @@ export class OfflineAudioChunker { if (samples.length === 0) { return [] } - this.buffered.push(samples) - this.bufferedSamples += samples.length + this.append(samples) const ready: Float32Array[] = [] while (this.bufferedSamples >= this.chunkSampleLimit) { - const combined = this.combineBuffered() - const splitIndex = this.findQuietSplitIndex(combined) - ready.push(combined.slice(0, splitIndex)) - const tail = combined.slice(splitIndex) - this.buffered = tail.length > 0 ? [tail] : [] - this.bufferedSamples = tail.length + const splitIndex = this.findQuietSplitIndex(this.buffered.subarray(0, this.bufferedSamples)) + ready.push(this.buffered.slice(0, splitIndex)) + this.buffered.copyWithin(0, splitIndex, this.bufferedSamples) + this.bufferedSamples -= splitIndex } + this.releaseOversizedCapacity() return ready } @@ -50,23 +48,32 @@ export class OfflineAudioChunker { if (this.bufferedSamples === 0) { return null } - const combined = this.combineBuffered() - this.buffered = [] + const combined = this.buffered.slice(0, this.bufferedSamples) + this.buffered = new Float32Array(0) this.bufferedSamples = 0 return combined } - private combineBuffered(): Float32Array { - if (this.buffered.length === 1) { - return this.buffered[0] + private append(samples: Float32Array): void { + const required = this.bufferedSamples + samples.length + if (required > this.buffered.length) { + const capacity = Math.max(required, Math.max(1_024, this.buffered.length * 2)) + const next = new Float32Array(capacity) + next.set(this.buffered.subarray(0, this.bufferedSamples)) + this.buffered = next } - const combined = new Float32Array(this.bufferedSamples) - let offset = 0 - for (const chunk of this.buffered) { - combined.set(chunk, offset) - offset += chunk.length + this.buffered.set(samples, this.bufferedSamples) + this.bufferedSamples = required + } + + private releaseOversizedCapacity(): void { + const retainedCapacity = Math.max(this.chunkSampleLimit, this.bufferedSamples) + if (this.buffered.length <= retainedCapacity * 2) { + return } - return combined + const compacted = new Float32Array(retainedCapacity) + compacted.set(this.buffered.subarray(0, this.bufferedSamples)) + this.buffered = compacted } private findQuietSplitIndex(samples: Float32Array): number { diff --git a/src/main/speech/stt-worker-model-config.ts b/src/main/speech/stt-worker-model-config.ts index de4687d9f46..c9f269c8908 100644 --- a/src/main/speech/stt-worker-model-config.ts +++ b/src/main/speech/stt-worker-model-config.ts @@ -1,5 +1,5 @@ -import { readdirSync } from 'node:fs' import { join } from 'node:path' +import { findSpeechModelBpeVocabFile } from './speech-model-directory-scanner' // Why: different models name their ONNX files differently (e.g. // encoder.int8.onnx vs tiny-encoder.onnx vs encoder-epoch-99-avg-1.onnx). @@ -29,16 +29,6 @@ export function resolveTokens(files: string[], modelDir: string): string { // Why: BPE models need a vocab file for hotwords token matching. The file // ships in the model archive but isn't listed in the manifest. We discover // it at runtime to avoid breaking existing downloads. -function discoverBpeVocab(modelDir: string): string | undefined { - try { - const entries = readdirSync(modelDir) - const vocabFile = entries.find((f) => f.endsWith('.vocab')) - return vocabFile ? join(modelDir, vocabFile) : undefined - } catch { - return undefined - } -} - export type HotwordsConfig = { decodingMethod: string hotwordsFile?: string @@ -59,7 +49,7 @@ export function buildHotwordsConfig(opts: { const unit = opts.modelingUnit if (unit?.includes('bpe')) { - const bpeVocab = discoverBpeVocab(opts.modelDir) + const bpeVocab = findSpeechModelBpeVocabFile(opts.modelDir) if (!bpeVocab) { return { decodingMethod: 'greedy_search' } } diff --git a/src/main/ssh/relay-protocol.test.ts b/src/main/ssh/relay-protocol.test.ts index 459eaef88df..3e2ada59323 100644 --- a/src/main/ssh/relay-protocol.test.ts +++ b/src/main/ssh/relay-protocol.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { HEADER_LENGTH, + MAX_BUFFERED_FRAME_CHUNKS, MessageType, encodeFrame, encodeJsonRpcFrame, @@ -12,6 +13,7 @@ import { type JsonRpcRequest, type DecodedFrame } from './relay-protocol' +import { RELAY_JSON_MAX_STRUCTURAL_TOKENS } from '../../shared/relay-json-admission' describe('git response stream marker', () => { it('accepts only complete non-negative integer metadata', () => { @@ -26,6 +28,15 @@ describe('git response stream marker', () => { __orcaGitResponseStream: { streamId: -1, totalBytes: 1024, chunkCount: 2 } }) ).toBe(false) + expect( + isGitResponseStreamMarker({ + __orcaGitResponseStream: { + streamId: 1, + totalBytes: Number.MAX_SAFE_INTEGER + 1, + chunkCount: 2 + } + }) + ).toBe(false) }) }) @@ -79,11 +90,37 @@ describe('frame encoding', () => { method: 'x', params: { data: 'a'.repeat(17 * 1024 * 1024) } } - expect(() => encodeJsonRpcFrame(bigPayload, 1, 0)).toThrow('Message too large') + const fromSpy = vi.spyOn(Buffer, 'from') + try { + expect(() => encodeJsonRpcFrame(bigPayload, 1, 0)).toThrow('Message too large') + expect(fromSpy).not.toHaveBeenCalled() + } finally { + fromSpy.mockRestore() + } }) }) describe('FrameDecoder', () => { + it('bounds retained chunk wrappers for a byte-fragmented frame', () => { + const frames: DecodedFrame[] = [] + const decoder = new FrameDecoder((frame) => frames.push(frame)) + const payloadLength = MAX_BUFFERED_FRAME_CHUNKS + 512 + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(payloadLength, 9) + decoder.feed(header) + + const byte = Buffer.from('x') + const state = decoder as unknown as { chunks: Buffer[] } + for (let index = 0; index < payloadLength; index += 1) { + decoder.feed(byte) + expect(state.chunks.length).toBeLessThanOrEqual(MAX_BUFFERED_FRAME_CHUNKS) + } + + expect(frames).toHaveLength(1) + expect(frames[0].payload.equals(Buffer.alloc(payloadLength, 0x78))).toBe(true) + }) + it('decodes a complete frame', () => { const frames: DecodedFrame[] = [] const decoder = new FrameDecoder((f) => frames.push(f)) @@ -163,6 +200,33 @@ describe('FrameDecoder', () => { expect(errors[0].message).toContain('discarded') }) + it('rejects an oversized header immediately without retaining its payload', () => { + const errors: Error[] = [] + const decoder = new FrameDecoder( + () => {}, + (err) => errors.push(err) + ) + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(0xffffffff, 9) + + decoder.feed(header) + const state = decoder as unknown as { + bufferedLength: number + oversizedPayloadBytesRemaining: number + } + expect(errors).toHaveLength(1) + expect(state.bufferedLength).toBe(0) + expect(state.oversizedPayloadBytesRemaining).toBe(0xffffffff) + + const payloadChunk = Buffer.alloc(64 * 1024) + for (let index = 0; index < 128; index += 1) { + decoder.feed(payloadChunk) + expect(state.bufferedLength).toBe(0) + } + expect(errors).toHaveLength(1) + }) + it('reset clears internal buffer', () => { const frames: DecodedFrame[] = [] const decoder = new FrameDecoder((f) => frames.push(f)) @@ -224,26 +288,31 @@ describe('FrameDecoder', () => { expect(frames[0].payload.toString()).toBe('after') }) - it('never rebuilds the buffered stream per feed while assembling a large frame', () => { - // Regression: feed() used Buffer.concat([buffered, chunk]) per data event, - // re-copying the whole backlog for every TCP chunk — O(n²) memcpy on the - // Electron main thread while fs.streamChunk frames arrive (SSH typing lag). + it('retains ordinary transport chunks without copying them while a frame is incomplete', () => { const frames: DecodedFrame[] = [] const decoder = new FrameDecoder((f) => frames.push(f)) - const frame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(512 * 1024, 0x61)) + const payloadLength = 1024 * 1024 + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(1, 1) + header.writeUInt32BE(payloadLength, 9) + decoder.feed(header) + const ordinaryChunks = Array.from({ length: 16 }, () => Buffer.alloc(32 * 1024, 0x61)) - const concatSpy = vi.spyOn(Buffer, 'concat') - try { - for (let i = 0; i < frame.length; i += 32 * 1024) { - decoder.feed(frame.subarray(i, i + 32 * 1024)) - } - } finally { - concatSpy.mockRestore() + for (const chunk of ordinaryChunks) { + decoder.feed(chunk) } + const state = decoder as unknown as { chunks: Buffer[] } + expect(state.chunks).toHaveLength(ordinaryChunks.length + 1) + for (const [index, chunk] of ordinaryChunks.entries()) { + expect(state.chunks[index + 1]).toBe(chunk) + } + + decoder.feed(Buffer.alloc(payloadLength / 2, 0x61)) + expect(frames).toHaveLength(1) - expect(frames[0].payload.length).toBe(512 * 1024) - expect(concatSpy).not.toHaveBeenCalled() + expect(frames[0].payload.length).toBe(payloadLength) }) }) @@ -270,6 +339,21 @@ describe('parseJsonRpcMessage', () => { const payload = Buffer.from('not json') expect(() => parseJsonRpcMessage(payload)).toThrow() }) + + it('rejects structurally amplified JSON before parsing', () => { + const payload = Buffer.from( + `{"jsonrpc":"2.0","id":1,"method":"x","params":{"values":[${'0,'.repeat( + RELAY_JSON_MAX_STRUCTURAL_TOKENS + )}0]}}` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + try { + expect(() => parseJsonRpcMessage(payload)).toThrow(/JSON structure exceeds/) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) }) describe('parseUnameToRelayPlatform', () => { diff --git a/src/main/ssh/relay-protocol.ts b/src/main/ssh/relay-protocol.ts index c05b6c33bfe..6dffbb1112f 100644 --- a/src/main/ssh/relay-protocol.ts +++ b/src/main/ssh/relay-protocol.ts @@ -3,6 +3,11 @@ // See design-ssh-support.md § JSON-RPC Protocol Specification. import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../../shared/node-bounded-json-stringify' +import { parseRelayJsonText } from '../../shared/relay-json-admission' export const RELAY_VERSION = '0.1.0' export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n` @@ -13,6 +18,8 @@ export const RELAY_REMOTE_DIR = '.orca-remote' export const HEADER_LENGTH = 13 export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 // 16 MB +export const MAX_BUFFERED_FRAME_CHUNKS = 4_096 +const MAX_COALESCED_TINY_FRAME_CHUNK_BYTES = 8 * 1024 /** Message type byte. */ export const MessageType = { @@ -36,7 +43,8 @@ export const RelayErrorCode = { PtyAllocationFailed: -33004, DiskFull: -33005, TooManyStreams: -33006, - StreamProtocolError: -33007 + StreamProtocolError: -33007, + MarkdownDocumentListingCapacity: -33008 } as const export const JsonRpcErrorCode = { @@ -68,6 +76,11 @@ export const GIT_RESPONSE_STREAM_THRESHOLD = 256 * 1024 * so it stays cross-version safe. */ export const GIT_RESPONSE_CHUNK_SIZE = 128 * 1024 +/** Bounds retained serialized Git results while preserving responses far above + * the per-file render limit and git.exec's output ceiling. */ +export const MAX_GIT_RESPONSE_STREAM_BYTES = 64 * 1024 * 1024 +export const MAX_GIT_RESPONSE_STREAM_CHUNKS = 4_096 + /** Sentinel the relay returns as the RPC result when the real payload streams * as git.responseChunk frames. Absent from old relays, so a new client falls * back to the plain result they return. */ @@ -85,11 +98,11 @@ export function isGitResponseStreamMarker(value: unknown): value is GitResponseS } const fields = marker as Record return ( - Number.isInteger(fields.streamId) && + Number.isSafeInteger(fields.streamId) && (fields.streamId as number) > 0 && - Number.isInteger(fields.totalBytes) && + Number.isSafeInteger(fields.totalBytes) && (fields.totalBytes as number) >= 0 && - Number.isInteger(fields.chunkCount) && + Number.isSafeInteger(fields.chunkCount) && (fields.chunkCount as number) >= 0 ) } @@ -144,10 +157,16 @@ export function encodeFrame( } export function encodeJsonRpcFrame(msg: JsonRpcMessage, id: number, ack: number): Buffer { - const payload = Buffer.from(JSON.stringify(msg), 'utf-8') - if (payload.length > MAX_MESSAGE_SIZE) { - throw new Error(`Message too large: ${payload.length} bytes (max ${MAX_MESSAGE_SIZE})`) + let serialized: string + try { + serialized = stringifyJsonWithinByteLimit(msg, MAX_MESSAGE_SIZE).serialized + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new Error(`Message too large: more than ${MAX_MESSAGE_SIZE} bytes`) + } + throw error } + const payload = Buffer.from(serialized, 'utf-8') return encodeFrame(MessageType.Regular, id, ack, payload) } @@ -173,6 +192,7 @@ export class FrameDecoder { // list assembles each frame exactly once instead. private chunks: Buffer[] = [] private bufferedLength = 0 + private oversizedPayloadBytesRemaining = 0 private onFrame: (frame: DecodedFrame) => void private onError: ((err: Error) => void) | null @@ -182,12 +202,18 @@ export class FrameDecoder { } feed(chunk: Buffer | Uint8Array): void { - const buf = Buffer.isBuffer(chunk) + let buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) + if (this.oversizedPayloadBytesRemaining > 0 && buf.length > 0) { + const discarded = Math.min(this.oversizedPayloadBytesRemaining, buf.length) + this.oversizedPayloadBytesRemaining -= discarded + buf = buf.subarray(discarded) + } if (buf.length > 0) { this.chunks.push(buf) this.bufferedLength += buf.length + this.coalesceBufferedChunks() } while (this.bufferedLength >= HEADER_LENGTH) { @@ -195,25 +221,26 @@ export class FrameDecoder { const length = header.readUInt32BE(9) const totalLength = HEADER_LENGTH + length - if (this.bufferedLength < totalLength) { - // Not fully received yet (also holds oversized frames until they can - // be skipped whole, keeping the decoder synchronized). - break - } - - // Why: throwing here would leave the buffer in a partially consumed - // state — subsequent feed() calls would try to parse leftover payload - // bytes as a new header, corrupting every future frame. Instead we - // skip the entire oversized frame so the decoder stays synchronized. if (length > MAX_MESSAGE_SIZE) { - this.discardBytes(totalLength) + // Why: retain only the header, then stream-discard the advertised + // payload so a 13-byte malicious header cannot pin up to 4 GiB. + this.discardBytes(HEADER_LENGTH) + this.oversizedPayloadBytesRemaining = length + this.discardBufferedOversizedPayload() const err = new Error(`Frame payload too large: ${length} bytes — discarded`) if (this.onError) { this.onError(err) } + if (this.oversizedPayloadBytesRemaining > 0) { + break + } continue } + if (this.bufferedLength < totalLength) { + break + } + const framed = this.takeBytes(totalLength) const frame: DecodedFrame = { type: framed[0], @@ -228,6 +255,7 @@ export class FrameDecoder { reset(): void { this.chunks = [] this.bufferedLength = 0 + this.oversizedPayloadBytesRemaining = 0 } /** View of the first `count` buffered bytes without consuming them. */ @@ -292,14 +320,40 @@ export class FrameDecoder { } this.bufferedLength -= count } + + private discardBufferedOversizedPayload(): void { + const discarded = Math.min(this.oversizedPayloadBytesRemaining, this.bufferedLength) + if (discarded === 0) { + return + } + this.discardBytes(discarded) + this.oversizedPayloadBytesRemaining -= discarded + } + + private coalesceBufferedChunks(): void { + // Why: merge adversarial tiny fragments without copying ordinary transport chunks. + while (this.chunks.length >= 2) { + const right = this.chunks.at(-1)! + const left = this.chunks.at(-2)! + if ( + left.length > right.length || + left.length + right.length > MAX_COALESCED_TINY_FRAME_CHUNK_BYTES + ) { + break + } + this.chunks.splice(-2, 2, Buffer.concat([left, right], left.length + right.length)) + } + if (this.chunks.length > MAX_BUFFERED_FRAME_CHUNKS) { + this.chunks = [Buffer.concat(this.chunks, this.bufferedLength)] + } + } } /** * Parse a JSON-RPC message from a frame payload. */ export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage { - const text = payload.toString('utf-8') - const msg = JSON.parse(text) as JsonRpcMessage + const msg = parseRelayJsonText(payload.toString('utf-8')) if (msg.jsonrpc !== '2.0') { throw new Error(`Invalid JSON-RPC version: ${(msg as Record).jsonrpc}`) } diff --git a/src/main/ssh/sftp-directory-removal.ts b/src/main/ssh/sftp-directory-removal.ts new file mode 100644 index 00000000000..546915e42bd --- /dev/null +++ b/src/main/ssh/sftp-directory-removal.ts @@ -0,0 +1,116 @@ +import type { SFTPWrapper } from 'ssh2' +import { SshDirectoryTransferBudget } from './ssh-directory-transfer-budget' + +type SftpDirectoryEntry = { + filename: string + attrs?: { + isDirectory?: () => boolean + } +} + +export async function removeDirectorySftp(sftp: SFTPWrapper, remoteDir: string): Promise { + const budget = new SshDirectoryTransferBudget() + budget.recordPath(remoteDir, 0, { countEntry: false }) + await removeDirectorySftpWithinBudget(sftp, remoteDir, budget, 0) +} + +async function removeDirectorySftpWithinBudget( + sftp: SFTPWrapper, + remoteDir: string, + budget: SshDirectoryTransferBudget, + depth: number +): Promise { + const handle = await opendirSftp(sftp, remoteDir) + const normalizedRemoteDir = remoteDir.replace(/\/+$/, '') + let completed = false + try { + for (;;) { + const entries = await readdirSftp(sftp, handle) + if (!entries) { + break + } + for (const entry of entries) { + if (entry.filename === '.' || entry.filename === '..') { + continue + } + const childPath = `${normalizedRemoteDir}/${entry.filename}` + budget.recordPath(childPath, depth + 1) + await (entry.attrs?.isDirectory?.() + ? removeDirectorySftpWithinBudget(sftp, childPath, budget, depth + 1) + : unlinkSftp(sftp, childPath)) + } + } + completed = true + } finally { + await closeSftpHandle(sftp, handle).catch((error: unknown) => { + if (completed) { + throw error + } + }) + } + await rmdirSftp(sftp, remoteDir) +} + +function opendirSftp(sftp: SFTPWrapper, remoteDir: string): Promise { + return new Promise((resolve, reject) => { + sftp.opendir(remoteDir, (err, handle) => { + if (err) { + reject(err) + return + } + resolve(handle) + }) + }) +} + +function readdirSftp(sftp: SFTPWrapper, handle: Buffer): Promise { + return new Promise((resolve, reject) => { + sftp.readdir(handle, (err, entries) => { + if ((err as { code?: number } | undefined)?.code === 1) { + resolve(null) + return + } + if (err) { + reject(err) + return + } + resolve((entries ?? []) as SftpDirectoryEntry[]) + }) + }) +} + +function closeSftpHandle(sftp: SFTPWrapper, handle: Buffer): Promise { + return new Promise((resolve, reject) => { + sftp.close(handle, (err) => { + if (err) { + reject(err) + return + } + resolve() + }) + }) +} + +function unlinkSftp(sftp: SFTPWrapper, remotePath: string): Promise { + return new Promise((resolve, reject) => { + sftp.unlink(remotePath, (err) => { + if (err) { + reject(err) + return + } + resolve() + }) + }) +} + +function rmdirSftp(sftp: SFTPWrapper, remoteDir: string): Promise { + return new Promise((resolve, reject) => { + sftp.rmdir(remoteDir, (err) => { + if (err) { + reject(err) + return + } + resolve() + }) + }) +} diff --git a/src/main/ssh/sftp-upload.test.ts b/src/main/ssh/sftp-upload.test.ts index c87c2a1b7f5..199ee585adf 100644 --- a/src/main/ssh/sftp-upload.test.ts +++ b/src/main/ssh/sftp-upload.test.ts @@ -5,6 +5,7 @@ import { Writable } from 'node:stream' import { describe, expect, it, vi } from 'vitest' import type { SFTPWrapper } from 'ssh2' import { removeDirectorySftp, uploadBuffer, uploadDirectory, uploadFile } from './sftp-upload' +import { SSH_DIRECTORY_TRANSFER_LIMITS } from './ssh-directory-transfer-budget' function createWritable(): Writable { return new Writable({ @@ -18,9 +19,13 @@ function createSftpMock(): SFTPWrapper { return { mkdir: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null)), createWriteStream: vi.fn(() => createWritable()), - readdir: vi.fn((_path: string, cb: (err?: Error | null, entries?: unknown[]) => void) => - cb(null, []) + opendir: vi.fn((path: string, cb: (err: Error | undefined, handle: Buffer) => void) => + cb(undefined, Buffer.from(path)) ), + readdir: vi.fn((_handle: Buffer, cb: (err?: Error | null, entries?: unknown[]) => void) => + cb(Object.assign(new Error('EOF'), { code: 1 }), []) + ), + close: vi.fn((_handle: Buffer, cb: (err?: Error | null) => void) => cb(null)), unlink: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null)), rmdir: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null)) } as unknown as SFTPWrapper @@ -94,6 +99,23 @@ describe('sftp-upload', () => { expect(sftp.createWriteStream).not.toHaveBeenCalled() }) + it('rejects excessive depth before creating remote entries', async () => { + const localDir = await mkdtemp(join(tmpdir(), 'orca-sftp-upload-')) + let nested = localDir + for (let depth = 0; depth <= SSH_DIRECTORY_TRANSFER_LIMITS.maximumDepth; depth += 1) { + nested = join(nested, 'd') + await mkdir(nested) + } + const sftp = createSftpMock() + + await expect( + uploadDirectory(sftp, localDir, '/remote/assets', await realpath(localDir)) + ).rejects.toMatchObject({ reason: 'depth' }) + + expect(sftp.mkdir).not.toHaveBeenCalled() + expect(sftp.createWriteStream).not.toHaveBeenCalled() + }) + it('does not create the remote file when the local source is a symlink', async () => { const localDir = await mkdtemp(join(tmpdir(), 'orca-sftp-upload-')) const targetPath = join(localDir, process.platform === 'win32' ? 'target-dir' : 'target.txt') @@ -116,8 +138,14 @@ describe('sftp-upload', () => { it('removes remote directory contents before removing the directory', async () => { const sftp = createSftpMock() - vi.mocked(sftp.readdir).mockImplementation((remotePath, cb) => { - const pathString = String(remotePath) + const readPaths = new Set() + vi.mocked(sftp.readdir).mockImplementation((handle, cb) => { + const pathString = String(handle) + if (readPaths.has(pathString)) { + cb(Object.assign(new Error('EOF'), { code: 1 }), [] as never) + return + } + readPaths.add(pathString) if (pathString === '/remote/assets') { cb(undefined, [ { filename: '.', attrs: { isDirectory: () => true } }, @@ -144,5 +172,6 @@ describe('sftp-upload', () => { expect(sftp.rmdir).toHaveBeenNthCalledWith(1, '/remote/assets/nested', expect.any(Function)) expect(sftp.unlink).toHaveBeenNthCalledWith(2, '/remote/assets/logo.png', expect.any(Function)) expect(sftp.rmdir).toHaveBeenNthCalledWith(2, '/remote/assets', expect.any(Function)) + expect(sftp.close).toHaveBeenCalledTimes(2) }) }) diff --git a/src/main/ssh/sftp-upload.ts b/src/main/ssh/sftp-upload.ts index 9ece3e2aaa9..e15beba93e0 100644 --- a/src/main/ssh/sftp-upload.ts +++ b/src/main/ssh/sftp-upload.ts @@ -1,8 +1,11 @@ import { constants } from 'node:fs' import type { ReadStream } from 'node:fs' -import { lstat, open, readdir, realpath } from 'node:fs/promises' +import { lstat, open, opendir, realpath } from 'node:fs/promises' import { isAbsolute, join as pathJoin, relative, sep } from 'node:path' import type { SFTPWrapper } from 'ssh2' +import { SshDirectoryTransferBudget } from './ssh-directory-transfer-budget' + +export { removeDirectorySftp } from './sftp-directory-removal' export function mkdirSftp( sftp: SFTPWrapper, @@ -132,87 +135,65 @@ export async function uploadDirectory( rootRealPath = localDir, options?: { exclusive?: boolean } ): Promise { + const entries = await collectLocalUploadEntries(localDir, remoteDir, rootRealPath) + for (const entry of entries) { + await assertLocalUploadPathInsideRoot(rootRealPath, entry.localPath) + const statResult = await lstat(entry.localPath) + if (statResult.isSymbolicLink() || statResult.isDirectory() !== (entry.kind === 'directory')) { + throw new Error(`Upload source changed during transfer: ${entry.localPath}`) + } + await (entry.kind === 'directory' + ? mkdirSftp(sftp, entry.remotePath, { allowExisting: !options?.exclusive }) + : uploadFile(sftp, entry.localPath, entry.remotePath, { + exclusive: options?.exclusive + })) + } +} + +type LocalUploadEntry = { + kind: 'directory' | 'file' + localPath: string + remotePath: string +} + +async function collectLocalUploadEntries( + localDir: string, + remoteDir: string, + rootRealPath: string +): Promise { + const budget = new SshDirectoryTransferBudget() + const collected: LocalUploadEntry[] = [] + const pending = [{ localDir, remoteDir, depth: 0 }] await assertLocalUploadPathInsideRoot(rootRealPath, localDir) - const entries = await readdir(localDir, { withFileTypes: true }) - for (const entry of entries) { - const localPath = pathJoin(localDir, entry.name) - const remotePath = `${remoteDir}/${entry.name}` - await assertLocalUploadPathInsideRoot(rootRealPath, localPath) - const statResult = await lstat(localPath) + budget.recordPath(remoteDir, 0, { countEntry: false }) + budget.recordPath(localDir, 0, { countEntry: false }) - // Why: skip symlinks and special files (sockets, FIFOs, devices) to - // prevent following symlinks that could exfiltrate local files to the - // remote. The caller's pre-scan catches symlinks up-front, but this - // guard closes the TOCTOU gap if one is created between scan and upload. - if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { - continue - } - - if (statResult.isDirectory()) { - await mkdirSftp(sftp, remotePath, { allowExisting: !options?.exclusive }) - await uploadDirectory(sftp, localPath, remotePath, rootRealPath, options) - } else { - await uploadFile(sftp, localPath, remotePath, { exclusive: options?.exclusive }) + while (pending.length > 0) { + const current = pending.pop()! + const handle = await opendir(current.localDir) + try { + for await (const entry of handle) { + const localPath = pathJoin(current.localDir, entry.name) + const remotePath = `${current.remoteDir}/${entry.name}` + const depth = current.depth + 1 + budget.recordPath(remotePath, depth) + budget.recordPath(localPath, depth, { countEntry: false }) + await assertLocalUploadPathInsideRoot(rootRealPath, localPath) + const statResult = await lstat(localPath) + if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { + continue + } + const kind = statResult.isDirectory() ? 'directory' : 'file' + collected.push({ kind, localPath, remotePath }) + if (kind === 'directory') { + pending.push({ localDir: localPath, remoteDir: remotePath, depth }) + } + } + } finally { + await handle.close().catch(() => undefined) } } -} - -export async function removeDirectorySftp(sftp: SFTPWrapper, remoteDir: string): Promise { - const entries = await readdirSftp(sftp, remoteDir) - const normalizedRemoteDir = remoteDir.replace(/\/+$/, '') - for (const entry of entries) { - if (entry.filename === '.' || entry.filename === '..') { - continue - } - const childPath = `${normalizedRemoteDir}/${entry.filename}` - await (entry.attrs?.isDirectory?.() - ? removeDirectorySftp(sftp, childPath) - : unlinkSftp(sftp, childPath)) - } - await rmdirSftp(sftp, remoteDir) -} - -type SftpDirectoryEntry = { - filename: string - attrs?: { - isDirectory?: () => boolean - } -} - -function readdirSftp(sftp: SFTPWrapper, remoteDir: string): Promise { - return new Promise((resolve, reject) => { - sftp.readdir(remoteDir, (err, entries) => { - if (err) { - reject(err) - return - } - resolve((entries ?? []) as SftpDirectoryEntry[]) - }) - }) -} - -function unlinkSftp(sftp: SFTPWrapper, remotePath: string): Promise { - return new Promise((resolve, reject) => { - sftp.unlink(remotePath, (err) => { - if (err) { - reject(err) - return - } - resolve() - }) - }) -} - -function rmdirSftp(sftp: SFTPWrapper, remoteDir: string): Promise { - return new Promise((resolve, reject) => { - sftp.rmdir(remoteDir, (err) => { - if (err) { - reject(err) - return - } - resolve() - }) - }) + return collected } async function assertLocalUploadPathInsideRoot( diff --git a/src/main/ssh/ssh-agent-identity-filter.test.ts b/src/main/ssh/ssh-agent-identity-filter.test.ts index 5b341c7ce63..3a26dafdf5e 100644 --- a/src/main/ssh/ssh-agent-identity-filter.test.ts +++ b/src/main/ssh/ssh-agent-identity-filter.test.ts @@ -4,11 +4,11 @@ import { join } from 'node:path' const mocks = vi.hoisted(() => ({ createAgent: vi.fn(), parseKey: vi.fn(), - readFileSync: vi.fn() + readSshKeyFile: vi.fn() })) -vi.mock('fs', () => ({ - readFileSync: (...args: unknown[]) => mocks.readFileSync(...args) +vi.mock('./ssh-key-file', () => ({ + readSshKeyFile: (...args: unknown[]) => mocks.readSshKeyFile(...args) })) vi.mock('os', () => ({ @@ -50,13 +50,13 @@ describe('createIdentityFilteredAgent', () => { beforeEach(() => { mocks.createAgent.mockReset() mocks.parseKey.mockReset() - mocks.readFileSync.mockReset() + mocks.readSshKeyFile.mockReset() }) it('offers only agent identities matching configured identity files', async () => { const allowedKey = makeKey('allowed') const otherKey = makeKey('other') - mocks.readFileSync.mockReturnValue('ssh-ed25519 AAAA allowed') + mocks.readSshKeyFile.mockReturnValue('ssh-ed25519 AAAA allowed') mocks.parseKey.mockReturnValue(allowedKey) mocks.createAgent.mockReturnValue({ getIdentities: vi.fn((callback) => callback(undefined, [allowedKey, otherKey])), @@ -75,12 +75,12 @@ describe('createIdentityFilteredAgent', () => { }) expect(identities).toEqual([allowedKey]) - expect(mocks.readFileSync).toHaveBeenCalledWith(testHomePath('.ssh', 'work_key.pub')) + expect(mocks.readSshKeyFile).toHaveBeenCalledWith(testHomePath('.ssh', 'work_key.pub')) }) it('expands Windows-style configured identity file paths before filtering', async () => { const allowedKey = makeKey('allowed') - mocks.readFileSync.mockReturnValue('ssh-ed25519 AAAA allowed') + mocks.readSshKeyFile.mockReturnValue('ssh-ed25519 AAAA allowed') mocks.parseKey.mockReturnValue(allowedKey) mocks.createAgent.mockReturnValue({ getIdentities: vi.fn((callback) => callback(undefined, [allowedKey])), @@ -99,7 +99,7 @@ describe('createIdentityFilteredAgent', () => { }) expect(identities).toEqual([allowedKey]) - expect(mocks.readFileSync).toHaveBeenCalledWith(testHomePath('.ssh', 'work_key.pub')) + expect(mocks.readSshKeyFile).toHaveBeenCalledWith(testHomePath('.ssh', 'work_key.pub')) }) it('filters nested public key entries returned by ssh2 agents', async () => { @@ -107,7 +107,7 @@ describe('createIdentityFilteredAgent', () => { const otherKey = makeKey('other') const allowedEntry = { pubKey: { pubKey: allowedKey, comment: 'allowed' } } const otherEntry = { pubKey: { pubKey: otherKey, comment: 'other' } } - mocks.readFileSync.mockReturnValue('ssh-ed25519 AAAA allowed') + mocks.readSshKeyFile.mockReturnValue('ssh-ed25519 AAAA allowed') mocks.parseKey.mockReturnValue(allowedKey) mocks.createAgent.mockReturnValue({ getIdentities: vi.fn((callback) => callback(undefined, [allowedEntry, otherEntry])), @@ -129,7 +129,7 @@ describe('createIdentityFilteredAgent', () => { }) it('does not create a broad agent when configured identity keys cannot be parsed', () => { - mocks.readFileSync.mockReturnValue('not-a-key') + mocks.readSshKeyFile.mockReturnValue('not-a-key') mocks.parseKey.mockReturnValue(new Error('parse failed')) expect(createIdentityFilteredAgent('/tmp/agent.sock', ['~/.ssh/work_key'])).toBeUndefined() @@ -141,7 +141,7 @@ describe('createIdentityFilteredAgent', () => { const sign = vi.fn() const options = { hash: 'sha256' as const } const callback = vi.fn() - mocks.readFileSync.mockReturnValue('ssh-ed25519 AAAA allowed') + mocks.readSshKeyFile.mockReturnValue('ssh-ed25519 AAAA allowed') mocks.parseKey.mockReturnValue(allowedKey) mocks.createAgent.mockReturnValue({ getIdentities: vi.fn(), diff --git a/src/main/ssh/ssh-agent-identity-filter.ts b/src/main/ssh/ssh-agent-identity-filter.ts index 35a3c0a7721..cd980be05a0 100644 --- a/src/main/ssh/ssh-agent-identity-filter.ts +++ b/src/main/ssh/ssh-agent-identity-filter.ts @@ -1,4 +1,3 @@ -import { readFileSync } from 'node:fs' import { BaseAgent, createAgent, @@ -10,6 +9,7 @@ import { type SigningRequestOptions } from 'ssh2' import { resolveSshConfigHomePath } from './ssh-config-path-expansion' +import { readSshKeyFile } from './ssh-key-file' type AgentPublicKey = ParsedKey | Buffer | string | PublicKeyEntry @@ -70,7 +70,7 @@ class IdentityFilteredAgent extends BaseAgent { function parseIdentityKeyFile(filePath: string): ParsedKey | undefined { try { - const parsed = utils.parseKey(readFileSync(filePath)) as ParsedKey | ParsedKey[] | Error + const parsed = utils.parseKey(readSshKeyFile(filePath)) as ParsedKey | ParsedKey[] | Error if (parsed instanceof Error) { return undefined } diff --git a/src/main/ssh/ssh-auth-resolution.ts b/src/main/ssh/ssh-auth-resolution.ts index a222ff2ee2d..2444d6bb619 100644 --- a/src/main/ssh/ssh-auth-resolution.ts +++ b/src/main/ssh/ssh-auth-resolution.ts @@ -1,9 +1,10 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { utils, type BaseAgent, type ParsedKey } from 'ssh2' import type { SshTarget } from '../../shared/ssh-types' import type { SshResolvedConfig } from './ssh-config-parser' import { createIdentityFilteredAgent } from './ssh-agent-identity-filter' import { resolveSshConfigHomePath } from './ssh-config-path-expansion' +import { readSshKeyFile } from './ssh-key-file' // Why: ssh2 only tries keys that are explicitly provided. Users with keys in // standard locations (e.g. ~/.ssh/id_ed25519) but no SSH agent running would @@ -26,7 +27,7 @@ export function findDefaultKeyFile(): PrivateKeyFile | undefined { if (!existsSync(resolved)) { continue } - const contents = readFileSync(resolved) + const contents = readSshKeyFile(resolved) return { path: keyPath, contents } } catch { continue @@ -96,7 +97,7 @@ function resolveExplicitPrivateKeyPath( function readPrivateKey(keyPath: string): PrivateKeyFile | undefined { try { const resolvedPath = resolveSshConfigHomePath(keyPath) - return { path: keyPath, contents: readFileSync(resolvedPath) } + return { path: keyPath, contents: readSshKeyFile(resolvedPath) } } catch { return undefined } diff --git a/src/main/ssh/ssh-channel-multiplexer.test.ts b/src/main/ssh/ssh-channel-multiplexer.test.ts index d17e756f75b..4f0faee8cc2 100644 --- a/src/main/ssh/ssh-channel-multiplexer.test.ts +++ b/src/main/ssh/ssh-channel-multiplexer.test.ts @@ -471,6 +471,25 @@ describe('SshChannelMultiplexer', () => { expect(internals.disposeHandlers).toHaveLength(0) }) + it('invokes every dispose handler when handlers unsubscribe themselves', () => { + const called: string[] = [] + let unsubscribeFirst = (): void => {} + let unsubscribeSecond = (): void => {} + unsubscribeFirst = mux.onDispose(() => { + called.push('first') + unsubscribeFirst() + }) + unsubscribeSecond = mux.onDispose(() => { + called.push('second') + unsubscribeSecond() + }) + + mux.dispose('connection_lost') + + expect(called).toEqual(['first', 'second']) + expect(getMuxInternals(mux).disposeHandlers).toHaveLength(0) + }) + it('does not retain handlers registered after dispose', () => { mux.dispose() diff --git a/src/main/ssh/ssh-channel-multiplexer.ts b/src/main/ssh/ssh-channel-multiplexer.ts index 9886b724755..1550eb5e210 100644 --- a/src/main/ssh/ssh-channel-multiplexer.ts +++ b/src/main/ssh/ssh-channel-multiplexer.ts @@ -305,14 +305,14 @@ export class SshChannelMultiplexer { this.decoder.reset() this.transport.close?.() - for (const handler of this.disposeHandlers) { + // Why: stream handlers unsubscribe while failing; detach first so one cleanup cannot skip the next. + for (const handler of this.disposeHandlers.splice(0)) { try { handler(reason) } catch { // Don't let a handler error prevent other handlers from running } } - this.disposeHandlers.length = 0 } isDisposed(): boolean { diff --git a/src/main/ssh/ssh-config-bounded-glob.test.ts b/src/main/ssh/ssh-config-bounded-glob.test.ts new file mode 100644 index 00000000000..fe849184a19 --- /dev/null +++ b/src/main/ssh/ssh-config-bounded-glob.test.ts @@ -0,0 +1,75 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { basename, join, posix, win32 } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { resolveBoundedSshConfigGlob } from './ssh-config-bounded-glob' + +const pathApi = process.platform === 'win32' ? win32 : posix +const tempRoots: string[] = [] +const SOURCE = readFileSync(new URL('./ssh-config-bounded-glob.ts', import.meta.url), 'utf8') + +function makeTempRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-ssh-config-glob-')) + tempRoots.push(root) + return root +} + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('bounded SSH config include globbing', () => { + it('retains only the first 256 sorted matches without materializing every result', () => { + const root = makeTempRoot() + const includeDir = join(root, 'conf.d') + mkdirSync(includeDir) + for (let index = 0; index < 2_000; index += 1) { + writeFileSync(join(includeDir, `${String(index).padStart(4, '0')}.conf`), '') + } + + const result = resolveBoundedSshConfigGlob(join(includeDir, '*.conf'), pathApi, 256) + + expect(result.totalMatches).toBe(2_000) + expect(result.truncated).toBe(true) + expect(result.matches).toHaveLength(256) + expect(result.matches.map((filePath) => basename(filePath))).toEqual( + Array.from({ length: 256 }, (_, index) => `${String(index).padStart(4, '0')}.conf`) + ) + expect(SOURCE).not.toContain('globSync') + }) + + it('scans past nonmatches and preserves sorted order below the cap', () => { + const root = makeTempRoot() + const includeDir = join(root, 'conf.d') + mkdirSync(includeDir) + for (let index = 0; index < 300; index += 1) { + writeFileSync(join(includeDir, `${index}.txt`), '') + } + writeFileSync(join(includeDir, 'zeta.conf'), '') + writeFileSync(join(includeDir, 'alpha.conf'), '') + + const result = resolveBoundedSshConfigGlob(join(includeDir, '*.conf'), pathApi, 256) + + expect(result).toMatchObject({ totalMatches: 2, truncated: false }) + expect(result.matches.map((filePath) => basename(filePath))).toEqual([ + 'alpha.conf', + 'zeta.conf' + ]) + }) + + it('rejects path patterns deep enough to threaten the traversal stack', () => { + const pattern = pathApi.join( + pathApi.parse(process.cwd()).root, + ...Array.from({ length: 65 }, () => '*') + ) + + expect(resolveBoundedSshConfigGlob(pattern, pathApi, 256)).toEqual({ + matches: [], + totalMatches: 0, + truncated: true, + patternTooDeep: true + }) + }) +}) diff --git a/src/main/ssh/ssh-config-bounded-glob.ts b/src/main/ssh/ssh-config-bounded-glob.ts new file mode 100644 index 00000000000..04b8dbc7e97 --- /dev/null +++ b/src/main/ssh/ssh-config-bounded-glob.ts @@ -0,0 +1,124 @@ +import { existsSync, opendirSync, statSync } from 'node:fs' +import type { posix, win32 } from 'node:path' + +export type SshConfigPathApi = typeof posix | typeof win32 + +export type BoundedSshConfigGlobResult = { + matches: string[] + totalMatches: number + truncated: boolean + patternTooDeep: boolean +} + +const MAX_GLOB_PATTERN_SEGMENTS = 64 + +export function resolveBoundedSshConfigGlob( + absolutePattern: string, + pathApi: SshConfigPathApi, + matchLimit: number +): BoundedSshConfigGlobResult { + const root = pathApi.parse(absolutePattern).root + const segments = absolutePattern.slice(root.length).split(pathApi.sep).filter(Boolean) + if (segments.length > MAX_GLOB_PATTERN_SEGMENTS) { + return { matches: [], totalMatches: 0, truncated: true, patternTooDeep: true } + } + + const matches: string[] = [] + let totalMatches = 0 + const retainMatch = (matchedPath: string): void => { + totalMatches += 1 + insertSortedWithinLimit(matches, matchedPath, matchLimit) + } + + const visit = (directoryPath: string, segmentIndex: number): void => { + const segment = segments[segmentIndex] + const isLast = segmentIndex === segments.length - 1 + if (!hasGlobPattern(segment)) { + const nextPath = pathApi.join(directoryPath, segment) + if (isLast) { + if (existsSync(nextPath)) { + retainMatch(nextPath) + } + return + } + visit(nextPath, segmentIndex + 1) + return + } + + let directory: ReturnType + try { + directory = opendirSync(directoryPath) + } catch { + return + } + try { + for (let entry = directory.readSync(); entry; entry = directory.readSync()) { + if (!pathApi.matchesGlob(entry.name, segment)) { + continue + } + const nextPath = pathApi.join(directoryPath, entry.name) + if (isLast) { + retainMatch(nextPath) + continue + } + if (entry.isDirectory() || isDirectorySymlink(nextPath, entry.isSymbolicLink())) { + visit(nextPath, segmentIndex + 1) + } + } + } finally { + directory.closeSync() + } + } + + if (segments.length === 0) { + if (existsSync(root)) { + retainMatch(root) + } + } else { + visit(root, 0) + } + return { + matches, + totalMatches, + truncated: totalMatches > matchLimit, + patternTooDeep: false + } +} + +function insertSortedWithinLimit(target: string[], value: string, limit: number): void { + if (limit <= 0) { + return + } + let low = 0 + let high = target.length + while (low < high) { + const middle = (low + high) >>> 1 + if (target[middle].localeCompare(value) <= 0) { + low = middle + 1 + } else { + high = middle + } + } + if (low >= limit) { + return + } + target.splice(low, 0, value) + if (target.length > limit) { + target.pop() + } +} + +function isDirectorySymlink(filePath: string, isSymbolicLink: boolean): boolean { + if (!isSymbolicLink) { + return false + } + try { + return statSync(filePath).isDirectory() + } catch { + return false + } +} + +function hasGlobPattern(input: string): boolean { + return /[*?[]/.test(input) +} diff --git a/src/main/ssh/ssh-config-expansion-budget.ts b/src/main/ssh/ssh-config-expansion-budget.ts new file mode 100644 index 00000000000..99e5cdc4ab6 --- /dev/null +++ b/src/main/ssh/ssh-config-expansion-budget.ts @@ -0,0 +1,153 @@ +import { statSync } from 'node:fs' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const SSH_CONFIG_INCLUDE_LIMITS = { + expandedBytes: 16 * 1024 * 1024, + expandedLines: 200_000, + fileBytes: 1024 * 1024, + files: 1024, + globMatches: 256, + nestingDepth: 16, + sourceBytes: 16 * 1024 * 1024 +} as const + +export type SshConfigExpansionBudget = { + cache: Map + expandedBytes: number + expandedLines: number + fileCount: number + outputTruncated: boolean + sourceBytes: number + warnedLimits: Set +} + +export function createSshConfigExpansionBudget(): SshConfigExpansionBudget { + return { + cache: new Map(), + expandedBytes: 0, + expandedLines: 0, + fileCount: 0, + outputTruncated: false, + sourceBytes: 0, + warnedLimits: new Set() + } +} + +export function admitSshConfigIncludeDepth( + budget: SshConfigExpansionBudget, + activeDepth: number +): boolean { + if (activeDepth < SSH_CONFIG_INCLUDE_LIMITS.nestingDepth) { + return true + } + warnOnce( + budget, + 'nesting-depth', + `[ssh] SSH config Include nesting exceeds ${SSH_CONFIG_INCLUDE_LIMITS.nestingDepth}; skipping deeper files` + ) + return false +} + +export function appendSshConfigExpandedLine( + target: string[], + line: string, + budget: SshConfigExpansionBudget +): void { + const nextBytes = Buffer.byteLength(line, 'utf8') + (target.length > 0 ? 1 : 0) + if ( + budget.expandedLines >= SSH_CONFIG_INCLUDE_LIMITS.expandedLines || + budget.expandedBytes + nextBytes > SSH_CONFIG_INCLUDE_LIMITS.expandedBytes + ) { + budget.outputTruncated = true + warnOnce( + budget, + 'expanded-output', + `[ssh] Expanded SSH config exceeds ${SSH_CONFIG_INCLUDE_LIMITS.expandedBytes} bytes or ${SSH_CONFIG_INCLUDE_LIMITS.expandedLines} lines; truncating` + ) + return + } + target.push(line) + budget.expandedBytes += nextBytes + budget.expandedLines += 1 +} + +export function readSshConfigSourceFile( + filePath: string, + budget: SshConfigExpansionBudget +): string | null { + const cached = budget.cache.get(filePath) + if (cached !== undefined) { + return cached + } + if (budget.fileCount >= SSH_CONFIG_INCLUDE_LIMITS.files) { + warnOnce( + budget, + 'file-count', + `[ssh] SSH config Include file count exceeds ${SSH_CONFIG_INCLUDE_LIMITS.files}; skipping additional files` + ) + return null + } + const fileBytes = getReadableRegularFileBytes(filePath) + if (fileBytes === null || !hasSourceCapacity(budget, fileBytes)) { + return null + } + + try { + const content = readNodeFileSyncWithinLimit( + filePath, + SSH_CONFIG_INCLUDE_LIMITS.fileBytes + ).buffer.toString('utf-8') + const actualBytes = Buffer.byteLength(content, 'utf8') + if ( + actualBytes > SSH_CONFIG_INCLUDE_LIMITS.fileBytes || + !hasSourceCapacity(budget, actualBytes) + ) { + return null + } + budget.cache.set(filePath, content) + budget.fileCount += 1 + budget.sourceBytes += actualBytes + return content + } catch { + return null + } +} + +function hasSourceCapacity(budget: SshConfigExpansionBudget, fileBytes: number): boolean { + if (budget.sourceBytes + fileBytes <= SSH_CONFIG_INCLUDE_LIMITS.sourceBytes) { + return true + } + warnOnce( + budget, + 'source-bytes', + `[ssh] SSH config Include sources exceed ${SSH_CONFIG_INCLUDE_LIMITS.sourceBytes} bytes; skipping additional files` + ) + return false +} + +function getReadableRegularFileBytes(filePath: string): number | null { + try { + const stats = statSync(filePath) + if (!stats.isFile()) { + console.warn(`[ssh] Skipping SSH config include "${filePath}": not a regular file`) + return null + } + if (stats.size > SSH_CONFIG_INCLUDE_LIMITS.fileBytes) { + console.warn( + `[ssh] Skipping SSH config include "${filePath}": size ${stats.size} exceeds ${SSH_CONFIG_INCLUDE_LIMITS.fileBytes} bytes` + ) + return null + } + return stats.size + } catch { + return null + } +} + +function warnOnce(budget: SshConfigExpansionBudget, key: string, message: string): void { + if (budget.warnedLimits.has(key)) { + return + } + budget.warnedLimits.add(key) + console.warn(message) +} diff --git a/src/main/ssh/ssh-config-include-expander.ts b/src/main/ssh/ssh-config-include-expander.ts index eb1aba64c17..1f49c426fad 100644 --- a/src/main/ssh/ssh-config-include-expander.ts +++ b/src/main/ssh/ssh-config-include-expander.ts @@ -1,11 +1,20 @@ -import { existsSync, globSync, readFileSync, realpathSync, statSync } from 'node:fs' +import { existsSync, realpathSync } from 'node:fs' import { homedir, hostname, userInfo } from 'node:os' import { posix, win32 } from 'node:path' +import { resolveBoundedSshConfigGlob, type SshConfigPathApi } from './ssh-config-bounded-glob' +import { + admitSshConfigIncludeDepth, + appendSshConfigExpandedLine, + createSshConfigExpansionBudget, + readSshConfigSourceFile, + SSH_CONFIG_INCLUDE_LIMITS, + type SshConfigExpansionBudget +} from './ssh-config-expansion-budget' -type PathApi = typeof posix | typeof win32 +type PathApi = SshConfigPathApi type IncludeExpansionContext = { - cache: Map + budget: SshConfigExpansionBudget home: string pathApi: PathApi rootDir: string @@ -14,8 +23,6 @@ type IncludeExpansionContext = { username: string } -const MAX_INCLUDE_GLOB_MATCHES = 256 -const MAX_INCLUDE_FILE_BYTES = 1024 * 1024 const TARGET_DEPENDENT_INCLUDE_TOKENS = new Set(['h', 'n', 'p', 'r', 'j', 'k', 'C']) export function expandSshConfigIncludes(configPath: string): string { @@ -25,7 +32,7 @@ export function expandSshConfigIncludes(configPath: string): string { const localHostname = hostname() const context: IncludeExpansionContext = { - cache: new Map(), + budget: createSshConfigExpansionBudget(), home, pathApi, rootDir: pathApi.dirname(configPath), @@ -34,69 +41,51 @@ export function expandSshConfigIncludes(configPath: string): string { username: currentUser } - return expandSshConfigFile(configPath, context, []).join('\n') + const expandedLines: string[] = [] + expandSshConfigFile(configPath, context, [], expandedLines) + return expandedLines.join('\n') } function expandSshConfigFile( filePath: string, context: IncludeExpansionContext, - activeStack: string[] -): string[] { + activeStack: string[], + expandedLines: string[] +): void { const canonicalPath = getCanonicalPath(filePath) if (!canonicalPath || activeStack.includes(canonicalPath)) { - return [] + return + } + if (!admitSshConfigIncludeDepth(context.budget, activeStack.length)) { + return } - const rawContent = readCachedFile(canonicalPath, context) + const rawContent = readSshConfigSourceFile(canonicalPath, context.budget) if (rawContent === null) { - return [] + return } - const expandedLines: string[] = [] const nextStack = [...activeStack, canonicalPath] for (const line of rawContent.split(/\r?\n/)) { + if (context.budget.outputTruncated) { + return + } const includeArgs = parseIncludeDirective(line) if (!includeArgs) { - expandedLines.push(line) + appendSshConfigExpandedLine(expandedLines, line, context.budget) continue } for (const includeArg of includeArgs) { for (const matchedPath of resolveIncludePaths(includeArg, context)) { - appendExpandedLines(expandedLines, expandSshConfigFile(matchedPath, context, nextStack)) + expandSshConfigFile(matchedPath, context, nextStack, expandedLines) + if (context.budget.outputTruncated) { + return + } } } } - - return expandedLines -} - -function appendExpandedLines(target: string[], lines: readonly string[]): void { - // Why: SSH config includes are user-controlled files, and a large included - // file can exceed the JavaScript call argument limit when spread into push. - for (const line of lines) { - target.push(line) - } -} - -function readCachedFile(filePath: string, context: IncludeExpansionContext): string | null { - const cached = context.cache.get(filePath) - if (cached !== undefined) { - return cached - } - - if (!isReadableRegularFile(filePath)) { - return null - } - - try { - const content = readFileSync(filePath, 'utf-8') - context.cache.set(filePath, content) - return content - } catch { - return null - } } function parseIncludeDirective(line: string): string[] | null { @@ -168,18 +157,23 @@ function resolveIncludePaths(pattern: string, context: IncludeExpansionContext): const absolutePattern = resolveIncludePatternPath(withTokens, context) if (hasGlobPattern(absolutePattern)) { - try { - const matches = globSync(absolutePattern).sort((left, right) => left.localeCompare(right)) - if (matches.length > MAX_INCLUDE_GLOB_MATCHES) { - console.warn( - `[ssh] Include pattern "${absolutePattern}" matched ${matches.length} files; processing first ${MAX_INCLUDE_GLOB_MATCHES}` - ) - return matches.slice(0, MAX_INCLUDE_GLOB_MATCHES) - } - return matches - } catch { + const result = resolveBoundedSshConfigGlob( + absolutePattern, + context.pathApi, + SSH_CONFIG_INCLUDE_LIMITS.globMatches + ) + if (result.patternTooDeep) { + console.warn( + `[ssh] Include pattern "${absolutePattern}" has too many path segments; skipping` + ) return [] } + if (result.truncated) { + console.warn( + `[ssh] Include pattern "${absolutePattern}" matched ${result.totalMatches} files; processing first ${SSH_CONFIG_INCLUDE_LIMITS.globMatches}` + ) + } + return result.matches } return existsSync(absolutePattern) ? [absolutePattern] : [] @@ -293,25 +287,6 @@ function getCanonicalPath(filePath: string): string | null { } } -function isReadableRegularFile(filePath: string): boolean { - try { - const stats = statSync(filePath) - if (!stats.isFile()) { - console.warn(`[ssh] Skipping SSH config include "${filePath}": not a regular file`) - return false - } - if (stats.size > MAX_INCLUDE_FILE_BYTES) { - console.warn( - `[ssh] Skipping SSH config include "${filePath}": size ${stats.size} exceeds ${MAX_INCLUDE_FILE_BYTES} bytes` - ) - return false - } - return true - } catch { - return false - } -} - function getCurrentUid(): string | undefined { try { const info = userInfo() diff --git a/src/main/ssh/ssh-config-include-expansion-bounds.test.ts b/src/main/ssh/ssh-config-include-expansion-bounds.test.ts new file mode 100644 index 00000000000..d1eda95eeb0 --- /dev/null +++ b/src/main/ssh/ssh-config-include-expansion-bounds.test.ts @@ -0,0 +1,82 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + appendSshConfigExpandedLine, + createSshConfigExpansionBudget, + readSshConfigSourceFile, + SSH_CONFIG_INCLUDE_LIMITS +} from './ssh-config-expansion-budget' +import { expandSshConfigIncludes } from './ssh-config-include-expander' + +const tempRoots: string[] = [] + +function makeTempRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-ssh-config-bounds-')) + tempRoots.push(root) + return root +} + +afterEach(() => { + vi.restoreAllMocks() + for (const root of tempRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('SSH config include expansion bounds', () => { + it('stops recursive includes at 16 active files', () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const root = makeTempRoot() + for (let depth = 0; depth < 20; depth += 1) { + writeFileSync( + join(root, `${depth}.conf`), + `Host depth-${depth}\n${depth < 19 ? `Include ${depth + 1}.conf\n` : ''}` + ) + } + + const expanded = expandSshConfigIncludes(join(root, '0.conf')) + + expect(expanded).toContain('Host depth-15') + expect(expanded).not.toContain('Host depth-16') + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('nesting exceeds 16')) + }) + + it('admits exactly 1,024 unique files and 16 MiB of source bytes', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const root = makeTempRoot() + const firstPath = join(root, 'first.conf') + const secondPath = join(root, 'second.conf') + writeFileSync(firstPath, 'a') + writeFileSync(secondPath, 'b') + const fileBudget = createSshConfigExpansionBudget() + fileBudget.fileCount = SSH_CONFIG_INCLUDE_LIMITS.files - 1 + + expect(readSshConfigSourceFile(firstPath, fileBudget)).toBe('a') + expect(readSshConfigSourceFile(secondPath, fileBudget)).toBeNull() + + const byteBudget = createSshConfigExpansionBudget() + byteBudget.sourceBytes = SSH_CONFIG_INCLUDE_LIMITS.sourceBytes - 1 + expect(readSshConfigSourceFile(firstPath, byteBudget)).toBe('a') + expect(readSshConfigSourceFile(secondPath, byteBudget)).toBeNull() + expect(warn).toHaveBeenCalledWith(expect.stringContaining('file count exceeds 1024')) + expect(warn).toHaveBeenCalledWith(expect.stringContaining('sources exceed 16777216 bytes')) + }) + + it('caps expanded output at 16 MiB or 200,000 lines', () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const byteBudget = createSshConfigExpansionBudget() + byteBudget.expandedBytes = SSH_CONFIG_INCLUDE_LIMITS.expandedBytes - 1 + const byteLines = [''] + appendSshConfigExpandedLine(byteLines, '', byteBudget) + expect(byteBudget.expandedBytes).toBe(SSH_CONFIG_INCLUDE_LIMITS.expandedBytes) + appendSshConfigExpandedLine(byteLines, '', byteBudget) + expect(byteBudget.outputTruncated).toBe(true) + + const lineBudget = createSshConfigExpansionBudget() + lineBudget.expandedLines = SSH_CONFIG_INCLUDE_LIMITS.expandedLines + appendSshConfigExpandedLine([], 'Host overflow', lineBudget) + expect(lineBudget.outputTruncated).toBe(true) + }) +}) diff --git a/src/main/ssh/ssh-config-loader-regression.test.ts b/src/main/ssh/ssh-config-loader-regression.test.ts index 089ee9d99bb..0dd35560883 100644 --- a/src/main/ssh/ssh-config-loader-regression.test.ts +++ b/src/main/ssh/ssh-config-loader-regression.test.ts @@ -8,6 +8,7 @@ afterEach(() => { vi.resetModules() vi.doUnmock('fs') vi.doUnmock('os') + vi.doUnmock('../../shared/node-bounded-file-reader') }) function normalizeWin(value: string): string { @@ -24,6 +25,24 @@ function platformSshPath(home: string, relativePath: string): string { : `${home}/${relativePath}` } +function mockDirectory(names: string[]) { + let index = 0 + return { + readSync: () => { + const name = names[index] + index += 1 + return name === undefined + ? null + : { + name, + isDirectory: () => false, + isSymbolicLink: () => false + } + }, + closeSync: () => {} + } +} + async function mockOs( home: string, username = 'testuser', @@ -42,6 +61,18 @@ async function mockOs( } async function loadUserSshConfig() { + vi.doMock('../../shared/node-bounded-file-reader', async () => { + const fs = await import('node:fs') + return { + readNodeFileSyncWithinLimit: (filePath: string, maxBytes: number) => { + const buffer = Buffer.from(fs.readFileSync(filePath, 'utf8')) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: fs.statSync(filePath) } + } + } + }) const mod = await import('./ssh-config-parser') return mod.loadUserSshConfig() } @@ -77,13 +108,12 @@ describe('loadUserSshConfig regressions', () => { return { ...actual, existsSync: (filePath: string) => files.has(normalizeWin(filePath)), - globSync: (pattern: string) => - normalizeWin(pattern) === normalizeWin('C:/Users/Test User/.ssh/conf.d/*.conf') - ? [ - normalizeWin('C:/Users/Test User/.ssh/conf.d/alpha.conf'), - normalizeWin('C:/Users/Test User/.ssh/conf.d/zeta.conf') - ] - : [], + opendirSync: (directoryPath: string) => { + if (normalizeWin(directoryPath) !== normalizeWin('C:/Users/Test User/.ssh/conf.d')) { + throw new Error(`ENOENT: ${directoryPath}`) + } + return mockDirectory(['zeta.conf', 'alpha.conf']) + }, readFileSync: (filePath: string) => { const content = files.get(normalizeWin(filePath)) if (content === undefined) { @@ -205,7 +235,8 @@ describe('loadUserSshConfig regressions', () => { ...actual, existsSync: (filePath: string) => filePath === configPath || includePaths.includes(filePath), - globSync: () => [...includePaths].toReversed(), + opendirSync: () => + mockDirectory(includePaths.map((filePath) => filePath.split(/[\\/]/).at(-1)!)), readFileSync: (filePath: string) => { if (filePath === configPath) { return 'Include conf.d/*.conf\n' diff --git a/src/main/ssh/ssh-connection-generation.test.ts b/src/main/ssh/ssh-connection-generation.test.ts index a6c995bcab2..3216794116a 100644 --- a/src/main/ssh/ssh-connection-generation.test.ts +++ b/src/main/ssh/ssh-connection-generation.test.ts @@ -1,9 +1,15 @@ import { afterEach, describe, expect, it } from 'vitest' import { + _internals, + MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES, + MAX_SSH_CONNECTION_GENERATION_RETAINED_TARGET_ID_BYTES, + MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES, + MAX_TRACKED_SSH_CONNECTION_GENERATIONS, advanceSshConnectionGeneration, assertSshMutationExpectation, getSshConnectionGeneration, resetSshConnectionGenerations, + retainSshConnectionGeneration, setSshConnectionGeneration } from './ssh-connection-generation' @@ -15,23 +21,31 @@ describe('SSH connection generation session scope', () => { it('does not reuse a target token when a restarted HUB reaches the same counter', () => { resetSshConnectionGenerations(41) - const beforeRestart = advanceSshConnectionGeneration('ssh-a') + const beforeRestartLease = retainSshConnectionGeneration('ssh-a') + const beforeRestart = getSshConnectionGeneration('ssh-a') resetSshConnectionGenerations(42) - const afterRestart = advanceSshConnectionGeneration('ssh-a') + const afterRestartLease = retainSshConnectionGeneration('ssh-a') + const afterRestart = getSshConnectionGeneration('ssh-a') expect(afterRestart).not.toBe(beforeRestart) expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', beforeRestart)).toThrow( 'SSH connection changed; refresh and try again' ) expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', afterRestart)).not.toThrow() + beforeRestartLease.release() + afterRestartLease.release() }) - it('keeps target counters independent within one HUB session', () => { + it('allocates globally unique target tokens within one HUB session', () => { resetSshConnectionGenerations(7) + const first = retainSshConnectionGeneration('ssh-a') + const second = retainSshConnectionGeneration('ssh-b') - expect(advanceSshConnectionGeneration('ssh-a')).toBe(advanceSshConnectionGeneration('ssh-b')) - expect(getSshConnectionGeneration('ssh-a')).toBe(getSshConnectionGeneration('ssh-b')) + expect(getSshConnectionGeneration('ssh-a')).not.toBe(getSshConnectionGeneration('ssh-b')) + + first.release() + second.release() }) it('rejects an SSH execution-host expectation when direct IPC resolves locally', () => { @@ -46,45 +60,165 @@ describe('SSH connection generation session scope', () => { ) }) - it('rolls the session scope after counter exhaustion and keeps rotating', () => { + it('rejects an unretained target even when its expected token equals the session base', () => { resetSshConnectionGenerations(7) - const exhaustedGeneration = 8 * SESSION_COUNTER_STRIDE - 1 - setSshConnectionGeneration('ssh-a', exhaustedGeneration) - const rolledGeneration = advanceSshConnectionGeneration('ssh-a') - - expect(rolledGeneration).toBe(8 * SESSION_COUNTER_STRIDE + 1) - expect(advanceSshConnectionGeneration('ssh-a')).toBe(rolledGeneration + 1) - expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', exhaustedGeneration)).toThrow( - 'SSH connection changed; refresh and try again' - ) + expect(() => + assertSshMutationExpectation('ssh-a', 'ssh-a', 7 * SESSION_COUNTER_STRIDE) + ).toThrow('SSH connection changed; refresh and try again') }) - it('invalidates other targets when exhaustion rolls the session scope', () => { + it('rotates allocation scopes without invalidating unrelated retained targets', () => { resetSshConnectionGenerations(11) - const otherTargetGeneration = advanceSshConnectionGeneration('ssh-b') + const otherLease = retainSshConnectionGeneration('ssh-b') + const otherTargetGeneration = getSshConnectionGeneration('ssh-b') + const rotatingLease = retainSshConnectionGeneration('ssh-a') setSshConnectionGeneration('ssh-a', 12 * SESSION_COUNTER_STRIDE - 1) - const rolledGeneration = advanceSshConnectionGeneration('ssh-a') + const rotatedGeneration = advanceSshConnectionGeneration('ssh-a') - expect(getSshConnectionGeneration('ssh-b')).toBe(12 * SESSION_COUNTER_STRIDE) - expect(rolledGeneration).toBe(12 * SESSION_COUNTER_STRIDE + 1) - expect(() => assertSshMutationExpectation('ssh-b', 'ssh-b', otherTargetGeneration)).toThrow( - 'SSH connection changed; refresh and try again' - ) - expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', rolledGeneration)).not.toThrow() + expect(rotatedGeneration).toBe(12 * SESSION_COUNTER_STRIDE + 1) + expect(getSshConnectionGeneration('ssh-b')).toBe(otherTargetGeneration) + expect(() => + assertSshMutationExpectation('ssh-b', 'ssh-b', otherTargetGeneration) + ).not.toThrow() + otherLease.release() + rotatingLease.release() }) - it('wraps the maximum safe numeric scope without reusing it', () => { + it('wraps the maximum numeric scope without reusing an allocated token', () => { resetSshConnectionGenerations(MAX_SESSION_SCOPE) + const lease = retainSshConnectionGeneration('ssh-a') setSshConnectionGeneration('ssh-a', Number.MAX_SAFE_INTEGER) const rolledGeneration = advanceSshConnectionGeneration('ssh-a') expect(rolledGeneration).toBe(1) expect(Number.isSafeInteger(rolledGeneration)).toBe(true) - expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', Number.MAX_SAFE_INTEGER)).toThrow( + expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', rolledGeneration!)).not.toThrow() + lease.release() + }) + + it('rejects only the 4,097th target without changing active or in-flight tokens', () => { + resetSshConnectionGenerations(7) + const leases = Array.from({ length: MAX_TRACKED_SSH_CONNECTION_GENERATIONS }, (_, index) => + retainSshConnectionGeneration(`ssh-${index}`) + ) + const activeGeneration = getSshConnectionGeneration('ssh-0') + const inFlightGeneration = getSshConnectionGeneration( + `ssh-${MAX_TRACKED_SSH_CONNECTION_GENERATIONS - 1}` + ) + + expect(() => retainSshConnectionGeneration('ssh-overflow')).toThrow( + 'SSH connection generation target capacity exhausted' + ) + expect(getSshConnectionGeneration('ssh-0')).toBe(activeGeneration) + expect(getSshConnectionGeneration(`ssh-${MAX_TRACKED_SSH_CONNECTION_GENERATIONS - 1}`)).toBe( + inFlightGeneration + ) + expect(() => assertSshMutationExpectation('ssh-0', 'ssh-0', activeGeneration)).not.toThrow() + + leases[1].release() + const overflowLease = retainSshConnectionGeneration('ssh-overflow') + expect(getSshConnectionGeneration('ssh-0')).toBe(activeGeneration) + for (const lease of leases) { + lease.release() + } + overflowLease.release() + expect(_internals.evidenceForTest()).toEqual({ + retainedGenerationLeases: 0, + retainedTargetIdBytes: 0, + trackedTargets: 0 + }) + }) + + it('keeps an entry until its final ownership lease releases', () => { + const activeLease = retainSshConnectionGeneration('ssh-a') + const inFlightLease = retainSshConnectionGeneration('ssh-a') + const generation = getSshConnectionGeneration('ssh-a') + + activeLease.release() + expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', generation)).not.toThrow() + + inFlightLease.release() + expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', generation)).toThrow( 'SSH connection changed; refresh and try again' ) }) + + it('never reuses a released target token during sequential churn', () => { + let previous = 0 + for (let index = 0; index < 10_000; index += 1) { + const lease = retainSshConnectionGeneration(`ssh-${index}`) + const generation = getSshConnectionGeneration(`ssh-${index}`) + expect(generation).toBeGreaterThan(previous) + previous = generation + lease.release() + } + + expect(_internals.evidenceForTest()).toEqual({ + retainedGenerationLeases: 0, + retainedTargetIdBytes: 0, + trackedTargets: 0 + }) + }) + + it('bounds outstanding leases for one hung target', () => { + const leases = Array.from({ length: MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES }, () => + retainSshConnectionGeneration('ssh-a') + ) + const generation = getSshConnectionGeneration('ssh-a') + + expect(() => retainSshConnectionGeneration('ssh-a')).toThrow( + 'SSH connection generation lease capacity exhausted' + ) + expect(getSshConnectionGeneration('ssh-a')).toBe(generation) + expect(() => assertSshMutationExpectation('ssh-a', 'ssh-a', generation)).not.toThrow() + expect(_internals.evidenceForTest()).toEqual({ + retainedGenerationLeases: MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES, + retainedTargetIdBytes: 5, + trackedTargets: 1 + }) + + for (const lease of leases) { + lease.release() + } + expect(_internals.evidenceForTest()).toEqual({ + retainedGenerationLeases: 0, + retainedTargetIdBytes: 0, + trackedTargets: 0 + }) + }) + + it('bounds individual and aggregate retained target-id bytes', () => { + expect(() => + retainSshConnectionGeneration('x'.repeat(MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES + 1)) + ).toThrow('SSH connection generation target id exceeds') + + const leases = Array.from( + { + length: + MAX_SSH_CONNECTION_GENERATION_RETAINED_TARGET_ID_BYTES / + MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES + }, + (_, index) => { + const prefix = `${index}:` + return retainSshConnectionGeneration( + prefix + 'x'.repeat(MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES - prefix.length) + ) + } + ) + expect(() => retainSshConnectionGeneration('overflow')).toThrow( + 'SSH connection generation target capacity exhausted' + ) + + for (const lease of leases) { + lease.release() + } + expect(_internals.evidenceForTest()).toEqual({ + retainedGenerationLeases: 0, + retainedTargetIdBytes: 0, + trackedTargets: 0 + }) + }) }) diff --git a/src/main/ssh/ssh-connection-generation.ts b/src/main/ssh/ssh-connection-generation.ts index 96f1ae04c83..e50285a7992 100644 --- a/src/main/ssh/ssh-connection-generation.ts +++ b/src/main/ssh/ssh-connection-generation.ts @@ -1,77 +1,184 @@ import { randomBytes } from 'node:crypto' import { toSshExecutionHostId } from '../../shared/execution-host' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' const SESSION_COUNTER_BITS = 13 const SESSION_COUNTER_STRIDE = 2 ** SESSION_COUNTER_BITS const MAX_SESSION_SCOPE = 2 ** (53 - SESSION_COUNTER_BITS) - 1 +export const MAX_TRACKED_SSH_CONNECTION_GENERATIONS = 4_096 +export const MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES = 8_192 +export const MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES = 64 * 1024 +export const MAX_SSH_CONNECTION_GENERATION_RETAINED_TARGET_ID_BYTES = 4 * 1024 * 1024 + +type SshConnectionGenerationEntry = { + generation: number + leaseCount: number + targetIdBytes: number +} + +export type SshConnectionGenerationLease = { + release: () => void +} function createSessionScope(): number { return randomBytes(5).readUIntBE(0, 5) } -let sessionGenerationBase = 0 +let allocationSessionBase = 0 +let allocationSessionCounter = 0 let sessionInitialized = false -const connectionGenerationByTarget = new Map() -const usedSessionScopes = new Set() +let allocatedSessionScopeCount = 1 +let generationRegistryEpoch = 0 +let retainedGenerationLeases = 0 +let retainedTargetIdBytes = 0 +const connectionGenerationByTarget = new Map() -function assertGenerationInCurrentSession(generation: number): void { - if ( - !Number.isSafeInteger(generation) || - generation < sessionGenerationBase || - generation - sessionGenerationBase >= SESSION_COUNTER_STRIDE - ) { +function rotateAllocationSessionScope(): void { + if (allocatedSessionScopeCount >= MAX_SESSION_SCOPE + 1) { throw new Error('SSH connection generation exhausted for this runtime session') } + const nextSessionScope = + (allocationSessionBase / SESSION_COUNTER_STRIDE + 1) % (MAX_SESSION_SCOPE + 1) + allocationSessionBase = nextSessionScope * SESSION_COUNTER_STRIDE + allocationSessionCounter = 0 + allocatedSessionScopeCount += 1 +} + +function allocateGeneration(): number { + if (allocationSessionCounter >= SESSION_COUNTER_STRIDE - 1) { + rotateAllocationSessionScope() + } + allocationSessionCounter += 1 + const generation = allocationSessionBase + allocationSessionCounter + if (!Number.isSafeInteger(generation)) { + throw new Error('SSH connection generation exhausted for this runtime session') + } + return generation +} + +function measureTargetId(targetId: string): number { + const measured = measureUtf8ByteLength(targetId, { + stopAfterBytes: MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES + }) + if (measured.exceededLimit) { + throw new Error( + `SSH connection generation target id exceeds ${MAX_SSH_CONNECTION_GENERATION_TARGET_ID_BYTES} UTF-8 bytes` + ) + } + return measured.byteLength +} + +export function retainSshConnectionGeneration(targetId: string): SshConnectionGenerationLease { + if (retainedGenerationLeases >= MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES) { + throw new Error('SSH connection generation lease capacity exhausted') + } + let entry = connectionGenerationByTarget.get(targetId) + if (!entry) { + const targetIdBytes = measureTargetId(targetId) + if ( + connectionGenerationByTarget.size >= MAX_TRACKED_SSH_CONNECTION_GENERATIONS || + retainedTargetIdBytes + targetIdBytes > MAX_SSH_CONNECTION_GENERATION_RETAINED_TARGET_ID_BYTES + ) { + throw new Error('SSH connection generation target capacity exhausted') + } + entry = { + generation: allocateGeneration(), + leaseCount: 0, + targetIdBytes + } + connectionGenerationByTarget.set(targetId, entry) + retainedTargetIdBytes += targetIdBytes + } + entry.leaseCount += 1 + retainedGenerationLeases += 1 + const retainedEntry = entry + const retainedEpoch = generationRegistryEpoch + let retained = true + return { + release: () => { + if (!retained) { + return + } + retained = false + retainedEntry.leaseCount -= 1 + if (generationRegistryEpoch === retainedEpoch) { + retainedGenerationLeases -= 1 + } + if ( + retainedEntry.leaseCount === 0 && + connectionGenerationByTarget.get(targetId) === retainedEntry + ) { + connectionGenerationByTarget.delete(targetId) + retainedTargetIdBytes -= retainedEntry.targetIdBytes + } + } + } } export function getSshConnectionGeneration(targetId: string): number { - return connectionGenerationByTarget.get(targetId) ?? sessionGenerationBase + return connectionGenerationByTarget.get(targetId)?.generation ?? allocationSessionBase } export function initializeSshConnectionGenerationSession(): void { if (sessionInitialized) { return } - const sessionScope = createSessionScope() - // Why: randomize the process scope so a replacement HUB does not predictably reuse the prior target/counter token. - sessionGenerationBase = sessionScope * SESSION_COUNTER_STRIDE - usedSessionScopes.add(sessionScope) + allocationSessionBase = createSessionScope() * SESSION_COUNTER_STRIDE + allocationSessionCounter = 0 + allocatedSessionScopeCount = 1 sessionInitialized = true } -export function advanceSshConnectionGeneration(targetId: string): number { - let next = getSshConnectionGeneration(targetId) + 1 - if (next - sessionGenerationBase >= SESSION_COUNTER_STRIDE) { - let nextSessionScope = - (sessionGenerationBase / SESSION_COUNTER_STRIDE + 1) % (MAX_SESSION_SCOPE + 1) - while (usedSessionScopes.has(nextSessionScope)) { - nextSessionScope = (nextSessionScope + 1) % (MAX_SESSION_SCOPE + 1) - } - usedSessionScopes.add(nextSessionScope) - sessionGenerationBase = nextSessionScope * SESSION_COUNTER_STRIDE - // Why: changing the scope must revoke tokens for every target, not only the target that exhausted its counter. - connectionGenerationByTarget.clear() - next = sessionGenerationBase + 1 +export function advanceSshConnectionGeneration(targetId: string): number | null { + const entry = connectionGenerationByTarget.get(targetId) + if (!entry) { + return null } - assertGenerationInCurrentSession(next) - connectionGenerationByTarget.set(targetId, next) - return next + entry.generation = allocateGeneration() + return entry.generation } export function setSshConnectionGeneration(targetId: string, generation: number): void { - assertGenerationInCurrentSession(generation) - connectionGenerationByTarget.set(targetId, generation) + const generationOffset = generation - allocationSessionBase + if ( + !Number.isSafeInteger(generation) || + generationOffset < 0 || + generationOffset >= SESSION_COUNTER_STRIDE + ) { + throw new Error('SSH connection generation exhausted for this runtime session') + } + let entry = connectionGenerationByTarget.get(targetId) + if (!entry) { + const targetIdBytes = measureTargetId(targetId) + if ( + retainedGenerationLeases >= MAX_RETAINED_SSH_CONNECTION_GENERATION_LEASES || + connectionGenerationByTarget.size >= MAX_TRACKED_SSH_CONNECTION_GENERATIONS || + retainedTargetIdBytes + targetIdBytes > MAX_SSH_CONNECTION_GENERATION_RETAINED_TARGET_ID_BYTES + ) { + throw new Error('SSH connection generation target capacity exhausted') + } + entry = { generation, leaseCount: 1, targetIdBytes } + connectionGenerationByTarget.set(targetId, entry) + retainedGenerationLeases += 1 + retainedTargetIdBytes += targetIdBytes + } else { + entry.generation = generation + } + allocationSessionCounter = Math.max(allocationSessionCounter, generationOffset) } export function resetSshConnectionGenerations(sessionScope = 0): void { if (!Number.isSafeInteger(sessionScope) || sessionScope < 0 || sessionScope > MAX_SESSION_SCOPE) { throw new Error('Invalid SSH connection generation session scope') } - sessionGenerationBase = sessionScope * SESSION_COUNTER_STRIDE + allocationSessionBase = sessionScope * SESSION_COUNTER_STRIDE + allocationSessionCounter = 0 sessionInitialized = true + allocatedSessionScopeCount = 1 + generationRegistryEpoch += 1 + retainedGenerationLeases = 0 + retainedTargetIdBytes = 0 connectionGenerationByTarget.clear() - usedSessionScopes.clear() - usedSessionScopes.add(sessionScope) } export function assertSshMutationExpectation( @@ -91,11 +198,27 @@ export function assertSshMutationExpectation( } return } + const current = connectionGenerationByTarget.get(connectionId) if ( expectedTargetId !== connectionId || expectedGeneration === undefined || - expectedGeneration !== getSshConnectionGeneration(connectionId) + !current || + expectedGeneration !== current.generation ) { throw new Error('SSH connection changed; refresh and try again') } } + +export const _internals = { + evidenceForTest(): { + retainedGenerationLeases: number + retainedTargetIdBytes: number + trackedTargets: number + } { + return { + retainedGenerationLeases, + retainedTargetIdBytes, + trackedTargets: connectionGenerationByTarget.size + } + } +} diff --git a/src/main/ssh/ssh-connection-utils.test.ts b/src/main/ssh/ssh-connection-utils.test.ts index 50f2eb66ff2..8ff44e65787 100644 --- a/src/main/ssh/ssh-connection-utils.test.ts +++ b/src/main/ssh/ssh-connection-utils.test.ts @@ -24,8 +24,11 @@ function testHomePath(...parts: string[]): string { } vi.mock('fs', () => ({ - existsSync: (...args: unknown[]) => mockExistsSync(...args), - readFileSync: (...args: unknown[]) => mockReadFileSync(...args) + existsSync: (...args: unknown[]) => mockExistsSync(...args) +})) + +vi.mock('./ssh-key-file', () => ({ + readSshKeyFile: (...args: unknown[]) => mockReadFileSync(...args) })) import { @@ -51,7 +54,10 @@ import type { SshResolvedConfig } from './ssh-config-parser' type MockProxyProcess = EventEmitter & { stdin: EventEmitter & { write: ReturnType } - stdout: EventEmitter + stdout: EventEmitter & { + pause: ReturnType + resume: ReturnType + } stderr: EventEmitter } @@ -60,7 +66,10 @@ function createMockProxyProcess(): MockProxyProcess { proc.stdin = Object.assign(new EventEmitter(), { write: vi.fn((_chunk, cb?: (error?: Error | null) => void) => cb?.()) }) - proc.stdout = new EventEmitter() + proc.stdout = Object.assign(new EventEmitter(), { + pause: vi.fn(), + resume: vi.fn() + }) proc.stderr = new EventEmitter() return proc } @@ -723,4 +732,23 @@ describe('spawnProxyCommand', () => { expect(proc.stdin.listenerCount('error')).toBe(0) expect(proc.listenerCount('error')).toBe(0) }) + + it('pauses proxy stdout at the socket high-water mark and resumes on demand', async () => { + const proc = createMockProxyProcess() + spawnMock.mockReturnValue(proc) + const { sock } = spawnProxyCommand( + { kind: 'jump-host', jumpHost: 'bastion.example.com' }, + 'target.example.com', + 22, + 'deploy' + ) + + proc.stdout.emit('data', Buffer.alloc(64 * 1024)) + + expect(proc.stdout.pause).toHaveBeenCalledOnce() + sock.on('data', () => {}) + await new Promise((resolve) => setImmediate(resolve)) + expect(proc.stdout.resume).toHaveBeenCalled() + sock.destroy() + }) }) diff --git a/src/main/ssh/ssh-connection-utils.ts b/src/main/ssh/ssh-connection-utils.ts index de9e0b6c63f..d51cda25bbb 100644 --- a/src/main/ssh/ssh-connection-utils.ts +++ b/src/main/ssh/ssh-connection-utils.ts @@ -311,7 +311,9 @@ export function spawnProxyCommand( proc.off('error', onProcessError) } const onStdoutData = (data: Buffer): void => { - stream.push(data) + if (!stream.push(data)) { + proc.stdout!.pause() + } } const onStdoutEnd = (): void => { stream.push(null) @@ -323,7 +325,9 @@ export function spawnProxyCommand( stream.destroy(err) } const stream = new Duplex({ - read() {}, + read() { + proc.stdout!.resume() + }, write(chunk, _encoding, cb) { proc.stdin!.write(chunk, cb) }, diff --git a/src/main/ssh/ssh-connection.test.ts b/src/main/ssh/ssh-connection.test.ts index 750932289d8..77df52a0677 100644 --- a/src/main/ssh/ssh-connection.test.ts +++ b/src/main/ssh/ssh-connection.test.ts @@ -168,6 +168,7 @@ import { } from './ssh-system-fallback' import { getRemoteHostPlatform } from './ssh-remote-platform' import type { SshTarget } from '../../shared/ssh-types' +import { SSH_CONNECTION_ERROR_MAX_UTF8_BYTES } from '../../shared/ssh-retained-payload-admission' function createTarget(overrides?: Partial): SshTarget { return { @@ -436,6 +437,16 @@ describe('SshConnection', () => { expect(conn.getState().status).toBe('error') }) + it('does not retain an oversized provider error in connection state', async () => { + connectBehavior = 'error' + connectErrorMessage = 'x'.repeat(SSH_CONNECTION_ERROR_MAX_UTF8_BYTES + 100) + const conn = new SshConnection(createTarget(), createCallbacks()) + + await conn.connect().catch(() => undefined) + + expect(conn.getState().error).toHaveLength(SSH_CONNECTION_ERROR_MAX_UTF8_BYTES) + }) + it('guards late ssh2 errors emitted while destroying a failed startup client', async () => { connectBehavior = 'error' connectErrorMessage = 'Connection lost before handshake' diff --git a/src/main/ssh/ssh-connection.ts b/src/main/ssh/ssh-connection.ts index baba633d79d..b24d5a35415 100644 --- a/src/main/ssh/ssh-connection.ts +++ b/src/main/ssh/ssh-connection.ts @@ -5,6 +5,7 @@ import { Client as SshClient } from 'ssh2' import type { ChildProcess } from 'node:child_process' import type { ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2' import type { SshTarget, SshConnectionState, SshConnectionStatus } from '../../shared/ssh-types' +import { clampSshConnectionError } from '../../shared/ssh-retained-payload-admission' import { getOrcaControlSocketPath, spawnSystemSsh, @@ -46,6 +47,7 @@ import { createLinkedSshFileTransferSignal, raceSftpFileTransferWithAbort } from './ssh-file-transfer-abort' +import { SystemSshOutputTail } from './system-ssh-output-tail' export type { SshConnectionCallbacks } from './ssh-connection-utils' type SshRemoteFileOptions = { @@ -813,8 +815,8 @@ export class SshConnection { }) try { await new Promise((resolve, reject) => { - let stdout = '' - let stderr = '' + const stdout = new SystemSshOutputTail() + const stderr = new SystemSshOutputTail() let settled = false const cleanup = (): void => { clearTimeout(timeout) @@ -833,10 +835,10 @@ export class SshConnection { callback() } const onStdoutData = (data: Buffer): void => { - stdout += data.toString('utf-8') + stdout.push(data) } const onStderrData = (data: Buffer): void => { - stderr += data.toString('utf-8') + stderr.push(data) } const onError = (err: Error): void => { settle(() => reject(err)) @@ -847,10 +849,11 @@ export class SshConnection { reject(new Error('SSH connection attempt was cancelled')) return } - if (code !== 0 || !stdout.includes('ORCA-SYSTEM-SSH-OK')) { + if (code !== 0 || !stdout.toString().includes('ORCA-SYSTEM-SSH-OK')) { + const stderrText = stderr.toString() reject( new Error( - `System SSH probe failed${code != null ? ` (exit ${code})` : ''}.${stderr ? ` stderr: ${stderr.trim()}` : ''}` + `System SSH probe failed${code != null ? ` (exit ${code})` : ''}.${stderrText ? ` stderr: ${stderrText.trim()}` : ''}` ) ) return @@ -1327,7 +1330,7 @@ export class SshConnection { this.state = { ...this.state, status, - error: error ?? null, + error: clampSshConnectionError(error ?? null), supportsFolderDownload: status === 'connected' && !this.useSystemSshTransport } this.callbacks.onStateChange(this.target.id, { ...this.state }) diff --git a/src/main/ssh/ssh-directory-transfer-budget.test.ts b/src/main/ssh/ssh-directory-transfer-budget.test.ts new file mode 100644 index 00000000000..0e2c94594f2 --- /dev/null +++ b/src/main/ssh/ssh-directory-transfer-budget.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { + SshDirectoryTransferBudget, + type SshDirectoryTransferLimits +} from './ssh-directory-transfer-budget' + +const limits: SshDirectoryTransferLimits = { + maximumEntries: 2, + maximumDepth: 2, + maximumPathBytes: 4, + maximumRetainedPathBytes: 8, + maximumFileBytes: 4, + maximumTotalFileBytes: 8 +} + +describe('SshDirectoryTransferBudget', () => { + it('accepts every exact boundary', () => { + const budget = new SshDirectoryTransferBudget(limits) + + budget.recordPath('1234', 2) + budget.recordPath('5678', 2) + budget.recordFile(4) + budget.recordFile(4) + }) + + it.each([ + ['entries', (budget: SshDirectoryTransferBudget) => budget.recordPath('', 0)], + ['depth', (budget: SshDirectoryTransferBudget) => budget.recordPath('', 3)], + ['path', (budget: SshDirectoryTransferBudget) => budget.recordPath('12345', 0)], + [ + 'paths', + (budget: SshDirectoryTransferBudget) => budget.recordPath('1', 0, { countEntry: false }) + ], + ['file', (budget: SshDirectoryTransferBudget) => budget.recordFile(5)], + ['files', (budget: SshDirectoryTransferBudget) => budget.recordFile(1)] + ] as const)('rejects one unit beyond the %s boundary', (reason, exceed) => { + const budget = new SshDirectoryTransferBudget(limits) + if (reason === 'entries') { + budget.recordPath('', 0) + budget.recordPath('', 0) + } else if (reason === 'paths') { + budget.recordPath('1234', 0, { countEntry: false }) + budget.recordPath('5678', 0, { countEntry: false }) + } else if (reason === 'files') { + budget.recordFile(4) + budget.recordFile(4) + } + + expect(() => exceed(budget)).toThrow(expect.objectContaining({ reason })) + }) +}) diff --git a/src/main/ssh/ssh-directory-transfer-budget.ts b/src/main/ssh/ssh-directory-transfer-budget.ts new file mode 100644 index 00000000000..18a965b06d5 --- /dev/null +++ b/src/main/ssh/ssh-directory-transfer-budget.ts @@ -0,0 +1,64 @@ +export type SshDirectoryTransferLimits = { + maximumEntries: number + maximumDepth: number + maximumPathBytes: number + maximumRetainedPathBytes: number + maximumFileBytes: number + maximumTotalFileBytes: number +} + +export const SSH_DIRECTORY_TRANSFER_LIMITS: SshDirectoryTransferLimits = { + maximumEntries: 4_096, + maximumDepth: 64, + maximumPathBytes: 16 * 1024, + maximumRetainedPathBytes: 4 * 1024 * 1024, + maximumFileBytes: 16 * 1024 * 1024, + maximumTotalFileBytes: 32 * 1024 * 1024 +} + +export const WINDOWS_SSH_UPLOAD_PACKAGE_MAX_BYTES = 48 * 1024 * 1024 + +export class SshDirectoryTransferCapacityError extends Error { + constructor(readonly reason: 'entries' | 'depth' | 'path' | 'paths' | 'file' | 'files') { + super(`SSH directory transfer exceeds the ${reason} limit`) + this.name = 'SshDirectoryTransferCapacityError' + } +} + +export class SshDirectoryTransferBudget { + private entries = 0 + private retainedPathBytes = 0 + private totalFileBytes = 0 + + constructor(private readonly limits = SSH_DIRECTORY_TRANSFER_LIMITS) {} + + recordPath(path: string, depth: number, options?: { countEntry?: boolean }): void { + if (depth > this.limits.maximumDepth) { + throw new SshDirectoryTransferCapacityError('depth') + } + if (options?.countEntry !== false) { + this.entries += 1 + if (this.entries > this.limits.maximumEntries) { + throw new SshDirectoryTransferCapacityError('entries') + } + } + const pathBytes = Buffer.byteLength(path, 'utf8') + if (pathBytes > this.limits.maximumPathBytes) { + throw new SshDirectoryTransferCapacityError('path') + } + this.retainedPathBytes += pathBytes + if (this.retainedPathBytes > this.limits.maximumRetainedPathBytes) { + throw new SshDirectoryTransferCapacityError('paths') + } + } + + recordFile(fileBytes: number): void { + if (fileBytes > this.limits.maximumFileBytes) { + throw new SshDirectoryTransferCapacityError('file') + } + this.totalFileBytes += fileBytes + if (this.totalFileBytes > this.limits.maximumTotalFileBytes) { + throw new SshDirectoryTransferCapacityError('files') + } + } +} diff --git a/src/main/ssh/ssh-file-stream-assembler.ts b/src/main/ssh/ssh-file-stream-assembler.ts new file mode 100644 index 00000000000..4a40c6adf21 --- /dev/null +++ b/src/main/ssh/ssh-file-stream-assembler.ts @@ -0,0 +1,196 @@ +import type { FileReadResult } from '../providers/types' +import { RelayErrorCode, STREAM_CHUNK_SIZE } from './relay-protocol' +import { + defaultSshStreamAssemblyBudget, + type SshStreamAssemblyBudget, + base64EncodedLength +} from './ssh-stream-reader-memory' +import type { RasterImageDimensions } from '../../shared/raster-image-dimensions' +import { isRasterImagePreviewDimensions } from '../../shared/raster-image-preview-limits' + +const MAX_PREVIEWABLE_BINARY_SIZE = 50 * 1024 * 1024 +const MAX_TEXT_FILE_SIZE = 10 * 1024 * 1024 + +type FileStreamMetadata = { + streamId?: number + totalSize: number + isBinary: boolean + isImage?: boolean + mimeType?: string + imageDimensions?: RasterImageDimensions + resultEncoding?: 'base64' | 'utf-8' + empty?: boolean +} + +export class StreamProtocolError extends Error { + readonly code = RelayErrorCode.StreamProtocolError + constructor(message: string) { + super(message) + } +} + +export type FileStreamSetup = + | { kind: 'empty'; result: FileReadResult } + | { kind: 'stream'; streamId: number; assembler: FileStreamAssembler } + +export function createFileStreamSetup( + rawMetadata: unknown, + assemblyBudget: SshStreamAssemblyBudget = defaultSshStreamAssemblyBudget +): FileStreamSetup { + const metadata = rawMetadata as FileStreamMetadata + if ( + typeof metadata !== 'object' || + metadata === null || + typeof metadata.isBinary !== 'boolean' || + !Number.isSafeInteger(metadata.totalSize) || + (metadata.empty !== undefined && typeof metadata.empty !== 'boolean') || + (metadata.isImage !== undefined && typeof metadata.isImage !== 'boolean') || + (metadata.mimeType !== undefined && typeof metadata.mimeType !== 'string') || + (metadata.imageDimensions !== undefined && + !isRasterImagePreviewDimensions(metadata.imageDimensions)) || + (metadata.resultEncoding !== undefined && + metadata.resultEncoding !== 'base64' && + metadata.resultEncoding !== 'utf-8') + ) { + throw new StreamProtocolError('Malformed file stream metadata') + } + if (metadata.empty) { + return { + kind: 'empty', + result: { + content: '', + isBinary: metadata.isBinary, + ...(metadata.isImage !== undefined ? { isImage: metadata.isImage } : {}), + ...(metadata.mimeType !== undefined ? { mimeType: metadata.mimeType } : {}), + ...(metadata.imageDimensions !== undefined + ? { imageDimensions: metadata.imageDimensions } + : {}) + } + } + } + if ( + typeof metadata.streamId !== 'number' || + !Number.isSafeInteger(metadata.streamId) || + metadata.streamId <= 0 + ) { + throw new StreamProtocolError('Metadata missing streamId for non-empty stream') + } + const cap = metadata.isBinary ? MAX_PREVIEWABLE_BINARY_SIZE : MAX_TEXT_FILE_SIZE + if (metadata.totalSize < 0 || metadata.totalSize > cap) { + throw new StreamProtocolError( + `Reported totalSize ${metadata.totalSize} exceeds client cap ${cap}` + ) + } + return { + kind: 'stream', + streamId: metadata.streamId, + assembler: new FileStreamAssembler(metadata, assemblyBudget) + } +} + +export class FileStreamAssembler { + private buffer: Buffer | null + private expectedSeq = 0 + private bytesReceived = 0 + private readonly totalChunks: number + private releaseBudget: (() => void) | null + + constructor( + private readonly metadata: FileStreamMetadata, + assemblyBudget: SshStreamAssemblyBudget + ) { + this.releaseBudget = assemblyBudget.reserve(metadata.totalSize) + if (!this.releaseBudget) { + throw new StreamProtocolError( + `Active SSH stream assembly would exceed ${assemblyBudget.maxBytes} bytes` + ) + } + try { + this.buffer = Buffer.alloc(metadata.totalSize) + } catch (error) { + this.releaseMemory() + throw new StreamProtocolError( + `Failed to allocate ${metadata.totalSize} bytes: ${(error as Error).message}` + ) + } + this.totalChunks = + metadata.totalSize === 0 ? 0 : Math.ceil(metadata.totalSize / STREAM_CHUNK_SIZE) + } + + acceptChunk(params: Record, streamId: number): number { + const seq = params.seq + const data = params.data + if (typeof seq !== 'number' || !Number.isSafeInteger(seq) || typeof data !== 'string') { + throw new StreamProtocolError(`Malformed chunk for stream ${streamId}`) + } + if (seq !== this.expectedSeq) { + throw new StreamProtocolError( + `Out-of-order chunk for stream ${streamId}: expected ${this.expectedSeq}, got ${seq}` + ) + } + if (seq >= this.totalChunks) { + throw new StreamProtocolError( + `File stream ${streamId} exceeded declared chunk count ${this.totalChunks}` + ) + } + const offset = seq * STREAM_CHUNK_SIZE + const expectedLength = Math.min(STREAM_CHUNK_SIZE, this.metadata.totalSize - offset) + if (expectedLength < 0 || data.length !== base64EncodedLength(expectedLength)) { + throw new StreamProtocolError( + `Encoded chunk length mismatch for stream ${streamId}: seq=${seq}` + ) + } + const decoded = Buffer.from(data, 'base64') + if (decoded.length !== expectedLength) { + throw new StreamProtocolError( + `Chunk length mismatch for stream ${streamId}: seq=${seq} expected=${expectedLength} got=${decoded.length}` + ) + } + if (!this.buffer) { + throw new StreamProtocolError(`Chunk arrived after completion for stream ${streamId}`) + } + decoded.copy(this.buffer, offset) + this.expectedSeq += 1 + this.bytesReceived += decoded.length + return seq + } + + finish(streamId: number): FileReadResult { + if (this.expectedSeq !== this.totalChunks || this.bytesReceived !== this.metadata.totalSize) { + throw new StreamProtocolError( + `File stream ${streamId} incomplete: chunks ${this.expectedSeq}/${this.totalChunks}, bytes ${this.bytesReceived}/${this.metadata.totalSize}` + ) + } + if (!this.buffer) { + throw new StreamProtocolError(`Stream ${streamId} already completed`) + } + const buffer = this.buffer + this.buffer = null + try { + return { + content: + (this.metadata.resultEncoding ?? 'base64') === 'base64' + ? buffer.toString('base64') + : buffer.toString('utf-8'), + isBinary: this.metadata.isBinary, + ...(this.metadata.isImage !== undefined ? { isImage: this.metadata.isImage } : {}), + ...(this.metadata.mimeType !== undefined ? { mimeType: this.metadata.mimeType } : {}), + ...(this.metadata.imageDimensions !== undefined + ? { imageDimensions: this.metadata.imageDimensions } + : {}) + } + } finally { + this.releaseMemory() + } + } + + release(): void { + this.buffer = null + this.releaseMemory() + } + + private releaseMemory(): void { + this.releaseBudget?.() + this.releaseBudget = null + } +} diff --git a/src/main/ssh/ssh-filesystem-stream-reader.ts b/src/main/ssh/ssh-filesystem-stream-reader.ts index 2147ed0ef4f..d05089abf80 100644 --- a/src/main/ssh/ssh-filesystem-stream-reader.ts +++ b/src/main/ssh/ssh-filesystem-stream-reader.ts @@ -1,23 +1,25 @@ import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' -import { STREAM_CHUNK_SIZE, JsonRpcErrorCode, RelayErrorCode } from './relay-protocol' +import { JsonRpcErrorCode } from './relay-protocol' import type { FileReadResult } from '../providers/types' +import { + PreMetadataStreamFrameBuffer, + STREAM_READER_INACTIVITY_TIMEOUT_MS, + defaultSshPreMetadataStreamBudget, + defaultSshStreamAssemblyBudget, + type SshPreMetadataStreamBudget, + type SshStreamAssemblyBudget, + createStreamInactivityDeadline +} from './ssh-stream-reader-memory' +import { + createFileStreamSetup, + type FileStreamAssembler, + StreamProtocolError +} from './ssh-file-stream-assembler' + +export { StreamProtocolError } from './ssh-file-stream-assembler' -const RESULT_ENCODING_BASE64 = 'base64' const SENTINEL_STREAM_ID = -1 -const MAX_PREVIEWABLE_BINARY_SIZE = 50 * 1024 * 1024 -const MAX_TEXT_FILE_SIZE = 10 * 1024 * 1024 - -type StreamMetadataResponse = { - streamId?: number - totalSize: number - isBinary: boolean - isImage?: boolean - mimeType?: string - resultEncoding?: 'base64' | 'utf-8' - empty?: boolean -} - export function isMethodNotFoundError(err: unknown): boolean { if (!err || typeof err !== 'object') { return false @@ -26,16 +28,12 @@ export function isMethodNotFoundError(err: unknown): boolean { return code === JsonRpcErrorCode.MethodNotFound } -export class StreamProtocolError extends Error { - readonly code = RelayErrorCode.StreamProtocolError - constructor(message: string) { - super(message) - } -} - export async function readFileViaStream( mux: SshChannelMultiplexer, - filePath: string + filePath: string, + options?: { inactivityTimeoutMs?: number }, + assemblyBudget: SshStreamAssemblyBudget = defaultSshStreamAssemblyBudget, + preMetadataBudget: SshPreMetadataStreamBudget = defaultSshPreMetadataStreamBudget ): Promise { // Why: subscribe BEFORE awaiting the metadata response so a chunk arriving // immediately after the response cannot beat the listener registration. @@ -55,42 +53,44 @@ export async function readFileViaStream( } return new Promise((resolve, reject) => { - let buffer: Buffer | null = null - let resultEncoding: 'base64' | 'utf-8' = RESULT_ENCODING_BASE64 - let isBinary = false - let isImage: boolean | undefined - let mimeType: string | undefined - let totalSize = 0 - let expectedSeq = 0 - let receivedChunks = 0 - let totalChunks = 0 - let bytesReceived = 0 + let assembler: FileStreamAssembler | null = null let settled = false // Why: chunk/end/error frames may arrive in the same dispatch tick as the // metadata response. Queue them until streamIdRef is set, then drain. - type PendingFrame = - | { kind: 'chunk'; params: Record } - | { kind: 'end'; params: Record } - | { kind: 'error'; params: Record } - const pending: PendingFrame[] = [] + const pending = new PreMetadataStreamFrameBuffer(preMetadataBudget) let metadataReady = false - const cancel = (): void => { - if (streamIdRef.current !== SENTINEL_STREAM_ID && !mux.isDisposed()) { + const inactivityMs = options?.inactivityTimeoutMs ?? STREAM_READER_INACTIVITY_TIMEOUT_MS + const inactivity = createStreamInactivityDeadline(inactivityMs, () => { + fail(new StreamProtocolError(`File stream stalled (>${inactivityMs}ms without data)`)) + }) + + const cancelStreamId = (streamId: number): void => { + if (!mux.isDisposed()) { try { - mux.notify('fs.cancelStream', { streamId: streamIdRef.current }) + mux.notify('fs.cancelStream', { streamId }) } catch { // Best-effort } } } + const cancel = (): void => { + if (streamIdRef.current !== SENTINEL_STREAM_ID) { + cancelStreamId(streamIdRef.current) + } + } + const fail = (err: Error): void => { if (settled) { return } settled = true + inactivity.clear() + pending.clear() + assembler?.release() + assembler = null cancel() cleanup() reject(err) @@ -101,6 +101,8 @@ export async function readFileViaStream( return } settled = true + inactivity.clear() + pending.clear() cleanup() resolve(value) } @@ -113,45 +115,26 @@ export async function readFileViaStream( if (id !== streamIdRef.current) { return } - const seq = params.seq as number - const data = params.data as string - if (typeof seq !== 'number' || typeof data !== 'string') { - fail(new StreamProtocolError(`Malformed chunk for stream ${id}`)) - return - } - if (seq !== expectedSeq) { - fail( - new StreamProtocolError( - `Out-of-order chunk for stream ${id}: expected ${expectedSeq}, got ${seq}` - ) - ) - return - } - const offset = seq * STREAM_CHUNK_SIZE - const decoded = Buffer.from(data, 'base64') - // Why: a short chunk would leave the pre-allocated buffer zero-filled and - // resolve as silently-corrupt data; validate each chunk's exact length. - const expectedLength = Math.min(STREAM_CHUNK_SIZE, totalSize - offset) - if (decoded.length !== expectedLength) { - fail( - new StreamProtocolError( - `Chunk length mismatch for stream ${id}: seq=${seq} expected=${expectedLength} got=${decoded.length}` - ) - ) - return - } - if (!buffer) { + if (!assembler) { fail(new StreamProtocolError(`Chunk arrived before metadata for stream ${id}`)) return } - decoded.copy(buffer, offset) - expectedSeq += 1 - receivedChunks += 1 - bytesReceived += decoded.length + let seq: number + try { + seq = assembler.acceptChunk(params, id) + } catch (error) { + fail(error as Error) + return + } + inactivity.reset() // Why: credit-based flow control — the relay caps unacked chunks so bulk // stream frames cannot queue unbounded ahead of interactive pty.data // frames on the shared SSH channel. Old relays ignore this notification. - mux.notify('fs.streamAck', { streamId: id, seq }) + try { + mux.notify('fs.streamAck', { streamId: id, seq }) + } catch { + // Disposal can race the write; teardown will settle the reader. + } } const handleEnd = (params: Record): void => { @@ -162,38 +145,15 @@ export async function readFileViaStream( if (id !== streamIdRef.current) { return } - if (receivedChunks !== totalChunks) { - fail( - new StreamProtocolError( - `Chunk count mismatch for stream ${id}: expected ${totalChunks}, received ${receivedChunks}` - ) - ) - return - } - // Why: redundant given the per-chunk length + count checks, but kept as a - // last-line invariant guard; never resolve with fewer bytes than declared. - if (bytesReceived !== totalSize) { - fail( - new StreamProtocolError( - `Byte count mismatch for stream ${id}: expected ${totalSize}, received ${bytesReceived}` - ) - ) - return - } - if (!buffer) { + if (!assembler) { fail(new StreamProtocolError(`Stream end before metadata for stream ${id}`)) return } - const content = - resultEncoding === RESULT_ENCODING_BASE64 - ? buffer.toString('base64') - : buffer.toString('utf-8') - succeed({ - content, - isBinary, - ...(isImage !== undefined ? { isImage } : {}), - ...(mimeType !== undefined ? { mimeType } : {}) - }) + try { + succeed(assembler.finish(id)) + } catch (error) { + fail(error as Error) + } } const handleStreamError = (params: Record): void => { @@ -213,7 +173,10 @@ export async function readFileViaStream( const drainPending = (): void => { while (!settled && pending.length > 0) { - const frame = pending.shift()! + const frame = pending.shift() + if (!frame) { + break + } if (frame.kind === 'chunk') { handleChunk(frame.params) } else if (frame.kind === 'end') { @@ -224,10 +187,18 @@ export async function readFileViaStream( } } + const pushPending = ( + kind: 'chunk' | 'end' | 'error', + params: Record + ): void => { + // Why: the stream id is unknown here; overload drops must not let one foreign frame fail every reader. + pending.push({ kind, params }) + } + unsubscribers.push( mux.onNotificationByMethod('fs.streamChunk', (params) => { if (!metadataReady) { - pending.push({ kind: 'chunk', params }) + pushPending('chunk', params) return } handleChunk(params) @@ -236,7 +207,7 @@ export async function readFileViaStream( unsubscribers.push( mux.onNotificationByMethod('fs.streamEnd', (params) => { if (!metadataReady) { - pending.push({ kind: 'end', params }) + pushPending('end', params) return } handleEnd(params) @@ -245,7 +216,7 @@ export async function readFileViaStream( unsubscribers.push( mux.onNotificationByMethod('fs.streamError', (params) => { if (!metadataReady) { - pending.push({ kind: 'error', params }) + pushPending('error', params) return } handleStreamError(params) @@ -269,51 +240,31 @@ export async function readFileViaStream( .request('fs.readFileStream', { filePath, flowControl: 'ack' }) .then((rawMetadata) => { if (settled) { + const streamId = (rawMetadata as { streamId?: unknown } | null)?.streamId + if (typeof streamId === 'number') { + cancelStreamId(streamId) + } return } - const metadata = rawMetadata as StreamMetadataResponse - isBinary = metadata.isBinary - isImage = metadata.isImage - mimeType = metadata.mimeType - resultEncoding = metadata.resultEncoding ?? RESULT_ENCODING_BASE64 - - if (metadata.empty) { - succeed({ - content: '', - isBinary: metadata.isBinary, - ...(metadata.isImage !== undefined ? { isImage: metadata.isImage } : {}), - ...(metadata.mimeType !== undefined ? { mimeType: metadata.mimeType } : {}) - }) - return - } - - if (typeof metadata.streamId !== 'number') { - fail(new StreamProtocolError('Metadata missing streamId for non-empty stream')) - return - } - - const cap = metadata.isBinary ? MAX_PREVIEWABLE_BINARY_SIZE : MAX_TEXT_FILE_SIZE - if (metadata.totalSize < 0 || metadata.totalSize > cap) { - streamIdRef.current = metadata.streamId - fail( - new StreamProtocolError( - `Reported totalSize ${metadata.totalSize} exceeds client cap ${cap}` - ) - ) - return - } - - totalSize = metadata.totalSize - totalChunks = totalSize === 0 ? 0 : Math.ceil(totalSize / STREAM_CHUNK_SIZE) + let setup try { - buffer = Buffer.alloc(totalSize) - } catch (err) { - streamIdRef.current = metadata.streamId - fail(new Error(`Failed to allocate ${totalSize} bytes: ${(err as Error).message}`)) + setup = createFileStreamSetup(rawMetadata, assemblyBudget) + } catch (error) { + const streamId = (rawMetadata as { streamId?: unknown } | null)?.streamId + if (typeof streamId === 'number') { + streamIdRef.current = streamId + } + fail(error as Error) return } - streamIdRef.current = metadata.streamId + if (setup.kind === 'empty') { + succeed(setup.result) + return + } + streamIdRef.current = setup.streamId + assembler = setup.assembler metadataReady = true + inactivity.reset() drainPending() }) .catch((err) => { diff --git a/src/main/ssh/ssh-git-response-stream-assembler.ts b/src/main/ssh/ssh-git-response-stream-assembler.ts new file mode 100644 index 00000000000..7153826ca80 --- /dev/null +++ b/src/main/ssh/ssh-git-response-stream-assembler.ts @@ -0,0 +1,108 @@ +import { RelayErrorCode } from './relay-protocol' +import { type SshStreamAssemblyBudget, base64EncodedLength } from './ssh-stream-reader-memory' + +export class GitResponseStreamError extends Error { + readonly code = RelayErrorCode.StreamProtocolError + constructor(message: string) { + super(message) + } +} + +export class GitResponseStreamAssembler { + private buffer: Buffer | null + private expectedSeq = 0 + private receivedBytes = 0 + private releaseBudget: (() => void) | null + + constructor( + private readonly streamId: number, + private readonly totalBytes: number, + private readonly chunkCount: number, + assemblyBudget: SshStreamAssemblyBudget + ) { + this.releaseBudget = assemblyBudget.reserve(totalBytes) + if (!this.releaseBudget) { + throw new GitResponseStreamError( + `Active SSH stream assembly would exceed ${assemblyBudget.maxBytes} bytes` + ) + } + try { + this.buffer = Buffer.alloc(totalBytes) + } catch (error) { + this.release() + throw new GitResponseStreamError( + `Failed to allocate ${totalBytes} bytes: ${(error as Error).message}` + ) + } + } + + acceptChunk(params: Record): number { + const seq = params.seq + const data = params.data + if ( + typeof seq !== 'number' || + !Number.isSafeInteger(seq) || + seq < 0 || + typeof data !== 'string' + ) { + throw new GitResponseStreamError(`Malformed chunk for git stream ${this.streamId}`) + } + if (seq !== this.expectedSeq) { + throw new GitResponseStreamError( + `Out-of-order chunk for git stream ${this.streamId}: expected ${this.expectedSeq}, got ${seq}` + ) + } + if (this.expectedSeq >= this.chunkCount) { + throw new GitResponseStreamError( + `Git stream ${this.streamId} exceeded declared chunk count ${this.chunkCount}` + ) + } + const remainingBytes = this.totalBytes - this.receivedBytes + if (data.length > base64EncodedLength(remainingBytes)) { + throw new GitResponseStreamError( + `Git stream ${this.streamId} chunk exceeds ${remainingBytes} remaining bytes` + ) + } + const decoded = Buffer.from(data, 'base64') + if (decoded.length > remainingBytes) { + throw new GitResponseStreamError( + `Git stream ${this.streamId} decoded beyond its declared size` + ) + } + if (!this.buffer) { + throw new GitResponseStreamError( + `Chunk arrived after completion for git stream ${this.streamId}` + ) + } + decoded.copy(this.buffer, this.receivedBytes) + this.receivedBytes += decoded.length + this.expectedSeq += 1 + return seq + } + + finish(): unknown { + if (this.expectedSeq !== this.chunkCount || this.receivedBytes !== this.totalBytes) { + throw new GitResponseStreamError( + `Git stream ${this.streamId} incomplete: chunks ${this.expectedSeq}/${this.chunkCount}, bytes ${this.receivedBytes}/${this.totalBytes}` + ) + } + if (!this.buffer) { + throw new GitResponseStreamError(`Git stream ${this.streamId} already completed`) + } + const serialized = this.buffer.toString('utf-8') + this.release() + try { + return JSON.parse(serialized) + } catch (error) { + throw new GitResponseStreamError( + `Git stream ${this.streamId} JSON parse failed: ${String(error)}` + ) + } + } + + release(): void { + this.buffer = null + this.releaseBudget?.() + this.releaseBudget = null + } +} diff --git a/src/main/ssh/ssh-git-response-stream-reader.ts b/src/main/ssh/ssh-git-response-stream-reader.ts index ef63c7a6dc0..94f9d5b8228 100644 --- a/src/main/ssh/ssh-git-response-stream-reader.ts +++ b/src/main/ssh/ssh-git-response-stream-reader.ts @@ -1,31 +1,26 @@ import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' -import { RelayErrorCode, isGitResponseStreamMarker } from './relay-protocol' +import { + MAX_GIT_RESPONSE_STREAM_BYTES, + MAX_GIT_RESPONSE_STREAM_CHUNKS, + isGitResponseStreamMarker +} from './relay-protocol' +import { + PreMetadataStreamFrameBuffer, + STREAM_READER_INACTIVITY_TIMEOUT_MS, + defaultSshPreMetadataStreamBudget, + defaultSshStreamAssemblyBudget, + type SshPreMetadataStreamBudget, + type SshStreamAssemblyBudget, + createStreamInactivityDeadline +} from './ssh-stream-reader-memory' +import { + GitResponseStreamAssembler, + GitResponseStreamError +} from './ssh-git-response-stream-assembler' const SENTINEL_STREAM_ID = -1 -/** Reject if no stream frame (chunk/end/error) arrives within this window, - * reset on each frame. mux.request's own timeout only bounds the fast sentinel - * response; without this, a relay pump that breaks on staleness (which sends no - * responseEnd) while the SSH channel stays up would hang the client forever. */ -const STREAM_INACTIVITY_TIMEOUT_MS = 30_000 - -/** Bound transient buffering of other concurrent streams' chunks while this - * reader awaits its sentinel: every reader sees all git.responseChunk frames - * and can't filter by streamId until its own sentinel resolves. Foreign frames - * are dropped on drain anyway; this just caps the pre-sentinel backlog. */ -const MAX_PENDING_FRAMES = 64 - -export class GitResponseStreamError extends Error { - readonly code = RelayErrorCode.StreamProtocolError - constructor(message: string) { - super(message) - } -} - -type PendingFrame = - | { kind: 'chunk'; params: Record } - | { kind: 'end'; params: Record } - | { kind: 'error'; params: Record } +export { GitResponseStreamError } from './ssh-git-response-stream-assembler' /** * Request a git method that may return a large payload, opting into response @@ -49,7 +44,9 @@ export function requestGitStreamable( timeoutMs?: number /** Bounds the post-sentinel reassembly stall; resets on each chunk. */ inactivityTimeoutMs?: number - } + }, + assemblyBudget: SshStreamAssemblyBudget = defaultSshStreamAssemblyBudget, + preMetadataBudget: SshPreMetadataStreamBudget = defaultSshPreMetadataStreamBudget ): Promise { // Why: subscribe to chunk/end/error BEFORE awaiting the sentinel response so a // chunk that lands in the same dispatch tick as the response is not dropped @@ -68,42 +65,22 @@ export function requestGitStreamable( } return new Promise((resolve, reject) => { - const parts: Buffer[] = [] - let expectedSeq = 0 - let receivedBytes = 0 - let totalBytes = 0 - let chunkCount = 0 + let assembler: GitResponseStreamAssembler | null = null let settled = false let metadataReady = false - const pending: PendingFrame[] = [] + const pending = new PreMetadataStreamFrameBuffer(preMetadataBudget) - const inactivityMs = options?.inactivityTimeoutMs ?? STREAM_INACTIVITY_TIMEOUT_MS - let inactivityTimer: ReturnType | null = null - const clearInactivity = (): void => { - if (inactivityTimer) { - clearTimeout(inactivityTimer) - inactivityTimer = null - } - } - // Why: reset on every stream frame so a legitimately long stream is not - // killed, but a wedged stream (no frames arriving) rejects instead of - // hanging the caller forever. - const armInactivity = (): void => { - clearInactivity() - inactivityTimer = setTimeout(() => { - fail( - new GitResponseStreamError( - `Git response stream stalled (>${inactivityMs}ms without data)` - ) - ) - }, inactivityMs) - inactivityTimer.unref?.() - } + const inactivityMs = options?.inactivityTimeoutMs ?? STREAM_READER_INACTIVITY_TIMEOUT_MS + const inactivity = createStreamInactivityDeadline(inactivityMs, () => { + fail( + new GitResponseStreamError(`Git response stream stalled (>${inactivityMs}ms without data)`) + ) + }) - const cancel = (): void => { - if (streamIdRef.current !== SENTINEL_STREAM_ID && !mux.isDisposed()) { + const cancelStreamId = (streamId: number): void => { + if (streamId !== SENTINEL_STREAM_ID && !mux.isDisposed()) { try { - mux.notify('git.cancelResponseStream', { streamId: streamIdRef.current }) + mux.notify('git.cancelResponseStream', { streamId }) } catch { // best-effort } @@ -114,8 +91,11 @@ export function requestGitStreamable( return } settled = true - clearInactivity() - cancel() + inactivity.clear() + pending.clear() + assembler?.release() + assembler = null + cancelStreamId(streamIdRef.current) cleanup() reject(err) } @@ -124,7 +104,8 @@ export function requestGitStreamable( return } settled = true - clearInactivity() + inactivity.clear() + pending.clear() cleanup() resolve(value) } @@ -133,25 +114,22 @@ export function requestGitStreamable( if (settled || p.streamId !== streamIdRef.current) { return } - const seq = p.seq as number - const data = p.data as string - if (typeof seq !== 'number' || typeof data !== 'string') { - fail(new GitResponseStreamError(`Malformed chunk for git stream ${streamIdRef.current}`)) - return - } - if (seq !== expectedSeq) { + if (!assembler) { fail( new GitResponseStreamError( - `Out-of-order chunk for git stream ${streamIdRef.current}: expected ${expectedSeq}, got ${seq}` + `Chunk arrived before metadata for git stream ${streamIdRef.current}` ) ) return } - const decoded = Buffer.from(data, 'base64') - parts.push(decoded) - receivedBytes += decoded.length - expectedSeq += 1 - armInactivity() + let seq: number + try { + seq = assembler.acceptChunk(p) + } catch (error) { + fail(error as Error) + return + } + inactivity.reset() // Why: credit-based flow control — the relay caps unacked chunks so a big // response cannot queue unbounded ahead of interactive pty.data frames. if (!mux.isDisposed()) { @@ -167,22 +145,18 @@ export function requestGitStreamable( if (settled || p.streamId !== streamIdRef.current) { return } - if (expectedSeq !== chunkCount || receivedBytes !== totalBytes) { + if (!assembler) { fail( new GitResponseStreamError( - `Git stream ${streamIdRef.current} incomplete: chunks ${expectedSeq}/${chunkCount}, bytes ${receivedBytes}/${totalBytes}` + `Stream end before metadata for git stream ${streamIdRef.current}` ) ) return } try { - succeed(JSON.parse(Buffer.concat(parts).toString('utf-8'))) - } catch (err) { - fail( - new GitResponseStreamError( - `Git stream ${streamIdRef.current} JSON parse failed: ${String(err)}` - ) - ) + succeed(assembler.finish()) + } catch (error) { + fail(error as Error) } } @@ -195,7 +169,10 @@ export function requestGitStreamable( const drainPending = (): void => { while (!settled && pending.length > 0) { - const frame = pending.shift()! + const frame = pending.shift() + if (!frame) { + break + } if (frame.kind === 'chunk') { handleChunk(frame.params) } else if (frame.kind === 'end') { @@ -206,22 +183,18 @@ export function requestGitStreamable( } } - // Why: pre-sentinel we cannot filter by streamId (our id is unknown yet), so - // every concurrent reader transiently buffers all readers' chunks. Cap the - // backlog by dropping the oldest; foreign frames are dropped on drain anyway, - // and if our own seq-0 were ever dropped the seq check fails loudly rather - // than corrupting. The sentinel normally resolves long before this cap. - const pushPending = (frame: PendingFrame): void => { - pending.push(frame) - if (pending.length > MAX_PENDING_FRAMES) { - pending.shift() - } + const pushPending = ( + kind: 'chunk' | 'end' | 'error', + params: Record + ): void => { + // Why: the stream id is unknown here; overload drops must not let one foreign frame fail every reader. + pending.push({ kind, params }) } unsubscribers.push( mux.onNotificationByMethod('git.responseChunk', (p) => { if (!metadataReady) { - pushPending({ kind: 'chunk', params: p }) + pushPending('chunk', p) return } handleChunk(p) @@ -230,7 +203,7 @@ export function requestGitStreamable( unsubscribers.push( mux.onNotificationByMethod('git.responseEnd', (p) => { if (!metadataReady) { - pushPending({ kind: 'end', params: p }) + pushPending('end', p) return } handleEnd(p) @@ -239,7 +212,7 @@ export function requestGitStreamable( unsubscribers.push( mux.onNotificationByMethod('git.responseError', (p) => { if (!metadataReady) { - pushPending({ kind: 'error', params: p }) + pushPending('error', p) return } handleStreamError(p) @@ -289,21 +262,52 @@ export function requestGitStreamable( void requestPromise .then((result) => { if (settled) { + if (isGitResponseStreamMarker(result)) { + cancelStreamId(result.__orcaGitResponseStream.streamId) + } return } // Old relay / small result: plain single-frame value, no stream follows. if (!isGitResponseStreamMarker(result)) { + if ( + typeof result === 'object' && + result !== null && + '__orcaGitResponseStream' in result + ) { + fail(new GitResponseStreamError('Malformed Git response stream metadata')) + return + } succeed(result) return } const marker = result.__orcaGitResponseStream - totalBytes = marker.totalBytes - chunkCount = marker.chunkCount streamIdRef.current = marker.streamId + if ( + marker.totalBytes > MAX_GIT_RESPONSE_STREAM_BYTES || + marker.chunkCount > MAX_GIT_RESPONSE_STREAM_CHUNKS + ) { + fail( + new GitResponseStreamError( + `Git response stream exceeds client limit (${marker.totalBytes} bytes, ${marker.chunkCount} chunks)` + ) + ) + return + } + try { + assembler = new GitResponseStreamAssembler( + marker.streamId, + marker.totalBytes, + marker.chunkCount, + assemblyBudget + ) + } catch (error) { + fail(error as Error) + return + } metadataReady = true // Why: start the inactivity deadline now — mux.request's timeout only // covered the sentinel; the reassembly phase needs its own guard. - armInactivity() + inactivity.reset() drainPending() }) .catch((err) => fail(err as Error)) diff --git a/src/main/ssh/ssh-key-file.test.ts b/src/main/ssh/ssh-key-file.test.ts new file mode 100644 index 00000000000..7a9212a22ce --- /dev/null +++ b/src/main/ssh/ssh-key-file.test.ts @@ -0,0 +1,37 @@ +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { readSshKeyFile, SSH_KEY_FILE_MAX_BYTES } from './ssh-key-file' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function makeKeyPath(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-ssh-key-')) + roots.push(root) + return join(root, 'id_ed25519') +} + +describe('SSH key file bounds', () => { + it('preserves ordinary key bytes', () => { + const keyPath = makeKeyPath() + writeFileSync(keyPath, 'private-key') + + expect(readSshKeyFile(keyPath)).toEqual(Buffer.from('private-key')) + }) + + it('rejects an oversized sparse key file', () => { + const keyPath = makeKeyPath() + const descriptor = openSync(keyPath, 'w') + ftruncateSync(descriptor, SSH_KEY_FILE_MAX_BYTES + 1) + closeSync(descriptor) + + expect(() => readSshKeyFile(keyPath)).toThrow('exceeds') + }) +}) diff --git a/src/main/ssh/ssh-key-file.ts b/src/main/ssh/ssh-key-file.ts new file mode 100644 index 00000000000..e04e5cfe1ac --- /dev/null +++ b/src/main/ssh/ssh-key-file.ts @@ -0,0 +1,7 @@ +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' + +export const SSH_KEY_FILE_MAX_BYTES = 1024 * 1024 + +export function readSshKeyFile(filePath: string): Buffer { + return readNodeFileSyncWithinLimit(filePath, SSH_KEY_FILE_MAX_BYTES).buffer +} diff --git a/src/main/ssh/ssh-port-forward.test.ts b/src/main/ssh/ssh-port-forward.test.ts index b49c627b8b3..a1f3f0c195e 100644 --- a/src/main/ssh/ssh-port-forward.test.ts +++ b/src/main/ssh/ssh-port-forward.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events' import { createServer } from 'node:net' import { describe, expect, it, vi, beforeEach } from 'vitest' -import { SshPortForwardManager } from './ssh-port-forward' +import { SSH_FORWARD_CLOSE_CONCURRENCY, SshPortForwardManager } from './ssh-port-forward' const { startSystemSshPortForwardProcessMock } = vi.hoisted(() => ({ startSystemSshPortForwardProcessMock: vi.fn() @@ -246,6 +246,43 @@ describe('SshPortForwardManager', () => { expect(resolved).toBe(true) }) + it.each([ + ['at the limit', SSH_FORWARD_CLOSE_CONCURRENCY], + ['above the limit', SSH_FORWARD_CLOSE_CONCURRENCY + 1] + ])('bounds forward closes %s', async (_, count) => { + let active = 0 + let peak = 0 + let started = 0 + const releases: (() => void)[] = [] + const forwards = Array.from({ length: count }, () => { + const forward = createFakeSystemSshForward() + forward.close.mockImplementation(async () => { + started++ + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + }) + return forward + }) + startSystemSshPortForwardProcessMock.mockImplementation(() => forwards.shift()) + const conn = createSystemSshConn() + for (let index = 0; index < count; index++) { + await manager.addForward('conn-1', conn as never, 3000 + index, '127.0.0.1', 8080) + } + + const removal = manager.removeAllForwards('conn-1') + await vi.waitFor(() => expect(started).toBe(Math.min(count, SSH_FORWARD_CLOSE_CONCURRENCY))) + if (count > SSH_FORWARD_CLOSE_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(started).toBe(count)) + } + releases.splice(0).forEach((release) => release()) + await removal + + expect(peak).toBe(Math.min(count, SSH_FORWARD_CLOSE_CONCURRENCY)) + }) + it('removes an unexpectedly exited system SSH forward and calls the close callback', async () => { const onForwardClosed = vi.fn() manager.setCallbacks({ onForwardClosed }) diff --git a/src/main/ssh/ssh-port-forward.ts b/src/main/ssh/ssh-port-forward.ts index bc151eaab1e..d296c27086e 100644 --- a/src/main/ssh/ssh-port-forward.ts +++ b/src/main/ssh/ssh-port-forward.ts @@ -7,6 +7,9 @@ import type { SshPortForwardProvider, StartedPortForward } from './ssh-port-forward-provider' +import { mapSettledWithConcurrency } from '../../shared/map-with-concurrency' + +export const SSH_FORWARD_CLOSE_CONCURRENCY = 8 export type { PortForwardEntry } export type { PortForwardCloseReason } @@ -179,7 +182,15 @@ export class SshPortForwardManager { const toRemove = [...this.forwards.entries()] .filter(([, { entry }]) => entry.connectionId === connectionId) .map(([id]) => id) - await Promise.all(toRemove.map((id) => this.removeForwardAsync(id))) + const results = await mapSettledWithConcurrency(toRemove, SSH_FORWARD_CLOSE_CONCURRENCY, (id) => + this.removeForwardAsync(id) + ) + const failedClose = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failedClose) { + throw failedClose.reason + } } dispose(): void { diff --git a/src/main/ssh/ssh-port-scanner.test.ts b/src/main/ssh/ssh-port-scanner.test.ts index 09be7c1f686..56dba7139e3 100644 --- a/src/main/ssh/ssh-port-scanner.test.ts +++ b/src/main/ssh/ssh-port-scanner.test.ts @@ -7,6 +7,10 @@ import { } from './ssh-port-scanner' import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' import type { DetectedPort } from '../../shared/ssh-types' +import { + SSH_DETECTED_PORTS_MAX_ENTRIES, + SSH_DETECTED_PORT_PROCESS_NAME_MAX_UTF8_BYTES +} from '../../shared/ssh-retained-payload-admission' type VisibilityHarness = { visibility: PortScannerWindowVisibility @@ -216,6 +220,26 @@ describe('PortScanner', () => { scanner.dispose() }) + it('bounds remote rows before retaining or publishing a scan', async () => { + const harness = createVisibilityHarness(true) + const remoteRows = Array.from({ length: SSH_DETECTED_PORTS_MAX_ENTRIES + 20 }, (_, index) => ({ + ...port(3000 + index), + processName: 'x'.repeat(SSH_DETECTED_PORT_PROCESS_NAME_MAX_UTF8_BYTES + 100) + })) + const { mux } = createMux(() => remoteRows) + const onChanged = vi.fn() + const scanner = new PortScanner(harness.visibility) + + scanner.startScanning('t1', mux, onChanged) + await vi.advanceTimersByTimeAsync(0) + + const retained = scanner.getDetectedPorts('t1') + expect(retained).toHaveLength(SSH_DETECTED_PORTS_MAX_ENTRIES) + expect(retained[0].processName).toHaveLength(SSH_DETECTED_PORT_PROCESS_NAME_MAX_UTF8_BYTES) + expect(onChanged).toHaveBeenCalledWith('t1', retained, 'linux') + scanner.dispose() + }) + it('stopScanning halts polling and detaches the visibility listener', async () => { const harness = createVisibilityHarness(true) const { mux, request } = createMux(() => [port(3000)]) diff --git a/src/main/ssh/ssh-port-scanner.ts b/src/main/ssh/ssh-port-scanner.ts index 6404d035f22..abb668095ce 100644 --- a/src/main/ssh/ssh-port-scanner.ts +++ b/src/main/ssh/ssh-port-scanner.ts @@ -1,5 +1,6 @@ import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' import type { DetectedPort } from '../../shared/ssh-types' +import { admitSshDetectedPorts } from '../../shared/ssh-retained-payload-admission' // Why: every tick walks /proc/*/fd on the remote relay, so cadence is remote // CPU, not just a local timer. 12s cuts steady-state request volume 4x vs the @@ -75,8 +76,10 @@ export class PortScanner { return } + // Why: relay responses cross a provider boundary and are retained between scans. + const admittedPorts = admitSshDetectedPorts(result.ports) const currentPorts = new Map() - for (const p of result.ports) { + for (const p of admittedPorts) { currentPorts.set(`${p.host}:${p.port}`, p) } @@ -87,7 +90,7 @@ export class PortScanner { if (!portsEqual(handle.previousPorts, currentPorts)) { handle.previousPorts = currentPorts handle.intervalMs = SSH_PORT_SCAN_BASE_INTERVAL_MS - onChanged(targetId, result.ports, result.platform) + onChanged(targetId, admittedPorts, result.platform) } else { handle.intervalMs = Math.min(handle.intervalMs * 2, SSH_PORT_SCAN_MAX_INTERVAL_MS) } diff --git a/src/main/ssh/ssh-relay-base-directory-listing.test.ts b/src/main/ssh/ssh-relay-base-directory-listing.test.ts new file mode 100644 index 00000000000..56d9fb676f2 --- /dev/null +++ b/src/main/ssh/ssh-relay-base-directory-listing.test.ts @@ -0,0 +1,121 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + getRelayBaseDirectoryListingCommand, + RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL, + RELAY_BASE_DIRECTORY_MAX_ENTRIES, + RELAY_BASE_DIRECTORY_MAX_UTF8_BYTES +} from './ssh-relay-base-directory-listing' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') + +function decodePowerShellCommand(command: string): string { + const match = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/) + return match ? Buffer.from(match[1], 'base64').toString('utf16le') : '' +} + +function runPosixListing( + directory: string, + options: { maxEntries: number; maxUtf8Bytes: number } +): string[] { + const result = spawnSync( + '/bin/sh', + ['-c', getRelayBaseDirectoryListingCommand(posix, directory, options)], + { encoding: 'utf8' } + ) + if (result.status !== 0) { + throw new Error(`listing exited ${result.status}: ${result.stderr}`) + } + return result.stdout + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) +} + +function withTempDirectory(run: (directory: string) => void): void { + const directory = mkdtempSync(join(tmpdir(), 'orca-relay-listing-')) + try { + run(directory) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +describe('relay base-directory listing', () => { + it('streams POSIX entries through fixed entry and byte budgets', () => { + const command = getRelayBaseDirectoryListingCommand(posix, '/home/u/.orca-remote') + + expect(command).toContain('find "$base" -mindepth 1 -maxdepth 1 -print') + expect(command).toContain(`entry_count>${RELAY_BASE_DIRECTORY_MAX_ENTRIES}`) + expect(command).toContain(`output_bytes+line_bytes>${RELAY_BASE_DIRECTORY_MAX_UTF8_BYTES}`) + expect(command).toContain('| sort') + expect(command).not.toContain('ls -1') + }) + + it('allows the exact POSIX entry and output-byte bounds', () => { + withTempDirectory((directory) => { + mkdirSync(join(directory, 'a')) + mkdirSync(join(directory, 'b')) + + expect( + runPosixListing(directory, { + maxEntries: 2, + maxUtf8Bytes: 4 + }).sort() + ).toEqual(['a', 'b']) + }) + }) + + it('emits a sentinel on the first POSIX entry beyond the bound', () => { + withTempDirectory((directory) => { + mkdirSync(join(directory, 'a')) + mkdirSync(join(directory, 'b')) + mkdirSync(join(directory, 'c')) + + expect( + runPosixListing(directory, { + maxEntries: 2, + maxUtf8Bytes: 100 + }) + ).toContain(RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL) + }) + }) + + it('emits a sentinel before exceeding the POSIX output-byte bound', () => { + withTempDirectory((directory) => { + mkdirSync(join(directory, 'aa')) + mkdirSync(join(directory, 'bb')) + + const lines = runPosixListing(directory, { + maxEntries: 2, + maxUtf8Bytes: 5 + }) + + expect(lines).toContain(RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL) + expect( + lines.filter((line) => line !== RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL) + ).toHaveLength(1) + }) + }) + + it('uses a disposable streaming iterator on Windows', () => { + const script = decodePowerShellCommand( + getRelayBaseDirectoryListingCommand(windows, 'C:/Users/u/.orca-remote') + ) + + expect(script).toContain('[System.IO.Directory]::EnumerateDirectories($base)') + expect(script).toContain('.GetEnumerator()') + expect(script).toContain('$iterator.MoveNext()') + expect(script).toContain('$iterator.Dispose()') + expect(script).toContain(`$maxEntries = ${RELAY_BASE_DIRECTORY_MAX_ENTRIES}`) + expect(script).toContain(`$maxBytes = ${RELAY_BASE_DIRECTORY_MAX_UTF8_BYTES}`) + expect(script).toContain('UTF8.GetByteCount($name) + 2') + expect(script).toContain(RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL) + expect(script).not.toContain('Get-ChildItem') + }) +}) diff --git a/src/main/ssh/ssh-relay-base-directory-listing.ts b/src/main/ssh/ssh-relay-base-directory-listing.ts new file mode 100644 index 00000000000..a20eb2897f7 --- /dev/null +++ b/src/main/ssh/ssh-relay-base-directory-listing.ts @@ -0,0 +1,93 @@ +import type { RemoteHostPlatform } from './ssh-remote-platform' +import { isWindowsRemoteHost } from './ssh-remote-platform' +import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' +import { shellEscape } from './ssh-connection-utils' + +export const RELAY_BASE_DIRECTORY_MAX_ENTRIES = 4_096 +export const RELAY_BASE_DIRECTORY_MAX_UTF8_BYTES = 256 * 1024 +export const RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL = + '__ORCA_RELAY_BASE_DIRECTORY_LISTING_TOO_LARGE__' + +export function isRelayBaseDirectoryListingLimited(listing: string): boolean { + return listing + .split(/\r?\n/) + .some((line) => line.trim() === RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL) +} + +type RelayBaseDirectoryListingOptions = { + maxEntries?: number + maxUtf8Bytes?: number +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback + if (!Number.isSafeInteger(resolved) || resolved < 1) { + throw new Error(`${name} must be a positive integer`) + } + return resolved +} + +export function getRelayBaseDirectoryListingCommand( + host: RemoteHostPlatform, + baseDir: string, + options?: RelayBaseDirectoryListingOptions +): string { + const maxEntries = positiveInteger( + options?.maxEntries, + RELAY_BASE_DIRECTORY_MAX_ENTRIES, + 'maxEntries' + ) + const maxUtf8Bytes = positiveInteger( + options?.maxUtf8Bytes, + RELAY_BASE_DIRECTORY_MAX_UTF8_BYTES, + 'maxUtf8Bytes' + ) + + if (!isWindowsRemoteHost(host)) { + const awkProgram = [ + '{', + 'entry_count++;', + `if(entry_count>${maxEntries}){print "${RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL}";exit}`, + 'name=$0;sub(/^.*\\//,"",name);', + 'if(substr(name,1,1)==".")next;', + 'line_bytes=length(name)+1;', + `if(output_bytes+line_bytes>${maxUtf8Bytes}){print "${RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL}";exit}`, + 'print name;output_bytes+=line_bytes', + '}' + ].join('') + return [ + `base=${shellEscape(baseDir)};`, + 'if [ -d "$base" ]; then', + 'LC_ALL=C find "$base" -mindepth 1 -maxdepth 1 -print 2>/dev/null', + `| LC_ALL=C awk ${shellEscape(awkProgram)} | sort;`, + 'fi' + ].join(' ') + } + + return powerShellCommand( + [ + `$base = ${powerShellLiteral(baseDir)}`, + `if (Test-Path -LiteralPath $base -PathType Container) {`, + `$maxEntries = ${maxEntries}`, + `$maxBytes = ${maxUtf8Bytes}`, + `$sentinel = ${powerShellLiteral(RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL)}`, + '$entryCount = 0', + '$outputBytes = 0', + '$iterator = ([System.IO.Directory]::EnumerateDirectories($base)).GetEnumerator()', + 'try {', + 'while ($iterator.MoveNext()) {', + 'if ($entryCount -ge $maxEntries) { Write-Output $sentinel; break }', + '$entryCount++', + '$name = [System.IO.Path]::GetFileName([string]$iterator.Current)', + '$lineBytes = [System.Text.Encoding]::UTF8.GetByteCount($name) + 2', + 'if (($outputBytes + $lineBytes) -gt $maxBytes) { Write-Output $sentinel; break }', + 'Write-Output $name', + '$outputBytes += $lineBytes', + '}', + '} finally {', + 'if ($null -ne $iterator) { $iterator.Dispose() }', + '}', + '}' + ].join('\n') + ) +} diff --git a/src/main/ssh/ssh-relay-bounded-marker-commands.test.ts b/src/main/ssh/ssh-relay-bounded-marker-commands.test.ts new file mode 100644 index 00000000000..50ab6e7235f --- /dev/null +++ b/src/main/ssh/ssh-relay-bounded-marker-commands.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { + SSH_RELAY_MARKER_MAX_BYTES, + posixReadRelayMarkerAssignment, + powerShellReadRelayMarkerAssignment +} from './ssh-relay-bounded-marker-commands' + +describe('bounded SSH relay marker commands', () => { + it('reads at most one byte past the Windows marker ceiling', () => { + const script = powerShellReadRelayMarkerAssignment('C:/Users/me/.orca-remote/.gc-owner') + + expect(script).toContain('[System.IO.File]::Open') + expect(script).toContain(`New-Object byte[] ${SSH_RELAY_MARKER_MAX_BYTES + 1}`) + expect(script).toContain(`-le ${SSH_RELAY_MARKER_MAX_BYTES}`) + expect(script).not.toContain('Get-Content') + expect(script).not.toContain('ReadAllText') + }) + + it('caps POSIX command substitution before comparing an owner token', () => { + const script = posixReadRelayMarkerAssignment('/home/u/.orca-remote/.gc-owner') + + expect(script).toContain(`bs=${SSH_RELAY_MARKER_MAX_BYTES + 1}`) + expect(script).toContain(`-gt ${SSH_RELAY_MARKER_MAX_BYTES}`) + expect(script).not.toContain('cat ') + }) +}) diff --git a/src/main/ssh/ssh-relay-bounded-marker-commands.ts b/src/main/ssh/ssh-relay-bounded-marker-commands.ts new file mode 100644 index 00000000000..4e052b6c161 --- /dev/null +++ b/src/main/ssh/ssh-relay-bounded-marker-commands.ts @@ -0,0 +1,23 @@ +import { shellEscape } from './ssh-connection-utils' +import { powerShellLiteral } from './ssh-remote-powershell' + +export const SSH_RELAY_MARKER_MAX_BYTES = 1024 + +export function powerShellReadRelayMarkerAssignment(markerPath: string): string { + return [ + '$orcaMarkerValue = $null', + '$orcaMarkerStream = $null', + `try { $orcaMarkerStream = [System.IO.File]::Open(${powerShellLiteral(markerPath)}, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, ([System.IO.FileShare]::ReadWrite -bor [System.IO.FileShare]::Delete))`, + `$orcaMarkerBuffer = New-Object byte[] ${SSH_RELAY_MARKER_MAX_BYTES + 1}`, + '$orcaMarkerCount = $orcaMarkerStream.Read($orcaMarkerBuffer, 0, $orcaMarkerBuffer.Length)', + `if ($orcaMarkerCount -le ${SSH_RELAY_MARKER_MAX_BYTES}) { $orcaMarkerValue = [System.Text.Encoding]::UTF8.GetString($orcaMarkerBuffer, 0, $orcaMarkerCount) }`, + '} catch {} finally { if ($null -ne $orcaMarkerStream) { $orcaMarkerStream.Dispose() } }' + ].join('; ') +} + +export function posixReadRelayMarkerAssignment(markerPath: string): string { + return [ + `orca_marker=$(dd if=${shellEscape(markerPath)} bs=${SSH_RELAY_MARKER_MAX_BYTES + 1} count=1 2>/dev/null) || orca_marker=`, + `if [ "\${#orca_marker}" -gt ${SSH_RELAY_MARKER_MAX_BYTES} ]; then orca_marker=; fi` + ].join('; ') +} diff --git a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts index 14f063e3bf1..82cccf1c0ff 100644 --- a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts +++ b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts @@ -14,8 +14,10 @@ vi.mock('electron', () => ({ })) vi.mock('fs', () => ({ - existsSync: vi.fn().mockReturnValue(true), - readFileSync: vi.fn().mockReturnValue('0.1.0+v2hash') + existsSync: vi.fn().mockReturnValue(true) +})) +vi.mock('../../shared/relay-version-marker', () => ({ + readRelayVersionMarkerSync: vi.fn().mockReturnValue('0.1.0+v2hash') })) vi.mock('./relay-protocol', () => ({ diff --git a/src/main/ssh/ssh-relay-deploy-helpers.test.ts b/src/main/ssh/ssh-relay-deploy-helpers.test.ts index 4df9c8ad28a..3e634492d25 100644 --- a/src/main/ssh/ssh-relay-deploy-helpers.test.ts +++ b/src/main/ssh/ssh-relay-deploy-helpers.test.ts @@ -3,7 +3,12 @@ import { describe, expect, it, vi } from 'vitest' import type { ClientChannel } from 'ssh2' import { execCommand, waitForSentinel } from './ssh-relay-deploy-helpers' import { shouldProbeBuildToolchainAfterNativeDepsFailure } from './ssh-relay-build-toolchain' -import { RELAY_SENTINEL } from './relay-protocol' +import { + HEADER_LENGTH, + MAX_BUFFERED_FRAME_CHUNKS, + MAX_MESSAGE_SIZE, + RELAY_SENTINEL +} from './relay-protocol' import { RelayVersionMismatchError, RELAY_EXIT_CODE_VERSION_MISMATCH @@ -204,6 +209,62 @@ describe('waitForSentinel', () => { expect(Buffer.concat(chunks)).toEqual(postSentinelPayload) expect(channel.close).not.toHaveBeenCalled() }) + + it('preserves the exact post-sentinel payload at the pending transport cap', async () => { + const channel = createMockChannel() + const transportPromise = waitForSentinel(channel) + const payload = Buffer.alloc((MAX_MESSAGE_SIZE + HEADER_LENGTH) * 2, 'p') + + channel.emit('data', Buffer.from(RELAY_SENTINEL)) + channel.emit('data', payload) + + const transport = await transportPromise + const chunks: Buffer[] = [] + transport.onData((chunk) => chunks.push(chunk)) + + expect(chunks).toHaveLength(1) + expect(chunks[0]?.equals(payload)).toBe(true) + expect(channel.close).not.toHaveBeenCalled() + }) + + it('closes before retaining post-sentinel bytes beyond the pending transport cap', async () => { + const channel = createMockChannel() + const transportPromise = waitForSentinel(channel) + + channel.emit('data', Buffer.from(RELAY_SENTINEL)) + channel.emit('data', Buffer.alloc((MAX_MESSAGE_SIZE + HEADER_LENGTH) * 2, 'p')) + channel.emit('data', Buffer.from('overflow')) + + const transport = await transportPromise + const onData = vi.fn() + const onClose = vi.fn() + transport.onData(onData) + transport.onClose(onClose) + + expect(onData).not.toHaveBeenCalled() + expect(onClose).toHaveBeenCalledOnce() + expect(channel.close).toHaveBeenCalledOnce() + }) + + it('closes before retaining too many post-sentinel fragments', async () => { + const channel = createMockChannel() + const transportPromise = waitForSentinel(channel) + + channel.emit('data', Buffer.from(RELAY_SENTINEL)) + for (let index = 0; index <= MAX_BUFFERED_FRAME_CHUNKS; index++) { + channel.emit('data', Buffer.from('p')) + } + + const transport = await transportPromise + const onData = vi.fn() + const onClose = vi.fn() + transport.onData(onData) + transport.onClose(onClose) + + expect(onData).not.toHaveBeenCalled() + expect(onClose).toHaveBeenCalledOnce() + expect(channel.close).toHaveBeenCalledOnce() + }) }) describe('execCommand', () => { diff --git a/src/main/ssh/ssh-relay-deploy-helpers.ts b/src/main/ssh/ssh-relay-deploy-helpers.ts index 80d2b78ced2..9294707af60 100644 --- a/src/main/ssh/ssh-relay-deploy-helpers.ts +++ b/src/main/ssh/ssh-relay-deploy-helpers.ts @@ -1,6 +1,12 @@ import type { ClientChannel } from 'ssh2' import { createSshOperationAbortError } from './ssh-connection-utils' -import { RELAY_SENTINEL, RELAY_SENTINEL_TIMEOUT_MS } from './relay-protocol' +import { + HEADER_LENGTH, + MAX_BUFFERED_FRAME_CHUNKS, + MAX_MESSAGE_SIZE, + RELAY_SENTINEL, + RELAY_SENTINEL_TIMEOUT_MS +} from './relay-protocol' import type { MultiplexerTransport } from './ssh-channel-multiplexer' import { buildRelayVersionMismatchError } from './ssh-relay-handshake-mismatch' @@ -10,6 +16,8 @@ export { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-exe // ── Sentinel detection ──────────────────────────────────────────────── const MAX_RELAY_STARTUP_BUFFER_BYTES = 64 * 1024 +// One maximum frame may be coalesced with the start of the next. +const MAX_PENDING_RELAY_BYTES = (MAX_MESSAGE_SIZE + HEADER_LENGTH) * 2 const RELAY_SENTINEL_BUFFER = Buffer.from(RELAY_SENTINEL, 'utf-8') export function waitForSentinel( @@ -162,19 +170,35 @@ export function waitForSentinel( notifyClosed() }) - // Why: data arriving in the same TCP chunk as the sentinel is buffered - // here. It's delivered on the first onData registration rather than - // immediately after resolve, because resolve schedules a microtask — - // the caller's `await` hasn't resumed yet, so no callbacks are - // registered when the synchronous code after resolve runs. - let pendingAfterSentinel: Buffer | null = null + // The caller cannot subscribe until the resolved promise resumes. + let pendingAfterSentinel: Buffer[] = [] + let pendingAfterSentinelBytes = 0 + + const bufferAfterSentinel = (data: Buffer): void => { + if (closedAfterSentinel) { + return + } + if ( + data.length > MAX_PENDING_RELAY_BYTES - pendingAfterSentinelBytes || + pendingAfterSentinel.length >= MAX_BUFFERED_FRAME_CHUNKS + ) { + pendingAfterSentinel = [] + pendingAfterSentinelBytes = 0 + notifyClosed() + channel.close() + return + } + pendingAfterSentinel.push(data) + pendingAfterSentinelBytes += data.length + } channel.on('data', (data: Buffer) => { if (sentinelReceived) { + if (closedAfterSentinel) { + return + } if (dataCallbacks.length === 0) { - pendingAfterSentinel = pendingAfterSentinel - ? Buffer.concat([pendingAfterSentinel, data]) - : data + bufferAfterSentinel(data) } else { for (const cb of dataCallbacks) { cb(data) @@ -211,19 +235,16 @@ export function waitForSentinel( const afterSentinel = data.subarray(Math.max(0, afterSentinelOffset)) if (afterSentinel.length > 0) { - pendingAfterSentinel = afterSentinel + bufferAfterSentinel(afterSentinel) } const transport: MultiplexerTransport = { write: (buf: Buffer) => channel.stdin.write(buf), onData: (cb) => { dataCallbacks.push(cb) - // Why: deliver buffered post-sentinel data to the first - // subscriber. This is the multiplexer constructor, which - // registers onData synchronously — the data is guaranteed - // to reach the decoder before any other frames arrive. - if (pendingAfterSentinel) { - const buf = pendingAfterSentinel - pendingAfterSentinel = null + if (pendingAfterSentinel.length > 0) { + const buf = Buffer.concat(pendingAfterSentinel, pendingAfterSentinelBytes) + pendingAfterSentinel = [] + pendingAfterSentinelBytes = 0 cb(buf) } }, diff --git a/src/main/ssh/ssh-relay-deploy.test.ts b/src/main/ssh/ssh-relay-deploy.test.ts index 28fe2a3e49c..6c608600ba8 100644 --- a/src/main/ssh/ssh-relay-deploy.test.ts +++ b/src/main/ssh/ssh-relay-deploy.test.ts @@ -101,7 +101,11 @@ function extractWindowsSockPath(script: string): string { } function extractWindowsMarkerPath(script: string): string { - return /-LiteralPath\s+'([^']*\.windows-active-pipe[^']*)'/.exec(script)?.[1] ?? '' + return ( + /(?:-LiteralPath\s+|\[System\.IO\.File\]::Open\()'([^']*\.windows-active-pipe[^']*)'/.exec( + script + )?.[1] ?? '' + ) } function makeMockConnection(): SshConnection { @@ -895,7 +899,11 @@ describe('deployAndLaunchRelay', () => { const markerPaths = mockExecCommand.mock.calls .map(([, command]) => decodePowerShellCommand(command)) - .filter((script): script is string => Boolean(script?.includes('Get-Content'))) + .filter((script): script is string => + Boolean( + script?.includes('[System.IO.File]::Open') && script.includes('.windows-active-pipe') + ) + ) .map(extractWindowsMarkerPath) expect(markerPaths).toHaveLength(2) diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index f2154d1eb13..136287ba918 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -64,6 +64,8 @@ import { MAX_SSH_RELAY_GRACE_PERIOD_SECONDS, MIN_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types' +import { powerShellReadRelayMarkerAssignment } from './ssh-relay-bounded-marker-commands' +import { windowsRelayTailLogCommand } from './ssh-windows-log-tail-command' export type RelayDeployResult = { transport: MultiplexerTransport @@ -1164,7 +1166,7 @@ async function readWindowsActiveRelayEndpoint( conn, hostPlatform, powerShellCommand( - `if (Test-Path -LiteralPath ${powerShellLiteral(markerPath)} -PathType Leaf) { Get-Content -LiteralPath ${powerShellLiteral(markerPath)} -Raw -ErrorAction SilentlyContinue }` + `${powerShellReadRelayMarkerAssignment(markerPath)}; if ($null -ne $orcaMarkerValue) { [Console]::Out.Write($orcaMarkerValue) }` ), { signal } ).catch(() => { @@ -1518,14 +1520,3 @@ function windowsRelayWaitCommand( ].join(' ') ) } - -function windowsRelayTailLogCommand(logFile: string, errFile: string): string { - const script = [ - `$out = if (Test-Path -LiteralPath ${powerShellLiteral(logFile)}) { Get-Content -LiteralPath ${powerShellLiteral(logFile)} -Tail 20 -ErrorAction SilentlyContinue } else { '(no stdout log)' }`, - `$err = if (Test-Path -LiteralPath ${powerShellLiteral(errFile)}) { Get-Content -LiteralPath ${powerShellLiteral(errFile)} -Tail 20 -ErrorAction SilentlyContinue } else { '(no stderr log)' }`, - 'Write-Output $out', - "Write-Output '--- stderr ---'", - 'Write-Output $err' - ].join('; ') - return powerShellCommand(script) -} diff --git a/src/main/ssh/ssh-relay-gc-claim.ts b/src/main/ssh/ssh-relay-gc-claim.ts index d0c8cc1221f..ddc6b178c5b 100644 --- a/src/main/ssh/ssh-relay-gc-claim.ts +++ b/src/main/ssh/ssh-relay-gc-claim.ts @@ -9,6 +9,10 @@ import { } from './ssh-relay-install-lock-commands' import { removeRemoteTreeCommand } from './ssh-remote-commands' import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' +import { + posixReadRelayMarkerAssignment, + powerShellReadRelayMarkerAssignment +} from './ssh-relay-bounded-marker-commands' import { getRemoteHostPlatform, isWindowsRemoteHost, @@ -131,9 +135,9 @@ export async function isRelayGcClaimOwned( const ownerPath = joinRemotePath(host, relayGcClaimPath(remoteRelayDir), RELAY_GC_OWNER_NAME) const command = isWindowsRemoteHost(host) ? powerShellCommand( - `if ((Get-Content -LiteralPath ${powerShellLiteral(ownerPath)} -Raw -ErrorAction SilentlyContinue) -ceq ${powerShellLiteral(token)}) { 'OWNED' } else { 'LOST' }` + `${powerShellReadRelayMarkerAssignment(ownerPath)}; if ($orcaMarkerValue -ceq ${powerShellLiteral(token)}) { 'OWNED' } else { 'LOST' }` ) - : `test "$(cat ${shellEscape(ownerPath)} 2>/dev/null)" = ${shellEscape(token)} && echo OWNED || echo LOST` + : `${posixReadRelayMarkerAssignment(ownerPath)}; test "$orca_marker" = ${shellEscape(token)} && echo OWNED || echo LOST` const output = await execHostCommand(conn, host, command).catch(() => 'LOST') return output.trim() === 'OWNED' } @@ -152,15 +156,17 @@ export async function releaseRelayGcClaim( ? powerShellCommand( `$claim = ${powerShellLiteral(claimPath)}; ` + `if (-not (Test-Path -LiteralPath $claim)) { 'RELEASED' } ` + - `elseif ((Get-Content -LiteralPath ${powerShellLiteral(ownerPath)} -Raw -ErrorAction SilentlyContinue) -cne ${powerShellLiteral(token)}) { 'LOST' } ` + + `else { ${powerShellReadRelayMarkerAssignment(ownerPath)}; ` + + `if ($orcaMarkerValue -cne ${powerShellLiteral(token)}) { 'LOST' } ` + 'else { try { Remove-Item -LiteralPath $claim -Recurse -Force -ErrorAction Stop } catch {}; ' + - "if (Test-Path -LiteralPath $claim) { 'UNKNOWN' } else { 'RELEASED' } }" + "if (Test-Path -LiteralPath $claim) { 'UNKNOWN' } else { 'RELEASED' } } }" ) : [ `if ! test -e ${shellEscape(claimPath)}; then echo RELEASED;`, - `elif test "$(cat ${shellEscape(ownerPath)} 2>/dev/null)" != ${shellEscape(token)}; then echo LOST;`, + `else ${posixReadRelayMarkerAssignment(ownerPath)};`, + `if test "$orca_marker" != ${shellEscape(token)}; then echo LOST;`, `else ${removeRemoteTreeCommand(host, claimPath)} 2>/dev/null;`, - `if test -e ${shellEscape(claimPath)}; then echo UNKNOWN; else echo RELEASED; fi; fi` + `if test -e ${shellEscape(claimPath)}; then echo UNKNOWN; else echo RELEASED; fi; fi; fi` ].join(' ') const output = await execHostCommand(conn, host, command).catch(() => 'UNKNOWN') switch (output.trim()) { diff --git a/src/main/ssh/ssh-relay-install-lock-commands.ts b/src/main/ssh/ssh-relay-install-lock-commands.ts index d6eab561a2d..05891683243 100644 --- a/src/main/ssh/ssh-relay-install-lock-commands.ts +++ b/src/main/ssh/ssh-relay-install-lock-commands.ts @@ -92,7 +92,10 @@ function posixStealInstallLockCommand(lockDir: string, staleAfterSeconds: number 'steal="$steal_root.$steal_generation";', 'done;', 'if [ "$owns_steal" = 1 ]; then', - `trap 'rm -rf "$steal_root".* 2>/dev/null || true; rm -rf "$lock_tombstone" 2>/dev/null || true' EXIT;`, + 'steal_parent=${steal_root%/*};', + 'steal_leaf=${steal_root##*/};', + '[ "$steal_parent" != "$steal_root" ] || steal_parent=.;', + `trap 'find "$steal_parent" -mindepth 1 -maxdepth 1 -name "$steal_leaf.*" -exec rm -rf {} + 2>/dev/null || true; rm -rf "$lock_tombstone" 2>/dev/null || true' EXIT;`, `${posixLockIdentityAssignment(lockDir, 'current_key')} && current_mtime=\${current_key%%:*} && current_now=$(date +%s) && current_age=$((current_now - current_mtime)) || current_age=0;`, `if [ "$current_key" = "$lock_key" ] && [ "\${current_age:-0}" -gt ${staleAfterSeconds} ] 2>/dev/null; then`, `lock_tombstone=${escapedLockDir}.tombstone.$$.$(date +%s);`, diff --git a/src/main/ssh/ssh-relay-liveness-probe-source.test.ts b/src/main/ssh/ssh-relay-liveness-probe-source.test.ts new file mode 100644 index 00000000000..7c74cf71ef9 --- /dev/null +++ b/src/main/ssh/ssh-relay-liveness-probe-source.test.ts @@ -0,0 +1,156 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { GENERATED_NODE_MANAGED_FILE_MAX_BYTES } from '../generated-node-bounded-file-reader' +import { + getWindowsRelayLivenessProbeSource, + WINDOWS_RELAY_LIVENESS_DIRECTORY_BUFFER_SIZE, + WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE, + WINDOWS_RELAY_LIVENESS_MAX_DIRECTORY_ENTRIES, + WINDOWS_RELAY_LIVENESS_MAX_PIPE_PATHS +} from './ssh-relay-liveness-probe-source' + +const PIPE_A = '\\\\.\\pipe\\orca-relay-00000000000000000000' +const PIPE_B = '\\\\.\\pipe\\orca-relay-11111111111111111111' +const PIPE_C = '\\\\.\\pipe\\orca-relay-22222222222222222222' + +function runProbe( + directory: string, + options: { + maxDirectoryEntries: number + maxPipePaths: number + seedPipePaths?: string[] + } +): string { + const source = getWindowsRelayLivenessProbeSource({ + maxDirectoryEntries: options.maxDirectoryEntries, + maxPipePaths: options.maxPipePaths, + directoryBufferSize: 1 + }) + const result = spawnSync( + process.execPath, + ['-e', source, directory, ...(options.seedPipePaths ?? [])], + { + encoding: 'utf8', + timeout: 5_000 + } + ) + if (result.status !== 0) { + throw new Error(`probe exited ${result.status}: ${result.stderr}`) + } + return result.stdout +} + +function withTempDirectory(run: (directory: string) => void): void { + const directory = mkdtempSync(join(tmpdir(), 'orca-relay-liveness-')) + try { + run(directory) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +describe('Windows relay liveness probe source', () => { + it('uses fixed-capacity incremental directory iteration', () => { + const source = getWindowsRelayLivenessProbeSource() + + expect(source).toContain( + `maxEntries=${WINDOWS_RELAY_LIVENESS_MAX_DIRECTORY_ENTRIES},maxPipes=${WINDOWS_RELAY_LIVENESS_MAX_PIPE_PATHS}` + ) + expect(source).toContain( + `opendirSync(dir,{bufferSize:${WINDOWS_RELAY_LIVENESS_DIRECTORY_BUFFER_SIZE}})` + ) + expect(source).toContain('directory.readSync()') + expect(source).not.toContain('readdirSync') + expect(source).not.toContain('pipes.includes') + }) + + it('allows exactly the directory-entry limit', () => { + withTempDirectory((directory) => { + writeFileSync(join(directory, 'one'), '') + writeFileSync(join(directory, 'two'), '') + + expect( + runProbe(directory, { + maxDirectoryEntries: 2, + maxPipePaths: 1, + seedPipePaths: [PIPE_A] + }) + ).toBe('WAITING') + }) + }) + + it('fails closed when the incremental iterator finds one entry beyond the limit', () => { + withTempDirectory((directory) => { + writeFileSync(join(directory, 'one'), '') + writeFileSync(join(directory, 'two'), '') + writeFileSync(join(directory, 'three'), '') + + expect( + runProbe(directory, { + maxDirectoryEntries: 2, + maxPipePaths: 1, + seedPipePaths: [PIPE_A] + }) + ).toBe(WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE) + }) + }) + + it('allows exactly the retained-pipe limit', () => { + withTempDirectory((directory) => { + writeFileSync(join(directory, '.windows-active-pipe-a'), PIPE_A) + writeFileSync(join(directory, '.windows-active-pipe-b'), PIPE_B) + + expect( + runProbe(directory, { + maxDirectoryEntries: 2, + maxPipePaths: 2 + }) + ).toBe('WAITING') + }) + }) + + it('fails closed before retaining a pipe beyond the limit', () => { + withTempDirectory((directory) => { + writeFileSync(join(directory, '.windows-active-pipe-a'), PIPE_A) + writeFileSync(join(directory, '.windows-active-pipe-b'), PIPE_B) + writeFileSync(join(directory, '.windows-active-pipe-c'), PIPE_C) + + expect( + runProbe(directory, { + maxDirectoryEntries: 3, + maxPipePaths: 2 + }) + ).toBe(WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE) + }) + }) + + it('fails closed when a marker exceeds its file-size limit', () => { + withTempDirectory((directory) => { + writeFileSync( + join(directory, '.windows-active-pipe-oversized'), + 'x'.repeat(GENERATED_NODE_MANAGED_FILE_MAX_BYTES + 1) + ) + + expect( + runProbe(directory, { + maxDirectoryEntries: 1, + maxPipePaths: 1 + }) + ).toBe(WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE) + }) + }) + + it('preserves the missing-directory liveness result', () => { + withTempDirectory((directory) => { + expect( + runProbe(join(directory, 'missing'), { + maxDirectoryEntries: 1, + maxPipePaths: 1 + }) + ).toBe('ALIVE') + }) + }) +}) diff --git a/src/main/ssh/ssh-relay-liveness-probe-source.ts b/src/main/ssh/ssh-relay-liveness-probe-source.ts new file mode 100644 index 00000000000..bc2e8063cc1 --- /dev/null +++ b/src/main/ssh/ssh-relay-liveness-probe-source.ts @@ -0,0 +1,91 @@ +import { getGeneratedNodeBoundedFileReaderSourceLines } from '../generated-node-bounded-file-reader' + +export const WINDOWS_RELAY_LIVENESS_MAX_DIRECTORY_ENTRIES = 4_096 +export const WINDOWS_RELAY_LIVENESS_MAX_PIPE_PATHS = 1_024 +export const WINDOWS_RELAY_LIVENESS_DIRECTORY_BUFFER_SIZE = 32 +export const WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE = 'INCONCLUSIVE' + +type WindowsRelayLivenessProbeSourceOptions = { + maxDirectoryEntries?: number + maxPipePaths?: number + directoryBufferSize?: number +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback + if (!Number.isSafeInteger(resolved) || resolved < 1) { + throw new Error(`${name} must be a positive integer`) + } + return resolved +} + +export function getWindowsRelayLivenessProbeSource( + options?: WindowsRelayLivenessProbeSourceOptions +): string { + const maxDirectoryEntries = positiveInteger( + options?.maxDirectoryEntries, + WINDOWS_RELAY_LIVENESS_MAX_DIRECTORY_ENTRIES, + 'maxDirectoryEntries' + ) + const maxPipePaths = positiveInteger( + options?.maxPipePaths, + WINDOWS_RELAY_LIVENESS_MAX_PIPE_PATHS, + 'maxPipePaths' + ) + const directoryBufferSize = positiveInteger( + options?.directoryBufferSize, + WINDOWS_RELAY_LIVENESS_DIRECTORY_BUFFER_SIZE, + 'directoryBufferSize' + ) + const boundedReadSource = getGeneratedNodeBoundedFileReaderSourceLines().join('') + + return [ + 'const fs=require("fs"),path=require("path"),net=require("net");', + boundedReadSource, + 'const [dir,...seed]=process.argv.slice(1);', + 'const valid=/^\\\\\\\\[.?]\\\\pipe\\\\orca-relay-[0-9a-f]{20}$/i;', + `const maxEntries=${maxDirectoryEntries},maxPipes=${maxPipePaths};`, + 'const pipes=[],seen=new Set();', + 'let markerCount=0,capacityExceeded=false;', + 'function addPipe(p){', + 'if(!valid.test(p)||seen.has(p))return;', + 'if(pipes.length>=maxPipes){capacityExceeded=true;return}', + 'seen.add(p);pipes.push(p)', + '}', + 'for(const p of seed){addPipe(p);if(capacityExceeded)break}', + 'let directory;', + 'if(!capacityExceeded)try{', + `directory=fs.opendirSync(dir,{bufferSize:${directoryBufferSize}});`, + 'let entryCount=0;', + 'while(true){', + 'const entry=directory.readSync();', + 'if(!entry)break;', + 'if(entryCount>=maxEntries){capacityExceeded=true;break}', + 'entryCount++;', + 'const name=entry.name;', + 'if(!name.startsWith(".windows-active-pipe-"))continue;', + 'markerCount++;', + 'const p=readOrcaManagedFileWithinLimit(fs,path.join(dir,name)).trim();', + 'addPipe(p);', + 'if(capacityExceeded)break', + '}', + '}catch(error){if(error&&error.code==="EFBIG")capacityExceeded=true}', + 'finally{if(directory)try{directory.closeSync()}catch{}}', + `if(capacityExceeded){process.stdout.write("${WINDOWS_RELAY_LIVENESS_INCONCLUSIVE_STATE}");process.exit(0)}`, + 'if(markerCount===0&&pipes.length===0){process.stdout.write("ALIVE");process.exit(0)}', + 'let i=0;', + 'function done(ok){process.stdout.write(ok?"ALIVE":"WAITING")}', + 'function next(){', + 'const pipe=pipes[i++];', + 'if(!pipe)return done(false);', + 'const s=net.connect(pipe);', + 'let settled=false;', + 'function finish(ok){if(settled)return;settled=true;s.destroy();if(ok)done(true);else next()}', + 's.setTimeout(200);', + 's.on("connect",()=>finish(true));', + 's.on("timeout",()=>finish(false));', + 's.on("error",()=>finish(false));', + '}', + 'next();' + ].join('') +} diff --git a/src/main/ssh/ssh-relay-session-data-delivery.test.ts b/src/main/ssh/ssh-relay-session-data-delivery.test.ts index 901f7940d31..61283ffae0e 100644 --- a/src/main/ssh/ssh-relay-session-data-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-data-delivery.test.ts @@ -56,6 +56,12 @@ vi.mock('../ipc/pty', () => ({ setPtyOwnership: vi.fn() })) +vi.mock('../ipc/pty-renderer-delivery-router', () => ({ + routeExternalPtyData: vi.fn(), + routeExternalPtyReplay: vi.fn(), + routeExternalPtyExit: vi.fn() +})) + vi.mock('../providers/ssh-filesystem-dispatch', () => ({ registerSshFilesystemProvider: vi.fn(), unregisterSshFilesystemProvider: vi.fn(), @@ -68,6 +74,7 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ })) const { registerSshPtyProvider } = await import('../ipc/pty') +const { routeExternalPtyData } = await import('../ipc/pty-renderer-delivery-router') describe('SshRelaySession data delivery', () => { beforeEach(() => { @@ -76,16 +83,9 @@ describe('SshRelaySession data delivery', () => { mockDeploySuccess() }) - it('delivers empty transformed relay spans with raw sequence metadata', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() - const runtime = { onPtyData: vi.fn(() => 17), onPtyExit: vi.fn() } - const session = new SshRelaySession( - 'target-1', - getMainWindow, - mockStore, - mockPortForward, - runtime as never - ) + it('routes empty transformed relay spans with raw sequence metadata', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) const ptyProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { onData: ReturnType @@ -99,14 +99,11 @@ describe('SshRelaySession data delivery', () => { onData({ id: 'ssh-pty-1', data: '', sequenceChars: 9, transformed: true }) - expect(runtime.onPtyData).toHaveBeenCalledWith('ssh-pty-1', '', expect.any(Number), 9, true) - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:data', { + expect(routeExternalPtyData).toHaveBeenCalledWith({ id: 'ssh-pty-1', data: '', sequenceChars: 9, - transformed: true, - seq: 17, - rawLength: 9 + transformed: true }) }) }) diff --git a/src/main/ssh/ssh-relay-session-incarnation.test.ts b/src/main/ssh/ssh-relay-session-incarnation.test.ts index a98180308b2..828de0fcef4 100644 --- a/src/main/ssh/ssh-relay-session-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-incarnation.test.ts @@ -51,6 +51,11 @@ vi.mock('../ipc/pty', () => ({ isCurrentPtyExit: vi.fn(() => true), answerStartupTerminalColorQueriesForPty: vi.fn((_id: string, data: string) => data) })) +vi.mock('../ipc/pty-renderer-delivery-router', () => ({ + routeExternalPtyData: vi.fn(), + routeExternalPtyReplay: vi.fn(), + routeExternalPtyExit: vi.fn() +})) vi.mock('../providers/ssh-filesystem-dispatch', () => ({ registerSshFilesystemProvider: vi.fn(), unregisterSshFilesystemProvider: vi.fn(), @@ -63,6 +68,7 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ const { registerSshPtyProvider, clearProviderPtyState, deletePtyOwnership, isCurrentPtyExit } = await import('../ipc/pty') +const { routeExternalPtyExit } = await import('../ipc/pty-renderer-delivery-router') describe('SSH relay PTY incarnation exits', () => { beforeEach(() => { @@ -73,7 +79,7 @@ describe('SSH relay PTY incarnation exits', () => { }) it('drops a stale exit before ownership cleanup and propagates a current incarnation', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const runtime = { onPtyData: vi.fn(), onPtyExit: vi.fn() } const session = new SshRelaySession( 'target-1', @@ -99,13 +105,19 @@ describe('SSH relay PTY incarnation exits', () => { expect(deletePtyOwnership).not.toHaveBeenCalled() expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalled() expect(runtime.onPtyExit).not.toHaveBeenCalled() - expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + expect(routeExternalPtyExit).not.toHaveBeenCalled() - onExit({ id: 'ssh:target-1@@pty-reused', code: 7, incarnationId: 'current-incarnation' }) + const currentExit = { + id: 'ssh:target-1@@pty-reused', + code: 7, + incarnationId: 'current-incarnation' + } + onExit(currentExit) expect(runtime.onPtyExit).toHaveBeenCalledWith( 'ssh:target-1@@pty-reused', 7, 'current-incarnation' ) + expect(routeExternalPtyExit).toHaveBeenCalledWith(currentExit) }) }) diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index 1a3e8721064..ce89762c602 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -53,6 +53,11 @@ vi.mock('../ipc/pty', () => ({ isCurrentPtyExit: vi.fn(() => true), answerStartupTerminalColorQueriesForPty: vi.fn((_id: string, data: string) => data) })) +vi.mock('../ipc/pty-renderer-delivery-router', () => ({ + routeExternalPtyData: vi.fn(), + routeExternalPtyReplay: vi.fn(), + routeExternalPtyExit: vi.fn() +})) vi.mock('../providers/ssh-filesystem-dispatch', () => ({ registerSshFilesystemProvider: vi.fn(), unregisterSshFilesystemProvider: vi.fn(), @@ -70,6 +75,7 @@ const { setPtyOwnership, restorePtyIncarnation } = await import('../ipc/pty') +const { routeExternalPtyReplay } = await import('../ipc/pty-renderer-delivery-router') const APP_PTY_ID = 'ssh:target-1@@pty-live' const INCARNATION_LEAF_ID = '11111111-1111-4111-8111-111111111111' @@ -146,7 +152,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { }) it('does not restore a PTY whose matching exit shares the attach reply batch', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const incarnationId = 'incarnation-exited-during-attach' const runtime = { acceptPtyIncarnationForExit: vi.fn(), @@ -186,15 +192,11 @@ describe('SshRelaySession reconnect incarnation ordering', () => { 'pty-live', 'terminated' ) - expect( - vi - .mocked(mockWindow.webContents.send) - .mock.calls.some(([channel]) => channel === 'pty:replay') - ).toBe(false) + expect(routeExternalPtyReplay).not.toHaveBeenCalled() }) it('ignores an older incarnation exit while reconnecting a reused PTY id', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const currentIncarnationId = 'incarnation-current' const runtime = { acceptPtyIncarnationForExit: vi.fn(), @@ -237,7 +239,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { expect(mockStore.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ ptyId: APP_PTY_ID, incarnationId: currentIncarnationId }) ) - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', { + expect(routeExternalPtyReplay).toHaveBeenCalledWith({ id: APP_PTY_ID, data: 'live-output' }) diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index 8fe7a292bcf..fc055643fc6 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -74,6 +74,12 @@ vi.mock('../ipc/pty', () => ({ isCurrentPtyExit: vi.fn(() => true) })) +vi.mock('../ipc/pty-renderer-delivery-router', () => ({ + routeExternalPtyData: vi.fn(), + routeExternalPtyReplay: vi.fn(), + routeExternalPtyExit: vi.fn() +})) + vi.mock('../providers/ssh-filesystem-dispatch', () => ({ registerSshFilesystemProvider: vi.fn(), unregisterSshFilesystemProvider: vi.fn(), @@ -86,12 +92,6 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ })) const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') -// Why: the hidden-delivery gate module is intentionally real (pure state, no -// electron deps) so the SSH parity tests exercise the same gate main uses. -const { markHiddenRendererPty, setRendererPtyDeliveryInterest } = - await import('../ipc/pty-hidden-delivery-gate') -const { _resetHiddenRendererPtyDeliveryGateForTest } = - await import('../ipc/pty-hidden-delivery-gate') const { execCommand } = await import('./ssh-relay-deploy-helpers') const { getRemoteHostPlatform } = await import('./ssh-remote-platform') const { @@ -106,6 +106,8 @@ const { registerSshFilesystemProvider, unregisterSshFilesystemProvider } = await import('../providers/ssh-filesystem-dispatch') const { registerSshGitProvider, unregisterSshGitProvider } = await import('../providers/ssh-git-dispatch') +const { routeExternalPtyData, routeExternalPtyReplay, routeExternalPtyExit } = + await import('../ipc/pty-renderer-delivery-router') describe('SshRelaySession', () => { beforeEach(() => { @@ -115,80 +117,10 @@ describe('SshRelaySession', () => { muxRequestMock.mockResolvedValue([]) mockDeploySuccess() vi.mocked(getPtyIdsForConnection).mockReturnValue([]) - _resetHiddenRendererPtyDeliveryGateForTest() }) - it('drops hidden-gated PTY data after runtime ingestion with one restore marker', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() - const runtime = { - onPtyData: vi.fn(() => 99), - onPtyExit: vi.fn() - } - const session = new SshRelaySession( - 'target-1', - getMainWindow, - mockStore, - mockPortForward, - runtime as never - ) - await session.establish(mockConn) - const ptyProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { - onData: ReturnType - } - const onData = ptyProvider.onData.mock.calls[0]?.[0] as (payload: { - id: string - data: string - }) => void - - markHiddenRendererPty('ssh-pty-1') - onData({ id: 'ssh-pty-1', data: 'hidden ssh output' }) - - // Runtime ingestion still ran; renderer delivery shrank to one marker. - expect(runtime.onPtyData).toHaveBeenCalledWith( - 'ssh-pty-1', - 'hidden ssh output', - expect.any(Number), - 'hidden ssh output'.length, - undefined - ) - expect(mockWindow.webContents.send).toHaveBeenCalledTimes(1) - // Why out-of-band: an in-band empty pty:data sentinel is ambiguous with - // chunks fully consumed by renderer OSC-9999 stripping. - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:modelRestoreNeeded', { - id: 'ssh-pty-1', - reason: 'hidden-drop', - markerSeq: 99 - }) - - onData({ id: 'ssh-pty-1', data: 'more hidden ssh output' }) - expect(mockWindow.webContents.send).toHaveBeenCalledTimes(1) - - // Delivery interest (renderer sidecars) suppresses the gate — parity with - // the local path in ipc/pty.ts. - setRendererPtyDeliveryInterest('ssh-pty-1', true) - onData({ id: 'ssh-pty-1', data: 'sidecar ssh bytes' }) - expect(mockWindow.webContents.send).toHaveBeenLastCalledWith('pty:data', { - id: 'ssh-pty-1', - data: 'sidecar ssh bytes', - seq: 99, - rawLength: 'sidecar ssh bytes'.length - }) - - // Non-hidden PTYs are unaffected. - onData({ id: 'ssh-pty-2', data: 'visible ssh output' }) - expect(mockWindow.webContents.send).toHaveBeenLastCalledWith('pty:data', { - id: 'ssh-pty-2', - data: 'visible ssh output', - seq: 99, - rawLength: 'visible ssh output'.length - }) - }) - - it('keeps hidden SSH delivery when the gate kill switch is off', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() - ;(mockStore as unknown as { getSettings: () => unknown }).getSettings = vi.fn(() => ({ - terminalHiddenDeliveryGate: false - })) + it('routes SSH PTY data and its captured producer credit through bounded delivery', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) const ptyProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { @@ -197,15 +129,16 @@ describe('SshRelaySession', () => { const onData = ptyProvider.onData.mock.calls[0]?.[0] as (payload: { id: string data: string + upstreamCredit?: { charCount: number; acknowledge(chars: number): void } }) => void + const upstreamCredit = { charCount: 16, acknowledge: vi.fn() } - markHiddenRendererPty('ssh-pty-1') - onData({ id: 'ssh-pty-1', data: 'still delivered' }) + onData({ id: 'ssh-pty-1', data: 'remote PTY bytes', upstreamCredit }) - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:data', { + expect(routeExternalPtyData).toHaveBeenCalledWith({ id: 'ssh-pty-1', - data: 'still delivered', - rawLength: 'still delivered'.length + data: 'remote PTY bytes', + upstreamCredit }) }) @@ -479,7 +412,7 @@ describe('SshRelaySession', () => { }) it('forwards reconnect replay after the attach attempt is still current', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) vi.clearAllMocks() @@ -495,14 +428,14 @@ describe('SshRelaySession', () => { await session.reconnect(mockConn) - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', { + expect(routeExternalPtyReplay).toHaveBeenCalledWith({ id: 'ssh:target-1@@pty-1', data: 'restored-output' }) }) it('drops identical reconnect replay payloads inside one reconnect burst', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) vi.clearAllMocks() @@ -519,11 +452,8 @@ describe('SshRelaySession', () => { await session.reconnect(mockConn) await session.reconnect(mockConn) - const replaySends = vi - .mocked(mockWindow.webContents.send) - .mock.calls.filter(([channel]) => channel === 'pty:replay') expect(mockAttach).toHaveBeenCalledTimes(2) - expect(replaySends).toHaveLength(1) + expect(routeExternalPtyReplay).toHaveBeenCalledTimes(1) }) it('establish re-attaches owned PTYs after explicit disconnect', async () => { @@ -612,7 +542,7 @@ describe('SshRelaySession', () => { }) it('does not expire a live reused relay id when attach rejects identity mismatch', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) vi.clearAllMocks() @@ -647,7 +577,7 @@ describe('SshRelaySession', () => { expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith('target-1', 'pty-1', 'expired') - expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', { + expect(routeExternalPtyExit).not.toHaveBeenCalledWith({ id: 'ssh:target-1@@pty-1', code: -1 }) @@ -718,7 +648,7 @@ describe('SshRelaySession', () => { }) it('invalidates and broadcasts remote PTYs that cannot reattach after relay reconnect', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) vi.clearAllMocks() @@ -741,7 +671,7 @@ describe('SshRelaySession', () => { expect(mockAttach).toHaveBeenCalledWith('pty-live') expect(clearProviderPtyState).toHaveBeenCalledWith('ssh:target-1@@pty-stale') expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:target-1@@pty-stale') - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { + expect(routeExternalPtyExit).toHaveBeenCalledWith({ id: 'ssh:target-1@@pty-stale', code: -1 }) diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index fdd0b115203..c3070ae4c1a 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -37,10 +37,10 @@ import { isCurrentPtyExit } from '../ipc/pty' import { - recordHiddenRendererPtyDataDrop, - shouldDropHiddenRendererPtyData -} from '../ipc/pty-hidden-delivery-gate' -import type { PtyModelRestoreNeededEvent } from '../../shared/pty-model-restore-marker' + routeExternalPtyData, + routeExternalPtyExit, + routeExternalPtyReplay +} from '../ipc/pty-renderer-delivery-router' import { registerSshFilesystemProvider, unregisterSshFilesystemProvider, @@ -919,45 +919,10 @@ export class SshRelaySession { private wireUpPtyEvents(ptyProvider: SshPtyProvider): void { ptyProvider.onData((payload) => { - const rawLength = payload.sequenceChars ?? payload.data.length - const seq = this.runtime?.onPtyData( - payload.id, - payload.data, - Date.now(), - rawLength, - payload.transformed - ) - const win = this.getMainWindow() - if (!win || win.isDestroyed()) { - return - } - // Why: hidden-delivery gate parity with ipc/pty.ts — latch model-restore out-of-band, never an in-band pty:data sentinel (OSC-9999-only chunks strip to empty). - const store = this.store as { getSettings?: Store['getSettings'] } - if (shouldDropHiddenRendererPtyData(payload.id, store.getSettings?.())) { - const drop = recordHiddenRendererPtyDataDrop(payload.id, rawLength) - if (drop.shouldEmitRestoreMarker) { - win.webContents.send('pty:modelRestoreNeeded', { - id: payload.id, - reason: 'hidden-drop', - ...(typeof seq === 'number' ? { markerSeq: seq } : {}) - } satisfies PtyModelRestoreNeededEvent) - } - return - } - if (payload.data.length > 0 || payload.transformed) { - win.webContents.send('pty:data', { - ...payload, - ...(typeof seq === 'number' ? { seq } : {}), - rawLength, - ...(payload.transformed ? { transformed: true } : {}) - }) - } + routeExternalPtyData(payload) }) ptyProvider.onReplay((payload) => { - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:replay', payload) - } + routeExternalPtyReplay(payload) }) ptyProvider.onExit((payload) => { const pendingReattach = this.pendingPtyReattaches.get(payload.id) @@ -980,10 +945,7 @@ export class SshRelaySession { this.forwardedReattachReplayByPty.delete(payload.id) this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'terminated') this.runtime?.onPtyExit(payload.id, payload.code, payload.incarnationId) - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', payload) - } + routeExternalPtyExit(payload) } private replayFingerprint(data: string): string { @@ -1008,10 +970,7 @@ export class SshRelaySession { if (!data || !this.shouldForwardReattachReplay(appPtyId, data)) { return } - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:replay', { id: appPtyId, data }) - } + routeExternalPtyReplay({ id: appPtyId, data }) } private async reattachKnownPtys(shouldContinue: () => boolean): Promise { @@ -1125,10 +1084,7 @@ export class SshRelaySession { this.forwardedReattachReplayByPty.delete(appPtyId) this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') // Why: reattach failure means the remote process is gone; tell the renderer to clear the stale pane. - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) - } + routeExternalPtyExit({ id: appPtyId, code: -1 }) } finally { if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) { this.pendingPtyReattaches.delete(appPtyId) diff --git a/src/main/ssh/ssh-relay-versioned-install.test.ts b/src/main/ssh/ssh-relay-versioned-install.test.ts index 23d57757b5b..1b6390879a8 100644 --- a/src/main/ssh/ssh-relay-versioned-install.test.ts +++ b/src/main/ssh/ssh-relay-versioned-install.test.ts @@ -1,8 +1,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('fs', () => ({ - existsSync: vi.fn(), - readFileSync: vi.fn() + existsSync: vi.fn() +})) + +const readRelayVersionMarkerSyncMock = vi.hoisted(() => vi.fn()) +vi.mock('../../shared/relay-version-marker', () => ({ + readRelayVersionMarkerSync: readRelayVersionMarkerSyncMock })) vi.mock('./ssh-relay-deploy-helpers', () => ({ @@ -13,7 +17,7 @@ vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { readLocalFullVersion, computeRemoteRelayDir, @@ -34,11 +38,11 @@ import { import { execCommand } from './ssh-relay-deploy-helpers' import { getRemoteHostPlatform } from './ssh-remote-platform' import type { SshConnection } from './ssh-connection' +import { RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL } from './ssh-relay-base-directory-listing' const conn = {} as SshConnection const mockExec = vi.mocked(execCommand) const mockExists = vi.mocked(existsSync) -const mockRead = vi.mocked(readFileSync) function decodePowerShellCommand(command: string): string { const match = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/) @@ -52,7 +56,7 @@ describe('readLocalFullVersion', () => { it('returns trimmed contents of the .version file', () => { mockExists.mockReturnValue(true) - mockRead.mockReturnValue('0.1.0+deadbeef\n') + readRelayVersionMarkerSyncMock.mockReturnValue('0.1.0+deadbeef') expect(readLocalFullVersion('/local/relay')).toBe('0.1.0+deadbeef') }) @@ -63,7 +67,7 @@ describe('readLocalFullVersion', () => { it('throws when the .version file is empty', () => { mockExists.mockReturnValue(true) - mockRead.mockReturnValue(' \n') + readRelayVersionMarkerSyncMock.mockReturnValue('') expect(() => readLocalFullVersion('/local/relay')).toThrow(/is empty/) }) }) @@ -509,7 +513,9 @@ describe('relay GC claim', () => { const releaseScript = decodePowerShellCommand(mockExec.mock.calls[2]?.[1] ?? '') expect(ownerScript).toContain('Set-Content -LiteralPath') expect(ownerScript).toContain('.gc-claim/.gc-owner') - expect(releaseScript).toContain('Get-Content -LiteralPath') + expect(releaseScript).toContain('[System.IO.File]::Open') + expect(releaseScript).toContain('New-Object byte[] 1025') + expect(releaseScript).not.toContain('Get-Content') expect(releaseScript).toContain('-cne') expect(releaseScript).toContain('Remove-Item -LiteralPath') expect(releaseScript).toContain("'RELEASED'") @@ -570,6 +576,16 @@ describe('gcOldRelayVersions', () => { ).toBe(false) }) + it('does not inspect or delete entries from a capacity-limited listing', async () => { + mockExec.mockResolvedValueOnce( + `relay-0.1.0+aaa\n${RELAY_BASE_DIRECTORY_LISTING_LIMIT_SENTINEL}\n` + ) + + await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb') + + expect(mockExec).toHaveBeenCalledTimes(1) + }) + it('cleans only strict POSIX orphan tombstones even with no relay candidates', async () => { mockExec .mockResolvedValueOnce( diff --git a/src/main/ssh/ssh-relay-versioned-install.ts b/src/main/ssh/ssh-relay-versioned-install.ts index 9945c997ef2..e35b9de6c80 100644 --- a/src/main/ssh/ssh-relay-versioned-install.ts +++ b/src/main/ssh/ssh-relay-versioned-install.ts @@ -5,7 +5,8 @@ // See: docs/ssh-relay-versioned-install-dirs.md import { join } from 'node:path' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' +import { readRelayVersionMarkerSync } from '../../shared/relay-version-marker' import type { SshConnection } from './ssh-connection' import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' @@ -26,6 +27,7 @@ import { removeRemoteTreeCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' +import { isRelayBaseDirectoryListingLimited } from './ssh-relay-base-directory-listing' import { getRemoteHostPlatform, isWindowsRemoteHost, @@ -77,7 +79,7 @@ export function readLocalFullVersion(localRelayDir: string): string { `This usually indicates a packaging or build problem; reinstall Orca.` ) } - const v = readFileSync(versionFile, 'utf-8').trim() + const v = readRelayVersionMarkerSync(versionFile) if (!v) { throw new Error( `Orca's local relay version marker at ${versionFile} is empty. ` + @@ -190,6 +192,9 @@ export async function gcOldRelayVersions( } catch { return } + if (isRelayBaseDirectoryListingLimited(listing)) { + return + } const entries = listing .split('\n') .map((s) => s.trim()) diff --git a/src/main/ssh/ssh-remote-cli-host-passthrough.ts b/src/main/ssh/ssh-remote-cli-host-passthrough.ts index 3ca55edfd69..9271a953ced 100644 --- a/src/main/ssh/ssh-remote-cli-host-passthrough.ts +++ b/src/main/ssh/ssh-remote-cli-host-passthrough.ts @@ -9,6 +9,7 @@ import { spawn as nodeSpawn } from 'node:child_process' import { existsSync } from 'node:fs' import { join } from 'node:path' import { getCanonicalUserDataPath } from '../persistence' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' export type RemoteOrcaCliRequest = { argv: string[] @@ -226,8 +227,7 @@ export async function runHostOrcaCliPassthrough( } class CappedOutputCollector { - private readonly chunks: Buffer[] = [] - private bytes = 0 + private readonly output = new GrowingByteBuffer() private truncated = false constructor(private readonly maxBytes: number) {} @@ -236,19 +236,17 @@ class CappedOutputCollector { if (this.truncated) { return } - const remaining = this.maxBytes - this.bytes + const remaining = this.maxBytes - this.output.byteLength if (chunk.length >= remaining) { - this.chunks.push(chunk.subarray(0, remaining)) - this.bytes = this.maxBytes + this.output.append(chunk.subarray(0, remaining)) this.truncated = true return } - this.chunks.push(chunk) - this.bytes += chunk.length + this.output.append(chunk) } toString(): string { - const text = Buffer.concat(this.chunks).toString('utf8') + const text = this.output.toString() return this.truncated ? `${text}\n[orca ssh cli] output truncated\n` : text } } diff --git a/src/main/ssh/ssh-remote-commands.test.ts b/src/main/ssh/ssh-remote-commands.test.ts index 594ed2864d5..ea269cd79d6 100644 --- a/src/main/ssh/ssh-remote-commands.test.ts +++ b/src/main/ssh/ssh-remote-commands.test.ts @@ -11,6 +11,7 @@ import { import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { GENERATED_NODE_MANAGED_FILE_MAX_BYTES } from '../generated-node-bounded-file-reader' import { lockAgeSecondsCommand, tryCreateInstallLockCommand, @@ -172,6 +173,11 @@ describe('ssh remote command builders', () => { expect(script).toContain('markerCount===0&&pipes.length===0') expect(script).toContain('C:\\Program Files\\nodejs') expect(script).not.toContain('Win32_Process') + expect(script).toContain( + `readOrcaManagedFileWithinLimit(fs, path, maxBytes = ${GENERATED_NODE_MANAGED_FILE_MAX_BYTES})` + ) + expect(script).toContain('readOrcaManagedFileWithinLimit(fs,path.join(dir,name)).trim()') + expect(script).not.toContain('readFileSync') expect(listRelayBaseDirsCommand(windows, 'C:/Users/me/.orca-remote')).toContain( '-EncodedCommand' ) @@ -243,6 +249,9 @@ describe('ssh remote command builders', () => { expect(posixCommand).toContain('steal_generation + 1') expect(posixCommand).not.toContain('.next.') expect(posixCommand).toContain('trap') + expect(posixCommand).toContain('find "$steal_parent" -mindepth 1 -maxdepth 1') + expect(posixCommand).toContain('-exec rm -rf {} +') + expect(posixCommand).not.toContain('rm -rf "$steal_root".*') expect(windowsScript).toContain('$lock.steal') expect(windowsScript).toContain('$stealGeneration++') expect(windowsScript).toContain('-gt 1200') diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 155fb5ba66f..4a2c244df3c 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -1,4 +1,6 @@ import type { RemoteHostPlatform } from './ssh-remote-platform' +import { getRelayBaseDirectoryListingCommand } from './ssh-relay-base-directory-listing' +import { getWindowsRelayLivenessProbeSource } from './ssh-relay-liveness-probe-source' import { isWindowsRemoteHost, joinRemotePath, remoteDirname } from './ssh-remote-platform' import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell' import { shellEscape } from './ssh-connection-utils' @@ -103,17 +105,7 @@ export function probeRelayInstalledCommand( } export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string { - if (!isWindowsRemoteHost(host)) { - return `ls -1 ${shellEscape(baseDir)} 2>/dev/null || true` - } - return powerShellCommand( - [ - `$base = ${powerShellLiteral(baseDir)}`, - 'if (Test-Path -LiteralPath $base -PathType Container) {', - 'Get-ChildItem -LiteralPath $base -Directory | ForEach-Object { $_.Name }', - '}' - ].join(' ') - ) + return getRelayBaseDirectoryListingCommand(host, baseDir) } export function probeDirectoryExistsCommand(host: RemoteHostPlatform, remotePath: string): string { @@ -154,35 +146,7 @@ export function relayLivenessProbeCommand( if (!windowsOptions) { return powerShellCommand("'ALIVE'") } - const js = [ - 'const fs=require("fs"),path=require("path"),net=require("net");', - 'const [dir,...seed]=process.argv.slice(1);', - 'const valid=/^\\\\\\\\[.?]\\\\pipe\\\\orca-relay-[0-9a-f]{20}$/i;', - 'const pipes=[];', - 'let markerCount=0;', - 'for(const p of seed){if(valid.test(p)&&!pipes.includes(p))pipes.push(p)}', - 'try{for(const name of fs.readdirSync(dir)){', - 'if(!name.startsWith(".windows-active-pipe-"))continue;', - 'markerCount++;', - 'const p=fs.readFileSync(path.join(dir,name),"utf8").trim();', - 'if(valid.test(p)&&!pipes.includes(p))pipes.push(p)', - '}}catch{}', - 'if(markerCount===0&&pipes.length===0){process.stdout.write("ALIVE");process.exit(0)}', - 'let i=0;', - 'function done(ok){process.stdout.write(ok?"ALIVE":"WAITING")}', - 'function next(){', - 'const pipe=pipes[i++];', - 'if(!pipe)return done(false);', - 'const s=net.connect(pipe);', - 'let settled=false;', - 'function finish(ok){if(settled)return;settled=true;s.destroy();if(ok)done(true);else next()}', - 's.setTimeout(200);', - 's.on("connect",()=>finish(true));', - 's.on("timeout",()=>finish(false));', - 's.on("error",()=>finish(false));', - '}', - 'next();' - ].join('') + const js = getWindowsRelayLivenessProbeSource() return commandWithNodePath( host, windowsOptions.nodePath, diff --git a/src/main/ssh/ssh-remote-node-candidate-probe.test.ts b/src/main/ssh/ssh-remote-node-candidate-probe.test.ts new file mode 100644 index 00000000000..ad9089a38f4 --- /dev/null +++ b/src/main/ssh/ssh-remote-node-candidate-probe.test.ts @@ -0,0 +1,91 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + buildPosixRemoteNodeCandidateProbe, + REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL, + REMOTE_NODE_CANDIDATE_MAX_COUNT, + REMOTE_NODE_CANDIDATE_MAX_UTF8_BYTES, + REMOTE_NODE_PROFILE_MAX_BYTES +} from './ssh-remote-node-candidate-probe' + +describe('POSIX remote Node candidate probe', () => { + it('uses bounded profile reads, batched directory iteration, and bounded output', () => { + const script = buildPosixRemoteNodeCandidateProbe() + + expect(script).toContain('dd if="$nvm_file" bs=1024 count=65') + expect(script).toContain(`-le ${REMOTE_NODE_PROFILE_MAX_BYTES}`) + expect(script).toContain(`find "$candidate_root" -mindepth 1 -maxdepth 1 ! -name '.*'`) + expect(script).toContain('-exec sh -c') + expect(script).toContain('{} +') + expect(script).toContain('| sort') + expect(script).toContain(`candidate_count>=${REMOTE_NODE_CANDIDATE_MAX_COUNT}`) + expect(script).toContain(`output_bytes+line_bytes>${REMOTE_NODE_CANDIDATE_MAX_UTF8_BYTES}`) + expect(script).toContain(REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL) + expect(script).not.toContain('/*/') + }) + + it.runIf(process.platform !== 'win32')( + 'accepts an exact-limit profile and ignores its first byte of overflow', + () => { + const home = mkdtempSync(join(tmpdir(), 'orca-node-profile-limit-')) + try { + const nodePath = join(home, 'tilde-nvm', 'versions', 'node', 'v20', 'bin', 'node') + mkdirSync(join(home, 'tilde-nvm', 'versions', 'node', 'v20', 'bin'), { + recursive: true + }) + writeFileSync(nodePath, '#!/bin/sh\nexit 0\n') + chmodSync(nodePath, 0o755) + const assignment = 'export NVM_DIR=~/tilde-nvm\n' + const exactProfile = + assignment + '#'.repeat(REMOTE_NODE_PROFILE_MAX_BYTES - assignment.length) + writeFileSync(join(home, '.profile'), exactProfile) + const script = buildPosixRemoteNodeCandidateProbe() + const environment = { ...process.env, HOME: home, NVM_DIR: '' } + + const exactOutput = execFileSync('/bin/sh', ['-c', script], { + encoding: 'utf8', + env: environment + }) + expect(exactOutput.split('\n')).toContain(nodePath) + + writeFileSync(join(home, '.profile'), `${exactProfile}x`) + const oversizedOutput = execFileSync('/bin/sh', ['-c', script], { + encoding: 'utf8', + env: environment + }) + expect(oversizedOutput.split('\n')).not.toContain(nodePath) + } finally { + rmSync(home, { recursive: true, force: true }) + } + } + ) + + it.runIf(process.platform !== 'win32')( + 'preserves lexical version ordering within each bounded manager directory', + () => { + const home = mkdtempSync(join(tmpdir(), 'orca-node-version-order-')) + try { + const nvmDir = join(home, 'nvm') + const newer = join(nvmDir, 'versions', 'node', 'v22', 'bin', 'node') + const older = join(nvmDir, 'versions', 'node', 'v18', 'bin', 'node') + for (const nodePath of [newer, older]) { + mkdirSync(join(nodePath, '..'), { recursive: true }) + writeFileSync(nodePath, '#!/bin/sh\nexit 0\n') + chmodSync(nodePath, 0o755) + } + + const output = execFileSync('/bin/sh', ['-c', buildPosixRemoteNodeCandidateProbe()], { + encoding: 'utf8', + env: { ...process.env, HOME: home, NVM_DIR: nvmDir } + }).split('\n') + + expect(output.indexOf(older)).toBeLessThan(output.indexOf(newer)) + } finally { + rmSync(home, { recursive: true, force: true }) + } + } + ) +}) diff --git a/src/main/ssh/ssh-remote-node-candidate-probe.ts b/src/main/ssh/ssh-remote-node-candidate-probe.ts new file mode 100644 index 00000000000..ca458712ee6 --- /dev/null +++ b/src/main/ssh/ssh-remote-node-candidate-probe.ts @@ -0,0 +1,133 @@ +export const REMOTE_NODE_PROFILE_MAX_BYTES = 64 * 1024 +export const REMOTE_NODE_NVM_DIR_MAX_BYTES = 4 * 1024 +export const REMOTE_NODE_CANDIDATE_MAX_COUNT = 256 +export const REMOTE_NODE_CANDIDATE_MAX_UTF8_BYTES = 64 * 1024 +export const REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL = '__ORCA_NODE_CANDIDATES_TOO_LARGE__' + +type RemoteNodeCandidateProbeOptions = { + maxCandidates?: number + maxUtf8Bytes?: number +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback + if (!Number.isSafeInteger(resolved) || resolved < 1) { + throw new Error(`${name} must be a positive integer`) + } + return resolved +} + +export function isRemoteNodeCandidateProbeLimited(output: string): boolean { + return output.split(/\r?\n/).some((line) => line.trim() === REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL) +} + +export function buildPosixRemoteNodeCandidateProbe( + options?: RemoteNodeCandidateProbeOptions +): string { + const maxCandidates = positiveInteger( + options?.maxCandidates, + REMOTE_NODE_CANDIDATE_MAX_COUNT, + 'maxCandidates' + ) + const maxUtf8Bytes = positiveInteger( + options?.maxUtf8Bytes, + REMOTE_NODE_CANDIDATE_MAX_UTF8_BYTES, + 'maxUtf8Bytes' + ) + + return ` +nvm_dirs=\${NVM_DIR:-"$HOME/.nvm"} +nvm_dirs_bytes=$(LC_ALL=C printf %s "$nvm_dirs" | wc -c | tr -d '[:space:]') +[ "\${nvm_dirs_bytes:-${REMOTE_NODE_NVM_DIR_MAX_BYTES + 1}}" -le ${REMOTE_NODE_NVM_DIR_MAX_BYTES} ] 2>/dev/null || nvm_dirs="$HOME/.nvm" +for nvm_file in "$HOME/.profile" "$HOME/.bash_profile" "$HOME/.bashrc" "$HOME/.zprofile" "$HOME/.zshrc" +do + [ -r "$nvm_file" ] || continue + profile_bytes=$(dd if="$nvm_file" bs=1024 count=65 2>/dev/null | wc -c | tr -d '[:space:]') + [ "\${profile_bytes:-${REMOTE_NODE_PROFILE_MAX_BYTES + 1}}" -le ${REMOTE_NODE_PROFILE_MAX_BYTES} ] 2>/dev/null || continue + nvm_dir_from_file=$(dd if="$nvm_file" bs=1024 count=64 2>/dev/null | sed -n 's/^[[:space:]]*export[[:space:]][[:space:]]*NVM_DIR[[:space:]]*=[[:space:]]*//p; s/^[[:space:]]*NVM_DIR[[:space:]]*=[[:space:]]*//p' | tail -n 1) + case "$nvm_dir_from_file" in + \\"*\\") nvm_dir_from_file=\${nvm_dir_from_file#\\"}; nvm_dir_from_file=\${nvm_dir_from_file%%\\"*} ;; + \\'*\\') nvm_dir_from_file=\${nvm_dir_from_file#\\'}; nvm_dir_from_file=\${nvm_dir_from_file%%\\'*} ;; + *) nvm_dir_from_file=\${nvm_dir_from_file%%[[:space:]]*} ;; + esac + case "$nvm_dir_from_file" in + '$HOME'*) nvm_dir_from_file="$HOME\${nvm_dir_from_file#'$HOME'}" ;; + "~/"*) nvm_dir_from_file="$HOME/\${nvm_dir_from_file#\\~/}" ;; + esac + nvm_dir_bytes=$(LC_ALL=C printf %s "$nvm_dir_from_file" | wc -c | tr -d '[:space:]') + [ "\${nvm_dir_bytes:-${REMOTE_NODE_NVM_DIR_MAX_BYTES + 1}}" -le ${REMOTE_NODE_NVM_DIR_MAX_BYTES} ] 2>/dev/null || continue + [ -n "$nvm_dir_from_file" ] && nvm_dirs="$nvm_dirs +$nvm_dir_from_file" +done + +find_node_candidates() { + candidate_root=$1 + candidate_suffix=$2 + [ -d "$candidate_root" ] || return 0 + find "$candidate_root" -mindepth 1 -maxdepth 1 ! -name '.*' -exec sh -c ' + candidate_suffix=$1 + shift + for candidate_dir + do + candidate=$candidate_dir/$candidate_suffix + [ -x "$candidate" ] && printf "%s\\n" "$candidate" + done + ' sh "$candidate_suffix" {} + | LC_ALL=C awk ' + { + line_bytes=length($0)+1 + if(candidate_count>=${maxCandidates} || output_bytes+line_bytes>${maxUtf8Bytes}) { + print "${REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL}" + exit + } + print + candidate_count++ + output_bytes+=line_bytes + } + ' | sort +} + +{ + command -v node 2>/dev/null + printf '%s\\n' "$nvm_dirs" | while IFS= read -r nvm_dir + do + [ -n "$nvm_dir" ] || continue + find_node_candidates "$nvm_dir/versions/node" "bin/node" + done + for candidate in \\ + /usr/local/bin/node \\ + /opt/homebrew/bin/node \\ + "$HOME/.local/bin/node" \\ + "$HOME/.fnm/aliases/default/bin/node" + do + [ -x "$candidate" ] && printf '%s\\n' "$candidate" + done + find_node_candidates "$HOME/.fnm/node-versions" "installation/bin/node" + find_node_candidates "$HOME/.local/share/fnm/node-versions" "installation/bin/node" + candidate="$HOME/.local/share/mise/shims/node"; [ -x "$candidate" ] && printf '%s\\n' "$candidate" + find_node_candidates "$HOME/.local/share/mise/installs/node" "bin/node" + candidate="$HOME/.asdf/shims/node"; [ -x "$candidate" ] && printf '%s\\n' "$candidate" + find_node_candidates "$HOME/.asdf/installs/nodejs" "bin/node" + candidate="$HOME/.volta/bin/node"; [ -x "$candidate" ] && printf '%s\\n' "$candidate" + find_node_candidates "/usr/local/n/versions/node" "bin/node" +} | LC_ALL=C awk ' + { + candidate=$0 + if(candidate=="${REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL}") { + print candidate + exit + } + if(candidate=="" || seen[candidate]) next + line_bytes=length(candidate)+1 + if(candidate_count>=${maxCandidates} || output_bytes+line_bytes>${maxUtf8Bytes}) { + print "${REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL}" + exit + } + seen[candidate]=1 + print candidate + candidate_count++ + output_bytes+=line_bytes + } +' +true +` +} diff --git a/src/main/ssh/ssh-remote-node-resolution.test.ts b/src/main/ssh/ssh-remote-node-resolution.test.ts index a48687284ad..ff8846eb3b2 100644 --- a/src/main/ssh/ssh-remote-node-resolution.test.ts +++ b/src/main/ssh/ssh-remote-node-resolution.test.ts @@ -4,6 +4,7 @@ import os from 'node:os' import path from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { SshConnection } from './ssh-connection' +import { REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL } from './ssh-remote-node-candidate-probe' import { getRemoteHostPlatform } from './ssh-remote-platform' const execCommandMock = vi.hoisted(() => vi.fn()) @@ -99,7 +100,9 @@ describe('resolveRemoteNodePath', () => { await resolveRemoteNodePath(conn) const callScript = execCommandMock.mock.calls[0]![1] as string - expect(callScript).toContain('"$HOME/.local/share/mise/installs/node"/*/bin/node') + expect(callScript).toContain( + 'find_node_candidates "$HOME/.local/share/mise/installs/node" "bin/node"' + ) }) it('probes asdf install directories', async () => { @@ -110,7 +113,7 @@ describe('resolveRemoteNodePath', () => { await resolveRemoteNodePath(conn) const callScript = execCommandMock.mock.calls[0]![1] as string - expect(callScript).toContain('"$HOME/.asdf/installs/nodejs"/*/bin/node') + expect(callScript).toContain('find_node_candidates "$HOME/.asdf/installs/nodejs" "bin/node"') }) it('probes volta bin directory', async () => { @@ -134,10 +137,10 @@ describe('resolveRemoteNodePath', () => { const callScript = execCommandMock.mock.calls[0]![1] as string expect(callScript).toContain('nvm_dirs=${NVM_DIR:-"$HOME/.nvm"}') expect(callScript).toContain('NVM_DIR[[:space:]]*=') - expect(callScript).toContain('"$nvm_dir"/versions/node/*/bin/node') + expect(callScript).toContain('find_node_candidates "$nvm_dir/versions/node" "bin/node"') }) - it('quotes version-manager directory prefixes while leaving globs active', async () => { + it('streams quoted version-manager directories without shell globs', async () => { execCommandMock .mockResolvedValueOnce('/home/u/.fnm/node-versions/v20.11.0/installation/bin/node\n') .mockResolvedValueOnce('v20.11.0\n') @@ -145,10 +148,17 @@ describe('resolveRemoteNodePath', () => { await resolveRemoteNodePath(conn) const callScript = execCommandMock.mock.calls[0]![1] as string - expect(callScript).toContain('"$HOME/.fnm/node-versions"/*/installation/bin/node') - expect(callScript).toContain('"$HOME/.local/share/fnm/node-versions"/*/installation/bin/node') - expect(callScript).toContain('"$HOME/.local/share/mise/installs/node"/*/bin/node') - expect(callScript).toContain('"$HOME/.asdf/installs/nodejs"/*/bin/node') + expect(callScript).toContain( + 'find_node_candidates "$HOME/.fnm/node-versions" "installation/bin/node"' + ) + expect(callScript).toContain( + 'find_node_candidates "$HOME/.local/share/fnm/node-versions" "installation/bin/node"' + ) + expect(callScript).toContain( + 'find_node_candidates "$HOME/.local/share/mise/installs/node" "bin/node"' + ) + expect(callScript).toContain('find_node_candidates "$HOME/.asdf/installs/nodejs" "bin/node"') + expect(callScript).not.toContain('/*/') }) it('probes fnm XDG data directory installs', async () => { @@ -276,6 +286,20 @@ describe('resolveRemoteNodePath', () => { }) }) + it('falls back without probing a partial capacity-limited candidate inventory', async () => { + execCommandMock + .mockResolvedValueOnce(`/old/node\n${REMOTE_NODE_CANDIDATE_LIMIT_SENTINEL}\n`) + .mockResolvedValueOnce('/bin/bash') + .mockResolvedValueOnce('/home/u/.nvm/versions/node/v20.11.0/bin/node\n') + .mockResolvedValueOnce('v20.11.0\n') + + await expect(resolveRemoteNodePath(conn)).resolves.toBe( + '/home/u/.nvm/versions/node/v20.11.0/bin/node' + ) + + expect(execCommandMock.mock.calls[1]?.[1]).toBe('echo "${SHELL:-/bin/sh}"') + }) + it('falls back to the login shell when every path-probe candidate is too old', async () => { execCommandMock .mockResolvedValueOnce('/old/node\n') // path probe diff --git a/src/main/ssh/ssh-remote-node-resolution.ts b/src/main/ssh/ssh-remote-node-resolution.ts index 1f2af057039..5ec0c0e2452 100644 --- a/src/main/ssh/ssh-remote-node-resolution.ts +++ b/src/main/ssh/ssh-remote-node-resolution.ts @@ -15,6 +15,10 @@ import { } from './ssh-remote-node-toolchain-probe' import { isSshSessionLimitError } from './ssh-session-limit-error' import { buildSshLoginShellCommand } from './ssh-login-shell-command' +import { + buildPosixRemoteNodeCandidateProbe, + isRemoteNodeCandidateProbeLimited +} from './ssh-remote-node-candidate-probe' // Why: the login-shell fallback catches custom PATH setups in ~/.profile that // the path probes don't cover. Interactive configs (conda prompts, etc.) can @@ -58,54 +62,13 @@ async function tryResolveViaKnownPaths( conn: SshConnection, options?: RemoteNodeResolutionOptions ): Promise { - const script = ` -command -v node 2>/dev/null -nvm_dirs=\${NVM_DIR:-"$HOME/.nvm"} -for nvm_file in "$HOME/.profile" "$HOME/.bash_profile" "$HOME/.bashrc" "$HOME/.zprofile" "$HOME/.zshrc" -do - [ -r "$nvm_file" ] || continue - nvm_dir_from_file=$(sed -n 's/^[[:space:]]*export[[:space:]][[:space:]]*NVM_DIR[[:space:]]*=[[:space:]]*//p; s/^[[:space:]]*NVM_DIR[[:space:]]*=[[:space:]]*//p' "$nvm_file" | tail -n 1) - case "$nvm_dir_from_file" in - \\"*\\") nvm_dir_from_file=\${nvm_dir_from_file#\\"}; nvm_dir_from_file=\${nvm_dir_from_file%%\\"*} ;; - \\'*\\') nvm_dir_from_file=\${nvm_dir_from_file#\\'}; nvm_dir_from_file=\${nvm_dir_from_file%%\\'*} ;; - *) nvm_dir_from_file=\${nvm_dir_from_file%%[[:space:]]*} ;; - esac - case "$nvm_dir_from_file" in - '$HOME'*) nvm_dir_from_file="$HOME\${nvm_dir_from_file#'$HOME'}" ;; - "~/"*) nvm_dir_from_file="$HOME/\${nvm_dir_from_file#\\~/}" ;; - esac - [ -n "$nvm_dir_from_file" ] && nvm_dirs="$nvm_dirs -$nvm_dir_from_file" -done -printf '%s\\n' "$nvm_dirs" | while IFS= read -r nvm_dir -do - [ -n "$nvm_dir" ] || continue - for candidate in "$nvm_dir"/versions/node/*/bin/node - do - [ -x "$candidate" ] && printf '%s\\n' "$candidate" - done -done -for candidate in \\ - /usr/local/bin/node \\ - /opt/homebrew/bin/node \\ - "$HOME/.local/bin/node" \\ - "$HOME/.fnm/aliases/default/bin/node" \\ - "$HOME/.fnm/node-versions"/*/installation/bin/node \\ - "$HOME/.local/share/fnm/node-versions"/*/installation/bin/node \\ - "$HOME/.local/share/mise/shims/node" \\ - "$HOME/.local/share/mise/installs/node"/*/bin/node \\ - "$HOME/.asdf/shims/node" \\ - "$HOME/.asdf/installs/nodejs"/*/bin/node \\ - "$HOME/.volta/bin/node" \\ - /usr/local/n/versions/node/*/bin/node -do - [ -x "$candidate" ] && printf '%s\\n' "$candidate" -done -true -` + const script = buildPosixRemoteNodeCandidateProbe() try { const result = await execCommandWithOptionalOptions(conn, script, signalOnlyOptions(options)) + if (isRemoteNodeCandidateProbeLimited(result)) { + return null + } const seen = new Set() for (const line of result.split('\n')) { const candidate = line.trim() diff --git a/src/main/ssh/ssh-remote-orca-cli.test.ts b/src/main/ssh/ssh-remote-orca-cli.test.ts index 3d1c3e60793..e99cc30cb69 100644 --- a/src/main/ssh/ssh-remote-orca-cli.test.ts +++ b/src/main/ssh/ssh-remote-orca-cli.test.ts @@ -70,9 +70,17 @@ describe('runRemoteOrcaCli', () => { getUnreadMessages: vi.fn((handle: string) => messages.filter((message) => message.to_handle === handle && message.read_at === null) ), + countUnreadMessages: vi.fn( + (handle: string) => + messages.filter((message) => message.to_handle === handle && message.read_at === null) + .length + ), getAllMessagesForHandle: vi.fn((handle: string) => messages.filter((message) => message.to_handle === handle) ), + countAllMessagesForHandle: vi.fn( + (handle: string) => messages.filter((message) => message.to_handle === handle).length + ), markAsRead: vi.fn((ids: string[]) => { for (const message of messages) { if (ids.includes(message.id)) { diff --git a/src/main/ssh/ssh-stream-reader-memory.test.ts b/src/main/ssh/ssh-stream-reader-memory.test.ts new file mode 100644 index 00000000000..8cbe291dc92 --- /dev/null +++ b/src/main/ssh/ssh-stream-reader-memory.test.ts @@ -0,0 +1,432 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + MAX_GIT_RESPONSE_STREAM_BYTES, + MAX_GIT_RESPONSE_STREAM_CHUNKS, + STREAM_CHUNK_SIZE, + MessageType, + encodeFrame +} from './relay-protocol' +import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer' +import { readFileViaStream } from './ssh-filesystem-stream-reader' +import { requestGitStreamable } from './ssh-git-response-stream-reader' +import { + MAX_PRE_METADATA_STREAM_ENCODED_BYTES, + PreMetadataStreamFrameBuffer, + SshPreMetadataStreamBudget, + SshStreamAssemblyBudget +} from './ssh-stream-reader-memory' + +type Notification = { method: string; params: Record | undefined } + +function createStreamMux(): { + mux: Record + emit: (method: string, params: Record) => void + resolveRequest: (value: unknown) => void + rejectRequest: (error: Error) => void + dispose: (reason: 'shutdown' | 'connection_lost') => void + notifications: Notification[] +} { + const handlers = new Map) => void>>() + const disposeHandlers = new Set<(reason: 'shutdown' | 'connection_lost') => void>() + const notifications: Notification[] = [] + let resolveRequest!: (value: unknown) => void + let rejectRequest!: (error: Error) => void + const requestPromise = new Promise((resolve, reject) => { + resolveRequest = resolve + rejectRequest = reject + }) + const mux = { + request: vi.fn(() => requestPromise), + notify: vi.fn((method: string, params?: Record) => { + notifications.push({ method, params }) + }), + onNotificationByMethod: vi.fn( + (method: string, handler: (params: Record) => void) => { + const current = handlers.get(method) ?? new Set() + current.add(handler) + handlers.set(method, current) + return () => current.delete(handler) + } + ), + onDispose: vi.fn((handler: (reason: 'shutdown' | 'connection_lost') => void) => { + disposeHandlers.add(handler) + return () => disposeHandlers.delete(handler) + }), + isDisposed: vi.fn(() => false) + } + return { + mux, + emit: (method, params) => { + for (const handler of handlers.get(method) ?? []) { + handler(params) + } + }, + resolveRequest, + rejectRequest, + dispose: (reason) => { + for (const handler of disposeHandlers) { + handler(reason) + } + }, + notifications + } +} + +async function flushRequestResolution(): Promise { + await Promise.resolve() + await Promise.resolve() +} + +function encodeMuxResponse(requestId: number, result: unknown, sequence: number): Buffer { + return encodeFrame( + MessageType.Regular, + sequence, + 0, + Buffer.from(JSON.stringify({ jsonrpc: '2.0', id: requestId, result })) + ) +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('SSH stream assembly aggregate budget', () => { + it('bounds concurrent reservations and releases them idempotently', () => { + const budget = new SshStreamAssemblyBudget(10) + const releaseSix = budget.reserve(6) + const releaseFour = budget.reserve(4) + + expect(releaseSix).not.toBeNull() + expect(releaseFour).not.toBeNull() + expect(budget.reserve(1)).toBeNull() + expect(budget.retainedBytes).toBe(10) + + releaseSix?.() + releaseSix?.() + expect(budget.retainedBytes).toBe(4) + expect(budget.reserve(6)).not.toBeNull() + }) +}) + +describe('SSH pre-metadata stream aggregate budget', () => { + it('bounds concurrent frame buffers by count and bytes', () => { + const budget = new SshPreMetadataStreamBudget(2, 5) + const first = new PreMetadataStreamFrameBuffer(budget) + const second = new PreMetadataStreamFrameBuffer(budget) + const third = new PreMetadataStreamFrameBuffer(budget) + + expect(first.push({ kind: 'chunk', params: { streamId: 1, seq: 0, data: 'abc' } })).toBe(true) + expect(second.push({ kind: 'chunk', params: { streamId: 2, seq: 0, data: 'de' } })).toBe(true) + expect(third.push({ kind: 'end', params: { streamId: 3 } })).toBe(false) + expect(budget.retainedFrames).toBe(2) + expect(budget.retainedBytes).toBe(5) + + first.clear() + expect(third.push({ kind: 'end', params: { streamId: 3 } })).toBe(true) + expect(budget.retainedFrames).toBe(2) + second.clear() + third.clear() + expect(budget.retainedFrames).toBe(0) + expect(budget.retainedBytes).toBe(0) + }) + + it('releases shared reservations on success, stream error, request timeout, and disconnect', async () => { + const budget = new SshPreMetadataStreamBudget(4, 64) + const fileHarness = createStreamMux() + const errorHarness = createStreamMux() + const gitHarness = createStreamMux() + const disconnectHarness = createStreamMux() + const fileResult = readFileViaStream( + fileHarness.mux as never, + 'empty.txt', + undefined, + new SshStreamAssemblyBudget(1), + budget + ) + const errorResult = readFileViaStream( + errorHarness.mux as never, + 'error.txt', + undefined, + new SshStreamAssemblyBudget(1), + budget + ) + const gitResult = requestGitStreamable( + gitHarness.mux as never, + 'git.diff', + {}, + undefined, + new SshStreamAssemblyBudget(1), + budget + ) + const disconnectResult = readFileViaStream( + disconnectHarness.mux as never, + 'disconnected.txt', + undefined, + new SshStreamAssemblyBudget(1), + budget + ) + fileHarness.emit('fs.streamEnd', { streamId: 1 }) + errorHarness.emit('fs.streamError', { streamId: 2, message: 'remote read failed' }) + gitHarness.emit('git.responseEnd', { streamId: 3 }) + disconnectHarness.emit('fs.streamEnd', { streamId: 4 }) + expect(budget.retainedFrames).toBe(4) + + const streamRejection = expect(errorResult).rejects.toThrow('remote read failed') + const gitRejection = expect(gitResult).rejects.toThrow(/timed out/) + const disconnectRejection = expect(disconnectResult).rejects.toMatchObject({ + code: 'CONNECTION_LOST' + }) + fileHarness.resolveRequest({ streamId: 1, totalSize: 0, isBinary: false }) + errorHarness.resolveRequest({ streamId: 2, totalSize: 1, isBinary: false }) + gitHarness.rejectRequest(new Error('Request "git.diff" timed out after 30ms')) + disconnectHarness.dispose('connection_lost') + + await expect(fileResult).resolves.toEqual({ content: '', isBinary: false }) + await streamRejection + await gitRejection + await disconnectRejection + expect(budget.retainedFrames).toBe(0) + expect(budget.retainedBytes).toBe(0) + }) +}) + +describe('SSH Git response stream memory bounds', () => { + it('rejects and cancels an oversized stream marker', async () => { + const harness = createStreamMux() + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}) + harness.resolveRequest({ + __orcaGitResponseStream: { + streamId: 7, + totalBytes: MAX_GIT_RESPONSE_STREAM_BYTES + 1, + chunkCount: 1 + } + }) + + await expect(result).rejects.toThrow(/exceeds client limit/) + expect(harness.notifications).toContainEqual({ + method: 'git.cancelResponseStream', + params: { streamId: 7 } + }) + }) + + it('rejects an excessive declared chunk count', async () => { + const harness = createStreamMux() + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}) + harness.resolveRequest({ + __orcaGitResponseStream: { + streamId: 8, + totalBytes: 1, + chunkCount: MAX_GIT_RESPONSE_STREAM_CHUNKS + 1 + } + }) + + await expect(result).rejects.toThrow(/exceeds client limit/) + }) + + it('drops an oversized foreign frame received before the stream marker', async () => { + const harness = createStreamMux() + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}) + harness.emit('git.responseChunk', { + streamId: 99, + seq: 0, + data: 'A'.repeat(MAX_PRE_METADATA_STREAM_ENCODED_BYTES + 1) + }) + const payload = Buffer.from('{"ok":true}') + harness.resolveRequest({ + __orcaGitResponseStream: { streamId: 9, totalBytes: payload.length, chunkCount: 1 } + }) + await flushRequestResolution() + harness.emit('git.responseChunk', { streamId: 9, seq: 0, data: payload.toString('base64') }) + harness.emit('git.responseEnd', { streamId: 9 }) + + await expect(result).resolves.toEqual({ ok: true }) + }) + + it('rejects an encoded chunk larger than the remaining declared bytes', async () => { + const harness = createStreamMux() + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}) + harness.resolveRequest({ + __orcaGitResponseStream: { streamId: 10, totalBytes: 1, chunkCount: 1 } + }) + await flushRequestResolution() + harness.emit('git.responseChunk', { streamId: 10, seq: 0, data: 'A'.repeat(1_000_000) }) + + await expect(result).rejects.toThrow(/exceeds 1 remaining bytes/) + }) + + it('reassembles a valid result without changing its value', async () => { + const harness = createStreamMux() + const expected = { answer: 'unchanged', count: 2 } + const payload = Buffer.from(JSON.stringify(expected)) + const budget = new SshStreamAssemblyBudget(payload.length) + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}, undefined, budget) + const first = payload.subarray(0, 7) + const second = payload.subarray(7) + harness.resolveRequest({ + __orcaGitResponseStream: { streamId: 11, totalBytes: payload.length, chunkCount: 2 } + }) + await flushRequestResolution() + expect(budget.retainedBytes).toBe(payload.length) + harness.emit('git.responseChunk', { streamId: 11, seq: 0, data: first.toString('base64') }) + harness.emit('git.responseChunk', { streamId: 11, seq: 1, data: second.toString('base64') }) + harness.emit('git.responseEnd', { streamId: 11 }) + + await expect(result).resolves.toEqual(expected) + expect(budget.retainedBytes).toBe(0) + }) + + it('rejects and cancels when the aggregate assembly budget is unavailable', async () => { + const harness = createStreamMux() + const budget = new SshStreamAssemblyBudget(1) + const held = budget.reserve(1) + const result = requestGitStreamable(harness.mux as never, 'git.diff', {}, undefined, budget) + harness.resolveRequest({ + __orcaGitResponseStream: { streamId: 16, totalBytes: 1, chunkCount: 1 } + }) + + await expect(result).rejects.toThrow(/Active SSH stream assembly would exceed/) + expect(harness.notifications).toContainEqual({ + method: 'git.cancelResponseStream', + params: { streamId: 16 } + }) + held?.() + }) +}) + +describe('SSH filesystem stream memory bounds', () => { + it('releases every active assembly immediately when the shared mux disconnects', async () => { + const dataHandlers: ((data: Buffer) => void)[] = [] + const transport: MultiplexerTransport = { + write: vi.fn(), + onData: (handler) => dataHandlers.push(handler), + onClose: vi.fn(), + close: vi.fn() + } + const mux = new SshChannelMultiplexer(transport) + const budget = new SshStreamAssemblyBudget(2) + const first = readFileViaStream(mux, 'first.txt', undefined, budget) + const second = readFileViaStream(mux, 'second.txt', undefined, budget) + const firstRejection = expect(first).rejects.toMatchObject({ code: 'CONNECTION_LOST' }) + const secondRejection = expect(second).rejects.toMatchObject({ code: 'CONNECTION_LOST' }) + + dataHandlers[0]( + Buffer.concat([ + encodeMuxResponse(1, { streamId: 21, totalSize: 1, isBinary: false }, 1), + encodeMuxResponse(2, { streamId: 22, totalSize: 1, isBinary: false }, 2) + ]) + ) + await flushRequestResolution() + expect(budget.retainedBytes).toBe(2) + + mux.dispose('connection_lost') + + expect(budget.retainedBytes).toBe(0) + await firstRejection + await secondRejection + }) + + it('drops an oversized foreign frame received before file metadata', async () => { + const harness = createStreamMux() + const result = readFileViaStream(harness.mux as never, 'tiny.txt') + harness.emit('fs.streamChunk', { + streamId: 90, + seq: 0, + data: 'A'.repeat(MAX_PRE_METADATA_STREAM_ENCODED_BYTES + 1) + }) + harness.resolveRequest({ + streamId: 12, + totalSize: 1, + isBinary: false, + resultEncoding: 'utf-8' + }) + await flushRequestResolution() + harness.emit('fs.streamChunk', { + streamId: 12, + seq: 0, + data: Buffer.from('a').toString('base64') + }) + harness.emit('fs.streamEnd', { streamId: 12 }) + + await expect(result).resolves.toEqual({ content: 'a', isBinary: false }) + }) + + it('validates encoded size before decoding a file chunk', async () => { + const harness = createStreamMux() + const result = readFileViaStream(harness.mux as never, 'tiny.txt') + harness.resolveRequest({ streamId: 13, totalSize: 1, isBinary: false }) + await flushRequestResolution() + harness.emit('fs.streamChunk', { streamId: 13, seq: 0, data: 'A'.repeat(1_000_000) }) + + await expect(result).rejects.toThrow(/Encoded chunk length mismatch/) + expect(harness.notifications).toContainEqual({ + method: 'fs.cancelStream', + params: { streamId: 13 } + }) + }) + + it('cancels a file stream that stops making progress', async () => { + vi.useFakeTimers() + const harness = createStreamMux() + const result = readFileViaStream(harness.mux as never, 'stalled.txt', { + inactivityTimeoutMs: 50 + }) + harness.resolveRequest({ streamId: 14, totalSize: 1, isBinary: false }) + await flushRequestResolution() + + const rejection = expect(result).rejects.toThrow(/stalled/) + await vi.advanceTimersByTimeAsync(51) + await rejection + expect(harness.notifications).toContainEqual({ + method: 'fs.cancelStream', + params: { streamId: 14 } + }) + }) + + it('resets the inactivity deadline on progress and preserves valid content', async () => { + vi.useFakeTimers() + const harness = createStreamMux() + const first = Buffer.alloc(STREAM_CHUNK_SIZE, 0x61) + const second = Buffer.from('z') + const budget = new SshStreamAssemblyBudget(first.length + second.length) + const result = readFileViaStream( + harness.mux as never, + 'progress.txt', + { inactivityTimeoutMs: 50 }, + budget + ) + harness.resolveRequest({ + streamId: 15, + totalSize: first.length + second.length, + isBinary: false, + resultEncoding: 'utf-8' + }) + await flushRequestResolution() + expect(budget.retainedBytes).toBe(first.length + second.length) + await vi.advanceTimersByTimeAsync(40) + harness.emit('fs.streamChunk', { streamId: 15, seq: 0, data: first.toString('base64') }) + await vi.advanceTimersByTimeAsync(40) + harness.emit('fs.streamChunk', { streamId: 15, seq: 1, data: second.toString('base64') }) + harness.emit('fs.streamEnd', { streamId: 15 }) + + await expect(result).resolves.toEqual({ + content: `${'a'.repeat(STREAM_CHUNK_SIZE)}z`, + isBinary: false + }) + expect(budget.retainedBytes).toBe(0) + }) + + it('rejects and cancels when file assembly would exceed the aggregate budget', async () => { + const harness = createStreamMux() + const budget = new SshStreamAssemblyBudget(1) + const held = budget.reserve(1) + const result = readFileViaStream(harness.mux as never, 'tiny.txt', undefined, budget) + harness.resolveRequest({ streamId: 17, totalSize: 1, isBinary: false }) + + await expect(result).rejects.toThrow(/Active SSH stream assembly would exceed/) + expect(harness.notifications).toContainEqual({ + method: 'fs.cancelStream', + params: { streamId: 17 } + }) + held?.() + }) +}) diff --git a/src/main/ssh/ssh-stream-reader-memory.ts b/src/main/ssh/ssh-stream-reader-memory.ts new file mode 100644 index 00000000000..c599a11a58b --- /dev/null +++ b/src/main/ssh/ssh-stream-reader-memory.ts @@ -0,0 +1,205 @@ +export const STREAM_READER_INACTIVITY_TIMEOUT_MS = 30_000 +export const MAX_PRE_METADATA_STREAM_FRAMES = 64 +export const MAX_PRE_METADATA_STREAM_ENCODED_BYTES = 8 * 1024 * 1024 +export const MAX_SSH_PRE_METADATA_STREAM_FRAMES = 256 +export const MAX_SSH_PRE_METADATA_STREAM_RETAINED_BYTES = 32 * 1024 * 1024 +export const MAX_SSH_STREAM_ASSEMBLY_BYTES = 128 * 1024 * 1024 + +export type BufferedStreamFrame = { + kind: 'chunk' | 'end' | 'error' + params: Record +} + +type RetainedStreamFrame = { + frame: BufferedStreamFrame + encodedBytes: number + release: () => void +} + +export class SshPreMetadataStreamBudget { + private frames = 0 + private bytes = 0 + + constructor( + readonly maxFrames: number, + readonly maxBytes: number + ) {} + + reserve(bytes: number): (() => void) | null { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + this.frames >= this.maxFrames || + bytes > this.maxBytes - this.bytes + ) { + return null + } + this.frames += 1 + this.bytes += bytes + let released = false + return () => { + if (released) { + return + } + released = true + this.frames -= 1 + this.bytes -= bytes + } + } + + get retainedFrames(): number { + return this.frames + } + + get retainedBytes(): number { + return this.bytes + } +} + +export const defaultSshPreMetadataStreamBudget = new SshPreMetadataStreamBudget( + MAX_SSH_PRE_METADATA_STREAM_FRAMES, + MAX_SSH_PRE_METADATA_STREAM_RETAINED_BYTES +) + +export class PreMetadataStreamFrameBuffer { + private frames: RetainedStreamFrame[] = [] + private encodedBytes = 0 + + constructor(private readonly budget: SshPreMetadataStreamBudget) {} + + push(frame: BufferedStreamFrame): boolean { + const retainedFrame = retainStreamFrameFields(frame) + const frameBytes = retainedStreamFrameBytes(retainedFrame) + if (frameBytes > MAX_PRE_METADATA_STREAM_ENCODED_BYTES) { + return false + } + while ( + this.frames.length >= MAX_PRE_METADATA_STREAM_FRAMES || + this.encodedBytes + frameBytes > MAX_PRE_METADATA_STREAM_ENCODED_BYTES + ) { + this.shift() + } + let release = this.budget.reserve(frameBytes) + while (!release && this.frames.length > 0) { + this.shift() + release = this.budget.reserve(frameBytes) + } + if (!release) { + return false + } + this.frames.push({ frame: retainedFrame, encodedBytes: frameBytes, release }) + this.encodedBytes += frameBytes + return true + } + + shift(): BufferedStreamFrame | undefined { + const retained = this.frames.shift() + if (retained) { + this.encodedBytes -= retained.encodedBytes + retained.release() + } + return retained?.frame + } + + get length(): number { + return this.frames.length + } + + clear(): void { + while (this.frames.length > 0) { + this.shift() + } + } +} + +function retainStreamFrameFields(frame: BufferedStreamFrame): BufferedStreamFrame { + const { params } = frame + if (frame.kind === 'chunk') { + return { + kind: frame.kind, + params: { + ...(typeof params.streamId === 'number' ? { streamId: params.streamId } : {}), + ...(typeof params.seq === 'number' ? { seq: params.seq } : {}), + ...(typeof params.data === 'string' ? { data: params.data } : {}) + } + } + } + if (frame.kind === 'error') { + return { + kind: frame.kind, + params: { + ...(typeof params.streamId === 'number' ? { streamId: params.streamId } : {}), + ...(typeof params.message === 'string' ? { message: params.message } : {}), + ...(typeof params.code === 'string' ? { code: params.code } : {}) + } + } + } + return { + kind: frame.kind, + params: typeof params.streamId === 'number' ? { streamId: params.streamId } : {} + } +} + +function retainedStreamFrameBytes(frame: BufferedStreamFrame): number { + let bytes = 0 + for (const value of Object.values(frame.params)) { + if (typeof value === 'string') { + bytes += Buffer.byteLength(value, 'utf-8') + } + } + return bytes +} + +export class SshStreamAssemblyBudget { + private retained = 0 + + constructor(readonly maxBytes: number) {} + + reserve(bytes: number): (() => void) | null { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.maxBytes - this.retained) { + return null + } + this.retained += bytes + let released = false + return () => { + if (released) { + return + } + released = true + this.retained -= bytes + } + } + + get retainedBytes(): number { + return this.retained + } +} + +export const defaultSshStreamAssemblyBudget = new SshStreamAssemblyBudget( + MAX_SSH_STREAM_ASSEMBLY_BYTES +) + +export function base64EncodedLength(rawBytes: number): number { + return Math.ceil(rawBytes / 3) * 4 +} + +export function createStreamInactivityDeadline( + timeoutMs: number, + onTimeout: () => void +): { reset: () => void; clear: () => void } { + let timer: ReturnType | null = null + const clear = (): void => { + if (timer) { + clearTimeout(timer) + timer = null + } + } + return { + reset: () => { + clear() + timer = setTimeout(onTimeout, timeoutMs) + timer.unref?.() + }, + clear + } +} diff --git a/src/main/ssh/ssh-system-fallback.test.ts b/src/main/ssh/ssh-system-fallback.test.ts index 71f94204d58..61721899554 100644 --- a/src/main/ssh/ssh-system-fallback.test.ts +++ b/src/main/ssh/ssh-system-fallback.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { EventEmitter } from 'node:events' @@ -37,6 +37,7 @@ import { spawnSystemSshPortForward } from './system-ssh-forward-process' import { getRemoteHostPlatform } from './ssh-remote-platform' import type { SshTarget } from '../../shared/ssh-types' import type { SystemSshResolvedConfig } from './ssh-control-socket' +import { SSH_DIRECTORY_TRANSFER_LIMITS } from './ssh-directory-transfer-budget' const SYSTEM_SSH_PATH = process.platform === 'win32' ? 'C:\\Windows\\System32\\OpenSSH\\ssh.exe' : '/usr/bin/ssh' @@ -798,6 +799,24 @@ describe('spawnSystemSsh', () => { ) }) + it('rejects an oversized Windows upload file before starting SSH', async () => { + const localDir = mkdtempSync(join(tmpdir(), 'orca-system-ssh-upload-')) + const oversized = join(localDir, 'oversized.bin') + writeFileSync(oversized, '') + truncateSync(oversized, SSH_DIRECTORY_TRANSFER_LIMITS.maximumFileBytes + 1) + + try { + await expect( + uploadDirectoryViaSystemSsh(createTarget(), localDir, 'C:/Users/me/.orca-remote/relay', { + hostPlatform: getRemoteHostPlatform('win32-x64') + }) + ).rejects.toMatchObject({ reason: 'file' }) + expect(spawnMock).not.toHaveBeenCalled() + } finally { + rmSync(localDir, { recursive: true, force: true }) + } + }) + it('forces standalone SSH for Windows upload packages when requested', async () => { const localDir = mkdtempSync(join(tmpdir(), 'orca-system-ssh-upload-')) writeFileSync(join(localDir, 'relay.js'), 'console.log("relay")') diff --git a/src/main/ssh/ssh-windows-log-tail-command.test.ts b/src/main/ssh/ssh-windows-log-tail-command.test.ts new file mode 100644 index 00000000000..ea3b9946df4 --- /dev/null +++ b/src/main/ssh/ssh-windows-log-tail-command.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { + windowsRelayTailLogCommand, + WINDOWS_RELAY_LOG_TAIL_MAX_BYTES +} from './ssh-windows-log-tail-command' + +function decodePowerShellCommand(command: string): string { + const match = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/) + return match ? Buffer.from(match[1], 'base64').toString('utf16le') : '' +} + +describe('Windows relay log tail command', () => { + it('reads a fixed byte tail without line materialization', () => { + const script = decodePowerShellCommand( + windowsRelayTailLogCommand('C:/Users/u/relay.log', 'C:/Users/u/relay.err.log') + ) + + expect(script).toContain( + `[Math]::Min([long]${WINDOWS_RELAY_LOG_TAIL_MAX_BYTES}, $stream.Length)` + ) + expect(script).toContain('$stream.Seek(-[long]$readLength') + expect(script).toContain('$stream.Read($buffer, $offset, $readLength - $offset)') + expect(script).toContain('[System.IO.FileShare]::ReadWrite') + expect(script).toContain('--- stderr ---') + expect(script).not.toContain('Get-Content') + expect(script).not.toContain('-Tail 20') + }) +}) diff --git a/src/main/ssh/ssh-windows-log-tail-command.ts b/src/main/ssh/ssh-windows-log-tail-command.ts new file mode 100644 index 00000000000..3d447e3f895 --- /dev/null +++ b/src/main/ssh/ssh-windows-log-tail-command.ts @@ -0,0 +1,31 @@ +import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' + +export const WINDOWS_RELAY_LOG_TAIL_MAX_BYTES = 64 * 1024 + +export function windowsRelayTailLogCommand(logFile: string, errFile: string): string { + const script = [ + 'function Read-OrcaLogTail {', + 'param([string]$Path, [string]$Missing)', + 'if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return $Missing }', + '$stream = $null', + 'try {', + '$stream = [System.IO.File]::Open($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite)', + `$readLength = [int][Math]::Min([long]${WINDOWS_RELAY_LOG_TAIL_MAX_BYTES}, $stream.Length)`, + '$buffer = New-Object byte[] $readLength', + '$null = $stream.Seek(-[long]$readLength, [System.IO.SeekOrigin]::End)', + '$offset = 0', + 'while ($offset -lt $readLength) {', + '$bytesRead = $stream.Read($buffer, $offset, $readLength - $offset)', + 'if ($bytesRead -eq 0) { break }', + '$offset += $bytesRead', + '}', + 'return [System.Text.Encoding]::UTF8.GetString($buffer, 0, $offset)', + "} catch { return '(unable to read log)' }", + 'finally { if ($null -ne $stream) { $stream.Dispose() } }', + '}', + `[Console]::Out.Write((Read-OrcaLogTail ${powerShellLiteral(logFile)} '(no stdout log)'))`, + '[Console]::Out.Write("`n--- stderr ---`n")', + `[Console]::Out.Write((Read-OrcaLogTail ${powerShellLiteral(errFile)} '(no stderr log)'))` + ].join('\n') + return powerShellCommand(script) +} diff --git a/src/main/ssh/system-ssh-file-transfer.ts b/src/main/ssh/system-ssh-file-transfer.ts index 72cb4425d11..0f508e28c07 100644 --- a/src/main/ssh/system-ssh-file-transfer.ts +++ b/src/main/ssh/system-ssh-file-transfer.ts @@ -1,9 +1,10 @@ import { spawn } from 'node:child_process' import { constants } from 'node:fs' -import { lstat, open, readdir } from 'node:fs/promises' +import { lstat, open, opendir } from 'node:fs/promises' import { join as pathJoin } from 'node:path' import { pipeline } from 'node:stream/promises' import type { SshTarget } from '../../shared/ssh-types' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' import { shellEscape, wrapRemoteCommandForPosixShell } from './ssh-connection-utils' import { findSystemSsh } from './system-ssh-binary' import { @@ -23,6 +24,10 @@ import { type ProcessResult } from './system-ssh-operation-lifecycle' import { writeBufferViaSystemSsh } from './system-ssh-file-binary-transfer' +import { + SshDirectoryTransferBudget, + WINDOWS_SSH_UPLOAD_PACKAGE_MAX_BYTES +} from './ssh-directory-transfer-budget' type SystemSshOperationOptions = SystemSshBuildArgsOptions & { signal?: AbortSignal @@ -113,7 +118,9 @@ async function uploadDirectoryViaSystemSshWindows( localDir, remoteDir, hostPlatform, - options.signal + options.signal, + new SshDirectoryTransferBudget(), + 0 ) await writeWindowsUploadPackageViaSystemSsh(target, entries, options) } @@ -133,26 +140,42 @@ async function collectWindowsUploadEntries( localDir: string, remoteDir: string, hostPlatform: RemoteHostPlatform, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + budget: SshDirectoryTransferBudget, + depth: number ): Promise { + budget.recordPath(remoteDir, depth, { countEntry: depth !== 0 }) const entries: WindowsUploadEntry[] = [{ kind: 'directory', path: remoteDir }] - const dirEntries = await readdir(localDir, { withFileTypes: true }) - for (const entry of dirEntries) { - throwIfAborted(signal) - const localPath = pathJoin(localDir, entry.name) - const remotePath = joinRemotePath(hostPlatform, remoteDir, entry.name) - const statResult = await lstat(localPath) - if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { - continue + const directory = await opendir(localDir) + try { + for await (const entry of directory) { + throwIfAborted(signal) + const localPath = pathJoin(localDir, entry.name) + const remotePath = joinRemotePath(hostPlatform, remoteDir, entry.name) + const statResult = await lstat(localPath) + if (statResult.isSymbolicLink() || (!statResult.isFile() && !statResult.isDirectory())) { + continue + } + if (statResult.isDirectory()) { + entries.push( + ...(await collectWindowsUploadEntries( + localPath, + remotePath, + hostPlatform, + signal, + budget, + depth + 1 + )) + ) + continue + } + budget.recordPath(remotePath, depth + 1) + budget.recordFile(statResult.size) + const buffer = await readLocalUploadFile(localPath, statResult) + entries.push({ kind: 'file', path: remotePath, contentsBase64: buffer.toString('base64') }) } - if (statResult.isDirectory()) { - entries.push( - ...(await collectWindowsUploadEntries(localPath, remotePath, hostPlatform, signal)) - ) - continue - } - const buffer = await readLocalUploadFile(localPath, statResult) - entries.push({ kind: 'file', path: remotePath, contentsBase64: buffer.toString('base64') }) + } finally { + await directory.close().catch(() => undefined) } return entries } @@ -163,6 +186,10 @@ async function writeWindowsUploadPackageViaSystemSsh( options: SystemSshOperationOptions ): Promise { throwIfAborted(options.signal) + const serialized = stringifyJsonWithinByteLimit( + entries, + WINDOWS_SSH_UPLOAD_PACKAGE_MAX_BYTES + ).serialized const channel = spawnSystemSshCommand(target, makeWindowsUploadPackageCommand(), { wrapCommand: false, ...getSystemSshBuildArgsFromOperationOptions(options) @@ -173,7 +200,7 @@ async function writeWindowsUploadPackageViaSystemSsh( waitForChannelClose(channel, 'windows relay upload') ) if (!options.signal?.aborted) { - channel.stdin.end(JSON.stringify(entries)) + channel.stdin.end(serialized) } await closePromise } @@ -193,7 +220,20 @@ async function readLocalUploadFile( ) { throw new Error(`File changed during upload: ${localPath}`) } - return await handle.readFile() + const buffer = Buffer.allocUnsafe(openedStat.size) + let offset = 0 + while (offset < buffer.length) { + const result = await handle.read(buffer, offset, buffer.length - offset, offset) + if (result.bytesRead === 0) { + throw new Error(`File changed during upload: ${localPath}`) + } + offset += result.bytesRead + } + const probe = Buffer.allocUnsafe(1) + if ((await handle.read(probe, 0, 1, offset)).bytesRead !== 0) { + throw new Error(`File changed during upload: ${localPath}`) + } + return buffer } finally { await handle.close() } diff --git a/src/main/ssh/system-ssh-forward-process.test.ts b/src/main/ssh/system-ssh-forward-process.test.ts index 4178d0e1796..b5e2fcaa3f5 100644 --- a/src/main/ssh/system-ssh-forward-process.test.ts +++ b/src/main/ssh/system-ssh-forward-process.test.ts @@ -34,6 +34,7 @@ import { waitForSystemSshForwardStop } from './system-ssh-forward-process' import type { SshTarget } from '../../shared/ssh-types' +import { SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES } from './system-ssh-output-tail' const SYSTEM_SSH_PATH = process.platform === 'win32' ? 'C:\\Windows\\System32\\OpenSSH\\ssh.exe' : '/usr/bin/ssh' @@ -247,6 +248,22 @@ describe('system SSH forward process', () => { await expect(pending).rejects.toThrow('bind: Address already in use') }) + it('keeps only a bounded stderr tail while waiting for forward startup', async () => { + vi.useFakeTimers() + const child = createFakeProcess() + connectMock.mockReturnValue(createFakeSocket()) + + const pending = waitForSystemSshForwardStartup(child as never, 3000) + child.stderr.emit( + 'data', + Buffer.from(`HEAD${'x'.repeat(SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES)}TAIL`) + ) + child.emit('exit', 255) + + await expect(pending).rejects.toThrow('TAIL') + await expect(pending).rejects.not.toThrow('HEAD') + }) + it('rejects startup when the ssh process emits an error', async () => { vi.useFakeTimers() const child = createFakeProcess() diff --git a/src/main/ssh/system-ssh-forward-process.ts b/src/main/ssh/system-ssh-forward-process.ts index 13486101bfd..65a4732d426 100644 --- a/src/main/ssh/system-ssh-forward-process.ts +++ b/src/main/ssh/system-ssh-forward-process.ts @@ -2,6 +2,7 @@ import { spawn, type ChildProcess } from 'node:child_process' import { connect, createServer } from 'node:net' import { buildSshArgs, findSystemSsh, type SystemSshBuildArgsOptions } from './ssh-system-fallback' import type { SshTarget } from '../../shared/ssh-types' +import { SystemSshOutputTail } from './system-ssh-output-tail' export const SYSTEM_SSH_FORWARD_STARTUP_GRACE_MS = 750 export const SYSTEM_SSH_FORWARD_LISTENER_PROBE_INTERVAL_MS = 50 @@ -103,7 +104,7 @@ export function waitForSystemSshForwardStartup( localPort: number ): Promise { return new Promise((resolve, reject) => { - let stderr = '' + const stderr = new SystemSshOutputTail() let settled = false let probeTimer: ReturnType | null = null let graceTimer: ReturnType | null = null @@ -127,13 +128,13 @@ export function waitForSystemSshForwardStartup( callback() } const onStderr = (chunk: Buffer): void => { - stderr += chunk.toString('utf-8') + stderr.push(chunk) } const onError = (error: Error): void => { finish(() => reject(error)) } const onExit = (code: number | null): void => { - finish(() => reject(systemSshForwardError(code, stderr))) + finish(() => reject(systemSshForwardError(code, stderr.toString()))) } const scheduleProbe = (): void => { probeTimer = setTimeout(() => { diff --git a/src/main/ssh/system-ssh-operation-lifecycle.test.ts b/src/main/ssh/system-ssh-operation-lifecycle.test.ts new file mode 100644 index 00000000000..bda9f6c9206 --- /dev/null +++ b/src/main/ssh/system-ssh-operation-lifecycle.test.ts @@ -0,0 +1,50 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it } from 'vitest' +import { waitForChannelClose, waitForProcess } from './system-ssh-operation-lifecycle' +import { SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES } from './system-ssh-output-tail' + +function createProcess(): EventEmitter & { + stderr: EventEmitter +} { + return Object.assign(new EventEmitter(), { stderr: new EventEmitter() }) +} + +function createChannel(): EventEmitter & { + stderr: EventEmitter +} { + return Object.assign(new EventEmitter(), { stderr: new EventEmitter() }) +} + +describe('system SSH operation output capture', () => { + it('returns a bounded stderr tail for successful child processes', async () => { + const process = createProcess() + const result = waitForProcess(process as never, 'test process') + + process.stderr.emit( + 'data', + Buffer.from(`HEAD${'x'.repeat(SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES)}TAIL`) + ) + process.emit('close', 0) + + await expect(result).resolves.toMatchObject({ + label: 'test process', + stderr: expect.stringContaining('TAIL') + }) + expect((await result).stderr).not.toContain('HEAD') + }) + + it('uses the bounded stderr tail in channel failure diagnostics', async () => { + const channel = createChannel() + const result = waitForChannelClose(channel as never, 'test channel') + + channel.stderr.emit( + 'data', + Buffer.from(`HEAD${'x'.repeat(SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES)}TAIL`) + ) + channel.emit('close', 1, null) + + await expect(result).rejects.toThrow('TAIL') + await expect(result).rejects.not.toThrow('HEAD') + expect(channel.stderr.listenerCount('data')).toBe(0) + }) +}) diff --git a/src/main/ssh/system-ssh-operation-lifecycle.ts b/src/main/ssh/system-ssh-operation-lifecycle.ts index c9c4424325b..864669ad326 100644 --- a/src/main/ssh/system-ssh-operation-lifecycle.ts +++ b/src/main/ssh/system-ssh-operation-lifecycle.ts @@ -1,5 +1,6 @@ import type { ChildProcess } from 'node:child_process' import type { SystemSshCommandChannel } from './system-ssh-command' +import { SystemSshOutputTail } from './system-ssh-output-tail' export type ProcessResult = { label: string; stderr: string } @@ -8,7 +9,7 @@ export function waitForChannelClose( label: string ): Promise { return new Promise((resolve, reject) => { - let stderr = '' + const stderr = new SystemSshOutputTail() const cleanup = (): void => { channel.stderr.off('data', onStderrData) channel.off('error', onError) @@ -19,7 +20,7 @@ export function waitForChannelClose( fn(val as never) } const onStderrData = (data: Buffer): void => { - stderr += data.toString('utf-8') + stderr.push(data) } const onError = (err: Error): void => { settle(reject, err) @@ -27,7 +28,7 @@ export function waitForChannelClose( const onClose = (code: number | null, signal?: NodeJS.Signals | null): void => { if (code !== 0) { const detail = code === null ? `signal ${signal ?? 'unknown'}` : `exit ${code}` - settle(reject, new Error(`${label} failed (${detail}): ${stderr.trim()}`)) + settle(reject, new Error(`${label} failed (${detail}): ${stderr.toString().trim()}`)) return } settle(resolve) @@ -41,7 +42,7 @@ export function waitForChannelClose( export function waitForProcess(proc: ChildProcess, label: string): Promise { return new Promise((resolve, reject) => { - let stderr = '' + const stderr = new SystemSshOutputTail() const cleanup = (): void => { proc.stderr?.off('data', onStderrData) proc.off('error', onError) @@ -52,17 +53,17 @@ export function waitForProcess(proc: ChildProcess, label: string): Promise { - stderr += data.toString('utf-8') + stderr.push(data) } const onError = (err: Error): void => { settle(reject, err) } const onClose = (code: number | null): void => { if (code !== 0) { - settle(reject, new Error(`${label} failed (exit ${code}): ${stderr.trim()}`)) + settle(reject, new Error(`${label} failed (exit ${code}): ${stderr.toString().trim()}`)) return } - settle(resolve, { label, stderr }) + settle(resolve, { label, stderr: stderr.toString() }) } proc.stderr?.on('data', onStderrData) diff --git a/src/main/ssh/system-ssh-output-tail.test.ts b/src/main/ssh/system-ssh-output-tail.test.ts new file mode 100644 index 00000000000..d47332eaad7 --- /dev/null +++ b/src/main/ssh/system-ssh-output-tail.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { SystemSshOutputTail } from './system-ssh-output-tail' + +describe('SystemSshOutputTail', () => { + it('preserves ordinary output exactly', () => { + const output = new SystemSshOutputTail(16) + output.push('first ') + output.push(Buffer.from('second')) + + expect(output.toString()).toBe('first second') + }) + + it('keeps only the newest bytes after many chunks', () => { + const output = new SystemSshOutputTail(8) + for (const chunk of ['123', '456', '789', 'tail']) { + output.push(chunk) + } + + expect(output.toString()).toBe('[earlier system SSH output truncated]\n6789tail') + }) + + it('copies a bounded slice instead of retaining a large chunk backing buffer', () => { + const output = new SystemSshOutputTail(4) + const large = Buffer.from('discard-prefix-tail') + output.push(large) + large.fill(0) + + expect(output.toString()).toBe('[earlier system SSH output truncated]\ntail') + }) +}) diff --git a/src/main/ssh/system-ssh-output-tail.ts b/src/main/ssh/system-ssh-output-tail.ts new file mode 100644 index 00000000000..89fa190f820 --- /dev/null +++ b/src/main/ssh/system-ssh-output-tail.ts @@ -0,0 +1,37 @@ +export const SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES = 64 * 1024 + +const TRUNCATION_NOTICE = '[earlier system SSH output truncated]\n' + +export class SystemSshOutputTail { + private buffer = Buffer.alloc(0) + private truncated = false + + constructor(private readonly maxBytes = SYSTEM_SSH_OUTPUT_TAIL_MAX_BYTES) { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('System SSH output tail limit must be a non-negative safe integer') + } + } + + push(value: Buffer | string): void { + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value) + if (chunk.byteLength >= this.maxBytes) { + const hadBufferedOutput = this.buffer.byteLength > 0 + this.buffer = Buffer.from(chunk.subarray(chunk.byteLength - this.maxBytes)) + this.truncated ||= hadBufferedOutput || chunk.byteLength > this.maxBytes + return + } + const combinedBytes = this.buffer.byteLength + chunk.byteLength + if (combinedBytes <= this.maxBytes) { + this.buffer = Buffer.concat([this.buffer, chunk], combinedBytes) + return + } + const discardBytes = combinedBytes - this.maxBytes + this.buffer = Buffer.concat([this.buffer.subarray(discardBytes), chunk], this.maxBytes) + this.truncated = true + } + + toString(): string { + const text = this.buffer.toString('utf8') + return this.truncated ? `${TRUNCATION_NOTICE}${text}` : text + } +} diff --git a/src/main/ssh/system-ssh-port-forward-provider.ts b/src/main/ssh/system-ssh-port-forward-provider.ts index e2b53abab7b..5570628b69c 100644 --- a/src/main/ssh/system-ssh-port-forward-provider.ts +++ b/src/main/ssh/system-ssh-port-forward-provider.ts @@ -8,6 +8,7 @@ import type { SshPortForwardProvider, StartedPortForward } from './ssh-port-forward-provider' +import { SystemSshOutputTail } from './system-ssh-output-tail' export class SystemSshPortForwardProvider implements SshPortForwardProvider { canHandle(conn: SshConnection): boolean { @@ -33,17 +34,9 @@ export class SystemSshPortForwardProvider implements SshPortForwardProvider { ) await forward.waitForStartup() - // Why: this stderr stays attached for the forward's whole lifetime but is - // only used to build the exit-error detail, so keep a bounded tail — a chatty - // remote sshd could otherwise grow it unbounded on a long-lived forward - // (mirrors MAX_RELAY_STARTUP_BUFFER_BYTES in ssh-relay-deploy-helpers). - const MAX_STDERR_TAIL_BYTES = 64 * 1024 - let stderr = '' + const stderr = new SystemSshOutputTail() const onStderr = (chunk: Buffer): void => { - stderr += chunk.toString('utf-8') - if (stderr.length > MAX_STDERR_TAIL_BYTES) { - stderr = stderr.slice(-MAX_STDERR_TAIL_BYTES) - } + stderr.push(chunk) } forward.process.stderr?.on('data', onStderr) @@ -60,7 +53,7 @@ export class SystemSshPortForwardProvider implements SshPortForwardProvider { forward.process.stderr?.off('data', onStderr) options.onUnexpectedClose?.(entry, { kind: 'unexpected-exit', - detail: systemSshForwardError(code, stderr).message + detail: systemSshForwardError(code, stderr.toString()).message }) }) diff --git a/src/main/startup/configure-process.test.ts b/src/main/startup/configure-process.test.ts index 7d582880f28..1471153953a 100644 --- a/src/main/startup/configure-process.test.ts +++ b/src/main/startup/configure-process.test.ts @@ -1,7 +1,8 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCA_PERSISTED_STATE_MAX_BYTES } from '../../shared/persisted-state-file-bounds' vi.mock('electron', () => { const paths = new Map([['appData', '/tmp/app-data']]) @@ -302,6 +303,14 @@ describe('configureElectronNetworkCompatibility', () => { expect(shouldDisableHttp2ForElectronNetworking({ env: {}, userDataPath })).toBe(false) }) + it('ignores an oversized persisted state file before early startup parsing', async () => { + const { shouldDisableHttp2ForElectronNetworking } = await import('./configure-process') + const userDataPath = createUserDataDir({ electronHttp1CompatibilityMode: true }) + truncateSync(join(userDataPath, 'orca-data.json'), ORCA_PERSISTED_STATE_MAX_BYTES + 1) + + expect(shouldDisableHttp2ForElectronNetworking({ env: {}, userDataPath })).toBe(false) + }) + it('lets the environment override force compatibility on', async () => { const { shouldDisableHttp2ForElectronNetworking } = await import('./configure-process') diff --git a/src/main/startup/configure-process.ts b/src/main/startup/configure-process.ts index a0f2f023678..5db98d1afbf 100644 --- a/src/main/startup/configure-process.ts +++ b/src/main/startup/configure-process.ts @@ -1,9 +1,10 @@ import { app } from 'electron' -import { existsSync, mkdirSync, readFileSync } from 'node:fs' +import { existsSync, mkdirSync } from 'node:fs' import { homedir } from 'node:os' import { join, resolve } from 'node:path' import { getVersionManagerBinPaths } from '../codex-cli/command' import { getMainE2EConfig } from '../e2e-config' +import { readPersistedStateJsonFileSync } from '../../shared/persisted-state-file-bounds' const DEV_PARENT_SHUTDOWN_GRACE_MS = 3000 const HTTP1_COMPATIBILITY_ENV_VAR = 'ORCA_DISABLE_HTTP2' @@ -37,9 +38,9 @@ function readPersistedHttp1CompatibilityMode(userDataPath: string): boolean { } try { - const parsed = JSON.parse(readFileSync(dataFile, 'utf-8')) as { + const { value: parsed } = readPersistedStateJsonFileSync<{ settings?: { electronHttp1CompatibilityMode?: unknown } - } + }>(dataFile) return parsed.settings?.electronHttp1CompatibilityMode === true } catch { return false diff --git a/src/main/startup/ensure-virtual-display.test.ts b/src/main/startup/ensure-virtual-display.test.ts index f66d79fefea..1b2e26e62ed 100644 --- a/src/main/startup/ensure-virtual-display.test.ts +++ b/src/main/startup/ensure-virtual-display.test.ts @@ -1,11 +1,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { spawnMock, spawnSyncMock, existsSyncMock, readFileSyncMock, rmSyncMock, appMock } = +const { spawnMock, spawnSyncMock, existsSyncMock, boundedReadMock, rmSyncMock, appMock } = vi.hoisted(() => ({ spawnMock: vi.fn(), spawnSyncMock: vi.fn(), existsSyncMock: vi.fn(), - readFileSyncMock: vi.fn(), + boundedReadMock: vi.fn(), rmSyncMock: vi.fn(), appMock: { disableHardwareAcceleration: vi.fn(), @@ -17,9 +17,11 @@ const { spawnMock, spawnSyncMock, existsSyncMock, readFileSyncMock, rmSyncMock, vi.mock('child_process', () => ({ spawn: spawnMock, spawnSync: spawnSyncMock })) vi.mock('fs', () => ({ existsSync: existsSyncMock, - readFileSync: readFileSyncMock, rmSync: rmSyncMock })) +vi.mock('../../shared/node-bounded-file-reader', () => ({ + readNodeFileSyncWithinLimit: boundedReadMock +})) vi.mock('electron', () => ({ app: appMock })) const ORIGINAL_PLATFORM = process.platform @@ -34,7 +36,7 @@ describe('ensureVirtualDisplayForHeadlessServe', () => { spawnMock.mockReset() spawnSyncMock.mockReset() existsSyncMock.mockReset() - readFileSyncMock.mockReset() + boundedReadMock.mockReset() rmSyncMock.mockReset() appMock.disableHardwareAcceleration.mockReset() appMock.commandLine.appendSwitch.mockReset() @@ -115,7 +117,7 @@ describe('ensureVirtualDisplayForHeadlessServe', () => { setPlatform('linux') spawnSyncMock.mockReturnValue({ status: 0 }) existsSyncMock.mockReturnValue(true) // :99 socket + lock present - readFileSyncMock.mockReturnValue('4321\n') // lock holds a PID + boundedReadMock.mockReturnValue({ buffer: Buffer.from('4321\n') }) // lock holds a PID const killSpy = vi.spyOn(process, 'kill').mockReturnValue(true as never) // PID alive const { ensureVirtualDisplayForHeadlessServe } = await import('./ensure-virtual-display') @@ -131,7 +133,7 @@ describe('ensureVirtualDisplayForHeadlessServe', () => { setPlatform('linux') spawnSyncMock.mockReturnValue({ status: 0 }) existsSyncMock.mockReturnValue(true) // orphan socket + lock present - readFileSyncMock.mockReturnValue('9999\n') + boundedReadMock.mockReturnValue({ buffer: Buffer.from('9999\n') }) // PID is gone: process.kill throws ESRCH. const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => { throw new Error('ESRCH') diff --git a/src/main/startup/ensure-virtual-display.ts b/src/main/startup/ensure-virtual-display.ts index a4adfed6113..bc03c083885 100644 --- a/src/main/startup/ensure-virtual-display.ts +++ b/src/main/startup/ensure-virtual-display.ts @@ -1,6 +1,7 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process' -import { existsSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, rmSync } from 'node:fs' import { app } from 'electron' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' // Why: headless `orca serve` backs browser panes with offscreen BrowserWindows. // On Linux, Electron has no display platform without an X server and segfaults @@ -12,6 +13,7 @@ const XVFB_STARTUP_TIMEOUT_MS = 5_000 const XVFB_POLL_INTERVAL_MS = 50 const VIRTUAL_DISPLAY_NUMBER = 99 const VIRTUAL_DISPLAY = `:${VIRTUAL_DISPLAY_NUMBER}` +export const X_DISPLAY_LOCK_MAX_BYTES = 64 let xvfbProcess: ChildProcess | null = null @@ -41,13 +43,8 @@ function isDisplayServerAlive(displayNumber: number): boolean { // No lock means no server claimed this display; the bare socket is stale. return false } - let pid: number - try { - pid = Number.parseInt(readFileSync(lockPath, 'utf8').trim(), 10) - } catch { - return false - } - if (!Number.isInteger(pid) || pid <= 0) { + const pid = readXDisplayLockPid(lockPath) + if (pid === null) { return false } try { @@ -59,6 +56,18 @@ function isDisplayServerAlive(displayNumber: number): boolean { } } +export function readXDisplayLockPid(lockPath: string): number | null { + try { + const raw = readNodeFileSyncWithinLimit(lockPath, X_DISPLAY_LOCK_MAX_BYTES) + .buffer.toString('utf8') + .trim() + const pid = Number.parseInt(raw, 10) + return Number.isInteger(pid) && pid > 0 ? pid : null + } catch { + return null + } +} + function removeStaleDisplayArtifacts(displayNumber: number): void { for (const path of [xDisplayLockPath(displayNumber), xvfbSocketPath(displayNumber)]) { try { diff --git a/src/main/startup/gpu-fallback-marker.test.ts b/src/main/startup/gpu-fallback-marker.test.ts index c52fc3c1f11..43b1d3fa99e 100644 --- a/src/main/startup/gpu-fallback-marker.test.ts +++ b/src/main/startup/gpu-fallback-marker.test.ts @@ -1,9 +1,10 @@ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import os from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { GPU_FALLBACK_MARKER_FILE, + MAX_GPU_FALLBACK_MARKER_FILE_BYTES, clearGpuFallbackMarker, readActiveGpuFallbackMarker, readGpuFallbackMarker, @@ -99,4 +100,13 @@ describe('gpu-fallback-marker', () => { clearGpuFallbackMarker(userDataPath) expect(readGpuFallbackMarker(userDataPath)).toBeNull() }) + + it('clears an oversized sparse marker without loading it', () => { + const path = join(userDataPath, GPU_FALLBACK_MARKER_FILE) + writeFileSync(path, '{"schemeVersion":2}') + truncateSync(path, MAX_GPU_FALLBACK_MARKER_FILE_BYTES + 1) + + expect(readActiveGpuFallbackMarker(userDataPath, environment)).toBeNull() + expect(existsSync(path)).toBe(false) + }) }) diff --git a/src/main/startup/gpu-fallback-marker.ts b/src/main/startup/gpu-fallback-marker.ts index e94e654b3ec..38bb4e62ad4 100644 --- a/src/main/startup/gpu-fallback-marker.ts +++ b/src/main/startup/gpu-fallback-marker.ts @@ -1,5 +1,6 @@ -import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' /** * Persisted "disable hardware acceleration for this build" marker. @@ -12,6 +13,7 @@ import { join } from 'node:path' export const GPU_FALLBACK_MARKER_FILE = 'gpu-fallback.json' export const GPU_FALLBACK_SCHEME_VERSION = 2 +export const MAX_GPU_FALLBACK_MARKER_FILE_BYTES = 8 * 1024 export type GpuFallbackEnvironment = { appVersion: string @@ -36,9 +38,12 @@ function markerPath(userDataPath: string): string { export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | null { try { - const parsed = JSON.parse(readFileSync(markerPath(userDataPath), 'utf-8')) as Partial< - Record - > + const parsed = JSON.parse( + readNodeFileSyncWithinLimit( + markerPath(userDataPath), + MAX_GPU_FALLBACK_MARKER_FILE_BYTES + ).buffer.toString('utf8') + ) as Partial> if (parsed.schemeVersion !== GPU_FALLBACK_SCHEME_VERSION) { return null } diff --git a/src/main/startup/hydrate-shell-path.test.ts b/src/main/startup/hydrate-shell-path.test.ts index 7ff6ee2ae81..e085506a597 100644 --- a/src/main/startup/hydrate-shell-path.test.ts +++ b/src/main/startup/hydrate-shell-path.test.ts @@ -5,6 +5,7 @@ import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { _resetHydrateShellPathCache, hydrateShellPath, + MAX_SHELL_PATH_STDOUT_BYTES, mergePathSegments, type HydrationResult } from './hydrate-shell-path' @@ -157,6 +158,45 @@ describe('hydrateShellPath', () => { vi.useRealTimers() } }) + + it('kills a login shell whose stdout exceeds the PATH capture limit', async () => { + const proc = createMockShellProcess() + spawnMock.mockReturnValue(proc) + + const resultPromise = hydrateShellPath({ shellOverride: '/bin/zsh', force: true }) + proc.stdout.emit('data', Buffer.alloc(1024 * 1024 + 1, 0x78)) + + await expect(resultPromise).resolves.toEqual({ + segments: [], + ok: false, + failureReason: 'empty_path' + }) + expect(proc.kill).toHaveBeenCalledWith('SIGKILL') + expect(proc.stdout.listenerCount('data')).toBe(0) + expect(proc.listenerCount('error')).toBe(0) + expect(proc.listenerCount('close')).toBe(0) + }) + + it('captures PATH after tens of thousands of tiny stdout chunks', async () => { + const proc = createMockShellProcess() + spawnMock.mockReturnValue(proc) + const delimiterText = '__ORCA_SHELL_PATH__' + const output = Buffer.from( + `${'x'.repeat(MAX_SHELL_PATH_STDOUT_BYTES - 128)}${delimiterText}/bin${delimiterText}` + ) + + const resultPromise = hydrateShellPath({ shellOverride: '/bin/zsh', force: true }) + for (let offset = 0; offset < output.byteLength; offset += 16) { + proc.stdout.emit('data', output.subarray(offset, offset + 16)) + } + proc.emit('close') + + await expect(resultPromise).resolves.toEqual({ + segments: ['/bin'], + ok: true, + failureReason: 'none' + }) + }) }) describe('mergePathSegments', () => { diff --git a/src/main/startup/hydrate-shell-path.ts b/src/main/startup/hydrate-shell-path.ts index 9411b0437d3..69bba7c0ce8 100644 --- a/src/main/startup/hydrate-shell-path.ts +++ b/src/main/startup/hydrate-shell-path.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process' import { delimiter } from 'node:path' import type { ShellHydrationFailureReason } from '../../shared/types' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' // Why: GUI-launched Electron on macOS/Linux inherits a minimal PATH from launchd // that does not include dirs appended by the user's shell rc files (~/.zshrc, @@ -16,6 +17,7 @@ import type { ShellHydrationFailureReason } from '../../shared/types' const DELIMITER = '__ORCA_SHELL_PATH__' const SPAWN_TIMEOUT_MS = 5000 +export const MAX_SHELL_PATH_STDOUT_BYTES = 1024 * 1024 // ANSI escape sequences can leak into the captured output when the user's rc // files print banners or set colored prompts. Strip them before parsing. @@ -85,7 +87,7 @@ function spawnShellAndReadPath(shell: string): Promise { // and .bashrc/.zshrc are sourced — matches what `which` in Terminal sees. const command = `printf '%s' '${DELIMITER}'; printf '%s' "$PATH"; printf '%s' '${DELIMITER}'` let finished = false - let stdout = '' + const stdout = new GrowingByteBuffer() let timer: ReturnType | null = null const child = spawn(shell, ['-ilc', command], { @@ -129,7 +131,15 @@ function spawnShellAndReadPath(shell: string): Promise { }, SPAWN_TIMEOUT_MS) const onStdoutData = (chunk: Buffer): void => { - stdout += chunk.toString('utf8') + if (chunk.byteLength > MAX_SHELL_PATH_STDOUT_BYTES - stdout.byteLength) { + try { + child.kill('SIGKILL') + } catch {} + stdout.clear() + finish({ segments: [], ok: false, failureReason: 'empty_path' }) + return + } + stdout.append(chunk) } const onError = (): void => { @@ -137,7 +147,7 @@ function spawnShellAndReadPath(shell: string): Promise { } const onClose = (): void => { - const segments = parseCapturedPath(stdout) + const segments = parseCapturedPath(stdout.toString('utf8')) if (segments.length === 0) { finish({ segments: [], ok: false, failureReason: 'empty_path' }) return diff --git a/src/main/startup/windows-user-data-acl.test.ts b/src/main/startup/windows-user-data-acl.test.ts index b49510b0bb1..ae860cb9916 100644 --- a/src/main/startup/windows-user-data-acl.test.ts +++ b/src/main/startup/windows-user-data-acl.test.ts @@ -1,11 +1,12 @@ import { EventEmitter } from 'node:events' -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs' import os from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { ensureWindowsUserDataAclGrant, WINDOWS_ACL_GRANT_MARKER_FILE, + WINDOWS_ACL_GRANT_MARKER_MAX_BYTES, WINDOWS_ACL_GRANT_SCHEME_VERSION, type WindowsAclGrantResult } from './windows-user-data-acl' @@ -85,6 +86,43 @@ describe('ensureWindowsUserDataAclGrant', () => { expect(fake.calls).toHaveLength(0) }) + it('accepts a matching ACL marker at the exact byte boundary', async () => { + const marker = JSON.stringify({ + schemeVersion: WINDOWS_ACL_GRANT_SCHEME_VERSION, + identity: 'testuser', + grantedAt: 1 + }) + writeFileSync( + join(userDataPath, WINDOWS_ACL_GRANT_MARKER_FILE), + marker + ' '.repeat(WINDOWS_ACL_GRANT_MARKER_MAX_BYTES - Buffer.byteLength(marker)) + ) + const fake = createFakeSpawn(0) + + expect( + await awaitResult(userDataPath, { + identity: 'testuser', + spawnFn: fake.spawnFn as never + }) + ).toEqual({ mode: 'marker-hit' }) + expect(fake.calls).toHaveLength(0) + }) + + it('re-grants and replaces a sparse ACL marker one byte over the boundary', async () => { + const markerPath = join(userDataPath, WINDOWS_ACL_GRANT_MARKER_FILE) + writeFileSync(markerPath, '{"schemeVersion":1,"identity":"testuser","grantedAt":1}') + truncateSync(markerPath, WINDOWS_ACL_GRANT_MARKER_MAX_BYTES + 1) + const fake = createFakeSpawn(0) + + expect( + await awaitResult(userDataPath, { + identity: 'testuser', + spawnFn: fake.spawnFn as never + }) + ).toEqual({ mode: 'granted' }) + expect(fake.calls).toHaveLength(2) + expect(statSync(markerPath).size).toBeLessThan(WINDOWS_ACL_GRANT_MARKER_MAX_BYTES) + }) + it('re-grants when the marker belongs to a different identity', async () => { writeFileSync( join(userDataPath, WINDOWS_ACL_GRANT_MARKER_FILE), diff --git a/src/main/startup/windows-user-data-acl.ts b/src/main/startup/windows-user-data-acl.ts index bf1e1d01f05..ef0f0834442 100644 --- a/src/main/startup/windows-user-data-acl.ts +++ b/src/main/startup/windows-user-data-acl.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process' -import { readFileSync, writeFileSync } from 'node:fs' +import { writeFileSync } from 'node:fs' import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' import { getIcaclsExePath, resolveCurrentWindowsIdentity } from '../win32-utils' /** @@ -32,6 +33,7 @@ import { getIcaclsExePath, resolveCurrentWindowsIdentity } from '../win32-utils' export const WINDOWS_ACL_GRANT_MARKER_FILE = 'windows-acl-grant.json' export const WINDOWS_ACL_GRANT_SCHEME_VERSION = 1 +export const WINDOWS_ACL_GRANT_MARKER_MAX_BYTES = 8 * 1024 const GRANT_TIMEOUT_MS = 120_000 @@ -58,7 +60,10 @@ type EnsureOptions = { function readMarker(userDataPath: string): WindowsAclGrantMarker | null { try { const parsed = JSON.parse( - readFileSync(join(userDataPath, WINDOWS_ACL_GRANT_MARKER_FILE), 'utf-8') + readNodeFileSyncWithinLimit( + join(userDataPath, WINDOWS_ACL_GRANT_MARKER_FILE), + WINDOWS_ACL_GRANT_MARKER_MAX_BYTES + ).buffer.toString('utf8') ) as Partial if ( parsed.schemeVersion === WINDOWS_ACL_GRANT_SCHEME_VERSION && diff --git a/src/main/startup/x-display-lock-reader.test.ts b/src/main/startup/x-display-lock-reader.test.ts new file mode 100644 index 00000000000..c6ffa400737 --- /dev/null +++ b/src/main/startup/x-display-lock-reader.test.ts @@ -0,0 +1,46 @@ +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { + commandLine: { appendSwitch: vi.fn() }, + disableHardwareAcceleration: vi.fn(), + once: vi.fn() + } +})) + +import { X_DISPLAY_LOCK_MAX_BYTES, readXDisplayLockPid } from './ensure-virtual-display' + +const roots: string[] = [] + +function createLock(contents: string): string { + const root = mkdtempSync(join(tmpdir(), 'orca-x-display-lock-')) + roots.push(root) + const lockPath = join(root, '.X99-lock') + writeFileSync(lockPath, contents) + return lockPath +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('X display lock reader', () => { + it('accepts a PID marker at the exact byte boundary', () => { + const pid = '4321' + const lockPath = createLock(pid + ' '.repeat(X_DISPLAY_LOCK_MAX_BYTES - pid.length)) + + expect(readXDisplayLockPid(lockPath)).toBe(4321) + }) + + it('rejects a sparse PID marker one byte over the boundary', () => { + const lockPath = createLock('4321') + truncateSync(lockPath, X_DISPLAY_LOCK_MAX_BYTES + 1) + + expect(readXDisplayLockPid(lockPath)).toBeNull() + }) +}) diff --git a/src/main/stats/collector-retention.test.ts b/src/main/stats/collector-retention.test.ts new file mode 100644 index 00000000000..d6858083b55 --- /dev/null +++ b/src/main/stats/collector-retention.test.ts @@ -0,0 +1,244 @@ +import { + closeSync, + ftruncateSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +let userDataDir: string +const statsPath = (): string => join(userDataDir, 'orca-stats.json') + +vi.mock('electron', () => ({ + app: { getPath: () => userDataDir } +})) + +async function importCollector() { + return import('./collector') +} + +describe('StatsCollector retention bounds', () => { + beforeEach(() => { + userDataDir = mkdtempSync(join(tmpdir(), 'orca-stats-retention-')) + vi.resetModules() + }) + + afterEach(() => { + rmSync(userDataDir, { recursive: true, force: true }) + vi.restoreAllMocks() + }) + + it('starts fresh without reading an oversized sparse stats file', async () => { + const { StatsCollector, STATS_FILE_MAX_BYTES, initStatsPath } = await importCollector() + const descriptor = openSync(statsPath(), 'w') + ftruncateSync(descriptor, STATS_FILE_MAX_BYTES + 1) + closeSync(descriptor) + const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) + initStatsPath() + + const collector = new StatsCollector() + + expect(collector.getSummary()).toEqual({ + totalAgentsSpawned: 0, + totalPRsCreated: 0, + totalAgentTimeMs: 0, + firstEventAt: null + }) + expect(error).toHaveBeenCalledWith( + '[stats] Failed to load stats, starting fresh:', + expect.objectContaining({ name: 'NodeFileReadTooLargeError' }) + ) + collector.flush() + }) + + it('normalizes loaded event and PR-dedup arrays to their persisted limits', async () => { + const { StatsCollector, STATS_COUNTED_PR_MAX_ENTRIES, STATS_EVENT_MAX_ENTRIES, initStatsPath } = + await importCollector() + writeFileSync( + statsPath(), + JSON.stringify({ + schemaVersion: 1, + events: Array.from({ length: STATS_EVENT_MAX_ENTRIES + 3 }, (_, at) => ({ + type: 'agent_start', + at + })), + aggregates: { + totalAgentsSpawned: 123, + totalPRsCreated: 456, + totalAgentTimeMs: 789, + countedPRs: Array.from( + { length: STATS_COUNTED_PR_MAX_ENTRIES + 3 }, + (_, index) => `https://example.test/pr/${index}` + ), + firstEventAt: 0 + } + }) + ) + initStatsPath() + const collector = new StatsCollector() + + collector.flush() + + const persisted = JSON.parse(readFileSync(statsPath(), 'utf8')) + expect(persisted.events).toHaveLength(STATS_EVENT_MAX_ENTRIES) + expect(persisted.events[0].at).toBe(3) + expect(persisted.aggregates.countedPRs).toHaveLength(STATS_COUNTED_PR_MAX_ENTRIES) + expect(persisted.aggregates.countedPRs[0]).toBe('https://example.test/pr/3') + expect(collector.getSummary()).toMatchObject({ + totalAgentsSpawned: 123, + totalPRsCreated: 456, + totalAgentTimeMs: 789 + }) + }) + + it('retains only the newest events immediately and persists them in order', async () => { + const { StatsCollector, STATS_EVENT_MAX_ENTRIES, initStatsPath } = await importCollector() + initStatsPath() + const collector = new StatsCollector() + const internals = collector as unknown as { events: unknown[] } + + for (let at = 0; at < STATS_EVENT_MAX_ENTRIES + 3; at++) { + collector.record({ type: 'agent_start', at }) + expect(internals.events.length).toBeLessThanOrEqual(STATS_EVENT_MAX_ENTRIES) + } + + collector.flush() + const persisted = JSON.parse(readFileSync(statsPath(), 'utf8')) + expect(persisted.events).toHaveLength(STATS_EVENT_MAX_ENTRIES) + expect(persisted.events[0].at).toBe(3) + expect(persisted.events.at(-1).at).toBe(STATS_EVENT_MAX_ENTRIES + 2) + }) + + it('accepts an exact-limit event and drops a one-byte-larger event', async () => { + const { StatsCollector, STATS_EVENT_MAX_BYTES, STATS_FILE_MAX_BYTES, initStatsPath } = + await importCollector() + initStatsPath() + const collector = new StatsCollector() + const exactEvent = { type: 'agent_start' as const, at: 1, meta: { payload: '' } } + const eventOverhead = Buffer.byteLength(JSON.stringify(exactEvent), 'utf8') + exactEvent.meta.payload = 'x'.repeat(STATS_EVENT_MAX_BYTES - eventOverhead) + const oversizedEvent = { + ...exactEvent, + meta: { payload: `${exactEvent.meta.payload}x` } + } + + expect(Buffer.byteLength(JSON.stringify(exactEvent), 'utf8')).toBe(STATS_EVENT_MAX_BYTES) + collector.record(exactEvent) + collector.record(oversizedEvent) + collector.flush() + + const persisted = JSON.parse(readFileSync(statsPath(), 'utf8')) + expect(persisted.events).toEqual([exactEvent]) + expect(collector.getSummary().totalAgentsSpawned).toBe(2) + expect(Buffer.byteLength(JSON.stringify(persisted), 'utf8')).toBeLessThanOrEqual( + STATS_FILE_MAX_BYTES + ) + }) + + it('evicts oldest events until the aggregate event-byte budget fits', async () => { + const { StatsCollector, STATS_EVENT_MAX_BYTES, STATS_EVENT_MAX_RETAINED_BYTES, initStatsPath } = + await importCollector() + initStatsPath() + const collector = new StatsCollector() + const eventCount = STATS_EVENT_MAX_RETAINED_BYTES / STATS_EVENT_MAX_BYTES + 1 + for (let at = 0; at < eventCount; at++) { + const event = { type: 'agent_start' as const, at, meta: { payload: '' } } + const overhead = Buffer.byteLength(JSON.stringify(event), 'utf8') + event.meta.payload = 'x'.repeat(STATS_EVENT_MAX_BYTES - overhead) + collector.record(event) + } + + collector.flush() + + const persisted = JSON.parse(readFileSync(statsPath(), 'utf8')) + expect(persisted.events).toHaveLength(eventCount - 1) + expect(persisted.events[0].at).toBe(1) + expect(persisted.events.at(-1).at).toBe(eventCount - 1) + }) + + it('closes the oldest live agent when the live-session ceiling is reached', async () => { + const { StatsCollector, STATS_LIVE_AGENT_MAX_ENTRIES, initStatsPath } = await importCollector() + initStatsPath() + const collector = new StatsCollector() + const internals = collector as unknown as { liveAgents: Map } + + for (let index = 0; index <= STATS_LIVE_AGENT_MAX_ENTRIES; index++) { + collector.onAgentStart(`pty-${index}`, 1_000 + index) + } + + expect(internals.liveAgents).toHaveLength(STATS_LIVE_AGENT_MAX_ENTRIES) + expect(internals.liveAgents.has('pty-0')).toBe(false) + expect(internals.liveAgents.has(`pty-${STATS_LIVE_AGENT_MAX_ENTRIES}`)).toBe(true) + expect(collector.getSummary().totalAgentTimeMs).toBe(STATS_LIVE_AGENT_MAX_ENTRIES) + + collector.onAgentStop('pty-0', 100_000) + expect(collector.getSummary().totalAgentTimeMs).toBe(STATS_LIVE_AGENT_MAX_ENTRIES) + collector.flush() + }) + + it('bounds live-agent ids by individual and aggregate bytes', async () => { + const { + StatsCollector, + STATS_LIVE_AGENT_ID_MAX_BYTES, + STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES, + initStatsPath + } = await importCollector() + initStatsPath() + const collector = new StatsCollector() + const internals = collector as unknown as { liveAgents: Map } + const retainedCount = STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES / STATS_LIVE_AGENT_ID_MAX_BYTES + const id = (index: number): string => + `${String(index).padStart(4, '0')}${'x'.repeat(STATS_LIVE_AGENT_ID_MAX_BYTES - 4)}` + + for (let index = 0; index <= retainedCount; index++) { + collector.onAgentStart(id(index), index) + } + collector.onAgentStart('x'.repeat(STATS_LIVE_AGENT_ID_MAX_BYTES + 1), 10_000) + + expect(internals.liveAgents).toHaveLength(retainedCount) + expect(internals.liveAgents.has(id(0))).toBe(false) + expect(internals.liveAgents.has(id(retainedCount))).toBe(true) + expect( + [...internals.liveAgents.keys()].reduce( + (bytes, ptyId) => bytes + Buffer.byteLength(ptyId, 'utf8'), + 0 + ) + ).toBe(STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES) + collector.flush() + }) + + it('bounds counted PR urls by individual and aggregate serialized bytes', async () => { + const { + StatsCollector, + STATS_COUNTED_PR_MAX_RETAINED_BYTES, + STATS_COUNTED_PR_URL_MAX_BYTES, + initStatsPath + } = await importCollector() + initStatsPath() + const collector = new StatsCollector() + const internals = collector as unknown as { aggregates: { countedPRs: string[] } } + const retainedCount = STATS_COUNTED_PR_MAX_RETAINED_BYTES / STATS_COUNTED_PR_URL_MAX_BYTES + const url = (index: number): string => { + const prefix = `https://example.test/${String(index).padStart(4, '0')}/` + return `${prefix}${'x'.repeat(STATS_COUNTED_PR_URL_MAX_BYTES - 2 - prefix.length)}` + } + + for (let index = 0; index <= retainedCount; index++) { + collector.record({ type: 'pr_created', at: index, meta: { prUrl: url(index) } }) + } + const oversizedUrl = 'x'.repeat(STATS_COUNTED_PR_URL_MAX_BYTES - 1) + collector.record({ type: 'pr_created', at: 10_000, meta: { prUrl: oversizedUrl } }) + + expect(Buffer.byteLength(JSON.stringify(url(0)), 'utf8')).toBe(STATS_COUNTED_PR_URL_MAX_BYTES) + expect(internals.aggregates.countedPRs).toHaveLength(retainedCount) + expect(collector.hasCountedPR(url(0))).toBe(false) + expect(collector.hasCountedPR(url(retainedCount))).toBe(true) + expect(collector.hasCountedPR(oversizedUrl)).toBe(false) + collector.flush() + }) +}) diff --git a/src/main/stats/collector.ts b/src/main/stats/collector.ts index 508aa0a172a..9cf71e74cb4 100644 --- a/src/main/stats/collector.ts +++ b/src/main/stats/collector.ts @@ -1,17 +1,25 @@ import { app } from 'electron' -import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync } from 'node:fs' +import { writeFileSync, mkdirSync, existsSync, renameSync } from 'node:fs' import { writeFile, mkdir, rm } from 'node:fs/promises' import { join, dirname } from 'node:path' import type { StatsSummary } from '../../shared/types' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { measureUtf8ByteLength } from '../../shared/utf8-byte-limits' import type { StatsEvent, StatsAggregates, StatsFile } from './types' +import { StatsAggregateTracker } from './stats-aggregate-tracker' +import { + createDefaultStatsFile, + parseLoadedStatsFile, + STATS_FILE_MAX_BYTES, + STATS_LIVE_AGENT_ID_MAX_BYTES, + STATS_LIVE_AGENT_MAX_ENTRIES, + STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES, + STATS_SCHEMA_VERSION, + StatsEventLog +} from './stats-retention' -const STATS_SCHEMA_VERSION = 1 -const MAX_EVENTS = 10_000 -// Why: countedPRs is a deduplication registry that grows with every PR created -// through Orca. Without a cap, a heavily-used instance accumulates thousands of -// URL strings across months. 2000 entries is about 6-12 months of active use -// for a power user, and at ~50 chars per URL the overhead is ~100KB max. -const MAX_COUNTED_PRS = 2_000 +export * from './stats-retention' // Why 5s instead of the main store's 300ms: stat events are infrequent // (a few per session) and not latency-sensitive for the UI. const DEBOUNCE_MS = 5_000 @@ -33,28 +41,11 @@ function getStatsFile(): string { return _statsFile } -function getDefaultAggregates(): StatsAggregates { - return { - totalAgentsSpawned: 0, - totalPRsCreated: 0, - totalAgentTimeMs: 0, - countedPRs: [], - firstEventAt: null - } -} - -function getDefaultStatsFile(): StatsFile { - return { - schemaVersion: STATS_SCHEMA_VERSION, - events: [], - aggregates: getDefaultAggregates() - } -} - export class StatsCollector { - private events: StatsEvent[] - private aggregates: StatsAggregates + private eventLog: StatsEventLog + private aggregateTracker: StatsAggregateTracker private liveAgents = new Map() // ptyId → startTimestamp + private liveAgentIdBytes = 0 private writeTimer: ReturnType | null = null // Monotonic id stamped on each prepared payload; the highest committed one // wins so a slow in-flight async write can't clobber a newer sync flush. @@ -67,8 +58,16 @@ export class StatsCollector { constructor() { const data = this.load() - this.events = data.events - this.aggregates = data.aggregates + this.eventLog = new StatsEventLog(data.events) + this.aggregateTracker = new StatsAggregateTracker(data.aggregates) + } + + private get aggregates(): StatsAggregates { + return this.aggregateTracker.aggregates + } + + private get events(): StatsEvent[] { + return this.eventLog.events } onAgentStarted(listener: (totalAgentsSpawned: number) => void): () => void { @@ -85,15 +84,36 @@ export class StatsCollector { // ── Recording ────────────────────────────────────────────────────── record(event: StatsEvent): void { - this.events.push(event) - this.updateAggregates(event) + this.eventLog.retain(event) + this.aggregateTracker.record(event, this.agentStartListeners) this.scheduleSave() } // ── Agent lifecycle (called by AgentDetector) ───────────────────── onAgentStart(ptyId: string, at: number, repoId?: string, worktreeId?: string): void { - this.liveAgents.set(ptyId, at) + const idMeasurement = measureUtf8ByteLength(ptyId, { + stopAfterBytes: STATS_LIVE_AGENT_ID_MAX_BYTES + }) + if (!idMeasurement.exceededLimit) { + if (!this.liveAgents.has(ptyId)) { + while ( + this.liveAgents.size >= STATS_LIVE_AGENT_MAX_ENTRIES || + this.liveAgentIdBytes + idMeasurement.byteLength > STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES + ) { + const oldest = this.liveAgents.entries().next() + if (oldest.done) { + break + } + const [oldestPtyId, oldestStartAt] = oldest.value + this.liveAgents.delete(oldestPtyId) + this.liveAgentIdBytes -= measureUtf8ByteLength(oldestPtyId).byteLength + this.recordAgentStop(oldestPtyId, oldestStartAt, at) + } + this.liveAgentIdBytes += idMeasurement.byteLength + } + this.liveAgents.set(ptyId, at) + } this.record({ type: 'agent_start', at, @@ -109,6 +129,11 @@ export class StatsCollector { return } this.liveAgents.delete(ptyId) + this.liveAgentIdBytes -= measureUtf8ByteLength(ptyId).byteLength + this.recordAgentStop(ptyId, startAt, at) + } + + private recordAgentStop(ptyId: string, startAt: number, at: number): void { const durationMs = Math.max(0, at - startAt) this.aggregates.totalAgentTimeMs += durationMs this.record({ @@ -127,12 +152,7 @@ export class StatsCollector { // ── Query ───────────────────────────────────────────────────────── getSummary(): StatsSummary { - return { - totalAgentsSpawned: this.aggregates.totalAgentsSpawned, - totalPRsCreated: this.aggregates.totalPRsCreated, - totalAgentTimeMs: this.aggregates.totalAgentTimeMs, - firstEventAt: this.aggregates.firstEventAt - } + return this.aggregateTracker.getSummary() } // ── Shutdown flush ──────────────────────────────────────────────── @@ -163,17 +183,10 @@ export class StatsCollector { try { const statsFile = getStatsFile() if (existsSync(statsFile)) { - const raw = readFileSync(statsFile, 'utf-8') - const parsed = JSON.parse(raw) as StatsFile - // Merge with defaults for forward compatibility - return { - ...getDefaultStatsFile(), - ...parsed, - aggregates: { - ...getDefaultAggregates(), - ...parsed.aggregates - } - } + const raw = readNodeFileSyncWithinLimit(statsFile, STATS_FILE_MAX_BYTES).buffer.toString( + 'utf8' + ) + return parseLoadedStatsFile(raw) } } catch (err) { // Why "start fresh" instead of crashing: lifetime aggregates are lost @@ -182,47 +195,7 @@ export class StatsCollector { // disk so it can be inspected for debugging. console.error('[stats] Failed to load stats, starting fresh:', err) } - return getDefaultStatsFile() - } - - private updateAggregates(event: StatsEvent): void { - if (this.aggregates.firstEventAt === null) { - this.aggregates.firstEventAt = event.at - } - - switch (event.type) { - case 'agent_start': - this.aggregates.totalAgentsSpawned++ - // Why: notify listeners synchronously AFTER increment so observers - // see the post-increment count. Listener errors are swallowed to - // keep stat recording robust — a buggy listener must not lose the - // event from the on-disk log. - for (const listener of this.agentStartListeners) { - try { - listener(this.aggregates.totalAgentsSpawned) - } catch (err) { - console.error('[stats] agent-start listener threw:', err) - } - } - break - case 'pr_created': - this.aggregates.totalPRsCreated++ - if (event.meta?.prUrl) { - this.aggregates.countedPRs.push(String(event.meta.prUrl)) - // Why: trim oldest entries so the dedup array does not grow without - // bound. The aggregate totalPRsCreated counter remains accurate; only - // the dedup lookup for very old PRs is lost, which is acceptable - // since PRs that old would never be re-counted in practice. - if (this.aggregates.countedPRs.length > MAX_COUNTED_PRS) { - this.aggregates.countedPRs = this.aggregates.countedPRs.slice(-MAX_COUNTED_PRS) - } - } - break - // agent_stop duration is handled directly in onAgentStop() to avoid - // double-counting — the duration is added to totalAgentTimeMs there. - case 'agent_stop': - break - } + return createDefaultStatsFile() } private scheduleSave(): void { @@ -250,9 +223,9 @@ export class StatsCollector { } } - // Serialize the current state and pick a unique temp path. Trimming mutates - // in-memory state and JSON.stringify must see a consistent snapshot, so both - // writers call this synchronously before any await to avoid a torn snapshot. + // Serialize the current state and pick a unique temp path. JSON.stringify + // must see a consistent snapshot, so both writers call this synchronously + // before any await to avoid a torn snapshot. // The monotonic generation lets a later write veto an earlier, still-in-flight // one so a stale rename can never win (see writeToDiskAsync). private prepareWritePayload(): { @@ -263,11 +236,6 @@ export class StatsCollector { } { const statsFile = getStatsFile() - // Trim events to bounded size before writing - if (this.events.length > MAX_EVENTS) { - this.events = this.events.slice(-MAX_EVENTS) - } - const data: StatsFile = { schemaVersion: STATS_SCHEMA_VERSION, events: this.events, @@ -278,7 +246,8 @@ export class StatsCollector { // Unique temp file so the async debounced writer and the sync shutdown // flush never write the same temp path (same pattern as persistence.ts). const tmpFile = `${statsFile}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp` - return { statsFile, tmpFile, json: JSON.stringify(data), generation } + const json = stringifyJsonWithinByteLimit(data, STATS_FILE_MAX_BYTES).serialized + return { statsFile, tmpFile, json, generation } } private writeToDiskSync(): void { diff --git a/src/main/stats/stats-aggregate-tracker.ts b/src/main/stats/stats-aggregate-tracker.ts new file mode 100644 index 00000000000..3ed168571ad --- /dev/null +++ b/src/main/stats/stats-aggregate-tracker.ts @@ -0,0 +1,71 @@ +import type { StatsSummary } from '../../shared/types' +import type { StatsAggregates, StatsEvent } from './types' +import { + jsonByteLengthWithinLimit, + STATS_COUNTED_PR_MAX_ENTRIES, + STATS_COUNTED_PR_MAX_RETAINED_BYTES, + STATS_COUNTED_PR_URL_MAX_BYTES +} from './stats-retention' + +export class StatsAggregateTracker { + private countedPrRetainedBytes: number + + constructor(readonly aggregates: StatsAggregates) { + this.countedPrRetainedBytes = aggregates.countedPRs.reduce( + (total, value) => + total + (jsonByteLengthWithinLimit(value, STATS_COUNTED_PR_URL_MAX_BYTES) ?? 0), + 0 + ) + } + + record(event: StatsEvent, agentStartListeners: ((total: number) => void)[]): void { + if (this.aggregates.firstEventAt === null) { + this.aggregates.firstEventAt = event.at + } + if (event.type === 'agent_start') { + this.aggregates.totalAgentsSpawned++ + for (const listener of agentStartListeners) { + try { + listener(this.aggregates.totalAgentsSpawned) + } catch (err) { + console.error('[stats] agent-start listener threw:', err) + } + } + return + } + if (event.type !== 'pr_created') { + return + } + this.aggregates.totalPRsCreated++ + if (!event.meta?.prUrl) { + return + } + const prUrl = String(event.meta.prUrl) + const prUrlBytes = jsonByteLengthWithinLimit(prUrl, STATS_COUNTED_PR_URL_MAX_BYTES) + if (prUrlBytes === null) { + return + } + this.aggregates.countedPRs.push(prUrl) + this.countedPrRetainedBytes += prUrlBytes + while ( + this.aggregates.countedPRs.length > STATS_COUNTED_PR_MAX_ENTRIES || + this.countedPrRetainedBytes > STATS_COUNTED_PR_MAX_RETAINED_BYTES + ) { + const oldestPrUrl = this.aggregates.countedPRs.shift() + if (oldestPrUrl === undefined) { + break + } + this.countedPrRetainedBytes -= + jsonByteLengthWithinLimit(oldestPrUrl, STATS_COUNTED_PR_URL_MAX_BYTES) ?? 0 + } + } + + getSummary(): StatsSummary { + return { + totalAgentsSpawned: this.aggregates.totalAgentsSpawned, + totalPRsCreated: this.aggregates.totalPRsCreated, + totalAgentTimeMs: this.aggregates.totalAgentTimeMs, + firstEventAt: this.aggregates.firstEventAt + } + } +} diff --git a/src/main/stats/stats-file-json-admission.test.ts b/src/main/stats/stats-file-json-admission.test.ts new file mode 100644 index 00000000000..3b4d3bc9f06 --- /dev/null +++ b/src/main/stats/stats-file-json-admission.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' +import { parseLoadedStatsFile } from './stats-retention' + +describe('stats file JSON admission', () => { + it('preserves ordinary stats under the structure limits', () => { + expect( + parseLoadedStatsFile('{"schemaVersion":1,"events":[],"aggregates":{}}', { + structuralTokens: 11, + nestingDepth: 2 + }) + ).toMatchObject({ schemaVersion: 1, events: [] }) + }) + + it('rejects structural amplification before parsing stats', () => { + expect(() => + parseLoadedStatsFile('{"events":[{},{}]}', { + structuralTokens: 7, + nestingDepth: 3 + }) + ).toThrow('JSON structure') + }) +}) diff --git a/src/main/stats/stats-retention.ts b/src/main/stats/stats-retention.ts new file mode 100644 index 00000000000..e750e65139d --- /dev/null +++ b/src/main/stats/stats-retention.ts @@ -0,0 +1,170 @@ +import { stringifyJsonWithinByteLimit } from '../../shared/node-bounded-json-stringify' +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from '../../shared/json-text-structure-limit' +import type { StatsAggregates, StatsEvent, StatsFile } from './types' + +export const STATS_SCHEMA_VERSION = 1 +export const STATS_FILE_MAX_BYTES = 16 * 1024 * 1024 +export const STATS_EVENT_MAX_ENTRIES = 10_000 +export const STATS_EVENT_MAX_BYTES = 1024 * 1024 +export const STATS_EVENT_MAX_RETAINED_BYTES = 12 * 1024 * 1024 +export const STATS_LIVE_AGENT_MAX_ENTRIES = 4_096 +export const STATS_LIVE_AGENT_ID_MAX_BYTES = 4 * 1024 +export const STATS_LIVE_AGENT_MAX_RETAINED_ID_BYTES = 1024 * 1024 +export const STATS_COUNTED_PR_MAX_ENTRIES = 2_000 +export const STATS_COUNTED_PR_URL_MAX_BYTES = 8 * 1024 +export const STATS_COUNTED_PR_MAX_RETAINED_BYTES = 1024 * 1024 +export const STATS_FILE_JSON_LIMITS: JsonTextStructureLimits = { + structuralTokens: 1_000_000, + nestingDepth: 128 +} + +export function jsonByteLengthWithinLimit(value: unknown, maxBytes: number): number | null { + try { + return stringifyJsonWithinByteLimit(value, maxBytes).byteLength + } catch { + return null + } +} + +function normalizeLoadedEvents(values: unknown[]): StatsEvent[] { + const newest: StatsEvent[] = [] + let retainedBytes = 0 + for ( + let index = values.length - 1; + index >= 0 && newest.length < STATS_EVENT_MAX_ENTRIES; + index-- + ) { + const value = values[index] + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + continue + } + const event = value as StatsEvent + const eventBytes = jsonByteLengthWithinLimit(event, STATS_EVENT_MAX_BYTES) + if (eventBytes === null) { + continue + } + if (retainedBytes + eventBytes > STATS_EVENT_MAX_RETAINED_BYTES) { + break + } + retainedBytes += eventBytes + newest.push(event) + } + return newest.toReversed() +} + +function normalizeCountedPrUrls(values: unknown[]): string[] { + const newest: string[] = [] + let retainedBytes = 0 + for ( + let index = values.length - 1; + index >= 0 && newest.length < STATS_COUNTED_PR_MAX_ENTRIES; + index-- + ) { + const value = values[index] + if (typeof value !== 'string') { + continue + } + const valueBytes = jsonByteLengthWithinLimit(value, STATS_COUNTED_PR_URL_MAX_BYTES) + if (valueBytes === null) { + continue + } + if (retainedBytes + valueBytes > STATS_COUNTED_PR_MAX_RETAINED_BYTES) { + break + } + retainedBytes += valueBytes + newest.push(value) + } + return newest.toReversed() +} + +export function createDefaultStatsFile(): StatsFile { + return { + schemaVersion: STATS_SCHEMA_VERSION, + events: [], + aggregates: { + totalAgentsSpawned: 0, + totalPRsCreated: 0, + totalAgentTimeMs: 0, + countedPRs: [], + firstEventAt: null + } + } +} + +export function normalizeLoadedStatsFile(parsed: unknown): StatsFile { + const candidate = + typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) + ? (parsed as Partial) + : {} + const aggregateCandidate: Partial = + typeof candidate.aggregates === 'object' && + candidate.aggregates !== null && + !Array.isArray(candidate.aggregates) + ? candidate.aggregates + : {} + const countedPRs = Array.isArray(aggregateCandidate.countedPRs) + ? normalizeCountedPrUrls(aggregateCandidate.countedPRs) + : [] + return { + schemaVersion: + typeof candidate.schemaVersion === 'number' ? candidate.schemaVersion : STATS_SCHEMA_VERSION, + events: Array.isArray(candidate.events) ? normalizeLoadedEvents(candidate.events) : [], + aggregates: { + totalAgentsSpawned: + typeof aggregateCandidate.totalAgentsSpawned === 'number' + ? aggregateCandidate.totalAgentsSpawned + : 0, + totalPRsCreated: + typeof aggregateCandidate.totalPRsCreated === 'number' + ? aggregateCandidate.totalPRsCreated + : 0, + totalAgentTimeMs: + typeof aggregateCandidate.totalAgentTimeMs === 'number' + ? aggregateCandidate.totalAgentTimeMs + : 0, + countedPRs, + firstEventAt: + typeof aggregateCandidate.firstEventAt === 'number' ? aggregateCandidate.firstEventAt : null + } + } +} + +export function parseLoadedStatsFile( + serialized: string, + structureLimits: JsonTextStructureLimits = STATS_FILE_JSON_LIMITS +): StatsFile { + assertJsonTextStructureWithinLimits(serialized, structureLimits) + return normalizeLoadedStatsFile(JSON.parse(serialized)) +} + +export class StatsEventLog { + private eventByteLengths: number[] + private retainedEventBytes: number + + constructor(readonly events: StatsEvent[]) { + this.eventByteLengths = events.map( + (event) => jsonByteLengthWithinLimit(event, STATS_EVENT_MAX_BYTES) ?? 0 + ) + this.retainedEventBytes = this.eventByteLengths.reduce((total, bytes) => total + bytes, 0) + } + + retain(event: StatsEvent): void { + const eventBytes = jsonByteLengthWithinLimit(event, STATS_EVENT_MAX_BYTES) + if (eventBytes === null) { + return + } + this.events.push(event) + this.eventByteLengths.push(eventBytes) + this.retainedEventBytes += eventBytes + while ( + this.events.length > STATS_EVENT_MAX_ENTRIES || + this.retainedEventBytes > STATS_EVENT_MAX_RETAINED_BYTES + ) { + this.events.shift() + this.retainedEventBytes -= this.eventByteLengths.shift() ?? 0 + } + } +} diff --git a/src/main/terminal-history-gc.test.ts b/src/main/terminal-history-gc.test.ts new file mode 100644 index 00000000000..15cfb3e972d --- /dev/null +++ b/src/main/terminal-history-gc.test.ts @@ -0,0 +1,213 @@ +import { existsSync } from 'node:fs' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + deleteWslWorktreeHistoryDirectories, + runTerminalHistoryGarbageCollection, + TERMINAL_HISTORY_GC_META_MAX_BYTES +} from './terminal-history-gc' + +let root = '' +let mainRoot = '' +let wslRoot = '' + +function oldMetadata(worktreeId: string): string { + return JSON.stringify({ + worktreeId, + createdAt: new Date(Date.now() - 10 * 60 * 1000).toISOString() + }) +} + +async function createHistoryDirectory( + historyRoot: string, + directoryName: string, + worktreeId = directoryName +): Promise { + const directory = join(historyRoot, directoryName) + await mkdir(directory, { recursive: true }) + await writeFile(join(directory, 'meta.json'), oldMetadata(worktreeId)) + return directory +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-terminal-history-gc-')) + mainRoot = join(root, 'terminal-history') + wslRoot = join(root, 'terminal-history-wsl') + await mkdir(mainRoot) + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) + +describe('terminal history GC memory limits', () => { + it('preserves ordinary live/orphan and age-guard behavior below the limits', async () => { + const liveDirectory = await createHistoryDirectory(mainRoot, 'live-dir', 'live-worktree') + const orphanDirectory = await createHistoryDirectory(mainRoot, 'orphan-dir', 'orphan-worktree') + const freshDirectory = await createHistoryDirectory(mainRoot, 'fresh-dir', 'fresh-worktree') + await writeFile( + join(freshDirectory, 'meta.json'), + JSON.stringify({ worktreeId: 'fresh-worktree', createdAt: new Date().toISOString() }) + ) + + const summary = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(['live-worktree']) + }) + + expect(summary).toMatchObject({ + capacityExceeded: false, + orphaned: 1, + pruned: 1, + totalDirs: 3 + }) + expect(existsSync(liveDirectory)).toBe(true) + expect(existsSync(freshDirectory)).toBe(true) + expect(existsSync(orphanDirectory)).toBe(false) + }) + + it('accepts metadata at the exact byte limit and skips one byte over', async () => { + const exactDirectory = await createHistoryDirectory(mainRoot, 'exact-dir', 'exact-worktree') + const exactMetaPath = join(exactDirectory, 'meta.json') + const exactJson = oldMetadata('exact-worktree') + await writeFile( + exactMetaPath, + `${exactJson}${' '.repeat(TERMINAL_HISTORY_GC_META_MAX_BYTES - exactJson.length)}` + ) + + expect( + runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set() + }).pruned + ).toBe(1) + + const oversizedDirectory = await createHistoryDirectory( + mainRoot, + 'oversized-dir', + 'oversized-worktree' + ) + const oversizedMetaPath = join(oversizedDirectory, 'meta.json') + await writeFile(oversizedMetaPath, oldMetadata('oversized-worktree')) + await truncate(oversizedMetaPath, TERMINAL_HISTORY_GC_META_MAX_BYTES + 8 * 1024 * 1024) + + expect( + runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set() + }).pruned + ).toBe(0) + expect(existsSync(oversizedDirectory)).toBe(true) + }) + + it('accepts the exact discovery-entry budget and stops before the next entry', async () => { + await createHistoryDirectory(mainRoot, 'one') + await createHistoryDirectory(mainRoot, 'two') + + const exact = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(), + limits: { maxDiscoveryEntries: 4 } + }) + expect(exact).toMatchObject({ capacityExceeded: false, pruned: 2 }) + + await createHistoryDirectory(mainRoot, 'one') + await createHistoryDirectory(mainRoot, 'two') + await createHistoryDirectory(mainRoot, 'three') + const capped = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(), + limits: { maxDiscoveryEntries: 4 } + }) + expect(capped.capacityExceeded).toBe(true) + expect( + [join(mainRoot, 'one'), join(mainRoot, 'two'), join(mainRoot, 'three')].some(existsSync) + ).toBe(true) + }) + + it('fails closed when a worktree directory exceeds its flat-file limit', async () => { + const directory = await createHistoryDirectory(mainRoot, 'many-files') + for (let index = 0; index < 100; index += 1) { + await writeFile(join(directory, `history-${index}`), 'x') + } + + const summary = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(), + limits: { maxFilesPerWorktree: 1 } + }) + + expect(summary).toMatchObject({ capacityExceeded: false, pruned: 0 }) + expect(existsSync(directory)).toBe(true) + }) + + it('does not recursively prune an unexpected nested history tree', async () => { + const directory = await createHistoryDirectory(mainRoot, 'nested-tree') + await mkdir(join(directory, 'unexpected', 'deep'), { recursive: true }) + + const summary = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set() + }) + + expect(summary).toMatchObject({ capacityExceeded: false, pruned: 0 }) + expect(existsSync(directory)).toBe(true) + }) + + it('caps WSL distro roots for both GC and direct worktree cleanup', async () => { + const first = join(wslRoot, 'Distro-A') + const second = join(wslRoot, 'Distro-B') + let firstHistory = await createHistoryDirectory(first, 'hash', 'first-worktree') + let secondHistory = await createHistoryDirectory(second, 'hash', 'second-worktree') + + const exactSummary = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(), + limits: { maxWslDistros: 2 } + }) + expect(exactSummary).toMatchObject({ capacityExceeded: false, pruned: 2 }) + + firstHistory = await createHistoryDirectory(first, 'hash', 'first-worktree') + secondHistory = await createHistoryDirectory(second, 'hash', 'second-worktree') + const cappedSummary = runTerminalHistoryGarbageCollection({ + mainRoot, + wslRoot, + liveWorktreeIds: new Set(), + limits: { maxWslDistros: 1 } + }) + expect(cappedSummary.capacityExceeded).toBe(true) + expect([firstHistory, secondHistory].some(existsSync)).toBe(true) + + await mkdir(firstHistory, { recursive: true }) + await mkdir(secondHistory, { recursive: true }) + deleteWslWorktreeHistoryDirectories({ + wslRoot, + worktreeHash: 'hash', + limits: { maxWslDistros: 2 } + }) + expect(existsSync(firstHistory)).toBe(false) + expect(existsSync(secondHistory)).toBe(false) + + await mkdir(firstHistory, { recursive: true }) + await mkdir(secondHistory, { recursive: true }) + deleteWslWorktreeHistoryDirectories({ + wslRoot, + worktreeHash: 'hash', + limits: { maxWslDistros: 1 } + }) + expect([firstHistory, secondHistory].some(existsSync)).toBe(true) + }) +}) diff --git a/src/main/terminal-history-gc.ts b/src/main/terminal-history-gc.ts new file mode 100644 index 00000000000..a84e907c749 --- /dev/null +++ b/src/main/terminal-history-gc.ts @@ -0,0 +1,305 @@ +import { existsSync, opendirSync, rmSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' + +export const TERMINAL_HISTORY_GC_MAX_DISCOVERY_ENTRIES = 100_000 +export const TERMINAL_HISTORY_GC_MAX_WSL_DISTROS = 256 +export const TERMINAL_HISTORY_GC_MAX_FILES_PER_WORKTREE = 64 +export const TERMINAL_HISTORY_GC_META_MAX_BYTES = 64 * 1024 + +export type TerminalHistoryGcLimits = { + maxDiscoveryEntries: number + maxFilesPerWorktree: number + maxMetaBytes: number + maxWslDistros: number +} + +export type TerminalHistoryGcSummary = { + capacityExceeded: boolean + orphaned: number + pruned: number + totalDirs: number + totalSizeKB: number +} + +const DEFAULT_LIMITS: TerminalHistoryGcLimits = { + maxDiscoveryEntries: TERMINAL_HISTORY_GC_MAX_DISCOVERY_ENTRIES, + maxFilesPerWorktree: TERMINAL_HISTORY_GC_MAX_FILES_PER_WORKTREE, + maxMetaBytes: TERMINAL_HISTORY_GC_META_MAX_BYTES, + maxWslDistros: TERMINAL_HISTORY_GC_MAX_WSL_DISTROS +} + +// Why: the live-worktree snapshot can predate a newly created history directory. +const GC_MIN_AGE_MS = 5 * 60 * 1000 + +class TerminalHistoryGcCapacityError extends Error { + constructor( + readonly resource: 'discovery entries' | 'WSL distros', + readonly limit: number + ) { + super(`Terminal history GC exceeded ${limit} ${resource}`) + this.name = 'TerminalHistoryGcCapacityError' + } +} + +class TerminalHistoryGcBudget { + private discoveryEntries = 0 + private wslDistros = 0 + + constructor(readonly limits: TerminalHistoryGcLimits) {} + + claimDiscoveryEntry(): void { + this.discoveryEntries += 1 + if (this.discoveryEntries > this.limits.maxDiscoveryEntries) { + throw new TerminalHistoryGcCapacityError('discovery entries', this.limits.maxDiscoveryEntries) + } + } + + claimWslDistro(): void { + this.wslDistros += 1 + if (this.wslDistros > this.limits.maxWslDistros) { + throw new TerminalHistoryGcCapacityError('WSL distros', this.limits.maxWslDistros) + } + } +} + +type MutableTerminalHistoryGcSummary = Omit + +export function runTerminalHistoryGarbageCollection(options: { + mainRoot: string + wslRoot: string + liveWorktreeIds: Set + limits?: Partial +}): TerminalHistoryGcSummary { + const limits = resolveTerminalHistoryGcLimits(options.limits) + const budget = new TerminalHistoryGcBudget(limits) + const summary: TerminalHistoryGcSummary = { + capacityExceeded: false, + orphaned: 0, + pruned: 0, + totalDirs: 0, + totalSizeKB: 0 + } + + try { + scanTerminalHistoryRoot(options.mainRoot, options.liveWorktreeIds, budget, limits, summary) + scanWslHistoryRoots(options.wslRoot, options.liveWorktreeIds, budget, limits, summary) + } catch (error) { + if (!(error instanceof TerminalHistoryGcCapacityError)) { + throw error + } + summary.capacityExceeded = true + console.warn(`[pty:history:gc] ${error.message}; remaining history will be scanned next run`) + } + return summary +} + +export function deleteWslWorktreeHistoryDirectories(options: { + wslRoot: string + worktreeHash: string + limits?: Partial +}): void { + if (!existsSync(options.wslRoot)) { + return + } + const limits = resolveTerminalHistoryGcLimits(options.limits) + const budget = new TerminalHistoryGcBudget(limits) + try { + forEachDirectoryEntry(options.wslRoot, (distro) => { + budget.claimDiscoveryEntry() + const distroRoot = join(options.wslRoot, distro) + if (!statSync(distroRoot).isDirectory()) { + return + } + budget.claimWslDistro() + const historyPath = join(distroRoot, options.worktreeHash) + if (existsSync(historyPath)) { + rmSync(historyPath, { recursive: true, force: true }) + } + }) + } catch (error) { + if (error instanceof TerminalHistoryGcCapacityError) { + console.warn(`[pty:history] ${error.message}; WSL cleanup stopped at the limit`) + return + } + throw error + } +} + +function scanWslHistoryRoots( + wslRoot: string, + liveWorktreeIds: Set, + budget: TerminalHistoryGcBudget, + limits: TerminalHistoryGcLimits, + summary: TerminalHistoryGcSummary +): void { + if (!existsSync(wslRoot)) { + return + } + try { + forEachDirectoryEntry(wslRoot, (distro) => { + budget.claimDiscoveryEntry() + const distroRoot = join(wslRoot, distro) + try { + if (!statSync(distroRoot).isDirectory()) { + return + } + budget.claimWslDistro() + scanTerminalHistoryRoot(distroRoot, liveWorktreeIds, budget, limits, summary) + } catch (error) { + if (error instanceof TerminalHistoryGcCapacityError) { + throw error + } + // One unavailable distro must not discard the main-root GC result. + } + }) + } catch (error) { + if (error instanceof TerminalHistoryGcCapacityError) { + throw error + } + // WSL history is optional and may disappear while distributions stop. + } +} + +function scanTerminalHistoryRoot( + root: string, + liveWorktreeIds: Set, + budget: TerminalHistoryGcBudget, + limits: TerminalHistoryGcLimits, + summary: MutableTerminalHistoryGcSummary +): void { + if (!existsSync(root)) { + return + } + const now = Date.now() + + forEachDirectoryEntry(root, (entry) => { + budget.claimDiscoveryEntry() + const entryPath = join(root, entry) + try { + if (!statSync(entryPath).isDirectory()) { + return + } + summary.totalDirs += 1 + const sizeEstimate = estimateHistoryDirectorySize(entryPath, budget, limits) + summary.totalSizeKB += sizeEstimate.totalSizeKB + if (!sizeEstimate.complete) { + return + } + + const meta = readTerminalHistoryMetadata(join(entryPath, 'meta.json'), limits.maxMetaBytes) + if (!meta || liveWorktreeIds.has(meta.worktreeId)) { + return + } + if (meta.createdAt) { + const ageMs = now - new Date(meta.createdAt).getTime() + if (ageMs < GC_MIN_AGE_MS) { + return + } + } + + summary.orphaned += 1 + rmSync(entryPath, { recursive: true, force: true }) + summary.pruned += 1 + console.log(`[pty:history:gc] Pruned orphaned history: ${meta.worktreeId}`) + } catch (error) { + if (error instanceof TerminalHistoryGcCapacityError) { + throw error + } + // One corrupt or concurrently removed history directory must not stop GC. + } + }) +} + +function estimateHistoryDirectorySize( + directoryPath: string, + budget: TerminalHistoryGcBudget, + limits: TerminalHistoryGcLimits +): { complete: boolean; totalSizeKB: number } { + let complete = true + let fileCount = 0 + let totalSizeKB = 0 + try { + forEachDirectoryEntry(directoryPath, (file) => { + budget.claimDiscoveryEntry() + fileCount += 1 + if (fileCount > limits.maxFilesPerWorktree) { + complete = false + return false + } + const fileStat = statSync(join(directoryPath, file)) + if (fileStat.isDirectory()) { + complete = false + return false + } + totalSizeKB += Math.ceil(fileStat.size / 1024) + return undefined + }) + } catch (error) { + if (error instanceof TerminalHistoryGcCapacityError) { + throw error + } + complete = false + } + return { complete, totalSizeKB } +} + +function readTerminalHistoryMetadata( + metaPath: string, + maxBytes: number +): { worktreeId: string; createdAt?: string } | null { + if (!existsSync(metaPath)) { + return null + } + try { + const parsed = JSON.parse( + readNodeFileSyncWithinLimit(metaPath, maxBytes).buffer.toString('utf8') + ) as unknown + if ( + parsed === null || + typeof parsed !== 'object' || + !('worktreeId' in parsed) || + typeof parsed.worktreeId !== 'string' || + parsed.worktreeId.length === 0 + ) { + return null + } + const createdAt = + 'createdAt' in parsed && typeof parsed.createdAt === 'string' ? parsed.createdAt : undefined + return { worktreeId: parsed.worktreeId, ...(createdAt ? { createdAt } : {}) } + } catch { + return null + } +} + +function forEachDirectoryEntry( + directoryPath: string, + visit: (entryName: string) => false | void +): void { + const directory = opendirSync(directoryPath) + try { + for (let entry = directory.readSync(); entry !== null; entry = directory.readSync()) { + if (visit(entry.name) === false) { + return + } + } + } finally { + try { + directory.closeSync() + } catch { + // The OS may have already closed a failed directory stream. + } + } +} + +function resolveTerminalHistoryGcLimits( + overrides: Partial = {} +): TerminalHistoryGcLimits { + const limits = { ...DEFAULT_LIMITS, ...overrides } + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`) + } + } + return limits +} diff --git a/src/main/terminal-history.test.ts b/src/main/terminal-history.test.ts index a9750bb60bc..1c66a77ef64 100644 --- a/src/main/terminal-history.test.ts +++ b/src/main/terminal-history.test.ts @@ -7,6 +7,7 @@ const { readFileSyncMock, rmSyncMock, readdirSyncMock, + opendirSyncMock, statSyncMock, getPathMock } = vi.hoisted(() => ({ @@ -16,6 +17,7 @@ const { readFileSyncMock: vi.fn(), rmSyncMock: vi.fn(), readdirSyncMock: vi.fn(), + opendirSyncMock: vi.fn(), statSyncMock: vi.fn(), getPathMock: vi.fn() })) @@ -27,9 +29,21 @@ vi.mock('fs', () => ({ readFileSync: readFileSyncMock, rmSync: rmSyncMock, readdirSync: readdirSyncMock, + opendirSync: opendirSyncMock, statSync: statSyncMock })) +vi.mock('../shared/node-bounded-file-reader', () => ({ + readNodeFileSyncWithinLimit: (filePath: string, maxBytes: number) => { + const value = readFileSyncMock(filePath, 'utf8') + const buffer = Buffer.isBuffer(value) ? value : Buffer.from(String(value)) + if (buffer.byteLength > maxBytes) { + throw new Error('File too large') + } + return { buffer, stats: statSyncMock(filePath) } + } +})) + vi.mock('electron', () => ({ app: { getPath: getPathMock @@ -66,7 +80,22 @@ describe('terminal-history', () => { vi.clearAllMocks() getPathMock.mockReturnValue('/fake/userData') existsSyncMock.mockReturnValue(true) - statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + statSyncMock.mockImplementation((filePath: string) => ({ + isDirectory: () => !filePath.endsWith('meta.json'), + size: 100 + })) + opendirSyncMock.mockImplementation((directoryPath: string) => { + const entries = (readdirSyncMock(directoryPath) ?? []) as string[] + let index = 0 + return { + closeSync: vi.fn(), + readSync: () => { + const name = entries[index] + index += 1 + return name === undefined ? null : { name } + } + } + }) }) describe('resolveShellKind', () => { @@ -302,7 +331,10 @@ describe('terminal-history', () => { } return ['meta.json'] }) - statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + statSyncMock.mockImplementation((filePath: string) => ({ + isDirectory: () => !filePath.endsWith('meta.json'), + size: 100 + })) readFileSyncMock.mockImplementation((p: string) => { // Use a createdAt old enough to pass the GC age threshold const oldDate = new Date(Date.now() - 10 * 60 * 1000).toISOString() @@ -336,7 +368,10 @@ describe('terminal-history', () => { } return ['meta.json'] }) - statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + statSyncMock.mockImplementation((filePath: string) => ({ + isDirectory: () => !filePath.endsWith('meta.json'), + size: 100 + })) // createdAt is just now — younger than the 5-minute GC threshold readFileSyncMock.mockReturnValue( JSON.stringify({ worktreeId: 'unknown-wt', createdAt: new Date().toISOString() }) diff --git a/src/main/terminal-history.ts b/src/main/terminal-history.ts index a6d09cfcaa2..833e82347cb 100644 --- a/src/main/terminal-history.ts +++ b/src/main/terminal-history.ts @@ -1,16 +1,12 @@ import { createHash } from 'node:crypto' import { join, basename } from 'node:path' -import { - mkdirSync, - existsSync, - readFileSync, - writeFileSync, - readdirSync, - rmSync, - statSync -} from 'node:fs' +import { mkdirSync, existsSync, writeFileSync, rmSync } from 'node:fs' import { app } from 'electron' import { parseWslPath, toLinuxPath } from './wsl' +import { + deleteWslWorktreeHistoryDirectories, + runTerminalHistoryGarbageCollection +} from './terminal-history-gc' // ─── Constants ───────────────────────────────────────────────────── @@ -227,14 +223,7 @@ export function deleteWorktreeHistoryDir(worktreeId: string): void { if (process.platform === 'win32') { try { const wslRoot = join(app.getPath('userData'), HISTORY_DIR_NAME_WSL) - if (existsSync(wslRoot)) { - for (const distro of readdirSync(wslRoot)) { - const wslDir = join(wslRoot, distro, worktreeHash) - if (existsSync(wslDir)) { - rmSync(wslDir, { recursive: true, force: true }) - } - } - } + deleteWslWorktreeHistoryDirectories({ wslRoot, worktreeHash }) } catch { // Non-fatal. } @@ -243,112 +232,19 @@ export function deleteWorktreeHistoryDir(worktreeId: string): void { // ─── Garbage Collection ──────────────────────────────────────────── -// Why 5 minutes: GC runs ~10s after startup, and the live-worktree snapshot is -// taken just before. A worktree created between the snapshot and GC execution -// won't appear in liveWorktreeIds, so without an age guard GC would delete its -// freshly-created history directory (TOCTOU race). 5 minutes is generous enough -// to cover any realistic snapshot-to-scan delay. -const GC_MIN_AGE_MS = 5 * 60 * 1000 - -/** Scan a single history root directory, pruning orphaned entries. - * Returns { totalDirs, orphaned, pruned, totalSizeKB }. */ -function gcScanRoot( - root: string, - liveWorktreeIds: Set -): { totalDirs: number; orphaned: number; pruned: number; totalSizeKB: number } { - const result = { totalDirs: 0, orphaned: 0, pruned: 0, totalSizeKB: 0 } - if (!existsSync(root)) { - return result - } - - const now = Date.now() - - for (const entry of readdirSync(root)) { - const entryPath = join(root, entry) - try { - const stat = statSync(entryPath) - if (!stat.isDirectory()) { - continue - } - result.totalDirs++ - - // Estimate directory size from meta.json + history files. - try { - for (const file of readdirSync(entryPath)) { - result.totalSizeKB += Math.ceil(statSync(join(entryPath, file)).size / 1024) - } - } catch { - // Skip size estimation on error. - } - - const metaPath = join(entryPath, 'meta.json') - if (!existsSync(metaPath)) { - // No meta.json — can't determine ownership, skip. - continue - } - - const meta = JSON.parse(readFileSync(metaPath, 'utf-8')) as { - worktreeId?: string - createdAt?: string - } - if (!meta.worktreeId) { - continue - } - - if (!liveWorktreeIds.has(meta.worktreeId)) { - // Why: avoid a TOCTOU race where a worktree is created after the - // live-ID snapshot but before GC runs. Directories younger than - // GC_MIN_AGE_MS are presumed still live and skipped. - if (meta.createdAt) { - const ageMs = now - new Date(meta.createdAt).getTime() - if (ageMs < GC_MIN_AGE_MS) { - continue - } - } - - result.orphaned++ - rmSync(entryPath, { recursive: true, force: true }) - result.pruned++ - console.log(`[pty:history:gc] Pruned orphaned history: ${meta.worktreeId}`) - } - } catch { - // Skip individual entries that fail. - } - } - return result -} - /** Run background GC to prune history directories for worktrees that are no * longer in Orca's known live-worktree set. */ export function runHistoryGc(liveWorktreeIds: Set): void { try { - const main = gcScanRoot(getHistoryRoot(), liveWorktreeIds) - - // Also scan WSL history directories (each distro has its own subdirectory). const wslRoot = join(app.getPath('userData'), HISTORY_DIR_NAME_WSL) - let wslTotals = { totalDirs: 0, orphaned: 0, pruned: 0, totalSizeKB: 0 } - if (existsSync(wslRoot)) { - try { - for (const distro of readdirSync(wslRoot)) { - const distroRoot = join(wslRoot, distro) - const r = gcScanRoot(distroRoot, liveWorktreeIds) - wslTotals.totalDirs += r.totalDirs - wslTotals.orphaned += r.orphaned - wslTotals.pruned += r.pruned - wslTotals.totalSizeKB += r.totalSizeKB - } - } catch { - // Non-fatal. - } - } - - const totalDirs = main.totalDirs + wslTotals.totalDirs - const orphaned = main.orphaned + wslTotals.orphaned - const pruned = main.pruned + wslTotals.pruned - const totalSizeKB = main.totalSizeKB + wslTotals.totalSizeKB + const summary = runTerminalHistoryGarbageCollection({ + mainRoot: getHistoryRoot(), + wslRoot, + liveWorktreeIds + }) console.log( - `[pty:history:gc] totalDirs=${totalDirs} orphaned=${orphaned} pruned=${pruned} totalSizeKB=${totalSizeKB}` + `[pty:history:gc] totalDirs=${summary.totalDirs} orphaned=${summary.orphaned} pruned=${summary.pruned} totalSizeKB=${summary.totalSizeKB}` ) } catch (err) { console.warn(`[pty:history:gc] GC failed: ${err instanceof Error ? err.message : String(err)}`) diff --git a/src/main/text-generation/commit-message-text-generation.test.ts b/src/main/text-generation/commit-message-text-generation.test.ts index 3b921daf65b..fe156150854 100644 --- a/src/main/text-generation/commit-message-text-generation.test.ts +++ b/src/main/text-generation/commit-message-text-generation.test.ts @@ -16,6 +16,7 @@ import { generateBranchNameFromContext, generateCommitMessageFromContext, generatePullRequestFieldsFromContext, + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS, resolveCommitMessageSettings, trimGeneratedCommitMessage } from './commit-message-text-generation' @@ -479,6 +480,119 @@ describe('discoverCommitMessageModelsLocal', () => { }) }) + it('shares one child process for concurrent identical model discovery', async () => { + const children: MockDiscoveryChild[] = [] + spawnMock.mockImplementation(() => { + const child = createMockDiscoveryChild() + children.push(child) + return child as never + }) + + const first = discoverCommitMessageModelsLocal('cursor', { TOKEN: 'secret' }) + const second = discoverCommitMessageModelsLocal('cursor', { TOKEN: 'secret' }) + + expect(spawnMock).toHaveBeenCalledTimes(1) + children[0]!.stdout.emit('data', Buffer.from('auto - Auto\n')) + children[0]!.emit('close', 0) + + const [firstResult, secondResult] = await Promise.all([first, second]) + expect(firstResult).toMatchObject({ success: true, defaultModelId: 'auto' }) + expect(secondResult).toEqual(firstResult) + + const retry = discoverCommitMessageModelsLocal('cursor', { TOKEN: 'secret' }) + expect(spawnMock).toHaveBeenCalledTimes(2) + children[1]!.stdout.emit('data', Buffer.from('auto - Auto\n')) + children[1]!.emit('close', 0) + await expect(retry).resolves.toEqual(firstResult) + }) + + it('keeps different agent parsers isolated when discovery commands match', async () => { + const children: MockDiscoveryChild[] = [] + spawnMock.mockImplementation(() => { + const child = createMockDiscoveryChild() + children.push(child) + return child as never + }) + + const cursor = discoverCommitMessageModelsLocal('cursor', undefined, 'custom-discovery') + const pi = discoverCommitMessageModelsLocal('pi', undefined, 'custom-discovery') + + expect(children).toHaveLength(2) + children[0]!.stdout.emit('data', Buffer.from('auto - Auto\n')) + children[0]!.emit('close', 0) + children[1]!.stderr.emit( + 'data', + Buffer.from( + [ + 'provider model context max-out thinking images', + 'github-copilot gpt-5.4-mini 400K 128K yes yes' + ].join('\n') + ) + ) + children[1]!.emit('close', 0) + + await expect(cursor).resolves.toMatchObject({ success: true, defaultModelId: 'auto' }) + await expect(pi).resolves.toMatchObject({ + success: true, + defaultModelId: 'github-copilot/gpt-5.4-mini' + }) + }) + + it('shares the local process cap across discovery and generation, then releases on close', async () => { + const children: MockDiscoveryChild[] = [] + spawnMock.mockImplementation(() => { + const child = createMockDiscoveryChild() + child.pid += children.length + children.push(child) + return child as never + }) + const active = Array.from({ length: MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS }, (_, index) => + discoverCommitMessageModelsLocal('cursor', undefined, undefined, { + cwd: `/repo-${index}` + }) + ) + + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + await expect( + discoverCommitMessageModelsLocal('cursor', undefined, undefined, { + cwd: '/repo-overflow' + }) + ).resolves.toEqual({ + success: false, + error: + 'Too many local AI generations are already running. Wait for one to finish and try again.' + }) + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M\tREADME.md', stagedPatch: '+hello' }, + { agentId: 'custom', model: '', customAgentCommand: 'agent' }, + { kind: 'local', cwd: '/generation-overflow' } + ) + ).resolves.toEqual({ + success: false, + error: + 'Too many local AI generations are already running. Wait for one to finish and try again.' + }) + expect(spawnMock).toHaveBeenCalledTimes(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + + children[0]!.stdout.emit('data', Buffer.from('auto - Auto\n')) + children[0]!.emit('close', 0) + await active[0] + + const retry = discoverCommitMessageModelsLocal('cursor', undefined, undefined, { + cwd: '/repo-overflow' + }) + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + 1) + + for (const child of children.slice(1)) { + child.stdout.emit('data', Buffer.from('auto - Auto\n')) + child.emit('close', 0) + } + await expect(Promise.all([...active, retry])).resolves.toHaveLength( + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + 1 + ) + }) + it('settles and detaches model discovery when timeout kill is ignored', async () => { vi.useFakeTimers() const child = createMockDiscoveryChild() @@ -498,6 +612,8 @@ describe('discoverCommitMessageModelsLocal', () => { expect(child.stdout.listenerCount('data')).toBe(0) expect(child.stderr.listenerCount('data')).toBe(0) expect(child.listenerCount('error')).toBe(0) + expect(child.listenerCount('close')).toBe(1) + child.emit('close', null) expect(child.listenerCount('close')).toBe(0) } finally { vi.useRealTimers() @@ -520,6 +636,8 @@ describe('discoverCommitMessageModelsLocal', () => { expect(child.stdout.listenerCount('data')).toBe(0) expect(child.stderr.listenerCount('data')).toBe(0) expect(child.listenerCount('error')).toBe(0) + expect(child.listenerCount('close')).toBe(1) + child.emit('close', null) expect(child.listenerCount('close')).toBe(0) }) }) @@ -1141,6 +1259,36 @@ describe('generateCommitMessageFromContext', () => { expectChildTerminated(child) }) + it('preserves local agent output delivered as 100,000 one-byte fragments', async () => { + const child = createMockDiscoveryChild() + spawnMock.mockReturnValue(child as never) + const pending = generateCommitMessageFromContext( + { + branch: 'main', + stagedSummary: 'M\tREADME.md', + stagedPatch: '+hello' + }, + { + agentId: 'custom', + model: '', + customAgentCommand: 'agent' + }, + { kind: 'local', cwd: '/fragmented-repo' } + ) + + for (let index = 0; index < 100_000; index += 1) { + child.stdout.emit('data', Buffer.from(' ')) + } + child.stdout.emit('data', Buffer.from('Update README\n')) + child.emit('close', 0) + + await expect(pending).resolves.toEqual({ + success: true, + message: 'Update README', + agentLabel: 'agent' + }) + }) + it('passes prepared provider environment to local agent subprocesses', async () => { const listeners = new Map void>() const child = { @@ -1338,6 +1486,207 @@ describe('generateCommitMessageFromContext', () => { expect(children[1]?.kill).not.toHaveBeenCalled() }) + it('cancels the previous process before replacing one local lane', async () => { + const children: { + kill: ReturnType + listeners: Map void> + }[] = [] + spawnMock.mockImplementation(() => { + const listeners = new Map void>() + const child = { + pid: 123 + children.length, + kill: vi.fn(), + stdout: { on: vi.fn((event, callback) => listeners.set(`stdout:${event}`, callback)) }, + stderr: { on: vi.fn((event, callback) => listeners.set(`stderr:${event}`, callback)) }, + stdin: { end: vi.fn() }, + on: vi.fn((event, callback) => listeners.set(event, callback)) + } + children.push({ kill: child.kill, listeners }) + return child as never + }) + const context = { branch: 'main', stagedSummary: 'M\tREADME.md', stagedPatch: '+hello' } + const params = { agentId: 'custom' as const, model: '', customAgentCommand: 'agent' } + const target = { kind: 'local' as const, cwd: '/same-repo' } + + const first = generateCommitMessageFromContext(context, params, target) + const second = generateCommitMessageFromContext(context, params, target) + + expectChildTerminated({ pid: 123, kill: children[0]!.kill }) + expect(children).toHaveLength(2) + children[0]?.listeners.get('close')?.(null) + children[1]?.listeners.get('stdout:data')?.(Buffer.from('Update README\n')) + children[1]?.listeners.get('close')?.(0) + await expect(first).resolves.toMatchObject({ success: false, canceled: true }) + await expect(second).resolves.toMatchObject({ success: true, message: 'Update README' }) + }) + + it('caps concurrent local text-generation child processes', async () => { + const children: Map void>[] = [] + spawnMock.mockImplementation(() => { + const listeners = new Map void>() + const child = { + pid: 200 + children.length, + kill: vi.fn(), + stdout: { on: vi.fn((event, callback) => listeners.set(`stdout:${event}`, callback)) }, + stderr: { on: vi.fn((event, callback) => listeners.set(`stderr:${event}`, callback)) }, + stdin: { end: vi.fn() }, + on: vi.fn((event, callback) => listeners.set(event, callback)) + } + children.push(listeners) + return child as never + }) + const context = { branch: 'main', stagedSummary: 'M\tREADME.md', stagedPatch: '+hello' } + const params = { agentId: 'custom' as const, model: '', customAgentCommand: 'agent' } + const active = Array.from({ length: MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS }, (_, index) => + generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: `/repo-${index}` + }) + ) + + await expect( + generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: '/repo-overflow' + }) + ).resolves.toEqual({ + success: false, + error: + 'Too many local AI generations are already running. Wait for one to finish and try again.' + }) + expect(spawnMock).toHaveBeenCalledTimes(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + + for (const listeners of children) { + listeners.get('stdout:data')?.(Buffer.from('Update README\n')) + listeners.get('close')?.(0) + } + await expect(Promise.all(active)).resolves.toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + }) + + it('waits for a full-capacity lane to close before spawning its replacement', async () => { + const children: { + pid: number + kill: ReturnType + listeners: Map void> + closed: boolean + }[] = [] + spawnMock.mockImplementation(() => { + const listeners = new Map void>() + const child = { + pid: 300 + children.length, + kill: vi.fn(), + stdout: { on: vi.fn((event, callback) => listeners.set(`stdout:${event}`, callback)) }, + stderr: { on: vi.fn((event, callback) => listeners.set(`stderr:${event}`, callback)) }, + stdin: { end: vi.fn() }, + on: vi.fn((event, callback) => listeners.set(event, callback)) + } + children.push({ pid: child.pid, kill: child.kill, listeners, closed: false }) + return child as never + }) + const context = { branch: 'main', stagedSummary: 'M\tREADME.md', stagedPatch: '+hello' } + const params = { agentId: 'custom' as const, model: '', customAgentCommand: 'agent' } + const active = Array.from({ length: MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS }, (_, index) => + generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: `/full-repo-${index}` + }) + ) + + const replacement = generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: '/full-repo-0' + }) + + expectChildTerminated(children[0]!) + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + expect(children.filter((child) => !child.closed)).toHaveLength( + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + ) + + children[0]!.closed = true + children[0]!.listeners.get('close')?.(null) + await Promise.resolve() + await Promise.resolve() + + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + 1) + expect(children.filter((child) => !child.closed)).toHaveLength( + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + ) + + for (const child of children.slice(1)) { + child.listeners.get('stdout:data')?.(Buffer.from('Update README\n')) + child.closed = true + child.listeners.get('close')?.(0) + } + await expect(active[0]).resolves.toMatchObject({ success: false, canceled: true }) + await expect(Promise.all([...active.slice(1), replacement])).resolves.toHaveLength( + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + ) + }) + + it('cancels and supersedes replacements queued behind a full lane', async () => { + const children: { + pid: number + kill: ReturnType + listeners: Map void> + }[] = [] + spawnMock.mockImplementation(() => { + const listeners = new Map void>() + const child = { + pid: 400 + children.length, + kill: vi.fn(), + stdout: { on: vi.fn((event, callback) => listeners.set(`stdout:${event}`, callback)) }, + stderr: { on: vi.fn((event, callback) => listeners.set(`stderr:${event}`, callback)) }, + stdin: { end: vi.fn() }, + on: vi.fn((event, callback) => listeners.set(event, callback)) + } + children.push({ pid: child.pid, kill: child.kill, listeners }) + return child as never + }) + const context = { branch: 'main', stagedSummary: 'M\tREADME.md', stagedPatch: '+hello' } + const params = { agentId: 'custom' as const, model: '', customAgentCommand: 'agent' } + const active = Array.from({ length: MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS }, (_, index) => + generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: `/queued-repo-${index}` + }) + ) + + const superseded = generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: '/queued-repo-0' + }) + const canceled = generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: '/queued-repo-0' + }) + + await expect(superseded).resolves.toMatchObject({ success: false, canceled: true }) + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + cancelGenerateCommitMessageLocal('/queued-repo-0') + await expect(canceled).resolves.toMatchObject({ success: false, canceled: true }) + + const replacement = generateCommitMessageFromContext(context, params, { + kind: 'local', + cwd: '/queued-repo-0' + }) + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) + + children[0]!.listeners.get('close')?.(null) + await Promise.resolve() + await Promise.resolve() + expect(children).toHaveLength(MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + 1) + + for (const child of children.slice(1)) { + child.listeners.get('stdout:data')?.(Buffer.from('Update README\n')) + child.listeners.get('close')?.(0) + } + await expect(active[0]).resolves.toMatchObject({ success: false, canceled: true }) + await expect(Promise.all([...active.slice(1), replacement])).resolves.toHaveLength( + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS + ) + }) + it('keeps local pull-request cancellation from stopping commit-message generation', async () => { const children: { pid: number @@ -1612,7 +1961,8 @@ describe('generateCommitMessageFromContext', () => { expect(listeners.has('stdout:data')).toBe(false) expect(listeners.has('stderr:data')).toBe(false) expect(listeners.has('error')).toBe(false) - expect(listeners.has('close')).toBe(false) + expect(listeners.has('close')).toBe(true) + listeners.get('close')?.(null) } finally { vi.useRealTimers() } diff --git a/src/main/text-generation/commit-message-text-generation.ts b/src/main/text-generation/commit-message-text-generation.ts index ff0c9e5fc24..0022ca03d6f 100644 --- a/src/main/text-generation/commit-message-text-generation.ts +++ b/src/main/text-generation/commit-message-text-generation.ts @@ -2,6 +2,8 @@ spawn failure handling, and output normalization; keeping them together prevents those paths from drifting. */ import { exec, spawn, type ChildProcess } from 'node:child_process' +import { createHash } from 'node:crypto' +import { GrowingByteBuffer } from '../../shared/growing-byte-buffer' import type { GlobalSettings, Repo, TuiAgent } from '../../shared/types' import { buildCommitMessagePrompt, @@ -53,6 +55,10 @@ import { } from '../win32-utils' import { withMacTailscaleDnsHint } from '../network/macos-tailscale-dns-diagnostic' import { wslAwareSpawn } from '../git/runner' +import { + MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS, + reserveLocalAiProcess +} from './local-ai-process-budget' const GENERATION_TIMEOUT_MS = 60_000 const MAX_AGENT_OUTPUT_BYTES = 4 * 1024 * 1024 @@ -296,6 +302,29 @@ function planModelDiscovery( } } +const inFlightLocalModelDiscoveries = new Map>() + +function localModelDiscoveryKey( + agentId: TuiAgent, + plan: CommitMessagePlan, + env: NodeJS.ProcessEnv, + options: CommitMessageModelDiscoveryLocalOptions +): string { + // Hash the full identity so environment and command secrets never remain in map keys. + return createHash('sha256') + .update( + JSON.stringify([ + agentId, + plan.binary, + plan.args, + options.cwd ?? null, + options.wslDistro ?? null, + Object.entries(env).sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)) + ]) + ) + .digest('hex') +} + export async function discoverCommitMessageModelsLocal( agentId: TuiAgent, env: NodeJS.ProcessEnv | undefined, @@ -311,19 +340,49 @@ export async function discoverCommitMessageModelsLocal( return toModelDiscoveryCapability(spec) } + const planned = planModelDiscovery(spec, agentCommandOverride) + if (!planned.ok) { + return { success: false, error: planned.error } + } + const spawnEnv = env ?? process.env + const discoveryKey = localModelDiscoveryKey(spec.id, planned.plan, spawnEnv, options) + const inFlight = inFlightLocalModelDiscoveries.get(discoveryKey) + if (inFlight) { + return inFlight + } + const pending = runLocalModelDiscovery(spec, planned.plan, env, spawnEnv, options) + inFlightLocalModelDiscoveries.set(discoveryKey, pending) + const clearPending = (): void => { + if (inFlightLocalModelDiscoveries.get(discoveryKey) === pending) { + inFlightLocalModelDiscoveries.delete(discoveryKey) + } + } + void pending.then(clearPending, clearPending) + return pending +} + +function runLocalModelDiscovery( + spec: NonNullable>, + plan: CommitMessagePlan, + explicitEnv: NodeJS.ProcessEnv | undefined, + spawnEnv: NodeJS.ProcessEnv, + options: CommitMessageModelDiscoveryLocalOptions +): Promise { + const reservation = reserveLocalAiProcess() + if (!reservation) { + return Promise.resolve({ + success: false, + error: + 'Too many local AI generations are already running. Wait for one to finish and try again.' + }) + } return new Promise((resolve) => { let child: ChildProcess - const spawnEnv = env ?? process.env try { - const planned = planModelDiscovery(spec, agentCommandOverride) - if (!planned.ok) { - resolve({ success: false, error: planned.error }) - return - } if (process.platform === 'win32' && options.wslDistro) { - child = wslAwareSpawn(planned.plan.binary, planned.plan.args, { + child = wslAwareSpawn(plan.binary, plan.args, { cwd: options.cwd, - env: buildWslLauncherEnv(env), + env: buildWslLauncherEnv(explicitEnv), stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, wslDistro: options.wslDistro, @@ -332,11 +391,11 @@ export async function discoverCommitMessageModelsLocal( } else { const resolvedBinary = process.platform === 'win32' - ? resolveCliCommand(planned.plan.binary, { + ? resolveCliCommand(plan.binary, { pathEnv: spawnEnv.PATH ?? spawnEnv.Path ?? null }) - : planned.plan.binary - const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(resolvedBinary, planned.plan.args) + : plan.binary + const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(resolvedBinary, plan.args) child = spawn(spawnCmd, spawnArgs, { env: spawnEnv, stdio: ['ignore', 'pipe', 'pipe'], @@ -344,6 +403,7 @@ export async function discoverCommitMessageModelsLocal( }) } } catch (error) { + reservation.release() console.error('[commit-message] Failed to spawn model discovery:', error) resolve({ success: false, @@ -351,9 +411,10 @@ export async function discoverCommitMessageModelsLocal( }) return } + const owner = reservation.register(child) - let stdout = '' - let stderr = '' + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() let outputLimitExceeded = false let settled = false let timer: ReturnType | null = null @@ -368,6 +429,8 @@ export async function discoverCommitMessageModelsLocal( timer = null } detachChildListeners() + stdout.clear() + stderr.clear() resolve(result) } timer = setTimeout(() => { @@ -378,18 +441,18 @@ export async function discoverCommitMessageModelsLocal( }) }, GENERATION_TIMEOUT_MS) - const onData = (chunk: Buffer, append: (text: string) => void): void => { - if (stdout.length + stderr.length + chunk.byteLength > MAX_AGENT_OUTPUT_BYTES) { + const onData = (chunk: Buffer, append: (value: Buffer) => void): void => { + if (stdout.byteLength + stderr.byteLength + chunk.byteLength > MAX_AGENT_OUTPUT_BYTES) { outputLimitExceeded = true killProcessTree(child) finish({ success: false, error: `${spec.label} returned too much model data.` }) return } - append(chunk.toString('utf-8')) + append(chunk) } - const onStdoutData = (chunk: Buffer): void => onData(chunk, (text) => (stdout += text)) - const onStderrData = (chunk: Buffer): void => onData(chunk, (text) => (stderr += text)) + const onStdoutData = (chunk: Buffer): void => onData(chunk, (value) => stdout.append(value)) + const onStderrData = (chunk: Buffer): void => onData(chunk, (value) => stderr.append(value)) const onError = (error: Error): void => { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { finish({ @@ -404,15 +467,17 @@ export async function discoverCommitMessageModelsLocal( }) } const onClose = (code: number | null): void => { + child.off?.('close', onClose) + owner.release() if (outputLimitExceeded) { finish({ success: false, error: `${spec.label} returned too much model data.` }) return } if (code !== 0) { - finish(finalizeModelDiscoveryOutput(spec, stdout, stderr, code)) + finish(finalizeModelDiscoveryOutput(spec, stdout.toString(), stderr.toString(), code)) return } - finish(finalizeModelDiscoveryOutput(spec, stdout, stderr, code)) + finish(finalizeModelDiscoveryOutput(spec, stdout.toString(), stderr.toString(), code)) } child.stdout?.on('data', onStdoutData) @@ -423,7 +488,6 @@ export async function discoverCommitMessageModelsLocal( child.stdout?.off?.('data', onStdoutData) child.stderr?.off?.('data', onStderrData) child.off?.('error', onError) - child.off?.('close', onClose) } }) } @@ -514,6 +578,8 @@ function killProcessTree(child: ChildProcess): void { // Keying by operation plus `local:${cwd}` keeps local cancellation independent // from SSH worktrees and from other generation features in the same worktree. const cancelTokensByLane = new Map void>() +const pendingReservationCancelTokensByLane = new Map void>() +export { MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS } const WSL_LAUNCHER_ENV_KEYS = [ 'ComSpec', 'COMSPEC', @@ -530,8 +596,17 @@ function localLaneKey(operation: TextGenerationOperation, cwd: string): string { return `${operation}:local:${cwd}` } +function cancelLocalGenerationLane(laneKey: string): void { + const cancelPendingReservation = pendingReservationCancelTokensByLane.get(laneKey) + if (cancelPendingReservation) { + cancelPendingReservation() + return + } + cancelTokensByLane.get(laneKey)?.() +} + export function cancelGenerateCommitMessageLocal(cwd: string): void { - cancelTokensByLane.get(localLaneKey('commit-message', cwd))?.() + cancelLocalGenerationLane(localLaneKey('commit-message', cwd)) } function buildWslLauncherEnv(explicitEnv: NodeJS.ProcessEnv | undefined): NodeJS.ProcessEnv { @@ -559,6 +634,46 @@ async function runLocalPlan( wslDistro?: string ): Promise { const { binary, args, stdinPayload, label } = plan + const laneKey = localLaneKey(operation, cwd) + pendingReservationCancelTokensByLane.get(laneKey)?.() + const reservation = reserveLocalAiProcess(laneKey) + if (!reservation) { + return { + success: false, + error: + 'Too many local AI generations are already running. Wait for one to finish and try again.' + } + } + const previousCancelToken = cancelTokensByLane.get(laneKey) + if (reservation.waitForClose) { + let canceledWhileWaiting = false + let resolvePendingCancellation = (): void => {} + const pendingCancellation = new Promise((resolve) => { + resolvePendingCancellation = resolve + }) + const cancelPendingReservation = (): void => { + if (canceledWhileWaiting) { + return + } + canceledWhileWaiting = true + reservation.release() + if (pendingReservationCancelTokensByLane.get(laneKey) === cancelPendingReservation) { + pendingReservationCancelTokensByLane.delete(laneKey) + } + resolvePendingCancellation() + } + pendingReservationCancelTokensByLane.set(laneKey, cancelPendingReservation) + previousCancelToken?.() + await Promise.race([reservation.waitForClose, pendingCancellation]) + if (pendingReservationCancelTokensByLane.get(laneKey) === cancelPendingReservation) { + pendingReservationCancelTokensByLane.delete(laneKey) + } + if (canceledWhileWaiting) { + return { success: false, error: 'Generation canceled.', canceled: true } + } + } else { + previousCancelToken?.() + } return new Promise((resolve) => { let child: ChildProcess try { @@ -586,6 +701,7 @@ async function runLocalPlan( }) } } catch (error) { + reservation.release() if (error instanceof UnsafeWindowsBatchArgumentsError) { resolve({ success: false, @@ -600,15 +716,13 @@ async function runLocalPlan( }) return } + const owner = reservation.register(child, laneKey) - let stdout = '' - let stderr = '' - let stdoutBytes = 0 - let stderrBytes = 0 + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() let outputLimitExceeded = false let settled = false let canceledByUser = false - const laneKey = localLaneKey(operation, cwd) let cancelToken: (() => void) | null = null let timer: ReturnType | null = null let detachChildListeners = (): void => {} @@ -625,6 +739,8 @@ async function runLocalPlan( if (cancelToken && cancelTokensByLane.get(laneKey) === cancelToken) { cancelTokensByLane.delete(laneKey) } + stdout.clear() + stderr.clear() resolve(result) } @@ -646,22 +762,20 @@ async function runLocalPlan( }, GENERATION_TIMEOUT_MS) const onStdoutData = (chunk: Buffer): void => { - stdoutBytes += chunk.byteLength - if (stdoutBytes > MAX_AGENT_OUTPUT_BYTES) { + if (stdout.byteLength + chunk.byteLength > MAX_AGENT_OUTPUT_BYTES) { outputLimitExceeded = true killProcessTree(child) return } - stdout += chunk.toString('utf-8') + stdout.append(chunk) } const onStderrData = (chunk: Buffer): void => { - stderrBytes += chunk.byteLength - if (stderrBytes > MAX_AGENT_OUTPUT_BYTES) { + if (stderr.byteLength + chunk.byteLength > MAX_AGENT_OUTPUT_BYTES) { outputLimitExceeded = true killProcessTree(child) return } - stderr += chunk.toString('utf-8') + stderr.append(chunk) } const onError = (error: Error): void => { const code = (error as NodeJS.ErrnoException).code @@ -679,6 +793,8 @@ async function runLocalPlan( }) } const onClose = (code: number | null): void => { + child.off?.('close', onClose) + owner.release() if (canceledByUser) { finalize({ success: false, error: 'Generation canceled.', canceled: true }) return @@ -692,8 +808,8 @@ async function runLocalPlan( } finalizeFromAgentOutput({ code, - stdout, - stderr, + stdout: stdout.toString(), + stderr: stderr.toString(), label, emptyResultName, finalize, @@ -708,7 +824,6 @@ async function runLocalPlan( child.stdout?.off?.('data', onStdoutData) child.stderr?.off?.('data', onStderrData) child.off?.('error', onError) - child.off?.('close', onClose) } child.stdin?.end(stdinPayload ?? undefined) @@ -899,7 +1014,7 @@ export async function generateCommitMessageFromContext( } export function cancelGeneratePullRequestFieldsLocal(cwd: string): void { - cancelTokensByLane.get(localLaneKey('pull-request-fields', cwd))?.() + cancelLocalGenerationLane(localLaneKey('pull-request-fields', cwd)) } function formatPullRequestFieldsGenerationResult( diff --git a/src/main/text-generation/local-ai-process-budget.ts b/src/main/text-generation/local-ai-process-budget.ts new file mode 100644 index 00000000000..9b51a6629ef --- /dev/null +++ b/src/main/text-generation/local-ai-process-budget.ts @@ -0,0 +1,110 @@ +import type { ChildProcess } from 'node:child_process' + +export const MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS = 8 + +type LocalAiProcessOwner = { + child: ChildProcess + closed: Promise + release: () => void +} + +export type LocalAiProcessReservation = { + waitForClose?: Promise + register: (child: ChildProcess, laneKey?: string) => LocalAiProcessOwner + release: () => void +} + +type ReplacementReservation = { + reservation: LocalAiProcessReservation + activate: () => void +} + +const activeOwners = new Set() +const reservedSlots = new Set() +const ownerByLane = new Map() +const replacementByOwner = new Map() + +export function reserveLocalAiProcess(laneKey?: string): LocalAiProcessReservation | null { + let predecessor: LocalAiProcessOwner | undefined + if (activeOwners.size + reservedSlots.size >= MAX_CONCURRENT_LOCAL_TEXT_GENERATIONS) { + predecessor = laneKey ? ownerByLane.get(laneKey) : undefined + if (!predecessor || replacementByOwner.has(predecessor)) { + return null + } + } + + let state: 'waiting' | 'reserved' | 'registered' | 'released' = predecessor + ? 'waiting' + : 'reserved' + let reservation: LocalAiProcessReservation + const activate = (): void => { + if (state !== 'waiting') { + return + } + state = 'reserved' + reservedSlots.add(reservation) + } + reservation = { + waitForClose: predecessor?.closed, + register: (child, ownerLaneKey) => { + if (state !== 'reserved') { + throw new Error('Local AI process reservation is not ready.') + } + reservedSlots.delete(reservation) + state = 'registered' + + let released = false + let resolveClosed = (): void => {} + const closed = new Promise((resolve) => { + resolveClosed = resolve + }) + const owner: LocalAiProcessOwner = { + child, + closed, + release: () => { + if (released) { + return + } + released = true + activeOwners.delete(owner) + if (ownerLaneKey && ownerByLane.get(ownerLaneKey) === owner) { + ownerByLane.delete(ownerLaneKey) + } + const replacement = replacementByOwner.get(owner) + if (replacement) { + replacementByOwner.delete(owner) + // Preserve the claimed slot before `closed` lets the replacement resume. + replacement.activate() + } + resolveClosed() + } + } + activeOwners.add(owner) + if (ownerLaneKey) { + ownerByLane.set(ownerLaneKey, owner) + } + return owner + }, + release: () => { + if (state === 'waiting' && predecessor) { + const replacement = replacementByOwner.get(predecessor) + if (replacement?.reservation === reservation) { + replacementByOwner.delete(predecessor) + } + state = 'released' + return + } + if (state === 'reserved') { + reservedSlots.delete(reservation) + state = 'released' + } + } + } + + if (predecessor) { + replacementByOwner.set(predecessor, { reservation, activate }) + } else { + reservedSlots.add(reservation) + } + return reservation +} diff --git a/src/main/updater-changelog.test.ts b/src/main/updater-changelog.test.ts index f6a348858d0..c9a9d306627 100644 --- a/src/main/updater-changelog.test.ts +++ b/src/main/updater-changelog.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { API_RESPONSE_MAX_BYTES, FetchResponseBodyTooLargeError } from './lib/fetch-response-body' const fetchMock = vi.fn() @@ -226,6 +227,18 @@ describe('fetchChangelog', () => { expect(result).toBeNull() }) + it('rejects an oversized successful Electron net response before reading it', async () => { + fetchMock.mockResolvedValue( + new Response('{', { + headers: { 'content-length': String(API_RESPONSE_MAX_BYTES + 1) } + }) + ) + + await expect(fetchChangelog('1.1.21', '1.1.19')).rejects.toBeInstanceOf( + FetchResponseBodyTooLargeError + ) + }) + it('prefers exact match over fallback when both have rich content', async () => { const entries = makeEntries([ { diff --git a/src/main/updater-changelog.ts b/src/main/updater-changelog.ts index a534075cfc6..ea52d9db4ff 100644 --- a/src/main/updater-changelog.ts +++ b/src/main/updater-changelog.ts @@ -1,6 +1,7 @@ import { net } from 'electron' import type { ChangelogData } from '../shared/types' import { compareVersions } from './updater-fallback' +import { readFetchResponseJsonWithinLimit } from './lib/fetch-response-body' type ChangelogEntry = { version: string @@ -48,7 +49,7 @@ export async function fetchChangelog( if (!res.ok) { return null } - const json: unknown = await res.json() + const json = await readFetchResponseJsonWithinLimit(res) // Why: the JSON endpoint is external and could serve malformed data. // Validate the shape before indexing into it to avoid runtime errors diff --git a/src/main/updater-nudge.ts b/src/main/updater-nudge.ts index 93ef8c1c105..1755c2ec021 100644 --- a/src/main/updater-nudge.ts +++ b/src/main/updater-nudge.ts @@ -1,5 +1,6 @@ import { net } from 'electron' import { compareVersions, isValidVersion } from './updater-fallback' +import { readFetchResponseJsonWithinLimit } from './lib/fetch-response-body' export type NudgeConfig = { id: string @@ -16,7 +17,7 @@ export async function fetchNudge(): Promise { return null } - const json: unknown = await res.json() + const json = await readFetchResponseJsonWithinLimit(res) if (!json || typeof json !== 'object' || Array.isArray(json)) { return null } diff --git a/src/main/updater-prerelease-feed.ts b/src/main/updater-prerelease-feed.ts index 21a1f397ff4..6bc5bca20ef 100644 --- a/src/main/updater-prerelease-feed.ts +++ b/src/main/updater-prerelease-feed.ts @@ -1,6 +1,7 @@ import { net } from 'electron' import { parse } from 'yaml' import { compareVersions, isPrereleaseVersion, isValidVersion } from './updater-fallback' +import { readFetchResponseTextWithinLimit } from './lib/fetch-response-body' const ATOM_FEED_URL = 'https://github.com/stablyai/orca/releases.atom' const RELEASES_DOWNLOAD_BASE = 'https://github.com/stablyai/orca/releases/download' @@ -61,7 +62,7 @@ async function fetchReleaseFeedTags(): Promise { if (!res.ok) { return null } - const body = await res.text() + const body = await readFetchResponseTextWithinLimit(res) const tags: ReleaseFeedTag[] = [] for (const match of body.matchAll(TAG_HREF_RE)) { @@ -128,7 +129,7 @@ async function hasReadyPlatformManifest(tag: string): Promise { if (!res.ok) { return false } - const assetNames = getManifestAssetNames(await res.text()) + const assetNames = getManifestAssetNames(await readFetchResponseTextWithinLimit(res)) if (assetNames.length === 0) { return false } diff --git a/src/main/usage-history-file-discovery.ts b/src/main/usage-history-file-discovery.ts new file mode 100644 index 00000000000..f16dfdc1d4d --- /dev/null +++ b/src/main/usage-history-file-discovery.ts @@ -0,0 +1,44 @@ +import { opendir } from 'node:fs/promises' +import { join } from 'node:path' +import type { UsageHistoryScanBudget } from './usage-history-scan-budget' + +export async function walkUsageHistoryJsonlFiles( + rootPath: string, + budget: UsageHistoryScanBudget +): Promise { + const pendingDirectories = [rootPath] + const files: string[] = [] + + while (pendingDirectories.length > 0) { + const directoryPath = pendingDirectories.pop()! + let directory: Awaited> + try { + directory = await opendir(directoryPath) + } catch (error) { + if (directoryPath !== rootPath && isVanishedDirectoryError(error)) { + continue + } + throw error + } + for await (const entry of directory) { + budget.claimDiscoveryEntry() + const fullPath = join(directoryPath, entry.name) + if (entry.isDirectory()) { + budget.claimPath(fullPath) + pendingDirectories.push(fullPath) + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + budget.claimFile(fullPath) + files.push(fullPath) + } + } + } + + return files +} + +function isVanishedDirectoryError(error: unknown): boolean { + if (error === null || typeof error !== 'object' || !('code' in error)) { + return false + } + return error.code === 'ENOENT' || error.code === 'ENOTDIR' +} diff --git a/src/main/usage-history-jsonl-reader.test.ts b/src/main/usage-history-jsonl-reader.test.ts new file mode 100644 index 00000000000..df2551b4628 --- /dev/null +++ b/src/main/usage-history-jsonl-reader.test.ts @@ -0,0 +1,63 @@ +import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + MAX_USAGE_HISTORY_JSONL_LINE_BYTES, + UsageHistoryJsonlLineCapacityError, + readUsageHistoryJsonlLines +} from './usage-history-jsonl-reader' + +describe('readUsageHistoryJsonlLines', () => { + const temporaryDirectories: string[] = [] + + afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true })) + ) + }) + + async function createPath(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'orca-usage-jsonl-')) + temporaryDirectories.push(directory) + return join(directory, 'history.jsonl') + } + + it('preserves ordinary empty, CRLF, and unterminated records', async () => { + const filePath = await createPath() + await writeFile(filePath, 'first\r\n\nlast') + + const lines: string[] = [] + for await (const line of readUsageHistoryJsonlLines(filePath)) { + lines.push(line) + } + + expect(lines).toEqual(['first', '', 'last']) + }) + + it('accepts a record exactly at the byte limit', async () => { + const filePath = await createPath() + await writeFile(filePath, Buffer.alloc(MAX_USAGE_HISTORY_JSONL_LINE_BYTES, 0x61)) + + const lines: string[] = [] + for await (const line of readUsageHistoryJsonlLines(filePath)) { + lines.push(line) + } + + expect(Buffer.byteLength(lines[0])).toBe(MAX_USAGE_HISTORY_JSONL_LINE_BYTES) + }) + + it('rejects a sparse unterminated record before reading the whole file', async () => { + const filePath = await createPath() + await writeFile(filePath, '') + await truncate(filePath, MAX_USAGE_HISTORY_JSONL_LINE_BYTES + 8 * 1024 * 1024) + + const consume = async (): Promise => { + for await (const _line of readUsageHistoryJsonlLines(filePath)) { + // The oversized record is unterminated, so no line should be yielded. + } + } + + await expect(consume()).rejects.toThrow(UsageHistoryJsonlLineCapacityError) + }) +}) diff --git a/src/main/usage-history-jsonl-reader.ts b/src/main/usage-history-jsonl-reader.ts new file mode 100644 index 00000000000..d9e5870c6ce --- /dev/null +++ b/src/main/usage-history-jsonl-reader.ts @@ -0,0 +1,57 @@ +import { createReadStream } from 'node:fs' + +export const MAX_USAGE_HISTORY_JSONL_LINE_BYTES = 4 * 1024 * 1024 + +export class UsageHistoryJsonlLineCapacityError extends Error { + constructor(readonly limit = MAX_USAGE_HISTORY_JSONL_LINE_BYTES) { + super(`Usage history JSONL record exceeded ${limit} bytes`) + this.name = 'UsageHistoryJsonlLineCapacityError' + } +} + +export async function* readUsageHistoryJsonlLines( + filePath: string, + options: { start?: number; maxLineBytes?: number } = {} +): AsyncGenerator { + const maxLineBytes = options.maxLineBytes ?? MAX_USAGE_HISTORY_JSONL_LINE_BYTES + const stream = createReadStream(filePath, { + start: options.start ?? 0, + highWaterMark: 64 * 1024 + }) + let fragments: Buffer[] = [] + let fragmentBytes = 0 + + for await (const rawChunk of stream) { + const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) + let offset = 0 + while (offset < chunk.length) { + const newlineIndex = chunk.indexOf(0x0a, offset) + const end = newlineIndex === -1 ? chunk.length : newlineIndex + const fragment = chunk.subarray(offset, end) + if (fragment.length > maxLineBytes - fragmentBytes) { + throw new UsageHistoryJsonlLineCapacityError(maxLineBytes) + } + if (fragment.length > 0) { + fragments.push(fragment) + fragmentBytes += fragment.length + } + if (newlineIndex === -1) { + break + } + yield decodeJsonlLine(fragments, fragmentBytes) + fragments = [] + fragmentBytes = 0 + offset = newlineIndex + 1 + } + } + + if (fragmentBytes > 0) { + yield decodeJsonlLine(fragments, fragmentBytes) + } +} + +function decodeJsonlLine(fragments: Buffer[], bytes: number): string { + const line = fragments.length === 1 ? fragments[0] : Buffer.concat(fragments, bytes) + const content = line.at(-1) === 0x0d ? line.subarray(0, -1) : line + return content.toString('utf8') +} diff --git a/src/main/usage-history-scan-budget.test.ts b/src/main/usage-history-scan-budget.test.ts new file mode 100644 index 00000000000..80cff368e22 --- /dev/null +++ b/src/main/usage-history-scan-budget.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_USAGE_HISTORY_DISCOVERY_ENTRIES, + MAX_USAGE_HISTORY_FILES, + MAX_USAGE_HISTORY_OWNERSHIP_KEYS, + MAX_USAGE_HISTORY_RECORDS, + MAX_USAGE_HISTORY_RETAINED_BYTES, + UsageHistoryScanBudget, + UsageHistoryScanCapacityError +} from './usage-history-scan-budget' + +describe('UsageHistoryScanBudget', () => { + it('publishes the production scan limits', () => { + expect({ + files: MAX_USAGE_HISTORY_FILES, + discoveryEntries: MAX_USAGE_HISTORY_DISCOVERY_ENTRIES, + records: MAX_USAGE_HISTORY_RECORDS, + ownershipKeys: MAX_USAGE_HISTORY_OWNERSHIP_KEYS, + retainedBytes: MAX_USAGE_HISTORY_RETAINED_BYTES + }).toEqual({ + files: 200_000, + discoveryEntries: 1_000_000, + records: 100_000, + ownershipKeys: 100_000, + retainedBytes: 256 * 1024 * 1024 + }) + }) + + it('accepts exact limits and fails closed before each collection can grow past them', () => { + const files = new UsageHistoryScanBudget({ files: 2, retainedBytes: 10_000 }) + files.claimFile('a') + files.claimFile('b') + expect(() => files.claimFile('c')).toThrowError( + expect.objectContaining({ resource: 'files', limit: 2 }) + ) + + const entries = new UsageHistoryScanBudget({ discoveryEntries: 2 }) + entries.claimDiscoveryEntry() + entries.claimDiscoveryEntry() + expect(() => entries.claimDiscoveryEntry()).toThrow(UsageHistoryScanCapacityError) + + const records = new UsageHistoryScanBudget({ records: 2, retainedBytes: 10_000 }) + records.claimRecord(1) + records.claimRecord(1) + expect(() => records.claimRecord(1)).toThrowError( + expect.objectContaining({ resource: 'records', limit: 2 }) + ) + + const keys = new UsageHistoryScanBudget({ ownershipKeys: 2, retainedBytes: 10_000 }) + keys.claimOwnershipKey('a') + keys.claimOwnershipKey('b') + expect(() => keys.claimOwnershipKey('c')).toThrowError( + expect.objectContaining({ resource: 'ownershipKeys', limit: 2 }) + ) + + const bytes = new UsageHistoryScanBudget({ retainedBytes: 2 }) + bytes.claimRetainedBytes(2) + expect(() => bytes.claimRetainedBytes(1)).toThrowError( + expect.objectContaining({ resource: 'retainedBytes', limit: 2 }) + ) + }) + + it('charges conservative collection overhead for many tiny values', () => { + const records = new UsageHistoryScanBudget({ records: 100, retainedBytes: 10 * 1024 }) + for (let index = 0; index < 10; index++) { + records.claimRecord(0) + } + expect(() => records.claimRecord(0)).toThrowError( + expect.objectContaining({ resource: 'retainedBytes', limit: 10 * 1024 }) + ) + + const paths = new UsageHistoryScanBudget({ files: 100, retainedBytes: 1_940 }) + for (let index = 0; index < 10; index++) { + paths.claimFile('a') + } + expect(() => paths.claimFile('b')).toThrowError( + expect.objectContaining({ resource: 'retainedBytes', limit: 1_940 }) + ) + }) +}) diff --git a/src/main/usage-history-scan-budget.ts b/src/main/usage-history-scan-budget.ts new file mode 100644 index 00000000000..d363da15f9b --- /dev/null +++ b/src/main/usage-history-scan-budget.ts @@ -0,0 +1,111 @@ +export const MAX_USAGE_HISTORY_FILES = 200_000 +export const MAX_USAGE_HISTORY_DISCOVERY_ENTRIES = 1_000_000 +export const MAX_USAGE_HISTORY_RECORDS = 100_000 +export const MAX_USAGE_HISTORY_OWNERSHIP_KEYS = 100_000 +export const MAX_USAGE_HISTORY_RETAINED_BYTES = 256 * 1024 * 1024 + +const FILE_COLLECTION_ENTRY_BYTES = 160 +const OWNERSHIP_COLLECTION_ENTRY_BYTES = 160 +const RECORD_PIPELINE_BYTES = 1024 +const PROJECTION_ENTRY_BYTES = 256 +const STRING_HEADER_BYTES = 32 + +type UsageHistoryScanLimits = { + files: number + discoveryEntries: number + records: number + ownershipKeys: number + retainedBytes: number +} + +const DEFAULT_LIMITS: UsageHistoryScanLimits = { + files: MAX_USAGE_HISTORY_FILES, + discoveryEntries: MAX_USAGE_HISTORY_DISCOVERY_ENTRIES, + records: MAX_USAGE_HISTORY_RECORDS, + ownershipKeys: MAX_USAGE_HISTORY_OWNERSHIP_KEYS, + retainedBytes: MAX_USAGE_HISTORY_RETAINED_BYTES +} + +export class UsageHistoryScanCapacityError extends Error { + constructor( + readonly resource: keyof UsageHistoryScanLimits, + readonly limit: number + ) { + super(`Usage history scan exceeded ${limit} ${resource}`) + this.name = 'UsageHistoryScanCapacityError' + } +} + +export class UsageHistoryScanBudget { + private readonly limits: UsageHistoryScanLimits + private readonly used: UsageHistoryScanLimits = { + files: 0, + discoveryEntries: 0, + records: 0, + ownershipKeys: 0, + retainedBytes: 0 + } + + constructor(limits: Partial = {}) { + this.limits = { ...DEFAULT_LIMITS, ...limits } + } + + claimFile(path: string): void { + this.claim('files', 1) + this.claimPath(path) + } + + claimDiscoveryEntry(): void { + this.claim('discoveryEntries', 1) + } + + claimRecord(retainedBytes: number): void { + this.claim('records', 1) + this.claim('retainedBytes', RECORD_PIPELINE_BYTES + retainedBytes) + } + + claimRecords(count: number): void { + this.claim('records', count) + this.claim('retainedBytes', RECORD_PIPELINE_BYTES * count) + } + + claimRetainedBytes(bytes: number): void { + this.claim('retainedBytes', bytes) + } + + claimPath(path: string): void { + this.claim('retainedBytes', FILE_COLLECTION_ENTRY_BYTES + getUsageHistoryRetainedBytes([path])) + } + + claimProjection(retainedStringBytes: number): void { + this.claim('retainedBytes', PROJECTION_ENTRY_BYTES + retainedStringBytes) + } + + claimOwnershipKey(key: string): void { + this.claim('ownershipKeys', 1) + this.claim( + 'retainedBytes', + OWNERSHIP_COLLECTION_ENTRY_BYTES + getUsageHistoryRetainedBytes([key]) + ) + } + + private claim(resource: keyof UsageHistoryScanLimits, amount: number): void { + if (!Number.isSafeInteger(amount) || amount < 0) { + throw new UsageHistoryScanCapacityError(resource, this.limits[resource]) + } + if (amount > this.limits[resource] - this.used[resource]) { + throw new UsageHistoryScanCapacityError(resource, this.limits[resource]) + } + this.used[resource] += amount + } +} + +export function getUsageHistoryRetainedBytes(values: readonly (string | null)[]): number { + let bytes = 0 + for (const value of values) { + if (value !== null) { + bytes += STRING_HEADER_BYTES + value.length * 2 + } + } + return bytes +} diff --git a/src/main/usage-projection-state-file.test.ts b/src/main/usage-projection-state-file.test.ts new file mode 100644 index 00000000000..9dd28c4a2d4 --- /dev/null +++ b/src/main/usage-projection-state-file.test.ts @@ -0,0 +1,96 @@ +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + readUsageProjectionStateFile, + serializeUsageProjectionState, + UsageProjectionStateCapacityError, + writeUsageProjectionStateFile, + writeUsageProjectionStateFileWithRecovery +} from './usage-projection-state-file' + +const tempRoots: string[] = [] + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('usage projection state files', () => { + it('accepts exact serialized bytes and rejects the next byte', () => { + expect(serializeUsageProjectionState('x'.repeat(30), 32)).toHaveLength(32) + expect(() => serializeUsageProjectionState('x'.repeat(31), 32)).toThrow( + UsageProjectionStateCapacityError + ) + }) + + it('checks the file size before allocating or decoding it', () => { + const root = createTempRoot() + const path = join(root, 'usage.json') + writeFileSync(path, '{"ok":true}') + expect(readUsageProjectionStateFile(path, 11)).toBe('{"ok":true}') + + truncateSync(path, 12) + expect(() => readUsageProjectionStateFile(path, 11)).toThrow('File too large') + expect(readUsageProjectionStateFile(join(root, 'absent.json'), 11)).toBeNull() + }) + + it('rejects structurally amplified caches before parsing them', () => { + const root = createTempRoot() + const path = join(root, 'usage.json') + const json = '{"rows":[{},{}]}' + writeFileSync(path, json) + + expect(() => + readUsageProjectionStateFile(path, Buffer.byteLength(json), { + structuralTokens: 7, + nestingDepth: 3 + }) + ).toThrow('JSON structure') + }) + + it('writes atomically and preserves ordinary JSON semantics', () => { + const root = createTempRoot() + const path = join(root, 'nested', 'usage.json') + const state = { enabled: true, rows: [{ id: 'one' }, { id: 'two' }] } + + writeUsageProjectionStateFile(path, state, 1024) + + expect(JSON.parse(readFileSync(path, 'utf8'))).toEqual(state) + expect(readdirSync(join(root, 'nested'))).toEqual(['usage.json']) + }) + + it('replaces an oversized rebuildable projection with a bounded error state', () => { + const root = createTempRoot() + const path = join(root, 'usage.json') + const recovered = writeUsageProjectionStateFileWithRecovery<{ + enabled: boolean + rows: string | unknown[] + error?: string + }>( + path, + { enabled: true, rows: 'x'.repeat(200) }, + (error) => ({ enabled: true, rows: [], error: error.message }), + 160 + ) + + expect(recovered).toMatchObject({ enabled: true, rows: [] }) + expect(JSON.parse(readFileSync(path, 'utf8'))).toEqual(recovered) + expect(readdirSync(root)).toEqual(['usage.json']) + }) +}) + +function createTempRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-usage-projection-')) + tempRoots.push(root) + return root +} diff --git a/src/main/usage-projection-state-file.ts b/src/main/usage-projection-state-file.ts new file mode 100644 index 00000000000..499a6debdb1 --- /dev/null +++ b/src/main/usage-projection-state-file.ts @@ -0,0 +1,98 @@ +import { mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { dirname } from 'node:path' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../shared/node-bounded-json-stringify' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from '../shared/json-text-structure-limit' + +export const MAX_USAGE_PROJECTION_STATE_FILE_BYTES = 64 * 1024 * 1024 +export const USAGE_PROJECTION_STATE_JSON_LIMITS: JsonTextStructureLimits = { + structuralTokens: 1_000_000, + nestingDepth: 256 +} + +export class UsageProjectionStateCapacityError extends Error { + constructor(readonly maxBytes = MAX_USAGE_PROJECTION_STATE_FILE_BYTES) { + super(`Usage analytics cache exceeds ${maxBytes} bytes and must be rebuilt.`) + this.name = 'UsageProjectionStateCapacityError' + } +} + +export function readUsageProjectionStateFile( + filePath: string, + maxBytes = MAX_USAGE_PROJECTION_STATE_FILE_BYTES, + structureLimits: JsonTextStructureLimits = USAGE_PROJECTION_STATE_JSON_LIMITS +): string | null { + try { + const serialized = readNodeFileSyncWithinLimit(filePath, maxBytes).buffer.toString('utf8') + assertJsonTextStructureWithinLimits(serialized, structureLimits) + return serialized + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return null + } + throw error + } +} + +export function serializeUsageProjectionState( + state: unknown, + maxBytes = MAX_USAGE_PROJECTION_STATE_FILE_BYTES +): string { + try { + return stringifyJsonWithinByteLimit(state, maxBytes).serialized + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new UsageProjectionStateCapacityError(maxBytes) + } + throw error + } +} + +export function writeUsageProjectionStateFile( + filePath: string, + state: unknown, + maxBytes = MAX_USAGE_PROJECTION_STATE_FILE_BYTES +): void { + const payload = serializeUsageProjectionState(state, maxBytes) + mkdirSync(dirname(filePath), { recursive: true }) + const tmpFile = `${filePath}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp` + let renamed = false + try { + writeFileSync(tmpFile, payload, 'utf8') + renameSync(tmpFile, filePath) + renamed = true + } finally { + if (!renamed) { + try { + unlinkSync(tmpFile) + } catch { + // The primary write error is more useful than a best-effort cleanup failure. + } + } + } +} + +export function writeUsageProjectionStateFileWithRecovery( + filePath: string, + state: T, + recover: (error: UsageProjectionStateCapacityError) => T, + maxBytes = MAX_USAGE_PROJECTION_STATE_FILE_BYTES +): T { + try { + writeUsageProjectionStateFile(filePath, state, maxBytes) + return state + } catch (error) { + if (!(error instanceof UsageProjectionStateCapacityError)) { + throw error + } + const recovered = recover(error) + writeUsageProjectionStateFile(filePath, recovered, maxBytes) + return recovered + } +} diff --git a/src/main/warp-themes/discovery.test.ts b/src/main/warp-themes/discovery.test.ts index 977a5b30de7..911fa40d4cc 100644 --- a/src/main/warp-themes/discovery.test.ts +++ b/src/main/warp-themes/discovery.test.ts @@ -7,10 +7,10 @@ type MockDirectoryEntry = { isDirectory: () => boolean } -const readdirSyncMock = vi.hoisted(() => vi.fn<() => MockDirectoryEntry[]>(() => [])) +const opendirSyncMock = vi.hoisted(() => vi.fn()) vi.mock('fs', () => ({ - readdirSync: readdirSyncMock + opendirSync: opendirSyncMock })) vi.mock('os', () => ({ @@ -34,13 +34,21 @@ function fileEntry(name: string): MockDirectoryEntry { } } +function mockDirectory(entries: MockDirectoryEntry[]) { + let index = 0 + const readSync = vi.fn(() => entries[index++] ?? null) + const closeSync = vi.fn() + opendirSyncMock.mockReturnValue({ readSync, closeSync }) + return { readSync, closeSync } +} + describe('getWarpThemeDirectories', () => { beforeEach(() => { vi.unstubAllEnvs() platformMock.mockReset() homedirMock.mockReturnValue('/Users/alice') - readdirSyncMock.mockReset() - readdirSyncMock.mockReturnValue([]) + opendirSyncMock.mockReset() + mockDirectory([]) }) it('returns macOS Warp channel theme directories in stable-first order', () => { @@ -57,11 +65,12 @@ describe('getWarpThemeDirectories', () => { it('adds dynamic macOS .warp directories after known channels', () => { platformMock.mockReturnValue('darwin') - readdirSyncMock.mockReturnValue([ - directoryEntry('.warp-future'), + mockDirectory([ + directoryEntry('.warp-z'), fileEntry('.warp-note'), directoryEntry('.not-warp'), - directoryEntry('.warp-preview') + directoryEntry('.warp-preview'), + directoryEntry('.warp-a') ]) expect(getWarpThemeDirectories()).toEqual([ @@ -71,7 +80,8 @@ describe('getWarpThemeDirectories', () => { '/Users/alice/.warp-dev/themes', '/Users/alice/.warp-local/themes', '/Users/alice/.warp-integration/themes', - '/Users/alice/.warp-future/themes' + '/Users/alice/.warp-a/themes', + '/Users/alice/.warp-z/themes' ]) }) @@ -91,7 +101,7 @@ describe('getWarpThemeDirectories', () => { it('adds dynamic Linux warp data directories', () => { platformMock.mockReturnValue('linux') vi.stubEnv('XDG_DATA_HOME', '/data/alice') - readdirSyncMock.mockReturnValue([ + mockDirectory([ directoryEntry('warp-future'), directoryEntry('warp-terminal'), directoryEntry('not-warp'), @@ -121,8 +131,8 @@ describe('getWarpThemeDirectories', () => { '/Users/alice/.local/share/warp-terminal-local/themes', '/Users/alice/.local/share/warp-terminal-integration/themes' ]) - expect(readdirSyncMock).toHaveBeenCalledWith('/Users/alice/.local/share', { - withFileTypes: true + expect(opendirSyncMock).toHaveBeenCalledWith('/Users/alice/.local/share', { + bufferSize: 32 }) }) @@ -143,7 +153,7 @@ describe('getWarpThemeDirectories', () => { it('adds dynamic Windows Warp app data directories', () => { platformMock.mockReturnValue('win32') vi.stubEnv('APPDATA', 'C:\\Users\\alice\\AppData\\Roaming') - readdirSyncMock.mockReturnValue([ + mockDirectory([ directoryEntry('WarpFuture'), directoryEntry('WarpPreview'), fileEntry('WarpNote') @@ -159,6 +169,34 @@ describe('getWarpThemeDirectories', () => { 'C:\\Users\\alice\\AppData\\Roaming\\warp\\WarpFuture\\data\\themes' ]) }) + + it('caps dynamic discovery while closing the streamed directory', () => { + platformMock.mockReturnValue('darwin') + const { readSync, closeSync } = mockDirectory( + Array.from({ length: 1_025 }, (_, index) => + directoryEntry(`.warp-dynamic-${String(index).padStart(4, '0')}`) + ) + ) + + const directories = getWarpThemeDirectories() + + expect(directories).toHaveLength(1_030) + expect(directories).toContain('/Users/alice/.warp-dynamic-1023/themes') + expect(directories).not.toContain('/Users/alice/.warp-dynamic-1024/themes') + expect(readSync).toHaveBeenCalledTimes(1_024) + expect(closeSync).toHaveBeenCalledOnce() + }) + + it('streams past unrelated entries without consuming the match budget', () => { + platformMock.mockReturnValue('darwin') + const { readSync } = mockDirectory([ + ...Array.from({ length: 1_024 }, (_, index) => fileEntry(`note-${index}`)), + directoryEntry('.warp-future') + ]) + + expect(getWarpThemeDirectories()).toContain('/Users/alice/.warp-future/themes') + expect(readSync).toHaveBeenCalledTimes(1_026) + }) }) describe('warpThemeSourceLabelForDirectory', () => { diff --git a/src/main/warp-themes/discovery.ts b/src/main/warp-themes/discovery.ts index 6f3f80b51da..f92699fc61e 100644 --- a/src/main/warp-themes/discovery.ts +++ b/src/main/warp-themes/discovery.ts @@ -1,8 +1,10 @@ -import { readdirSync } from 'node:fs' -import type { Dirent } from 'node:fs' +import { opendirSync, type Dir, type Dirent } from 'node:fs' import { homedir, platform } from 'node:os' import path from 'node:path' +const MAX_DYNAMIC_DISCOVERY_MATCHES = 1_024 +const MAX_DYNAMIC_DISCOVERY_SCANNED_ENTRIES = 100_000 + const WARP_CHANNELS = [ { macName: '.warp', linuxName: 'warp-terminal', windowsName: 'Warp' }, { macName: '.warp-preview', linuxName: 'warp-terminal-preview', windowsName: 'WarpPreview' }, @@ -16,13 +18,39 @@ const WARP_CHANNELS = [ } ] -function readDirectoryEntries(directoryPath: string): Dirent[] { +function readDirectoryEntries( + directoryPath: string, + includeEntry: (entry: Dirent) => boolean +): Dirent[] { + let directory: Dir | undefined try { - return readdirSync(directoryPath, { withFileTypes: true }).sort((left, right) => + directory = opendirSync(directoryPath, { bufferSize: 32 }) + const entries: Dirent[] = [] + let scannedEntries = 0 + while ( + entries.length < MAX_DYNAMIC_DISCOVERY_MATCHES && + scannedEntries < MAX_DYNAMIC_DISCOVERY_SCANNED_ENTRIES + ) { + const entry = directory.readSync() + if (entry === null) { + break + } + scannedEntries += 1 + if (includeEntry(entry)) { + entries.push(entry) + } + } + return entries.sort((left, right) => left.name.localeCompare(right.name, undefined, { sensitivity: 'base' }) ) } catch { return [] + } finally { + try { + directory?.closeSync() + } catch { + // A failed close cannot make already bounded discovery unsafe. + } } } @@ -57,9 +85,10 @@ function getMacWarpThemeDirectories(home: string): string[] { return warpThemeDirectoriesFromDataHomes( [ ...WARP_CHANNELS.map((channel) => pathImpl.join(home, channel.macName)), - ...readDirectoryEntries(home) - .filter((entry) => entry.isDirectory() && entry.name.startsWith('.warp')) - .map((entry) => pathImpl.join(home, entry.name)) + ...readDirectoryEntries( + home, + (entry) => entry.isDirectory() && entry.name.startsWith('.warp') + ).map((entry) => pathImpl.join(home, entry.name)) ], pathImpl ) @@ -77,13 +106,11 @@ function getLinuxWarpThemeDirectories(home: string): string[] { return warpThemeDirectoriesFromDataHomes( [ ...WARP_CHANNELS.map((channel) => pathImpl.join(dataHome, channel.linuxName)), - ...readDirectoryEntries(dataHome) - .filter( - (entry) => - entry.isDirectory() && - (entry.name === 'warp-terminal' || entry.name.startsWith('warp-')) - ) - .map((entry) => pathImpl.join(dataHome, entry.name)) + ...readDirectoryEntries( + dataHome, + (entry) => + entry.isDirectory() && (entry.name === 'warp-terminal' || entry.name.startsWith('warp-')) + ).map((entry) => pathImpl.join(dataHome, entry.name)) ], pathImpl ) @@ -102,10 +129,7 @@ function getWindowsWarpThemeDirectories(home: string): string[] { path.win32 ) } - for (const entry of readDirectoryEntries(warpAppData)) { - if (!entry.isDirectory()) { - continue - } + for (const entry of readDirectoryEntries(warpAppData, (candidate) => candidate.isDirectory())) { addDedupeDirectory( directories, seenDirectories, diff --git a/src/main/warp-themes/index.test.ts b/src/main/warp-themes/index.test.ts index f70154b4079..a24e5212300 100644 --- a/src/main/warp-themes/index.test.ts +++ b/src/main/warp-themes/index.test.ts @@ -1,9 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import path from 'node:path' +import type * as BoundedFileReader from '../../shared/node-bounded-file-reader' import type * as WarpThemeDiscovery from './discovery' const opendirMock = vi.hoisted(() => vi.fn()) -const readFileMock = vi.hoisted(() => vi.fn()) +const readNodeFileWithinLimitMock = vi.hoisted(() => vi.fn()) const realpathMock = vi.hoisted(() => vi.fn((filePath: string) => Promise.resolve(filePath))) const statMock = vi.hoisted(() => vi.fn()) const getWarpThemeDirectoriesMock = vi.hoisted(() => vi.fn(() => ['/Users/alice/.warp/themes'])) @@ -17,11 +18,15 @@ vi.mock('electron', () => ({ vi.mock('fs/promises', () => ({ opendir: opendirMock, - readFile: readFileMock, realpath: realpathMock, stat: statMock })) +vi.mock('../../shared/node-bounded-file-reader', async (importOriginal) => ({ + ...(await importOriginal()), + readNodeFileWithinLimit: readNodeFileWithinLimitMock +})) + vi.mock('./discovery', async (importOriginal) => { const actual = await importOriginal() return { @@ -36,6 +41,7 @@ vi.mock('./parser-runner', () => ({ import { previewWarpThemeImport } from './index' import { parseWarpThemeYaml } from './parser' +import { NodeFileReadTooLargeError } from '../../shared/node-bounded-file-reader' import type { Store } from '../persistence' const VALID_THEME = ` @@ -97,12 +103,19 @@ function mockStat(filePath: string) { : { isFile: () => true, size: VALID_THEME.length } } +function fileRead(content: string, size = Buffer.byteLength(content)) { + return { + buffer: Buffer.from(content), + stats: { isFile: () => true, size } + } +} + describe('previewWarpThemeImport', () => { beforeEach(() => { vi.clearAllMocks() getWarpThemeDirectoriesMock.mockReturnValue(['/Users/alice/.warp/themes']) statMock.mockImplementation(mockStat) - readFileMock.mockResolvedValue(VALID_THEME) + readNodeFileWithinLimitMock.mockResolvedValue(fileRead(VALID_THEME)) realpathMock.mockImplementation((filePath: string) => Promise.resolve(filePath)) opendirMock.mockResolvedValue(mockDirectory([fileEntry('z.yml'), fileEntry('a.yml')])) parseWarpThemeYamlWithTimeoutMock.mockImplementation(parseWarpThemeYaml) @@ -115,10 +128,9 @@ describe('previewWarpThemeImport', () => { 'warp:duplicate:a-yml', 'warp:duplicate:z-yml' ]) - expect(readFileMock.mock.calls.map(([filePath]) => path.basename(filePath as string))).toEqual([ - 'a.yml', - 'z.yml' - ]) + expect( + readNodeFileWithinLimitMock.mock.calls.map(([filePath]) => path.basename(filePath as string)) + ).toEqual(['a.yml', 'z.yml']) }) it('returns an empty errorless preview when no local Warp theme folder exists', async () => { @@ -135,7 +147,7 @@ describe('previewWarpThemeImport', () => { skippedFiles: [] }) expect(preview.error).toBeUndefined() - expect(readFileMock).not.toHaveBeenCalled() + expect(readNodeFileWithinLimitMock).not.toHaveBeenCalled() }) it('returns an empty errorless preview for an empty readable local Warp theme folder', async () => { @@ -150,7 +162,7 @@ describe('previewWarpThemeImport', () => { skippedFiles: [] }) expect(preview.error).toBeUndefined() - expect(readFileMock).not.toHaveBeenCalled() + expect(readNodeFileWithinLimitMock).not.toHaveBeenCalled() }) it('merges themes from multiple readable Warp directories', async () => { @@ -170,7 +182,7 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) - expect(readFileMock.mock.calls.map(([filePath]) => filePath)).toEqual([ + expect(readNodeFileWithinLimitMock.mock.calls.map(([filePath]) => filePath)).toEqual([ path.join('/Users/alice/.warp/themes', 'stable.yaml'), path.join('/Users/alice/.warp-preview/themes', 'preview.yaml') ]) @@ -195,7 +207,7 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.found).toBe(true) - expect(readFileMock.mock.calls.map(([filePath]) => filePath)).toEqual([ + expect(readNodeFileWithinLimitMock.mock.calls.map(([filePath]) => filePath)).toEqual([ path.join('/Users/alice/.warp-oss/themes', 'oss.yaml') ]) expect(preview.themes.map((theme) => theme.sourceLabel)).toEqual(['.warp-oss']) @@ -220,9 +232,9 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.themes).toHaveLength(1) - expect(readFileMock).toHaveBeenCalledWith( + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith( path.join('/Users/alice/.warp/themes', 'shared.yaml'), - 'utf-8' + 1_000_000 ) expect(preview.themes[0]?.sourceLabel).toBe('.warp') expect(preview.skippedFiles).not.toContainEqual({ @@ -237,9 +249,9 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.found).toBe(true) - expect(readFileMock).toHaveBeenCalledWith( + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith( path.join('/Users/alice/.warp/themes', 'linked.yaml'), - 'utf-8' + 1_000_000 ) }) @@ -254,7 +266,7 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.themes).toHaveLength(1) - expect(readFileMock).toHaveBeenCalledTimes(1) + expect(readNodeFileWithinLimitMock).toHaveBeenCalledTimes(1) }) it('applies the theme file cap globally across merged auto-discovery directories', async () => { @@ -283,7 +295,7 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.themes).toHaveLength(200) - expect(readFileMock).toHaveBeenCalledTimes(200) + expect(readNodeFileWithinLimitMock).toHaveBeenCalledTimes(200) expect(preview.skippedFiles).toContainEqual({ label: 'Warp themes', reason: 'Only the first 200 theme files were scanned.' @@ -312,9 +324,9 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.themes).toHaveLength(200) - expect(readFileMock).not.toHaveBeenCalledWith( + expect(readNodeFileWithinLimitMock).not.toHaveBeenCalledWith( path.join('/Users/alice/.warp-preview/themes', 'preview.yaml'), - 'utf-8' + 1_000_000 ) expect(preview.skippedFiles).toContainEqual({ label: 'Warp themes', @@ -358,7 +370,10 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.themes).toHaveLength(200) - expect(readFileMock).toHaveBeenCalledWith(path.join(previewDirectory, 'unique.yaml'), 'utf-8') + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith( + path.join(previewDirectory, 'unique.yaml'), + 1_000_000 + ) }) it('reports bounded skips when local Warp folders are unreadable', async () => { @@ -466,10 +481,12 @@ describe('previewWarpThemeImport', () => { it('keeps same-basename manual file ids stable independent of dialog order', async () => { const firstPath = path.join('/Users/alice/light', 'duplicate.yaml') const secondPath = path.join('/Users/alice/dark', 'duplicate.yaml') - readFileMock.mockImplementation((filePath: string) => - filePath === firstPath - ? VALID_THEME.replace("background: '#111111'", "background: '#222222'") - : VALID_THEME.replace("background: '#111111'", "background: '#333333'") + readNodeFileWithinLimitMock.mockImplementation((filePath: string) => + fileRead( + filePath === firstPath + ? VALID_THEME.replace("background: '#111111'", "background: '#222222'") + : VALID_THEME.replace("background: '#111111'", "background: '#333333'") + ) ) showOpenDialogMock.mockResolvedValueOnce({ canceled: false, @@ -560,7 +577,7 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) expect(preview.found).toBe(true) - expect(readFileMock.mock.calls.map(([filePath]) => filePath)).toEqual([ + expect(readNodeFileWithinLimitMock.mock.calls.map(([filePath]) => filePath)).toEqual([ path.join('/Users/alice/.warp/themes', 'standard', 'tokyo-night.yaml'), path.join('/Users/alice/.warp/themes', 'warp_bundled', 'dracula.yml') ]) @@ -711,8 +728,8 @@ describe('previewWarpThemeImport', () => { const preview = await previewWarpThemeImport({} as Store, { kind: 'chooseFile' }) expect(preview.themes).toHaveLength(200) - expect(readFileMock.mock.calls[0]?.[0]).toBe(themePaths[0]) - expect(readFileMock.mock.calls.at(-1)?.[0]).toBe(themePaths[199]) + expect(readNodeFileWithinLimitMock.mock.calls[0]?.[0]).toBe(themePaths[0]) + expect(readNodeFileWithinLimitMock.mock.calls.at(-1)?.[0]).toBe(themePaths[199]) expect(preview.skippedFiles).toContainEqual({ label: 'Selected Warp themes', reason: 'Only the first 200 theme files were scanned.' @@ -779,4 +796,41 @@ describe('previewWarpThemeImport', () => { expect(preview.skippedFiles).toEqual([{ label: 'private.yml', reason: 'Could not read file.' }]) }) + + it('accepts a theme at the exact byte limit', async () => { + const themePath = path.join('/Users/alice/.warp/themes', 'boundary.yml') + opendirMock.mockResolvedValue(mockDirectory([fileEntry('boundary.yml')])) + statMock.mockImplementation((filePath: string) => + filePath === themePath ? { isFile: () => true, size: 1_000_000 } : mockStat(filePath) + ) + readNodeFileWithinLimitMock.mockResolvedValue(fileRead(VALID_THEME, 1_000_000)) + + const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) + + expect(preview.found).toBe(true) + expect(readNodeFileWithinLimitMock).toHaveBeenCalledWith(themePath, 1_000_000) + expect(parseWarpThemeYamlWithTimeoutMock).toHaveBeenCalledOnce() + }) + + it('rejects theme growth beyond the limit before parsing', async () => { + const themePath = path.join('/Users/alice/.warp/themes', 'growing.yml') + opendirMock.mockResolvedValue(mockDirectory([fileEntry('growing.yml')])) + statMock.mockImplementation((filePath: string) => + filePath === themePath ? { isFile: () => true, size: 128 } : mockStat(filePath) + ) + readNodeFileWithinLimitMock.mockRejectedValue( + new NodeFileReadTooLargeError(1_000_001, 1_000_000) + ) + + const preview = await previewWarpThemeImport({} as Store, { kind: 'auto' }) + + expect(preview.found).toBe(false) + expect(preview.skippedFiles).toEqual([ + { + label: 'growing.yml', + reason: 'File is too large to import (1000001 bytes, limit 1000000).' + } + ]) + expect(parseWarpThemeYamlWithTimeoutMock).not.toHaveBeenCalled() + }) }) diff --git a/src/main/warp-themes/index.ts b/src/main/warp-themes/index.ts index d192d322773..f99148f6569 100644 --- a/src/main/warp-themes/index.ts +++ b/src/main/warp-themes/index.ts @@ -1,6 +1,10 @@ -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import type { WebContents } from 'electron' import type { Store } from '../persistence' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../../shared/node-bounded-file-reader' import type { WarpThemeImportPreview, WarpThemeImportSource @@ -109,8 +113,20 @@ export async function previewWarpThemeImport( }) continue } - content = await readFile(file.path, 'utf-8') - } catch { + const result = await readNodeFileWithinLimit(file.path, MAX_THEME_FILE_BYTES) + if (!result.stats.isFile()) { + skippedFiles.push({ label: file.label, reason: 'Not a file.' }) + continue + } + content = result.buffer.toString('utf8') + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + skippedFiles.push({ + label: file.label, + reason: `File is too large to import (${error.observedBytes} bytes, limit ${MAX_THEME_FILE_BYTES}).` + }) + continue + } skippedFiles.push({ label: file.label, reason: sanitizeReadError('Could not read file.') diff --git a/src/main/window/attach-main-window-services.test.ts b/src/main/window/attach-main-window-services.test.ts index 1ae323607c6..31b004c2bfc 100644 --- a/src/main/window/attach-main-window-services.test.ts +++ b/src/main/window/attach-main-window-services.test.ts @@ -202,17 +202,15 @@ describe('attachMainWindowServices', () => { it('reloads the app renderer through main and marks expected renderer teardown', async () => { const onBeforeRendererReload = vi.fn() const mainWindow = createMainWindow() + const store = createStore() + const runtime = createRuntime() - attachMainWindowServices( - mainWindow as never, - createStore(), - createRuntime() as never, - undefined, - undefined, - { onBeforeRendererReload } - ) + attachMainWindowServices(mainWindow as never, store, runtime as never, undefined, undefined, { + onBeforeRendererReload + }) expect(removeHandlerMock).toHaveBeenCalledWith('app:reload') + expect(registerRepoHandlersMock).toHaveBeenCalledWith(mainWindow, store, runtime) const reloadHandler = handleMock.mock.calls.find(([channel]) => channel === 'app:reload')?.[1] expect(reloadHandler).toBeTypeOf('function') diff --git a/src/main/window/attach-main-window-services.ts b/src/main/window/attach-main-window-services.ts index 0a481b209bc..cc9ea9f8280 100644 --- a/src/main/window/attach-main-window-services.ts +++ b/src/main/window/attach-main-window-services.ts @@ -87,7 +87,7 @@ export function attachMainWindowServices( } ): void { registerAppReloadHandler(mainWindow, options?.onBeforeRendererReload) - registerRepoHandlers(mainWindow, store) + registerRepoHandlers(mainWindow, store, runtime) registerWorktreeHandlers(mainWindow, store, runtime) // Why: repo/settings mutations resync watchers through this attached main-window context. setWorktreeBaseDirectoryWatcherSyncContext(store, mainWindow) diff --git a/src/main/window/clipboard-ipc-handlers.test.ts b/src/main/window/clipboard-ipc-handlers.test.ts index b1465beb82d..63d089584d0 100644 --- a/src/main/window/clipboard-ipc-handlers.test.ts +++ b/src/main/window/clipboard-ipc-handlers.test.ts @@ -13,6 +13,7 @@ const { childStdinEndMock, resolveAuthorizedPathMock, fsMkdirMock, + fsOpendirMock, fsReaddirMock, fsRmMock, fsWriteFileMock, @@ -46,6 +47,7 @@ const { }), resolveAuthorizedPathMock: vi.fn(), fsMkdirMock: vi.fn(), + fsOpendirMock: vi.fn(), fsReaddirMock: vi.fn(), fsRmMock: vi.fn(), fsWriteFileMock: vi.fn(), @@ -69,7 +71,7 @@ vi.mock('node:child_process', () => ({ vi.mock('node:fs/promises', () => ({ mkdir: fsMkdirMock, - readdir: fsReaddirMock, + opendir: fsOpendirMock, rm: fsRmMock, open: fsOpenMock, stat: fsStatMock, @@ -194,6 +196,13 @@ describe('registerClipboardHandlers', () => { resolveAuthorizedPathMock.mockImplementation(async (path: string) => path) fsMkdirMock.mockReset() fsMkdirMock.mockResolvedValue(undefined) + fsOpendirMock.mockReset() + fsOpendirMock.mockImplementation(async () => ({ + async *[Symbol.asyncIterator]() { + yield* await fsReaddirMock() + }, + close: vi.fn().mockResolvedValue(undefined) + })) fsReaddirMock.mockReset() fsReaddirMock.mockResolvedValue([]) fsRmMock.mockReset() diff --git a/src/main/window/clipboard-remote-file-cleanup-memory.test.ts b/src/main/window/clipboard-remote-file-cleanup-memory.test.ts new file mode 100644 index 00000000000..4127b4006b6 --- /dev/null +++ b/src/main/window/clipboard-remote-file-cleanup-memory.test.ts @@ -0,0 +1,56 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { opendirMock, rmMock, statMock } = vi.hoisted(() => ({ + opendirMock: vi.fn(), + rmMock: vi.fn(), + statMock: vi.fn() +})) + +vi.mock('node:fs/promises', () => ({ + mkdir: vi.fn(), + opendir: opendirMock, + rm: rmMock, + stat: statMock +})) +vi.mock('electron', () => ({ app: { getPath: () => '/tmp' } })) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + requireSshFilesystemProvider: vi.fn() +})) +vi.mock('./clipboard-file-copy', () => ({ writeFileToClipboard: vi.fn() })) + +import { cleanupExpiredRemoteClipboardFiles } from './clipboard-remote-file-copy' + +describe('remote clipboard cleanup memory bounds', () => { + beforeEach(() => { + vi.clearAllMocks() + rmMock.mockResolvedValue(undefined) + }) + + it('streams all entries with at most eight cleanups in flight', async () => { + const nowMs = 1_760_000_000_000 + const entries = Array.from({ length: 257 }, (_, index) => ({ + name: `orca-clipboard-file-expired-${index}`, + isDirectory: () => true + })) + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + yield* entries + }, + close: vi.fn().mockResolvedValue(undefined) + }) + let active = 0 + let peak = 0 + statMock.mockImplementation(async () => { + active += 1 + peak = Math.max(peak, active) + await new Promise((resolve) => setImmediate(resolve)) + active -= 1 + return { mtimeMs: nowMs - 60 * 60 * 1000 - 1 } + }) + + await cleanupExpiredRemoteClipboardFiles(nowMs) + + expect(rmMock).toHaveBeenCalledTimes(entries.length) + expect(peak).toBe(8) + }) +}) diff --git a/src/main/window/clipboard-remote-file-copy.ts b/src/main/window/clipboard-remote-file-copy.ts index 052fce31ef3..fdb63eb11a0 100644 --- a/src/main/window/clipboard-remote-file-copy.ts +++ b/src/main/window/clipboard-remote-file-copy.ts @@ -1,6 +1,5 @@ import { randomUUID } from 'node:crypto' -import type { Dirent } from 'node:fs' -import { mkdir, readdir, rm, stat } from 'node:fs/promises' +import { mkdir, opendir, rm, stat } from 'node:fs/promises' import { join } from 'node:path' import { app } from 'electron' @@ -17,6 +16,7 @@ type RemoteClipboardFileDeps = Omit const REMOTE_CLIPBOARD_FILE_TTL_MS = 60 * 60 * 1000 const REMOTE_CLIPBOARD_FILE_PREFIX = 'orca-clipboard-file-' +const REMOTE_CLIPBOARD_CLEANUP_CONCURRENCY = 8 const WINDOWS_RESERVED_LOCAL_BASENAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/i const LOCAL_FILENAME_REPLACEMENT_CHARS = new Set(['<', '>', ':', '"', '/', '\\', '|', '?', '*']) @@ -81,30 +81,44 @@ export async function writeRemoteFileToClipboard({ export async function cleanupExpiredRemoteClipboardFiles(nowMs = Date.now()): Promise { const tempRoot = app.getPath('temp') - let entries: Dirent[] + let directory: Awaited> try { - entries = await readdir(tempRoot, { withFileTypes: true }) + directory = await opendir(tempRoot) } catch { return } - await Promise.all( - entries.map(async (entry) => { + const pending = new Set>() + try { + for await (const entry of directory) { if (!entry.isDirectory() || !entry.name.startsWith(REMOTE_CLIPBOARD_FILE_PREFIX)) { - return + continue } const tempDir = join(tempRoot, entry.name) - try { - const tempStats = await stat(tempDir) - if (nowMs - tempStats.mtimeMs < REMOTE_CLIPBOARD_FILE_TTL_MS) { - return - } - await rm(tempDir, { recursive: true, force: true }) - } catch { - // Why: stale staged SSH files should not make startup cleanup noisy. + const cleanup = cleanupRemoteClipboardDirectory(tempDir, nowMs) + pending.add(cleanup) + void cleanup.finally(() => pending.delete(cleanup)) + if (pending.size >= REMOTE_CLIPBOARD_CLEANUP_CONCURRENCY) { + await Promise.race(pending) } - }) - ) + } + } catch { + // A partial best-effort startup sweep is sufficient. + } finally { + await directory.close().catch(() => undefined) + } + await Promise.all(pending) +} + +async function cleanupRemoteClipboardDirectory(tempDir: string, nowMs: number): Promise { + try { + const tempStats = await stat(tempDir) + if (nowMs - tempStats.mtimeMs >= REMOTE_CLIPBOARD_FILE_TTL_MS) { + await rm(tempDir, { recursive: true, force: true }) + } + } catch { + // Why: stale staged SSH files should not make startup cleanup noisy. + } } function sanitizeLocalClipboardFilename(remoteBasename: string): string { diff --git a/src/main/workspace-space-analysis-capacity.test.ts b/src/main/workspace-space-analysis-capacity.test.ts new file mode 100644 index 00000000000..2f6b9b1d8cd --- /dev/null +++ b/src/main/workspace-space-analysis-capacity.test.ts @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as NodeProcess from 'node:process' +import type { Repo } from '../shared/types' +import { WORKSPACE_SPACE_MAX_SCANNED_ENTRIES } from '../shared/workspace-space-scan-budget' +import type { Store } from './persistence' + +const { lstatMock, opendirMock, listRepoWorktreesMock } = vi.hoisted(() => ({ + lstatMock: vi.fn(), + opendirMock: vi.fn(), + listRepoWorktreesMock: vi.fn() +})) + +vi.mock('node:fs/promises', () => ({ + lstat: lstatMock, + opendir: opendirMock +})) + +vi.mock('node:process', async () => { + const actual = await vi.importActual('node:process') + return { ...actual, platform: 'win32' } +}) + +vi.mock('./repo-worktrees', () => ({ + createFolderWorktree: (repo: Repo) => ({ + path: repo.path, + head: '', + branch: '', + isBare: false, + isMainWorktree: true + }), + listRepoWorktrees: listRepoWorktreesMock +})) + +vi.mock('./providers/ssh-filesystem-dispatch', () => ({ + getSshFilesystemProvider: vi.fn() +})) + +vi.mock('./providers/ssh-git-dispatch', () => ({ + getSshGitProvider: vi.fn() +})) + +import { analyzeWorkspaceSpace } from './workspace-space-analysis' + +function createStore(repo: Repo): Store { + return { + getRepos: () => [repo], + getWorktreeMeta: () => undefined + } as unknown as Store +} + +describe('analyzeWorkspaceSpace portable scan capacity', () => { + beforeEach(() => { + lstatMock.mockReset() + opendirMock.mockReset() + listRepoWorktreesMock.mockReset() + }) + + it('surfaces an over-cap Windows directory as an unavailable row', async () => { + const repo: Repo = { + id: 'repo-1', + path: 'C:\\repo', + displayName: 'orca', + badgeColor: '#000', + addedAt: 0 + } + listRepoWorktreesMock.mockResolvedValue([ + { + path: repo.path, + head: 'a', + branch: 'refs/heads/main', + isBare: false, + isMainWorktree: true + } + ]) + lstatMock.mockResolvedValue({ + size: 1, + isDirectory: () => true, + isSymbolicLink: () => false + }) + opendirMock.mockResolvedValue({ + async *[Symbol.asyncIterator]() { + const entry = { name: 'repeated-entry' } + for (let index = 0; index <= WORKSPACE_SPACE_MAX_SCANNED_ENTRIES; index += 1) { + yield entry + } + } + }) + + const result = await analyzeWorkspaceSpace(createStore(repo)) + + expect(result).toMatchObject({ + scannedWorktreeCount: 0, + unavailableWorktreeCount: 1, + worktrees: [ + { + status: 'error', + error: expect.stringContaining('too large to scan safely'), + sizeBytes: 0 + } + ] + }) + expect(lstatMock).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/workspace-space-analysis.ts b/src/main/workspace-space-analysis.ts index cc792202fb8..c8307b8c08d 100644 --- a/src/main/workspace-space-analysis.ts +++ b/src/main/workspace-space-analysis.ts @@ -1,24 +1,33 @@ /* eslint-disable max-lines -- Why: this module keeps local and SSH directory-walk semantics paired so reclaimable-byte, symlink, and partial-failure behavior cannot drift. */ -import { lstat, readdir } from 'node:fs/promises' +import { lstat, opendir } from 'node:fs/promises' import { execFile } from 'node:child_process' import { posix, win32 } from 'node:path' import { platform } from 'node:process' import type { Dirent } from 'node:fs' import type { Store } from './persistence' import { isFolderRepo } from '../shared/repo-kind' -import type { GitWorktreeInfo, Repo, Worktree } from '../shared/types' +import type { DirEntry, GitWorktreeInfo, Repo, Worktree } from '../shared/types' import type { WorkspaceSpaceAnalysis, WorkspaceSpaceDirectoryScanResult, WorkspaceSpaceItem, - WorkspaceSpaceItemKind, WorkspaceSpaceRepoSummary, WorkspaceSpaceScanProgress, WorkspaceSpaceScanStatus, WorkspaceSpaceWorktree } from '../shared/workspace-space-types' import { compactWorkspaceSpaceItems } from '../shared/workspace-space-compaction' +import { mapWithConcurrency } from '../shared/map-with-concurrency' +import { + scanWorkspaceSpaceEntryTree, + type WorkspaceSpaceEntryScan +} from '../shared/workspace-space-entry-traversal' +import { + collectWorkspaceSpaceDirectoryEntries, + createWorkspaceSpaceScanBudget, + WorkspaceSpaceScanCapacityError +} from '../shared/workspace-space-scan-budget' import type { IFilesystemProvider } from './providers/types' import { getSshFilesystemProvider } from './providers/ssh-filesystem-dispatch' import { getSshGitProvider } from './providers/ssh-git-dispatch' @@ -31,16 +40,7 @@ const REMOTE_FS_CONCURRENCY = 10 const DU_TIMEOUT_MS = 120_000 const DU_MAX_BUFFER_BYTES = 16 * 1024 * 1024 -type AsyncLimiter = (task: () => Promise) => Promise - -type ScanStats = { - name: string - path: string - kind: WorkspaceSpaceItemKind - sizeBytes: number - skippedEntryCount: number - children?: ScanStats[] -} +type ScanStats = WorkspaceSpaceEntryScan type WorktreeListResult = | { ok: true; worktrees: GitWorktreeInfo[] } @@ -86,67 +86,6 @@ function isRelayMethodNotFoundError(error: unknown): boolean { return (error as { code?: unknown }).code === -32601 } -function createAsyncLimiter(maxConcurrent: number, signal?: AbortSignal): AsyncLimiter { - let active = 0 - const queue: { resolve: () => void; reject: (error: Error) => void }[] = [] - - const acquire = async (): Promise => { - throwIfAborted(signal) - if (active < maxConcurrent) { - active += 1 - return - } - await new Promise((resolve, reject) => { - let onAbort: (() => void) | null = null - const waiter = { - resolve: () => { - if (onAbort) { - signal?.removeEventListener('abort', onAbort) - } - resolve() - }, - reject - } - onAbort = () => { - const index = queue.indexOf(waiter) - if (index !== -1) { - queue.splice(index, 1) - } - reject(new WorkspaceSpaceScanCancelledError()) - } - queue.push(waiter) - if (signal) { - signal.addEventListener('abort', onAbort, { once: true }) - if (signal.aborted) { - onAbort() - } - } - }) - throwIfAborted(signal) - active += 1 - } - - return async (task: () => Promise): Promise => { - await acquire() - try { - return await task() - } finally { - active -= 1 - const next = queue.shift() - next?.resolve() - } - } -} - -async function mapLimit( - items: readonly T[], - maxConcurrent: number, - mapper: (item: T) => Promise -): Promise { - const limit = createAsyncLimiter(maxConcurrent) - return Promise.all(items.map((item) => limit(() => mapper(item)))) -} - function looksLikeWindowsPath(pathValue: string): boolean { return /^[A-Za-z]:[\\/]/.test(pathValue) || pathValue.startsWith('\\\\') } @@ -355,195 +294,73 @@ function createScannedWorktreeRow( async function scanLocalEntry( entryPath: string, name: string, - limit: AsyncLimiter, signal?: AbortSignal ): Promise { - throwIfAborted(signal) - const stats = await limit(() => lstat(entryPath)) - throwIfAborted(signal) - - if (stats.isSymbolicLink()) { - return { - name, - path: entryPath, - kind: 'symlink', - // Why: symlink targets may be shared outside the worktree. Counting the - // link itself reflects what deleting this worktree can actually reclaim. - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - if (!stats.isDirectory()) { - return { - name, - path: entryPath, - kind: 'file', - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - let entries: Dirent[] - try { - entries = await limit(() => readdir(entryPath, { withFileTypes: true })) - } catch { - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes: stats.size, - skippedEntryCount: 1 - } - } - - const childStats = await Promise.all( - entries.map(async (entry): Promise => { - try { - return await scanLocalEntry( - joinFilesystemPath(entryPath, entry.name), - entry.name, - limit, - signal - ) - } catch (error) { - if (error instanceof WorkspaceSpaceScanCancelledError) { - throw error - } - return null + return scanWorkspaceSpaceEntryTree({ + rootPath: entryPath, + rootName: name, + concurrency: LOCAL_FS_CONCURRENCY, + signal, + entryName: (entry) => entry.name, + joinPath: joinFilesystemPath, + classifyEntry: async (path) => { + const stats = await lstat(path) + throwIfAborted(signal) + if (stats.isSymbolicLink()) { + return { kind: 'symlink', sizeBytes: stats.size } } - }) - ) - - let sizeBytes = stats.size - let skippedEntryCount = 0 - for (const child of childStats) { - if (!child) { - skippedEntryCount += 1 - continue - } - sizeBytes += child.sizeBytes - skippedEntryCount += child.skippedEntryCount - } - - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes, - skippedEntryCount, - children: childStats.filter((child): child is ScanStats => child !== null) - } + return stats.isDirectory() + ? { kind: 'directory', sizeBytes: stats.size } + : { kind: 'file', sizeBytes: stats.size } + }, + readDirectory: (path) => opendir(path), + checkCancelled: () => throwIfAborted(signal), + createCancellationError: () => new WorkspaceSpaceScanCancelledError(), + isCancellationError: (error) => error instanceof WorkspaceSpaceScanCancelledError + }) } async function scanRemoteEntry( entryPath: string, name: string, provider: IFilesystemProvider, - limit: AsyncLimiter, - signal?: AbortSignal, - knownSymlink = false + signal?: AbortSignal ): Promise { - throwIfAborted(signal) - if (knownSymlink) { - return { - name, - path: entryPath, - kind: 'symlink', - sizeBytes: 0, - skippedEntryCount: 0 - } - } - - const stats = await limit(() => provider.stat(entryPath)) - throwIfAborted(signal) - if (stats.type === 'symlink') { - return { - name, - path: entryPath, - kind: 'symlink', - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - if (stats.type !== 'directory') { - return { - name, - path: entryPath, - kind: 'file', - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - let entries - try { - entries = await limit(() => provider.readDir(entryPath)) - throwIfAborted(signal) - } catch (error) { - if (error instanceof WorkspaceSpaceScanCancelledError) { - throw error - } - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes: stats.size, - skippedEntryCount: 1 - } - } - - const childStats = await Promise.all( - entries.map(async (entry): Promise => { - try { - return await scanRemoteEntry( - joinFilesystemPath(entryPath, entry.name), - entry.name, - provider, - limit, - signal, - entry.isSymlink - ) - } catch (error) { - if (error instanceof WorkspaceSpaceScanCancelledError) { - throw error - } - return null + return scanWorkspaceSpaceEntryTree({ + rootPath: entryPath, + rootName: name, + concurrency: REMOTE_FS_CONCURRENCY, + signal, + entryName: (entry) => entry.name, + joinPath: joinFilesystemPath, + classifyEntry: async (path, sourceEntry) => { + if (sourceEntry?.isSymlink) { + return { kind: 'symlink', sizeBytes: 0 } } - }) - ) - - let sizeBytes = stats.size - let skippedEntryCount = 0 - for (const child of childStats) { - if (!child) { - skippedEntryCount += 1 - continue - } - sizeBytes += child.sizeBytes - skippedEntryCount += child.skippedEntryCount - } - - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes, - skippedEntryCount, - children: childStats.filter((child): child is ScanStats => child !== null) - } + const stats = await provider.stat(path) + throwIfAborted(signal) + if (stats.type === 'symlink') { + return { kind: 'symlink', sizeBytes: stats.size } + } + return stats.type === 'directory' + ? { kind: 'directory', sizeBytes: stats.size } + : { kind: 'file', sizeBytes: stats.size } + }, + readDirectory: (path) => provider.readDir(path), + checkCancelled: () => throwIfAborted(signal), + createCancellationError: () => new WorkspaceSpaceScanCancelledError(), + isCancellationError: (error) => error instanceof WorkspaceSpaceScanCancelledError + }) } async function scanLocalTopLevelEntry( entryPath: string, name: string, duSizes: Map, - limit: AsyncLimiter, signal?: AbortSignal ): Promise { throwIfAborted(signal) - const stats = await limit(() => lstat(entryPath)) + const stats = await lstat(entryPath) throwIfAborted(signal) if (stats.isSymbolicLink()) { @@ -584,13 +401,7 @@ async function scanLocalWorktreeWithDu( throwIfAborted(signal) const rootStats = await lstat(worktree.path) if (!rootStats.isDirectory() || rootStats.isSymbolicLink()) { - const limit = createAsyncLimiter(LOCAL_FS_CONCURRENCY, signal) - const root = await scanLocalEntry( - worktree.path, - basenameFilesystemPath(worktree.path), - limit, - signal - ) + const root = await scanLocalEntry(worktree.path, basenameFilesystemPath(worktree.path), signal) const compact = compactWorkspaceSpaceItems((root.children ?? []).map(toWorkspaceSpaceItem)) return { ...createBaseWorktreeRow(repo, worktree, scannedAt), @@ -604,19 +415,27 @@ async function scanLocalWorktreeWithDu( } const [entries, duSizes] = await Promise.all([ - readdir(worktree.path, { withFileTypes: true }), + opendir(worktree.path).then((directory) => + collectWorkspaceSpaceDirectoryEntries( + directory, + worktree.path, + (entry) => entry.name, + createWorkspaceSpaceScanBudget(), + () => throwIfAborted(signal) + ) + ), readLocalDuDepthOne(worktree.path, signal) ]) throwIfAborted(signal) - const limit = createAsyncLimiter(LOCAL_FS_CONCURRENCY, signal) - const childStats = await Promise.all( - entries.map(async (entry): Promise => { + const childStats = await mapWithConcurrency( + entries, + LOCAL_FS_CONCURRENCY, + async (entry): Promise => { try { return await scanLocalTopLevelEntry( joinFilesystemPath(worktree.path, entry.name), entry.name, duSizes, - limit, signal ) } catch (error) { @@ -625,7 +444,7 @@ async function scanLocalWorktreeWithDu( } return null } - }) + } ) const children = childStats.filter((child): child is ScanStats => child !== null) const skippedEntryCount = childStats.length - children.length @@ -652,13 +471,7 @@ async function scanLocalWorktreeWithNode( signal?: AbortSignal ): Promise { try { - const limit = createAsyncLimiter(LOCAL_FS_CONCURRENCY, signal) - const root = await scanLocalEntry( - worktree.path, - basenameFilesystemPath(worktree.path), - limit, - signal - ) + const root = await scanLocalEntry(worktree.path, basenameFilesystemPath(worktree.path), signal) const compact = compactWorkspaceSpaceItems((root.children ?? []).map(toWorkspaceSpaceItem)) return { ...createBaseWorktreeRow(repo, worktree, scannedAt), @@ -701,6 +514,16 @@ async function scanLocalWorktree( if (error instanceof WorkspaceSpaceScanCancelledError) { throw error } + if (error instanceof WorkspaceSpaceScanCapacityError) { + const classified = classifyError(error) + return createUnavailableWorktreeRow( + repo, + worktree, + scannedAt, + classified.status, + classified.message + ) + } // Fall through to the portable scanner so unsupported du variants or // permission edge cases still produce partial rows instead of failing. } @@ -733,12 +556,10 @@ async function scanRemoteWorktree( } } - const limit = createAsyncLimiter(REMOTE_FS_CONCURRENCY, signal) const root = await scanRemoteEntry( worktree.path, basenameFilesystemPath(worktree.path), provider, - limit, signal ) const compact = compactWorkspaceSpaceItems((root.children ?? []).map(toWorkspaceSpaceItem)) @@ -862,7 +683,7 @@ async function scanRepo( options.onProgress ) const remoteProvider = repo.connectionId ? getSshFilesystemProvider(repo.connectionId) : undefined - const rows = await mapLimit(worktrees, WORKTREE_SCAN_CONCURRENCY, async (worktree) => { + const rows = await mapWithConcurrency(worktrees, WORKTREE_SCAN_CONCURRENCY, async (worktree) => { throwIfAborted(options.signal) reportProgress( progress, @@ -937,7 +758,7 @@ export async function analyzeWorkspaceSpace( currentWorktreeDisplayName: null } options.onProgress?.({ ...progress }) - const repoResults = await mapLimit(reposToScan, 2, (repo) => + const repoResults = await mapWithConcurrency(reposToScan, 2, (repo) => scanRepo(repo, scannedAt, store, progress, options) ) throwIfAborted(options.signal) diff --git a/src/main/worktree-orphan-gitdir-proof.ts b/src/main/worktree-orphan-gitdir-proof.ts index afa73041777..503e6870b88 100644 --- a/src/main/worktree-orphan-gitdir-proof.ts +++ b/src/main/worktree-orphan-gitdir-proof.ts @@ -1,5 +1,7 @@ import { type posix, win32 } from 'node:path' +export const MAX_WORKTREE_GIT_POINTER_BYTES = 64 * 1024 + type PathOps = typeof posix export type StatPath = (path: string) => Promise export type ReadPath = (path: string) => Promise @@ -41,13 +43,15 @@ function isGitFileStat(stat: unknown): boolean { function readFileResultToText(result: unknown): string | null { if (typeof result === 'string') { - return result + return Buffer.byteLength(result, 'utf8') <= MAX_WORKTREE_GIT_POINTER_BYTES ? result : null } if (Buffer.isBuffer(result)) { - return result.toString('utf8') + return result.byteLength <= MAX_WORKTREE_GIT_POINTER_BYTES ? result.toString('utf8') : null } if (result instanceof Uint8Array) { - return Buffer.from(result).toString('utf8') + return result.byteLength <= MAX_WORKTREE_GIT_POINTER_BYTES + ? Buffer.from(result).toString('utf8') + : null } if (!result || typeof result !== 'object') { return null @@ -56,7 +60,9 @@ function readFileResultToText(result: unknown): string | null { if (remoteRead.isBinary === true || typeof remoteRead.content !== 'string') { return null } - return remoteRead.content + return Buffer.byteLength(remoteRead.content, 'utf8') <= MAX_WORKTREE_GIT_POINTER_BYTES + ? remoteRead.content + : null } function resolveGitdirPath(gitdirPath: string, basePath: string, pathOps: PathOps): string { diff --git a/src/main/worktree-removal-safety.test.ts b/src/main/worktree-removal-safety.test.ts index 5a64b8c90e5..e1d7e97ae79 100644 --- a/src/main/worktree-removal-safety.test.ts +++ b/src/main/worktree-removal-safety.test.ts @@ -190,6 +190,22 @@ describe('canSafelyRemoveOrphanedWorktreeDirectory', () => { ).resolves.toBe(false) }) + it('rejects oversized git pointer metadata from filesystem providers', async () => { + await expect( + canSafelyRemoveOrphanedWorktreeDirectory( + '/workspaces/orphan', + '/repo', + makeStatPath(['/workspaces/orphan/.git'], ['/repo/.git']), + makeReadPath([ + [ + '/workspaces/orphan/.git', + `gitdir: /repo/.git/worktrees/orphan\n${'x'.repeat(64 * 1024)}` + ] + ]) + ) + ).resolves.toBe(false) + }) + it('rejects a copied .git file when the admin entry points at another candidate path', async () => { await expect( canSafelyRemoveOrphanedWorktreeDirectory( diff --git a/src/main/worktree-removal-safety.ts b/src/main/worktree-removal-safety.ts index a589bce9e10..01c3294e8ba 100644 --- a/src/main/worktree-removal-safety.ts +++ b/src/main/worktree-removal-safety.ts @@ -1,4 +1,4 @@ -import { lstat, readFile } from 'node:fs/promises' +import { lstat } from 'node:fs/promises' import { homedir } from 'node:os' import { posix, win32 } from 'node:path' import { isWindowsAbsolutePathLike } from '../shared/cross-platform-path' @@ -6,9 +6,11 @@ import type { GitWorktreeInfo, Repo, WorktreeMeta } from '../shared/types' import { areWorktreePathsEqual } from './ipc/worktree-logic' import { gitFileProvesOrphanedWorktreeDirectory, + MAX_WORKTREE_GIT_POINTER_BYTES, type ReadPath, type StatPath } from './worktree-orphan-gitdir-proof' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' type PathOps = typeof posix @@ -151,7 +153,8 @@ export async function canSafelyRemoveOrphanedWorktreeDirectory( worktreePath: string, repoPath: string, statPath: StatPath = lstat, - readPath: ReadPath = (path) => readFile(path, 'utf8') + readPath: ReadPath = async (path) => + (await readNodeFileWithinLimit(path, MAX_WORKTREE_GIT_POINTER_BYTES)).buffer.toString('utf8') ): Promise { if (isDangerousWorktreeRemovalPath(worktreePath, repoPath)) { return false diff --git a/src/main/worktree-root-preparation.test.ts b/src/main/worktree-root-preparation.test.ts index 16adf035efe..316df5e69d5 100644 --- a/src/main/worktree-root-preparation.test.ts +++ b/src/main/worktree-root-preparation.test.ts @@ -14,7 +14,11 @@ vi.mock('./ipc/filesystem-auth', () => ({ authorizeExternalPath: authorizeExternalPathMock })) -import { prepareLocalWorktreeRootForRepo } from './worktree-root-preparation' +import { + LOCAL_WORKTREE_ROOT_PREPARATION_CONCURRENCY, + prepareLocalWorktreeRootForRepo, + prepareLocalWorktreeRootsForRepos +} from './worktree-root-preparation' const repo: Repo = { id: 'repo-1', @@ -76,4 +80,28 @@ describe('prepareLocalWorktreeRootForRepo', () => { await expect(prepareLocalWorktreeRootForRepo(store as never, repo)).resolves.toBeUndefined() expect(authorizeExternalPathMock).not.toHaveBeenCalled() }) + + it('prepares every repo with a fixed-size worker pool', async () => { + const repos = Array.from({ length: 10_000 }, (_, index) => ({ + ...repo, + id: `repo-${index}`, + path: `/projects/repo-${index}` + })) + let inFlight = 0 + let peak = 0 + mkdirMock.mockImplementation(async () => { + inFlight += 1 + peak = Math.max(peak, inFlight) + await Promise.resolve() + inFlight -= 1 + }) + + await prepareLocalWorktreeRootsForRepos({ + ...store, + getRepos: () => repos + } as never) + + expect(mkdirMock).toHaveBeenCalledTimes(repos.length) + expect(peak).toBe(LOCAL_WORKTREE_ROOT_PREPARATION_CONCURRENCY) + }) }) diff --git a/src/main/worktree-root-preparation.ts b/src/main/worktree-root-preparation.ts index 3edcf311012..95afa74fdf1 100644 --- a/src/main/worktree-root-preparation.ts +++ b/src/main/worktree-root-preparation.ts @@ -1,6 +1,7 @@ import { mkdir } from 'node:fs/promises' import type { GlobalSettings, Repo } from '../shared/types' import { getRepoExecutionHostId, LOCAL_EXECUTION_HOST_ID } from '../shared/execution-host' +import { forEachWithConcurrency } from '../shared/map-with-concurrency' import { isFolderRepo } from '../shared/repo-kind' import { computeWorkspaceRoot, getWorktreePathSettings } from './ipc/worktree-logic' @@ -10,6 +11,8 @@ type WorktreeRootPreparationStore = { getRepos: () => Repo[] } +export const LOCAL_WORKTREE_ROOT_PREPARATION_CONCURRENCY = 8 + export async function prepareLocalWorktreeRootForRepo( store: Pick, repo: Repo @@ -31,5 +34,9 @@ export async function prepareLocalWorktreeRootForRepo( export async function prepareLocalWorktreeRootsForRepos( store: WorktreeRootPreparationStore ): Promise { - await Promise.all(store.getRepos().map((repo) => prepareLocalWorktreeRootForRepo(store, repo))) + await forEachWithConcurrency( + store.getRepos(), + LOCAL_WORKTREE_ROOT_PREPARATION_CONCURRENCY, + (repo) => prepareLocalWorktreeRootForRepo(store, repo) + ) } diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index eb4b6f1e542..ce2a7eb08b1 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -33,6 +33,7 @@ import type { import type { MobileRelayStatus } from '../shared/mobile-relay-status' import type { MobilePairingConnectionMode } from '../shared/mobile-pairing-connection-mode' import type { SshMutationExpectation } from '../shared/ssh-types' +import type { RasterImageDimensions } from '../shared/raster-image-dimensions' import type { CreateLocalOrcaProfileArgs, CreateLocalOrcaProfileResult, @@ -1373,6 +1374,9 @@ export type PreloadApi = { publishTerminalViewAttributes: (attributes: TerminalViewAttributes) => void hasChildProcesses: (id: string) => Promise getForegroundProcess: (id: string) => Promise + inspectProcess: ( + id: string + ) => Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> confirmForegroundProcess: (id: string) => Promise getCwd: (id: string) => Promise getSize: (id: string) => Promise<{ cols: number; rows: number } | null> @@ -2144,6 +2148,9 @@ export type PreloadApi = { /** Synchronous persisted-settings read for startup decisions that can't wait for async hydration. Blocking IPC — call sparingly. */ getSync: () => GlobalSettings | null set: (args: Partial) => Promise + setActiveRuntimeEnvironmentPreference: (args: { + environmentId: string | null + }) => Promise updatePRBotAuthorOverride: (args: { author: string; isBot: boolean }) => Promise listFonts: () => Promise previewGhosttyImport: () => Promise @@ -2474,6 +2481,7 @@ export type PreloadApi = { isBinary: boolean isImage?: boolean mimeType?: string + imageDimensions?: RasterImageDimensions fileIdentity?: string }> readLocalLogTail: (args: LocalLogTailReadArgs) => Promise diff --git a/src/preload/emulator-stream-frame-listeners.test.ts b/src/preload/emulator-stream-frame-listeners.test.ts new file mode 100644 index 00000000000..ac2f6c1b72f --- /dev/null +++ b/src/preload/emulator-stream-frame-listeners.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from 'vitest' +import { + subscribeFrameStreamFrames, + subscribeVideoStreamFrames +} from './emulator-stream-frame-listeners' + +type Listener = (event: Electron.IpcRendererEvent, data: never) => void + +function makeIpcRenderer() { + const listeners = new Map() + return { + ipcRenderer: { + on: vi.fn((channel: string, listener: Listener) => { + listeners.set(channel, listener) + }), + removeListener: vi.fn((channel: string, listener: Listener) => { + if (listeners.get(channel) === listener) { + listeners.delete(channel) + } + }), + send: vi.fn() + }, + listeners + } +} + +describe('emulator stream frame listeners', () => { + it('acknowledges an MJPEG frame after the renderer callback', () => { + const { ipcRenderer, listeners } = makeIpcRenderer() + const callback = vi.fn() + const unsubscribe = subscribeFrameStreamFrames(ipcRenderer as never, callback) + const bytes = new ArrayBuffer(2) + + listeners.get('emulator:frameStreamFrame')!( + {} as Electron.IpcRendererEvent, + { + streamId: 'frame-stream', + deliveryId: 7, + bytes + } as never + ) + + expect(callback).toHaveBeenCalledWith({ streamId: 'frame-stream', bytes }) + expect(ipcRenderer.send).toHaveBeenCalledWith('emulator:frameStreamFrameAck', { + streamId: 'frame-stream', + deliveryId: 7 + }) + unsubscribe() + expect(listeners.has('emulator:frameStreamFrame')).toBe(false) + }) + + it('acknowledges an H.264 frame even when the renderer callback throws', () => { + const { ipcRenderer, listeners } = makeIpcRenderer() + const callback = vi.fn(() => { + throw new Error('decode failed') + }) + subscribeVideoStreamFrames(ipcRenderer as never, callback) + + expect(() => + listeners.get('emulator:videoStreamFrame')!( + {} as Electron.IpcRendererEvent, + { + streamId: 'video-stream', + deliveryToken: 'delivery-token', + deliveryId: 9, + deviceId: 'emulator-5554', + config: false, + keyFrame: true, + bytes: new ArrayBuffer(1) + } as never + ) + ).toThrow('decode failed') + expect(ipcRenderer.send).toHaveBeenCalledWith('emulator:videoStreamFrameAck', { + streamId: 'video-stream', + deliveryToken: 'delivery-token', + deliveryId: 9 + }) + }) +}) diff --git a/src/preload/emulator-stream-frame-listeners.ts b/src/preload/emulator-stream-frame-listeners.ts new file mode 100644 index 00000000000..4d26d42ef20 --- /dev/null +++ b/src/preload/emulator-stream-frame-listeners.ts @@ -0,0 +1,60 @@ +type FrameStreamData = { + streamId: string + deliveryId: number + bytes: ArrayBuffer +} + +type VideoStreamData = { + streamId: string + deliveryToken: string + deliveryId: number + deviceId: string + config: boolean + keyFrame: boolean + bytes: ArrayBuffer +} + +type EmulatorStreamIpcRenderer = Pick + +export function subscribeFrameStreamFrames( + ipcRenderer: EmulatorStreamIpcRenderer, + callback: (data: Omit) => void +): () => void { + const listener = (_event: Electron.IpcRendererEvent, data: FrameStreamData): void => { + try { + callback({ streamId: data.streamId, bytes: data.bytes }) + } finally { + ipcRenderer.send('emulator:frameStreamFrameAck', { + streamId: data.streamId, + deliveryId: data.deliveryId + }) + } + } + ipcRenderer.on('emulator:frameStreamFrame', listener) + return () => ipcRenderer.removeListener('emulator:frameStreamFrame', listener) +} + +export function subscribeVideoStreamFrames( + ipcRenderer: EmulatorStreamIpcRenderer, + callback: (data: Omit) => void +): () => void { + const listener = (_event: Electron.IpcRendererEvent, data: VideoStreamData): void => { + try { + callback({ + streamId: data.streamId, + deviceId: data.deviceId, + config: data.config, + keyFrame: data.keyFrame, + bytes: data.bytes + }) + } finally { + ipcRenderer.send('emulator:videoStreamFrameAck', { + streamId: data.streamId, + deliveryToken: data.deliveryToken, + deliveryId: data.deliveryId + }) + } + } + ipcRenderer.on('emulator:videoStreamFrame', listener) + return () => ipcRenderer.removeListener('emulator:videoStreamFrame', listener) +} diff --git a/src/preload/index.ts b/src/preload/index.ts index dbf8ad7278e..1de8efbfed4 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -3,6 +3,10 @@ import { contextBridge, ipcRenderer, webFrame, webUtils } from 'electron' import { electronAPI } from '@electron-toolkit/preload' import { preloadE2EConfig } from './e2e-config' import { glApi } from './gitlab' +import { + subscribeFrameStreamFrames, + subscribeVideoStreamFrames +} from './emulator-stream-frame-listeners' import type { AppIdentity } from '../shared/app-identity' import type { DashboardSnapshot, DashboardRevealAgentArgs } from '../shared/dashboard-snapshot' import type { @@ -981,6 +985,10 @@ const api = { /** Return the PTY foreground process basename when available (e.g. "codex"). */ getForegroundProcess: (id: string): Promise => ipcRenderer.invoke('pty:getForegroundProcess', { id }), + inspectProcess: ( + id: string + ): Promise<{ foregroundProcess: string | null; hasChildProcesses: boolean }> => + ipcRenderer.invoke('pty:inspectProcess', { id }), confirmForegroundProcess: (id: string): Promise => ipcRenderer.invoke('pty:confirmForegroundProcess', { id }), @@ -1839,6 +1847,11 @@ const api = { set: (args: Record): Promise => ipcRenderer.invoke('settings:set', args), + setActiveRuntimeEnvironmentPreference: (args: { + environmentId: string | null + }): Promise => + ipcRenderer.invoke('settings:set-active-runtime-environment-preference', args), + updatePRBotAuthorOverride: (args: { author: string; isBot: boolean }): Promise => ipcRenderer.invoke('settings:update-pr-bot-author-override', args), @@ -2573,14 +2586,7 @@ const api = { ipcRenderer.invoke('emulator:frameStreamStop', args), onFrameStreamFrame: ( callback: (data: { streamId: string; bytes: ArrayBuffer }) => void - ): (() => void) => { - const listener = ( - _event: Electron.IpcRendererEvent, - data: { streamId: string; bytes: ArrayBuffer } - ) => callback(data) - ipcRenderer.on('emulator:frameStreamFrame', listener) - return () => ipcRenderer.removeListener('emulator:frameStreamFrame', listener) - }, + ): (() => void) => subscribeFrameStreamFrames(ipcRenderer, callback), onFrameStreamError: ( callback: (data: { streamId: string; message: string }) => void ): (() => void) => { @@ -2623,20 +2629,7 @@ const api = { keyFrame: boolean bytes: ArrayBuffer }) => void - ): (() => void) => { - const listener = ( - _event: Electron.IpcRendererEvent, - data: { - streamId: string - deviceId: string - config: boolean - keyFrame: boolean - bytes: ArrayBuffer - } - ) => callback(data) - ipcRenderer.on('emulator:videoStreamFrame', listener) - return () => ipcRenderer.removeListener('emulator:videoStreamFrame', listener) - }, + ): (() => void) => subscribeVideoStreamFrames(ipcRenderer, callback), onPaneFocus: (callback: (data: { worktreeId: string }) => void): (() => void) => { const listener = (_event: Electron.IpcRendererEvent, data: { worktreeId: string }) => callback(data) diff --git a/src/relay/agent-exec-handler.test.ts b/src/relay/agent-exec-handler.test.ts index c884c499dac..2f9b5ce5d96 100644 --- a/src/relay/agent-exec-handler.test.ts +++ b/src/relay/agent-exec-handler.test.ts @@ -2,6 +2,7 @@ import { execFile, spawn } from 'node:child_process' import { beforeEach, describe, expect, it, vi } from 'vitest' import type * as ChildProcess from 'node:child_process' import { createFakeChild, createHandlers, requestContext } from './agent-exec-handler-test-harness' +import { MAX_CONCURRENT_AGENT_EXECS } from './agent-exec-handler' import { TERMINAL_GIT_CREDENTIAL_GUARD_POLICY_ENV } from '../shared/terminal-git-credential-guard' vi.mock('child_process', async (importOriginal) => { @@ -16,7 +17,7 @@ vi.mock('child_process', async (importOriginal) => { const spawnMock = vi.mocked(spawn) const execFileMock = vi.mocked(execFile) -type AgentExecResult = { exitCode: number | null; timedOut: boolean } +type AgentExecResult = { stdout: string; exitCode: number | null; timedOut: boolean } describe('AgentExecHandler', () => { beforeEach(() => { @@ -64,6 +65,68 @@ describe('AgentExecHandler', () => { expect(child.stdin.end).toHaveBeenCalledWith('PROMPT') }) + it('preserves output delivered as 100,000 one-byte fragments', async () => { + const child = createFakeChild() + spawnMock.mockReturnValue(child as never) + const handlers = createHandlers() + const pending = handlers.get('agent.execNonInteractive')!( + { binary: 'agent', cwd: '/repo', timeoutMs: 5_000 }, + requestContext() + ) as Promise + + for (let index = 0; index < 100_000; index += 1) { + child.stdout.emit('data', Buffer.from(index % 2 === 0 ? 'a' : 'b')) + } + child.emit('close', 0) + + const result = await pending + expect(result.stdout).toHaveLength(100_000) + expect(result.stdout.slice(0, 4)).toBe('abab') + expect(result.stdout.slice(-4)).toBe('abab') + }) + + it('caps physical agent children until close confirms release', async () => { + const children = Array.from({ length: MAX_CONCURRENT_AGENT_EXECS + 1 }, (_, index) => { + const child = createFakeChild() + child.pid += index + return child + }) + let spawnIndex = 0 + spawnMock.mockImplementation(() => children[spawnIndex++] as never) + const handlers = createHandlers() + const active = children + .slice(0, MAX_CONCURRENT_AGENT_EXECS) + .map((_, index) => + handlers.get('agent.execNonInteractive')!( + { binary: 'agent', cwd: `/repo-${index}`, timeoutMs: 5_000 }, + requestContext() + ) + ) + + await expect( + handlers.get('agent.execNonInteractive')!( + { binary: 'agent', cwd: '/overflow', timeoutMs: 5_000 }, + requestContext() + ) + ).resolves.toMatchObject({ spawnError: 'Remote agent execution capacity reached' }) + expect(spawnMock).toHaveBeenCalledTimes(MAX_CONCURRENT_AGENT_EXECS) + + children[0]!.emit('close', 0) + await active[0] + const retry = handlers.get('agent.execNonInteractive')!( + { binary: 'agent', cwd: '/retry', timeoutMs: 5_000 }, + requestContext() + ) + expect(spawnMock).toHaveBeenCalledTimes(MAX_CONCURRENT_AGENT_EXECS + 1) + + for (const child of children.slice(1)) { + child.emit('close', 0) + } + await expect(Promise.all([...active.slice(1), retry])).resolves.toHaveLength( + MAX_CONCURRENT_AGENT_EXECS + ) + }) + it('merges caller-supplied provider environment into the spawned command environment', async () => { const child = createFakeChild() spawnMock.mockReturnValue(child as never) @@ -450,6 +513,8 @@ describe('AgentExecHandler', () => { expect(child.stdout.listenerCount('data')).toBe(0) expect(child.stderr.listenerCount('data')).toBe(0) expect(child.listenerCount('error')).toBe(0) + expect(child.listenerCount('close')).toBe(1) + child.emit('close', null) expect(child.listenerCount('close')).toBe(0) } finally { vi.useRealTimers() diff --git a/src/relay/agent-exec-handler.ts b/src/relay/agent-exec-handler.ts index a82791dcf45..9adb3d0b6c4 100644 --- a/src/relay/agent-exec-handler.ts +++ b/src/relay/agent-exec-handler.ts @@ -3,12 +3,14 @@ import { existsSync } from 'node:fs' import { delimiter, join } from 'node:path' import type { RelayDispatcher, RequestContext } from './dispatcher' import { applyTerminalGitCredentialPromptGuard } from '../shared/terminal-git-credential-guard' +import { GrowingByteBuffer } from '../shared/growing-byte-buffer' import { mergeGitConfigEnvProtocol } from '../shared/git-credential-prompt-env' import { terminateRelaySubprocessTree } from './subprocess-tree-termination' const DEFAULT_TIMEOUT_MS = 60_000 const MAX_TIMEOUT_MS = 5 * 60 * 1000 const MAX_OUTPUT_BYTES = 4 * 1024 * 1024 +export const MAX_CONCURRENT_AGENT_EXECS = 8 const WINDOWS_BATCH_UNSAFE_ARGUMENTS_ERROR = 'UNSAFE_WINDOWS_BATCH_ARGUMENTS' function getCmdExePath(): string { @@ -111,6 +113,7 @@ export class AgentExecHandler { // Why: commit-message and PR-field generation can run together for one cwd; // operation lanes let cancel target only the user-visible job that stopped. private inFlightByLane = new Map() + private activeChildren = new Set() private laneKey(cwd: string, operation: unknown): string { return laneKeyFor(cwd, operation) @@ -159,6 +162,16 @@ export class AgentExecHandler { platform: process.platform }) + if (this.activeChildren.size >= MAX_CONCURRENT_AGENT_EXECS) { + return { + stdout: '', + stderr: '', + exitCode: null, + timedOut: false, + spawnError: 'Remote agent execution capacity reached' + } + } + return new Promise((resolve) => { let child try { @@ -179,11 +192,13 @@ export class AgentExecHandler { }) return } + this.activeChildren.add(child) + child.once('close', () => { + this.activeChildren.delete(child) + }) - let stdout = '' - let stderr = '' - let stdoutBytes = 0 - let stderrBytes = 0 + const stdout = new GrowingByteBuffer() + const stderr = new GrowingByteBuffer() let timedOut = false let canceled = false let settled = false @@ -206,6 +221,8 @@ export class AgentExecHandler { if (laneKey && entry && this.inFlightByLane.get(laneKey) === entry) { this.inFlightByLane.delete(laneKey) } + stdout.clear() + stderr.clear() resolve(result) } const cancelCurrent = (): void => { @@ -231,36 +248,46 @@ export class AgentExecHandler { // also Windows wraps `.cmd` shims in cmd.exe, so the immediate child // is not the real node.exe process. terminateRelaySubprocessTree(child) - finish({ stdout, stderr, exitCode: null, timedOut, canceled }) + finish({ + stdout: stdout.toString(), + stderr: stderr.toString(), + exitCode: null, + timedOut, + canceled + }) }, timeoutMs) const onStdoutData = (chunk: Buffer): void => { - stdoutBytes += chunk.byteLength - if (stdoutBytes > MAX_OUTPUT_BYTES) { + if (stdout.byteLength + chunk.byteLength > MAX_OUTPUT_BYTES) { terminateRelaySubprocessTree(child) return } - stdout += chunk.toString('utf-8') + stdout.append(chunk) } const onStderrData = (chunk: Buffer): void => { - stderrBytes += chunk.byteLength - if (stderrBytes > MAX_OUTPUT_BYTES) { + if (stderr.byteLength + chunk.byteLength > MAX_OUTPUT_BYTES) { terminateRelaySubprocessTree(child) return } - stderr += chunk.toString('utf-8') + stderr.append(chunk) } const onError = (error: Error): void => { finish({ - stdout, - stderr, + stdout: stdout.toString(), + stderr: stderr.toString(), exitCode: null, timedOut, spawnError: error.message }) } const onClose = (code: number | null): void => { - finish({ stdout, stderr, exitCode: code, timedOut, canceled }) + finish({ + stdout: stdout.toString(), + stderr: stderr.toString(), + exitCode: code, + timedOut, + canceled + }) } child.stdout?.on('data', onStdoutData) child.stderr?.on('data', onStderrData) diff --git a/src/relay/agent-hook-server.test.ts b/src/relay/agent-hook-server.test.ts index 3d73506c7bf..68f6c7c1e4c 100644 --- a/src/relay/agent-hook-server.test.ts +++ b/src/relay/agent-hook-server.test.ts @@ -2,7 +2,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' import { join } from 'node:path' -import { endpointDirForRelaySocket, RelayAgentHookServer } from './agent-hook-server' +import { + endpointDirForRelaySocket, + MAX_RELAY_AGENT_HOOK_STATUS_CACHE_PANES, + RelayAgentHookServer +} from './agent-hook-server' import type { AgentHookRelayEnvelope } from '../shared/agent-hook-relay' import { makePaneKey } from '../shared/stable-pane-id' import * as agentHookListener from '../shared/agent-hook-listener' @@ -391,10 +395,8 @@ describe('RelayAgentHookServer', () => { } }) - it('caps the replay cache at 256 panes, evicting the least-recently-updated', async () => { - // Mirrors the server's private MAX_CACHED_PANES. The WSL relay never gets a - // per-pane teardown signal, so the cache is recency-capped instead. - const CAP = 256 + it('caps the replay cache, evicting the least-recently-updated', async () => { + const CAP = MAX_RELAY_AGENT_HOOK_STATUS_CACHE_PANES const forward = vi.fn<(envelope: AgentHookRelayEnvelope) => void>() const server = new RelayAgentHookServer({ endpointDir: dir, forward }) await server.start() diff --git a/src/relay/agent-hook-server.ts b/src/relay/agent-hook-server.ts index 40775b5bd6a..69bacaf3544 100644 --- a/src/relay/agent-hook-server.ts +++ b/src/relay/agent-hook-server.ts @@ -29,6 +29,7 @@ import { type AgentHookRelayEnvelope, type AgentHookSource } from '../shared/agent-hook-relay' +import { upsertBoundedAgentHookStatus } from '../shared/agent-hook-status-cache' export type RelayHookForward = (envelope: AgentHookRelayEnvelope) => void @@ -40,9 +41,8 @@ const ASSISTANT_MESSAGE_RETRY_MS = 50 // Why: cap env/version at 64 chars so a misbehaving agent CLI can't grow the meta cache unboundedly; canonical values are short. const MAX_HOOK_META_LEN = 64 - -// Why: WSL relay has no per-pane teardown (PTYs live on the Windows host), so the replay cache would grow forever without a recency cap. -const MAX_CACHED_PANES = 256 +// Why: preserve the relay's existing replay-memory ceiling while main uses the renderer-aligned 500-pane cap. +export const MAX_RELAY_AGENT_HOOK_STATUS_CACHE_PANES = 256 function defaultEndpointDir(): string { return join(homedir(), RELAY_HOOKS_DIR_NAME, RELAY_HOOKS_SUBDIR) @@ -326,17 +326,14 @@ export class RelayAgentHookServer { if (event.payload.state !== 'done' || event.payload.lastAssistantMessage) { this.clearAssistantMessageRetry(event.paneKey) } - // Why: delete-then-set makes Map insertion order = recency, so the cap below evicts the longest-idle pane. - this.state.lastStatusByPaneKey.delete(event.paneKey) - this.state.lastStatusByPaneKey.set(event.paneKey, event) + const evicted = upsertBoundedAgentHookStatus(this.state, event, { + maxPanes: MAX_RELAY_AGENT_HOOK_STATUS_CACHE_PANES + }) this.lastEnvelopeMetaByPaneKey.delete(event.paneKey) this.lastEnvelopeMetaByPaneKey.set(event.paneKey, { source, env, version }) - while (this.state.lastStatusByPaneKey.size > MAX_CACHED_PANES) { - const oldest = this.state.lastStatusByPaneKey.keys().next().value - if (oldest === undefined) { - break - } - this.clearPaneState(oldest) + for (const { paneKey } of evicted) { + this.clearAssistantMessageRetry(paneKey) + this.lastEnvelopeMetaByPaneKey.delete(paneKey) } this.forwardEvent(event, source, env, version) } diff --git a/src/relay/client-request-aborts.test.ts b/src/relay/client-request-aborts.test.ts new file mode 100644 index 00000000000..3b4cdac75fb --- /dev/null +++ b/src/relay/client-request-aborts.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' +import { + ClientRequestAborts, + MAX_ACTIVE_RELAY_REQUEST_BYTES, + MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT, + MAX_ACTIVE_RELAY_REQUESTS, + MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT +} from './client-request-aborts' + +describe('ClientRequestAborts', () => { + it('accepts the per-client request boundary, rejects overflow, and recovers', () => { + const requests = new ClientRequestAborts() + const registrations = Array.from( + { length: MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT }, + (_, requestId) => requests.create(1, requestId) + ) + + expect(() => requests.create(1, MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT)).toThrow( + `Relay client active request limit of ${MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT} reached` + ) + + requests.delete(registrations[0].key) + expect(() => requests.create(1, MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT)).not.toThrow() + }) + + it('accepts the aggregate request boundary without evicting active clients', () => { + const requests = new ClientRequestAborts() + for (let index = 0; index < MAX_ACTIVE_RELAY_REQUESTS; index += 1) { + const clientId = Math.floor(index / MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT) + 1 + requests.create(clientId, index) + } + + expect(() => requests.create(99, MAX_ACTIVE_RELAY_REQUESTS)).toThrow( + `Relay active request limit of ${MAX_ACTIVE_RELAY_REQUESTS} reached` + ) + expect(requests.get(1, 0)?.signal.aborted).toBe(false) + + requests.delete({ clientId: 1, requestId: 0 }) + expect(() => requests.create(99, MAX_ACTIVE_RELAY_REQUESTS)).not.toThrow() + }) + + it('bounds retained payload bytes per client and releases the budget on delete', () => { + const requests = new ClientRequestAborts() + const halfBudget = MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT / 2 + const first = requests.create(1, 1, halfBudget) + requests.create(1, 2, halfBudget) + + expect(() => requests.create(1, 3, 1)).toThrow( + `Relay client active request payload limit of ${MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT} bytes exceeded` + ) + + requests.delete(first.key) + expect(() => requests.create(1, 3, 1)).not.toThrow() + }) + + it('bounds aggregate retained payload bytes and recovers after client abort', () => { + const requests = new ClientRequestAborts() + requests.create(1, 1, MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT) + requests.create( + 2, + 2, + MAX_ACTIVE_RELAY_REQUEST_BYTES - MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT + ) + + expect(() => requests.create(3, 3, 1)).toThrow( + `Relay active request payload limit of ${MAX_ACTIVE_RELAY_REQUEST_BYTES} bytes exceeded` + ) + + requests.abortClient(1) + expect(() => requests.create(3, 3, 1)).not.toThrow() + }) + + it('rejects duplicate ids and releases only the requested owners', () => { + const requests = new ClientRequestAborts() + const first = requests.create(1, 7) + const second = requests.create(2, 7) + + expect(() => requests.create(1, 7)).toThrow('Duplicate active relay request id 7') + expect(requests.get(1, 7)).toBe(first.controller) + + requests.abortClient(1) + expect(first.controller.signal.aborted).toBe(true) + expect(second.controller.signal.aborted).toBe(false) + expect(requests.get(1, 7)).toBeUndefined() + expect(requests.get(2, 7)).toBe(second.controller) + + requests.abortAll() + expect(second.controller.signal.aborted).toBe(true) + expect(requests.get(2, 7)).toBeUndefined() + }) +}) diff --git a/src/relay/client-request-aborts.ts b/src/relay/client-request-aborts.ts index 12d44363d0d..b2b984ebf08 100644 --- a/src/relay/client-request-aborts.ts +++ b/src/relay/client-request-aborts.ts @@ -1,40 +1,118 @@ +export const MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT = 256 +export const MAX_ACTIVE_RELAY_REQUESTS = 1024 +// Why: handlers retain parsed params until settlement, so cardinality alone +// cannot bound a burst of near-maximum relay frames. +export const MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT = 32 * 1024 * 1024 +export const MAX_ACTIVE_RELAY_REQUEST_BYTES = 64 * 1024 * 1024 + +type ClientRequestAbortEntry = { + controller: AbortController + retainedBytes: number +} + +export type ClientRequestAbortKey = { + clientId: number + requestId: number +} + +export class RelayRequestAdmissionError extends Error { + constructor(message: string) { + super(message) + this.name = 'RelayRequestAdmissionError' + } +} + export class ClientRequestAborts { - private readonly controllers = new Map() + private readonly entriesByClient = new Map>() + private activeRequestCount = 0 + private activeRequestBytes = 0 + + create( + clientId: number, + requestId: number, + retainedBytes = 0 + ): { key: ClientRequestAbortKey; controller: AbortController } { + const clientEntries = this.entriesByClient.get(clientId) + if (clientEntries?.has(requestId)) { + throw new RelayRequestAdmissionError(`Duplicate active relay request id ${requestId}`) + } + if ((clientEntries?.size ?? 0) >= MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT) { + throw new RelayRequestAdmissionError( + `Relay client active request limit of ${MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT} reached` + ) + } + if (this.activeRequestCount >= MAX_ACTIVE_RELAY_REQUESTS) { + throw new RelayRequestAdmissionError( + `Relay active request limit of ${MAX_ACTIVE_RELAY_REQUESTS} reached` + ) + } + const normalizedBytes = Math.max(0, retainedBytes) + const clientBytes = sumRetainedBytes(clientEntries?.values()) + if (clientBytes + normalizedBytes > MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT) { + throw new RelayRequestAdmissionError( + `Relay client active request payload limit of ${MAX_ACTIVE_RELAY_REQUEST_BYTES_PER_CLIENT} bytes exceeded` + ) + } + if (this.activeRequestBytes + normalizedBytes > MAX_ACTIVE_RELAY_REQUEST_BYTES) { + throw new RelayRequestAdmissionError( + `Relay active request payload limit of ${MAX_ACTIVE_RELAY_REQUEST_BYTES} bytes exceeded` + ) + } - create(clientId: number, requestId: number): { key: string; controller: AbortController } { - const key = this.key(clientId, requestId) const controller = new AbortController() - this.controllers.set(key, controller) - return { key, controller } + const entries = clientEntries ?? new Map() + entries.set(requestId, { controller, retainedBytes: normalizedBytes }) + this.entriesByClient.set(clientId, entries) + this.activeRequestCount += 1 + this.activeRequestBytes += normalizedBytes + return { key: { clientId, requestId }, controller } } get(clientId: number, requestId: number): AbortController | undefined { - return this.controllers.get(this.key(clientId, requestId)) + return this.entriesByClient.get(clientId)?.get(requestId)?.controller } - delete(key: string): void { - this.controllers.delete(key) + delete(key: ClientRequestAbortKey): void { + const clientEntries = this.entriesByClient.get(key.clientId) + const entry = clientEntries?.get(key.requestId) + if (!clientEntries || !entry) { + return + } + clientEntries.delete(key.requestId) + this.activeRequestCount -= 1 + this.activeRequestBytes -= entry.retainedBytes + if (clientEntries.size === 0) { + this.entriesByClient.delete(key.clientId) + } } abortClient(clientId: number): void { - const prefix = `${clientId}:` - for (const [key, controller] of this.controllers) { - if (!key.startsWith(prefix)) { - continue - } - controller.abort() - this.controllers.delete(key) + const clientEntries = this.entriesByClient.get(clientId) + if (!clientEntries) { + return + } + for (const [requestId, entry] of clientEntries) { + entry.controller.abort() + this.delete({ clientId, requestId }) } } abortAll(): void { - for (const [, controller] of this.controllers) { - controller.abort() + for (const clientEntries of this.entriesByClient.values()) { + for (const entry of clientEntries.values()) { + entry.controller.abort() + } } - this.controllers.clear() - } - - private key(clientId: number, requestId: number): string { - return `${clientId}:${requestId}` + this.entriesByClient.clear() + this.activeRequestCount = 0 + this.activeRequestBytes = 0 } } + +function sumRetainedBytes(entries?: Iterable): number { + let total = 0 + for (const entry of entries ?? []) { + total += entry.retainedBytes + } + return total +} diff --git a/src/relay/dispatcher.test.ts b/src/relay/dispatcher.test.ts index 87f5cb69053..66b1f81468b 100644 --- a/src/relay/dispatcher.test.ts +++ b/src/relay/dispatcher.test.ts @@ -1,11 +1,17 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import { RelayDispatcher } from './dispatcher' +import { + MAX_PENDING_RELAY_REQUESTS, + MAX_RELAY_DISPATCHER_CLIENTS, + RelayDispatcher +} from './dispatcher' +import { MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT } from './client-request-aborts' import { encodeJsonRpcFrame, encodeKeepAliveFrame, MessageType, type JsonRpcRequest, - type JsonRpcNotification + type JsonRpcNotification, + type JsonRpcResponse } from './protocol' function decodeFirstFrame(buf: Buffer): { type: number; id: number; ack: number; payload: Buffer } { @@ -118,6 +124,48 @@ describe('RelayDispatcher', () => { expect(resp.id).toBe(5) }) + it('rejects incoming request overflow without evicting active work', async () => { + const resolutions: ((value: unknown) => void)[] = [] + const handler = vi.fn( + (_params, context) => + new Promise((resolve) => { + resolutions.push(resolve) + context.signal?.addEventListener('abort', () => resolve(null), { once: true }) + }) + ) + dispatcher.onRequest('slow.method', handler) + + for (let id = 1; id <= MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT + 1; id += 1) { + dispatcher.feed(encodeJsonRpcFrame({ jsonrpc: '2.0', id, method: 'slow.method' }, id, 0)) + } + await vi.advanceTimersByTimeAsync(0) + + expect(handler).toHaveBeenCalledTimes(MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT) + const overflowId = MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT + 1 + const overflow = written + .map(decodeFirstFrame) + .filter((frame) => frame.type === MessageType.Regular) + .map((frame) => JSON.parse(frame.payload.toString('utf-8')) as JsonRpcResponse) + .find((response) => response.id === overflowId) + expect(overflow?.error?.message).toBe( + `Relay client active request limit of ${MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT} reached` + ) + expect(resolutions).toHaveLength(MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT) + + resolutions[0]('done') + await vi.advanceTimersByTimeAsync(0) + dispatcher.feed( + encodeJsonRpcFrame( + { jsonrpc: '2.0', id: overflowId + 1, method: 'slow.method' }, + overflowId + 1, + 0 + ) + ) + await vi.advanceTimersByTimeAsync(0) + + expect(handler).toHaveBeenCalledTimes(MAX_ACTIVE_RELAY_REQUESTS_PER_CLIENT + 1) + }) + it('sends method-not-found for unknown methods', async () => { const req: JsonRpcRequest = { jsonrpc: '2.0', @@ -201,6 +249,21 @@ describe('RelayDispatcher', () => { expect(socketWritten).toHaveLength(1) }) + it('accepts the exact client cap, rejects aggregate overflow, and recovers after detach', () => { + const attachedIds = Array.from({ length: MAX_RELAY_DISPATCHER_CLIENTS - 1 }, () => + dispatcher.attachClient(() => undefined) + ) + + expect(dispatcher.connectedClientIds()).toHaveLength(MAX_RELAY_DISPATCHER_CLIENTS) + expect(() => dispatcher.attachClient(() => undefined)).toThrow( + `Relay client limit of ${MAX_RELAY_DISPATCHER_CLIENTS} reached` + ) + + dispatcher.detachClient(attachedIds[0]) + expect(() => dispatcher.attachClient(() => undefined)).not.toThrow() + expect(dispatcher.connectedClientIds()).toHaveLength(MAX_RELAY_DISPATCHER_CLIENTS) + }) + it('targets terminal ownership notifications to one attached client', () => { const firstWritten: Buffer[] = [] const secondWritten: Buffer[] = [] @@ -218,6 +281,76 @@ describe('RelayDispatcher', () => { expect(secondWritten).toHaveLength(0) }) + it('reports targeted sink saturation and waits for its drain signal', async () => { + const ownerWritten: Buffer[] = [] + let signalDrain: (() => void) | undefined + const ownerId = dispatcher.attachClient( + (data) => { + ownerWritten.push(Buffer.from(data)) + return false + }, + { + waitWriteDrain: (callback) => { + signalDrain = callback + } + } + ) + const otherWritten: Buffer[] = [] + dispatcher.attachClient((data) => { + otherWritten.push(Buffer.from(data)) + }) + + const result = dispatcher.notifyClientWithBackpressure(ownerId, 'pty.data', { + id: 'pty-1', + data: 'output' + }) + + expect(result.delivered).toBe(true) + expect(result.saturated).toBe(true) + expect(ownerWritten).toHaveLength(1) + expect(written).toHaveLength(0) + expect(otherWritten).toHaveLength(0) + let drained = false + void result.drained.then(() => { + drained = true + }) + await Promise.resolve() + expect(drained).toBe(false) + + signalDrain?.() + await expect(result.drained).resolves.toBeUndefined() + expect(drained).toBe(true) + }) + + it('reports a detached targeted client without writing', async () => { + const ownerId = dispatcher.attachClient(() => true) + dispatcher.detachClient(ownerId) + + const result = dispatcher.notifyClientWithBackpressure(ownerId, 'pty.data', { + id: 'pty-1', + data: 'output' + }) + + expect(result).toMatchObject({ delivered: false, saturated: false }) + await expect(result.drained).resolves.toBeUndefined() + expect(written).toHaveLength(0) + }) + + it('allows a targeted drain waiter to be canceled during owner cleanup', async () => { + const ownerId = dispatcher.attachClient(() => false, { + waitWriteDrain: () => {} + }) + const result = dispatcher.notifyClientWithBackpressure(ownerId, 'pty.data', { + id: 'pty-1', + data: 'output' + }) + + expect(result.saturated).toBe(true) + result.cancelDrain?.() + + await expect(result.drained).resolves.toBeUndefined() + }) + it('forwards relay-originated requests to an owning socket client instead of the caller', async () => { dispatcher.invalidateClient() const ownerWritten: Buffer[] = [] @@ -273,6 +406,48 @@ describe('RelayDispatcher', () => { await expect(pending).resolves.toEqual({ exitCode: 0 }) }) + it('bounds relay-originated pending requests and recovers after a response', async () => { + const pending = Array.from({ length: MAX_PENDING_RELAY_REQUESTS }, (_, index) => { + const request = dispatcher.requestPrimary('orca.cli', { index }) + void request.catch(() => undefined) + return request + }) + + expect(written).toHaveLength(MAX_PENDING_RELAY_REQUESTS) + await expect(dispatcher.requestPrimary('orca.cli', { overflow: true })).rejects.toThrow( + `Relay pending request limit of ${MAX_PENDING_RELAY_REQUESTS} reached` + ) + expect(written).toHaveLength(MAX_PENDING_RELAY_REQUESTS) + + const firstRequest = JSON.parse( + decodeFirstFrame(written[0]).payload.toString('utf-8') + ) as JsonRpcRequest + dispatcher.feed( + encodeJsonRpcFrame({ jsonrpc: '2.0', id: firstRequest.id, result: 'done' }, 1, 0) + ) + await expect(pending[0]).resolves.toBe('done') + + const recovered = dispatcher.requestPrimary('orca.cli', { recovered: true }) + expect(written).toHaveLength(MAX_PENDING_RELAY_REQUESTS + 1) + const recoveredRequest = JSON.parse( + decodeFirstFrame(written.at(-1)!).payload.toString('utf-8') + ) as JsonRpcRequest + dispatcher.feed( + encodeJsonRpcFrame({ jsonrpc: '2.0', id: recoveredRequest.id, result: 'recovered' }, 2, 0) + ) + await expect(recovered).resolves.toBe('recovered') + }) + + it('rejects relay-originated requests when their owning client detaches', async () => { + const ownerId = dispatcher.attachClient(() => undefined) + dispatcher.invalidateClient() + const pending = dispatcher.requestAnyClient('orca.cli') + + dispatcher.detachClient(ownerId) + + await expect(pending).rejects.toThrow('Relay client disconnected') + }) + it('isolates failed socket-client writes from other clients', () => { const goodSocketWritten: Buffer[] = [] const failingClientId = dispatcher.attachClient(() => { diff --git a/src/relay/dispatcher.ts b/src/relay/dispatcher.ts index 1f1445da2f1..4b1ac665121 100644 --- a/src/relay/dispatcher.ts +++ b/src/relay/dispatcher.ts @@ -11,7 +11,7 @@ import { type JsonRpcNotification, type JsonRpcResponse } from './protocol' -import { ClientRequestAborts } from './client-request-aborts' +import { ClientRequestAborts, RelayRequestAdmissionError } from './client-request-aborts' export type RequestContext = { clientId: number @@ -32,6 +32,15 @@ export type RelayClientWrite = (data: Buffer) => boolean | void export type RelayClientSinkOptions = { /** One-shot: invoke `cb` when the sink can accept more data (drain) or is permanently dead, so waiters never hang. */ waitWriteDrain?: (cb: () => void) => void + /** Close only this transport when bounded per-client delivery can no longer retain its backlog. */ + disconnect?: () => void +} + +export type RelayNotificationWriteResult = { + delivered: boolean + saturated: boolean + drained: Promise + cancelDrain?: () => void } type RelayClient = { @@ -39,6 +48,7 @@ type RelayClient = { decoder: FrameDecoder write: RelayClientWrite waitWriteDrain?: (cb: () => void) => void + disconnect?: () => void /** Resolvers for bulk sends stalled on sink saturation; flushed so no pump hangs. */ drainWaiters: Set<() => void> /** Serializes bulk-lane sends so only one bulk frame is admitted past the sink high-water mark at a time. */ @@ -50,12 +60,16 @@ type RelayClient = { } type PendingRelayRequest = { + clientId: number resolve: (result: unknown) => void reject: (error: Error) => void timer: ReturnType } const RELAY_TO_CLIENT_REQUEST_TIMEOUT_MS = 30_000 +export const MAX_RELAY_DISPATCHER_CLIENTS = 16 +export const MAX_RELAY_SOCKET_CONNECTIONS = MAX_RELAY_DISPATCHER_CLIENTS - 1 +export const MAX_PENDING_RELAY_REQUESTS = 256 export class RelayDispatcher { private readonly primaryClient: RelayClient @@ -80,8 +94,10 @@ export class RelayDispatcher { // Why: the new client's multiplexer restarts at seq=1, so reset seq/decoder state or acks stall and fire a false connection-dead signal. setWrite(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions): void { this.requestAborts.abortClient(this.primaryClient.id) + this.rejectPendingRequestsForClient(this.primaryClient.id, 'Relay client reconnected') this.primaryClient.write = write this.primaryClient.waitWriteDrain = sinkOptions?.waitWriteDrain + this.primaryClient.disconnect = sinkOptions?.disconnect this.primaryClient.closed = false // Why: the old sink is gone; wake stalled bulk senders to re-evaluate against the new one. this.flushDrainWaiters(this.primaryClient) @@ -91,6 +107,7 @@ export class RelayDispatcher { // Why: mark in-flight requests stale on disconnect so a late pty.spawn/fs.watch can't create unowned remote state. invalidateClient(): void { this.requestAborts.abortClient(this.primaryClient.id) + this.rejectPendingRequestsForClient(this.primaryClient.id, 'Relay client disconnected') this.primaryClient.generation++ this.primaryClient.closed = true this.flushDrainWaiters(this.primaryClient) @@ -99,6 +116,12 @@ export class RelayDispatcher { // Why: seq numbers and request ids are per SSH channel, so each attached client needs independent protocol state. attachClient(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions): number { + if (this.disposed) { + throw new Error('Relay dispatcher is disposed') + } + if (this.clients.size >= MAX_RELAY_DISPATCHER_CLIENTS) { + throw new Error(`Relay client limit of ${MAX_RELAY_DISPATCHER_CLIENTS} reached`) + } const client = this.createClient(write, sinkOptions) this.clients.set(client.id, client) return client.id @@ -110,6 +133,7 @@ export class RelayDispatcher { return } this.requestAborts.abortClient(clientId) + this.rejectPendingRequestsForClient(clientId, 'Relay client disconnected') client.generation++ client.closed = true this.flushDrainWaiters(client) @@ -117,6 +141,33 @@ export class RelayDispatcher { this.notifyClientDetached(clientId) } + connectedClientIds(): number[] { + if (this.disposed) { + return [] + } + return Array.from(this.clients.values(), (client) => client) + .filter((client) => !client.closed) + .map((client) => client.id) + } + + evictClient(clientId: number): void { + const client = this.clients.get(clientId) + if (!client || client.closed) { + return + } + const disconnect = client.disconnect + if (client === this.primaryClient) { + this.invalidateClient() + } else { + this.detachClient(clientId) + } + try { + disconnect?.() + } catch { + // Why: disconnect failures cannot restore an already-detached dispatcher. + } + } + feedClient(clientId: number, data: Buffer): void { const client = this.clients.get(clientId) if (!client) { @@ -184,6 +235,35 @@ export class RelayDispatcher { }) } + notifyClientWithBackpressure( + clientId: number, + method: string, + params?: Record + ): RelayNotificationWriteResult { + const client = this.clients.get(clientId) + if (this.disposed || !client || client.closed) { + return { delivered: false, saturated: false, drained: Promise.resolve() } + } + const accepted = this.sendFrame(client, { + jsonrpc: '2.0', + method, + ...(params !== undefined ? { params } : {}) + }) + if (client.closed || !this.clients.has(clientId)) { + return { delivered: false, saturated: false, drained: Promise.resolve() } + } + if (accepted !== false) { + return { delivered: true, saturated: false, drained: Promise.resolve() } + } + const drain = this.waitForClientDrainCancelable(client) + return { + delivered: true, + saturated: true, + drained: drain.drained, + cancelDrain: drain.cancel + } + } + /** * Bulk-lane notification: sends are serialized per client and the promise * resolves only after the sink accepted the frame (backpressure), so bulk @@ -233,10 +313,18 @@ export class RelayDispatcher { } private waitForClientDrain(client: RelayClient): Promise { + return this.waitForClientDrainCancelable(client).drained + } + + private waitForClientDrainCancelable(client: RelayClient): { + drained: Promise + cancel: () => void + } { if (this.disposed || client.closed || !client.waitWriteDrain) { - return Promise.resolve() + return { drained: Promise.resolve(), cancel: () => {} } } - return new Promise((resolve) => { + let cancel = (): void => {} + const drained = new Promise((resolve) => { let settled = false const finish = (): void => { if (settled) { @@ -246,6 +334,7 @@ export class RelayDispatcher { client.drainWaiters.delete(finish) resolve() } + cancel = finish client.drainWaiters.add(finish) try { client.waitWriteDrain!(finish) @@ -253,6 +342,7 @@ export class RelayDispatcher { finish() } }) + return { drained, cancel } } private flushDrainWaiters(client: RelayClient): void { @@ -295,6 +385,11 @@ export class RelayDispatcher { if (this.disposed || !client || client.closed) { return Promise.reject(new Error('Relay client is not connected')) } + if (this.pendingRelayRequests.size >= MAX_PENDING_RELAY_REQUESTS) { + return Promise.reject( + new Error(`Relay pending request limit of ${MAX_PENDING_RELAY_REQUESTS} reached`) + ) + } const id = this.nextRequestId++ const msg: JsonRpcRequest = { jsonrpc: '2.0', @@ -308,11 +403,27 @@ export class RelayDispatcher { this.pendingRelayRequests.delete(id) reject(new Error(`Request "${method}" timed out after ${timeoutMs}ms`)) }, timeoutMs) - this.pendingRelayRequests.set(id, { resolve, reject, timer }) + this.pendingRelayRequests.set(id, { clientId, resolve, reject, timer }) this.sendFrame(client, msg) + if (client.closed) { + clearTimeout(timer) + this.pendingRelayRequests.delete(id) + reject(new Error('Relay client disconnected')) + } }) } + private rejectPendingRequestsForClient(clientId: number, message: string): void { + for (const [id, pending] of this.pendingRelayRequests) { + if (pending.clientId !== clientId) { + continue + } + clearTimeout(pending.timer) + this.pendingRelayRequests.delete(id) + pending.reject(new Error(message)) + } + } + dispose(): void { if (this.disposed) { return @@ -341,6 +452,7 @@ export class RelayDispatcher { decoder: new FrameDecoder((frame) => this.handleFrame(client, frame)), write, waitWriteDrain: sinkOptions?.waitWriteDrain, + disconnect: sinkOptions?.disconnect, drainWaiters: new Set(), bulkChain: Promise.resolve(), nextOutgoingSeq: 1, @@ -371,7 +483,7 @@ export class RelayDispatcher { if (frame.type === MessageType.Regular) { try { const msg = parseJsonRpcMessage(frame.payload) - this.handleMessage(client, msg) + this.handleMessage(client, msg, frame.payload.byteLength) } catch (err) { process.stderr.write( `[relay] Parse error: ${err instanceof Error ? err.message : String(err)}\n` @@ -382,10 +494,11 @@ export class RelayDispatcher { private handleMessage( client: RelayClient, - msg: JsonRpcRequest | JsonRpcNotification | JsonRpcResponse + msg: JsonRpcRequest | JsonRpcNotification | JsonRpcResponse, + retainedBytes: number ): void { if ('id' in msg && 'method' in msg) { - void this.handleRequest(client, msg as JsonRpcRequest) + void this.handleRequest(client, msg as JsonRpcRequest, retainedBytes) } else if ('id' in msg && ('result' in msg || 'error' in msg)) { this.handleResponse(msg as JsonRpcResponse) } else if ('method' in msg && !('id' in msg)) { @@ -410,7 +523,11 @@ export class RelayDispatcher { pending.resolve(msg.result) } - private async handleRequest(client: RelayClient, req: JsonRpcRequest): Promise { + private async handleRequest( + client: RelayClient, + req: JsonRpcRequest, + retainedBytes: number + ): Promise { const handler = this.requestHandlers.get(req.method) if (!handler) { this.sendResponse(client, req.id, undefined, { @@ -422,10 +539,17 @@ export class RelayDispatcher { // Why: snapshot generation before the await to detect if the client disconnected mid-flight. const gen = client.generation - const { key: abortKey, controller: abortController } = this.requestAborts.create( - client.id, - req.id - ) + let registration: ReturnType + try { + registration = this.requestAborts.create(client.id, req.id, retainedBytes) + } catch (error) { + if (error instanceof RelayRequestAdmissionError) { + this.sendResponse(client, req.id, undefined, { code: -32000, message: error.message }) + return + } + throw error + } + const { key: abortKey, controller: abortController } = registration const context: RequestContext = { clientId: client.id, isStale: () => @@ -518,6 +642,7 @@ export class RelayDispatcher { client.closed = true client.generation++ this.requestAborts.abortClient(client.id) + this.rejectPendingRequestsForClient(client.id, 'Relay client disconnected') this.flushDrainWaiters(client) // Why: frames have no retransmit buffer; detach now so reconnect/PTY-reattach runs instead of waiting the ~20s keepalive timeout. if (client !== this.primaryClient) { diff --git a/src/relay/external-automations-handler.test.ts b/src/relay/external-automations-handler.test.ts index d2a19d79c2d..993131f27ab 100644 --- a/src/relay/external-automations-handler.test.ts +++ b/src/relay/external-automations-handler.test.ts @@ -1,4 +1,7 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ExternalAutomationsHandler } from './external-automations-handler' import type { RelayDispatcher } from './dispatcher' @@ -15,6 +18,7 @@ const execFileMock = vi.hoisted(() => vi.mock('child_process', () => ({ execFile: execFileMock })) type CapturedHandler = (params?: Record) => Promise +const tempDirs: string[] = [] function createHandlerHarness(): { handler: ExternalAutomationsHandler @@ -34,6 +38,10 @@ beforeEach(() => { execFileMock.mockClear() }) +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + describe('ExternalAutomationsHandler', () => { it('runs external lifecycle actions without shell wrapping', async () => { const { requestHandlers } = createHandlerHarness() @@ -55,26 +63,32 @@ describe('ExternalAutomationsHandler', () => { it('paginates remote Hermes run history after ref lookup', async () => { const { handler, requestHandlers } = createHandlerHarness() const handlerInternals = handler as unknown as { - readHermesRunRefs: (jobId: string) => Promise<{ id: string; run_at: string }[]> + readHermesRunRefs: (jobId: string) => Promise<{ + refs: { id: string; run_at: string }[] + saturated: boolean + }> hydrateHermesRunRef: ( jobId: string, ref: { id: string; run_at: string } ) => Promise<{ id: string; run_at: string }> } - handlerInternals.readHermesRunRefs = vi.fn().mockResolvedValue([ - { - id: 'cron_job-1_20260516_090000', - run_at: '2026-05-16T09:00:00' - }, - { - id: 'job-1:2026-05-15_09-00-00.md', - run_at: '2026-05-15T09:00:00' - }, - { - id: 'job-1:2026-05-14_09-00-00.md', - run_at: '2026-05-14T09:00:00' - } - ]) + handlerInternals.readHermesRunRefs = vi.fn().mockResolvedValue({ + refs: [ + { + id: 'cron_job-1_20260516_090000', + run_at: '2026-05-16T09:00:00' + }, + { + id: 'job-1:2026-05-15_09-00-00.md', + run_at: '2026-05-15T09:00:00' + }, + { + id: 'job-1:2026-05-14_09-00-00.md', + run_at: '2026-05-14T09:00:00' + } + ], + saturated: false + }) handlerInternals.hydrateHermesRunRef = vi.fn(async (_jobId, ref) => ref) const result = (await requestHandlers.get('externalAutomations.runs')?.({ @@ -91,12 +105,80 @@ describe('ExternalAutomationsHandler', () => { ]) }) + it('bounds remote hydration concurrency and omits page-wide output overflow', async () => { + const { handler, requestHandlers } = createHandlerHarness() + const refs = Array.from({ length: 3 }, (_, index) => ({ + id: `run-${index}`, + run_at: `2026-05-15T09:00:0${index}` + })) + let active = 0 + let maxActive = 0 + const handlerInternals = handler as unknown as { + readHermesRunRefs: () => Promise<{ refs: typeof refs; saturated: boolean }> + hydrateHermesRunRef: ( + jobId: string, + ref: (typeof refs)[number] + ) => Promise<{ id: string; output_content: string }> + } + handlerInternals.readHermesRunRefs = vi.fn().mockResolvedValue({ refs, saturated: false }) + handlerInternals.hydrateHermesRunRef = vi.fn(async (_jobId, ref) => { + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 1)) + active -= 1 + return { id: ref.id, output_content: 'x'.repeat(16 * 1024 * 1024 + 1) } + }) + + const result = (await requestHandlers.get('externalAutomations.runs')?.({ + provider: 'hermes', + jobId: 'job-1', + page: 1, + pageSize: 3 + })) as { + runs: { id: string; output_content: string | null; error?: string }[] + } + + expect(maxActive).toBe(2) + expect(result.runs.map((run) => run.id)).toEqual(refs.map((ref) => ref.id)) + expect(result.runs[0]?.output_content).toHaveLength(16 * 1024 * 1024 + 1) + expect(result.runs.slice(1)).toEqual([ + expect.objectContaining({ output_content: null, error: expect.stringContaining('omitted') }), + expect.objectContaining({ output_content: null, error: expect.stringContaining('omitted') }) + ]) + }) + + it('omits an oversized sparse remote markdown output without reading it wholesale', async () => { + const tempDir = await mkdtemp(join(tmpdir(), 'orca-relay-hermes-output-')) + tempDirs.push(tempDir) + const outputPath = join(tempDir, '2026-05-15_09-00-00.md') + await writeFile(outputPath, '', 'utf-8') + await truncate(outputPath, 64 * 1024 * 1024) + const { handler } = createHandlerHarness() + const handlerInternals = handler as unknown as { + readHermesOutputFileRun: (ref: Record) => Promise + } + + const run = await handlerInternals.readHermesOutputFileRun({ + kind: 'output', + id: 'job-1:2026-05-15_09-00-00.md', + job_id: 'job-1', + run_at: '2026-05-15T09:00:00', + run_key: '20260515_090000', + output_path: outputPath + }) + + expect(run).toMatchObject({ + output_content: null, + error: expect.stringContaining('File too large') + }) + }) + it('uses a count-only path for remote Hermes manager listing run counts', async () => { const { handler, requestHandlers } = createHandlerHarness() const handlerInternals = handler as unknown as { - readHermesRunCount: (jobId: string) => Promise + readHermesRunCount: (jobId: string) => Promise<{ total: number }> } - handlerInternals.readHermesRunCount = vi.fn().mockResolvedValue(42) + handlerInternals.readHermesRunCount = vi.fn().mockResolvedValue({ total: 42 }) const result = (await requestHandlers.get('externalAutomations.runs')?.({ provider: 'hermes', @@ -108,12 +190,34 @@ describe('ExternalAutomationsHandler', () => { expect(result).toEqual({ total: 42, runs: [] }) }) + it('reports when the remote run total is saturated', async () => { + const { handler, requestHandlers } = createHandlerHarness() + const handlerInternals = handler as unknown as { + readHermesRunCount: () => Promise<{ total: number; totalSaturated: true }> + } + handlerInternals.readHermesRunCount = vi + .fn() + .mockResolvedValue({ total: 10_000, totalSaturated: true }) + + await expect( + requestHandlers.get('externalAutomations.runs')?.({ + provider: 'hermes', + jobId: 'job-1', + page: 1, + pageSize: 0 + }) + ).resolves.toEqual({ total: 10_000, totalSaturated: true, runs: [] }) + }) + it('deduplicates concurrent remote Hermes count reads', async () => { const { handler, requestHandlers } = createHandlerHarness() - let resolveRefs: (refs: { id: string; run_at: string }[]) => void = () => {} + let resolveRefs: (result: { + refs: { id: string; run_at: string }[] + saturated: boolean + }) => void = () => {} const readHermesRunRefs = vi.fn( () => - new Promise<{ id: string; run_at: string }[]>((resolve) => { + new Promise<{ refs: { id: string; run_at: string }[]; saturated: boolean }>((resolve) => { resolveRefs = resolve }) ) @@ -136,10 +240,13 @@ describe('ExternalAutomationsHandler', () => { }) expect(readHermesRunRefs).toHaveBeenCalledTimes(1) - resolveRefs([ - { id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }, - { id: 'job-1:2026-05-16_09-00-00.md', run_at: '2026-05-16T09:00:00' } - ]) + resolveRefs({ + refs: [ + { id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }, + { id: 'job-1:2026-05-16_09-00-00.md', run_at: '2026-05-16T09:00:00' } + ], + saturated: false + }) await expect(Promise.all([first, second])).resolves.toEqual([ { total: 2, runs: [] }, @@ -151,13 +258,17 @@ describe('ExternalAutomationsHandler', () => { const { handler, requestHandlers } = createHandlerHarness() const readHermesRunRefs = vi .fn() - .mockResolvedValueOnce([ - { id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' } - ]) - .mockResolvedValueOnce([ - { id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }, - { id: 'job-1:2026-05-16_09-00-00.md', run_at: '2026-05-16T09:00:00' } - ]) + .mockResolvedValueOnce({ + refs: [{ id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }], + saturated: false + }) + .mockResolvedValueOnce({ + refs: [ + { id: 'job-1:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }, + { id: 'job-1:2026-05-16_09-00-00.md', run_at: '2026-05-16T09:00:00' } + ], + saturated: false + }) const handlerInternals = handler as unknown as { readHermesRunRefs: typeof readHermesRunRefs } @@ -200,8 +311,11 @@ describe('ExternalAutomationsHandler', () => { const { handler, requestHandlers } = createHandlerHarness() const readHermesRunRefs = vi.fn(async (jobId: string) => jobId === 'job-0' - ? [{ id: 'job-0:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }] - : [] + ? { + refs: [{ id: 'job-0:2026-05-15_09-00-00.md', run_at: '2026-05-15T09:00:00' }], + saturated: false + } + : { refs: [], saturated: false } ) const handlerInternals = handler as unknown as { readHermesRunRefs: typeof readHermesRunRefs diff --git a/src/relay/external-automations-handler.ts b/src/relay/external-automations-handler.ts index baaaa99481e..34d7486a675 100644 --- a/src/relay/external-automations-handler.ts +++ b/src/relay/external-automations-handler.ts @@ -2,11 +2,28 @@ * run history, and actions must stay co-located behind one relay request handler. */ import { execFile } from 'node:child_process' import { existsSync } from 'node:fs' -import { open, readdir, readFile, realpath, stat } from 'node:fs/promises' +import { open, opendir, realpath, stat } from 'node:fs/promises' import { createRequire } from 'node:module' import { homedir } from 'node:os' import { isAbsolute, join, relative, resolve, sep } from 'node:path' import { promisify } from 'node:util' +import { readExternalAutomationJobsFile } from '../shared/external-automation-jobs-file' +import { + type BoundedHermesRunRefs, + HERMES_RUN_REF_MAX_ENTRIES, + HERMES_SESSION_RUN_REFS_SELECT_SQL, + HermesRunRefRetainer +} from '../shared/hermes-run-ref-retention' +import { + formatHermesSessionMessagesWithinLimits, + HERMES_PRIMARY_OUTPUT_MAX_BYTES, + HERMES_RUN_PAGE_MAX_RUNS, + HERMES_SESSION_TRANSCRIPT_SELECT_SQL, + HERMES_SESSION_TRANSCRIPT_TRUNCATED_ERROR, + hydrateHermesRunPageWithinLimits +} from '../shared/hermes-run-output-limits' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' +import { mapWithConcurrency } from '../shared/map-with-concurrency' import type { RelayDispatcher } from './dispatcher' const execFileAsync = promisify(execFile) @@ -23,6 +40,7 @@ const HERMES_RUN_KEY_PATTERN = /^(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})$/ const MAX_SESSION_OUTPUT_GAP_MS = 24 * 60 * 60 * 1000 const MAX_REFERENCED_LOG_BYTES = 5 * 1024 * 1024 const HERMES_RUN_COUNT_CACHE_MAX_ENTRIES = 200 +const HERMES_JOB_RUN_COUNT_CONCURRENCY = 4 const FULL_SESSION_LOG_HEADING = '## Full session log' const REFERENCED_LOG_HEADING = '## Latest log file' const LATEST_LOG_PATH_PATTERN = @@ -30,6 +48,7 @@ const LATEST_LOG_PATH_PATTERN = type SqliteStatement = { get: (...args: unknown[]) => Record | undefined all: (...args: unknown[]) => Record[] + iterate?: (...args: unknown[]) => Iterable> } type SqliteDatabase = { prepare: (sql: string) => SqliteStatement @@ -71,9 +90,10 @@ type HermesMergedRunRef = { session: HermesSessionRunRef | null } type HermesRunCountCacheEntry = { - promise: Promise + promise: Promise expiresAt: number } +type HermesRunCount = { total: number; totalSaturated?: true } const HERMES_RUN_COUNT_CACHE_TTL_MS = 2000 @@ -106,37 +126,27 @@ export class ExternalAutomationsHandler { if (!existsSync(jobsFile)) { return [] } - const content = await readFile(jobsFile, 'utf-8') - const parsed = JSON.parse(content) as unknown - const jobs = Array.isArray(parsed) - ? parsed - : typeof parsed === 'object' && - parsed !== null && - !Array.isArray(parsed) && - Array.isArray((parsed as { jobs?: unknown }).jobs) - ? (parsed as { jobs: unknown[] }).jobs - : [] + const jobs = await readExternalAutomationJobsFile(jobsFile, { allowRootArray: true }) if (provider !== 'hermes') { return jobs } - return Promise.all( - jobs.map(async (job) => { - if (!this.isRecord(job) || typeof job.id !== 'string') { - return job - } - const runsPage = await this.listRuns({ - provider: 'hermes', - jobId: job.id, - page: 1, - pageSize: 0 - }) - return { - ...job, - run_count: runsPage.total, - runs: runsPage.runs - } + return mapWithConcurrency(jobs, HERMES_JOB_RUN_COUNT_CONCURRENCY, async (job) => { + if (!this.isRecord(job) || typeof job.id !== 'string') { + return job + } + const runsPage = await this.listRuns({ + provider: 'hermes', + jobId: job.id, + page: 1, + pageSize: 0 }) - ) + return { + ...job, + run_count: runsPage.total, + ...(runsPage.totalSaturated ? { run_count_saturated: true } : {}), + runs: runsPage.runs + } + }) } private isRecord(value: unknown): value is Record { @@ -259,9 +269,10 @@ export class ExternalAutomationsHandler { return null } if (logStat.size <= MAX_REFERENCED_LOG_BYTES) { + const { buffer } = await readNodeFileWithinLimit(logPath, MAX_REFERENCED_LOG_BYTES) return { path: logPath, - content: await readFile(logPath, 'utf-8'), + content: buffer.toString('utf-8'), truncated: false } } @@ -313,31 +324,6 @@ export class ExternalAutomationsHandler { .join('\n\n') } - private formatSessionMessages(messages: Record[]): string | null { - if (messages.length === 0) { - return null - } - return messages - .map((message) => { - const role = typeof message.role === 'string' ? message.role : 'message' - const content = typeof message.content === 'string' ? message.content.trim() : '' - const toolName = typeof message.tool_name === 'string' ? message.tool_name.trim() : '' - const reasoning = - typeof message.reasoning_content === 'string' - ? message.reasoning_content.trim() - : typeof message.reasoning === 'string' - ? message.reasoning.trim() - : '' - const parts = [ - `## ${role}${toolName ? ` / ${toolName}` : ''}`, - reasoning ? `### Reasoning\n\n${reasoning}` : null, - content || '(empty)' - ].filter(Boolean) - return parts.join('\n\n') - }) - .join('\n\n---\n\n') - } - private getRunKey(run: unknown): string | null { return this.isRecord(run) && typeof run.run_key === 'string' && run.run_key.trim() ? run.run_key @@ -535,19 +521,20 @@ export class ExternalAutomationsHandler { return databaseConstructor } - private async readHermesRunRefs(jobId: string): Promise { + private async readHermesRunRefs( + jobId: string + ): Promise> { const outputRuns = await this.readHermesOutputFileRunRefs(jobId) - return this.mergeHermesOutputAndSessionRunRefs( - outputRuns, - this.readHermesSessionDbRunRefs(jobId) - ).sort((a, b) => { - const aTime = this.getRawRunTime(a) - const bTime = this.getRawRunTime(b) - if (Number.isFinite(aTime) && Number.isFinite(bTime)) { - return bTime - aTime - } - return this.getRawRunId(b).localeCompare(this.getRawRunId(a)) - }) + const sessionRuns = this.readHermesSessionDbRunRefs(jobId) + const mergedRetainer = new HermesRunRefRetainer() + for (const ref of this.mergeHermesOutputAndSessionRunRefs(outputRuns.refs, sessionRuns.refs)) { + mergedRetainer.add(ref) + } + const merged = mergedRetainer.finish() + return { + refs: merged.refs, + saturated: outputRuns.saturated || sessionRuns.saturated || merged.saturated + } } private async hydrateHermesRunRef(jobId: string, ref: HermesMergedRunRef): Promise { @@ -564,9 +551,9 @@ export class ExternalAutomationsHandler { ) } - private async readHermesRunCount(jobId: string): Promise { + private async readHermesRunCount(jobId: string): Promise { if (!EXTERNAL_JOB_ID_PATTERN.test(jobId)) { - return 0 + return { total: 0 } } const now = Date.now() const cached = this.hermesRunCountCache.get(jobId) @@ -580,7 +567,10 @@ export class ExternalAutomationsHandler { // processes are long-lived, so stale job ids need both TTL and a hard cap. this.pruneHermesRunCountCache(now) const entry: HermesRunCountCacheEntry = { - promise: this.readHermesRunRefs(jobId).then((refs) => refs.length), + promise: this.readHermesRunRefs(jobId).then((result) => ({ + total: result.refs.length, + ...(result.saturated ? { totalSaturated: true as const } : {}) + })), expiresAt: Number.POSITIVE_INFINITY } this.hermesRunCountCache.set(jobId, entry) @@ -621,6 +611,7 @@ export class ExternalAutomationsHandler { private async listRuns(params: Record = {}): Promise<{ total: number + totalSaturated?: true runs: unknown[] }> { const provider = params.provider === 'openclaw' ? 'openclaw' : 'hermes' @@ -631,7 +622,7 @@ export class ExternalAutomationsHandler { : 1 const pageSize = typeof params.pageSize === 'number' && Number.isFinite(params.pageSize) - ? Math.min(100, Math.max(0, Math.floor(params.pageSize))) + ? Math.min(HERMES_RUN_PAGE_MAX_RUNS, Math.max(0, Math.floor(params.pageSize))) : 25 if (provider !== 'hermes') { return { total: 0, runs: [] } @@ -642,53 +633,52 @@ export class ExternalAutomationsHandler { if (pageSize === 0) { // Why: manager listing only needs a badge count; hydrating markdown logs // and full session transcripts can make opening Automations very slow. - return { total: await this.readHermesRunCount(jobId), runs: [] } + return { ...(await this.readHermesRunCount(jobId)), runs: [] } } const runRefs = await this.readHermesRunRefs(jobId) const start = (page - 1) * pageSize return { - total: runRefs.length, - runs: await Promise.all( - runRefs.slice(start, start + pageSize).map((ref) => this.hydrateHermesRunRef(jobId, ref)) + total: runRefs.refs.length, + ...(runRefs.saturated ? { totalSaturated: true } : {}), + runs: await hydrateHermesRunPageWithinLimits( + runRefs.refs.slice(start, start + pageSize), + (ref) => this.hydrateHermesRunRef(jobId, ref) ) } } - private getRawRunId(run: unknown): string { - if (this.isRecord(run) && 'id' in run) { - return String(run.id) - } - return '' - } - - private getRawRunTime(run: unknown): number { - if (!this.isRecord(run) || !('run_at' in run)) { - return Number.NaN - } - return typeof run.run_at === 'string' ? Date.parse(run.run_at) : Number.NaN - } - - private async readHermesOutputFileRunRefs(jobId: string): Promise { + private async readHermesOutputFileRunRefs( + jobId: string + ): Promise> { const outputDir = join(HERMES_OUTPUT_DIR, jobId) if (!existsSync(outputDir)) { - return [] + return { refs: [], saturated: false } } - const entries = await readdir(outputDir, { withFileTypes: true }) - return entries - .filter((entry) => entry.isFile() && HERMES_OUTPUT_FILE_PATTERN.test(entry.name)) - .map((entry) => ({ + const retainer = new HermesRunRefRetainer() + const directory = await opendir(outputDir) + for await (const entry of directory) { + if (!entry.isFile() || !HERMES_OUTPUT_FILE_PATTERN.test(entry.name)) { + continue + } + retainer.add({ kind: 'output' as const, id: `${jobId}:${entry.name}`, job_id: jobId, run_at: this.runAtFromHermesOutputFile(entry.name), run_key: this.runKeyFromHermesOutputFile(entry.name), output_path: join(outputDir, entry.name) - })) + }) + } + return retainer.finish() } private async readHermesOutputFileRun(ref: HermesOutputRunRef): Promise { try { - const content = await readFile(ref.output_path, 'utf-8') + const { buffer } = await readNodeFileWithinLimit( + ref.output_path, + HERMES_PRIMARY_OUTPUT_MAX_BYTES + ) + const content = buffer.toString('utf-8') const parsed = this.parseHermesOutput(content) const outputContent = await this.appendReferencedLogFile(parsed.outputContent) return { @@ -717,41 +707,45 @@ export class ExternalAutomationsHandler { } } - private readHermesSessionDbRunRefs(jobId: string): HermesSessionRunRef[] { + private readHermesSessionDbRunRefs(jobId: string): BoundedHermesRunRefs { if (!existsSync(HERMES_STATE_DB)) { - return [] + return { refs: [], saturated: false } } const Database = this.getDatabaseConstructor() if (!Database) { - return [] + return { refs: [], saturated: false } } try { const db = new Database(HERMES_STATE_DB, { readonly: true, fileMustExist: true }) try { const pattern = `cron\\_${this.escapeSqlLike(jobId)}\\_%` - const rows = db - .prepare( - `SELECT id, started_at - FROM sessions - WHERE id LIKE ? ESCAPE '\\' - ORDER BY started_at DESC` - ) - .all(pattern) as Record[] - return rows.map((row) => { + const statement = db.prepare(HERMES_SESSION_RUN_REFS_SELECT_SQL) + const rows = + typeof statement.iterate === 'function' + ? statement.iterate(pattern) + : statement.all(pattern) + const retainer = new HermesRunRefRetainer() + let read = 0 + for (const row of rows) { + if (read >= HERMES_RUN_REF_MAX_ENTRIES + 1) { + break + } + read += 1 const runId = typeof row.id === 'string' ? row.id : `${jobId}:${String(row.started_at)}` - return { + retainer.add({ kind: 'session', id: runId, job_id: jobId, run_at: this.runAtFromUnixSeconds(row.started_at), run_key: runId.split(`${jobId}_`).at(-1) ?? null - } - }) + }) + } + return retainer.finish() } finally { db.close() } } catch { - return [] + return { refs: [], saturated: false } } } @@ -777,14 +771,12 @@ export class ExternalAutomationsHandler { if (!row) { return null } - const messages = db - .prepare( - `SELECT role, content, tool_name, reasoning, reasoning_content - FROM messages - WHERE session_id = ? - ORDER BY timestamp, id` - ) - .all(runId) as Record[] + const messageStatement = db.prepare(HERMES_SESSION_TRANSCRIPT_SELECT_SQL) + const messages = + typeof messageStatement.iterate === 'function' + ? messageStatement.iterate(runId) + : messageStatement.all(runId) + const formattedMessages = formatHermesSessionMessagesWithinLimits(messages) const title = typeof row.title === 'string' && row.title.trim() ? row.title.trim() : null const model = typeof row.model === 'string' && row.model.trim() ? row.model.trim() : null const messageCount = typeof row.message_count === 'number' ? row.message_count : null @@ -804,8 +796,8 @@ export class ExternalAutomationsHandler { run_key: runId.split(`${jobId}_`).at(-1) ?? null, status: typeof row.ended_at === 'number' ? 'completed' : 'unknown', output_preview: summaryParts.join(' · ') || null, - output_content: this.formatSessionMessages(messages), - error: null, + output_content: formattedMessages.content, + error: formattedMessages.truncated ? HERMES_SESSION_TRANSCRIPT_TRUNCATED_ERROR : null, output_path: HERMES_STATE_DB } } finally { diff --git a/src/relay/filesystem-directory-reader.test.ts b/src/relay/filesystem-directory-reader.test.ts new file mode 100644 index 00000000000..6a7d4387a64 --- /dev/null +++ b/src/relay/filesystem-directory-reader.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' +import { FILESYSTEM_DIRECTORY_LIMIT_MESSAGE } from '../shared/filesystem-directory-listing-limit' +import { collectRelayFilesystemDirectoryEntries } from './filesystem-directory-reader' + +function entry(name: string, options?: { directory?: boolean; symlink?: boolean }) { + return { + name, + isDirectory: () => options?.directory ?? false, + isSymbolicLink: () => options?.symlink ?? false + } +} + +describe('relay filesystem directory reader', () => { + it('preserves directory-first ordering and symlink-directory classification', async () => { + const classify = vi.fn(async (_dirPath, source) => + source.isSymbolicLink() ? true : source.isDirectory() + ) + + await expect( + collectRelayFilesystemDirectoryEntries( + '/repo', + [ + entry('z.txt'), + entry('linked', { symlink: true }), + entry('beta', { directory: true }), + entry('alpha', { directory: true }) + ], + undefined, + classify + ) + ).resolves.toEqual([ + { name: 'alpha', isDirectory: true, isSymlink: false }, + { name: 'beta', isDirectory: true, isSymlink: false }, + { name: 'linked', isDirectory: true, isSymlink: true }, + { name: 'z.txt', isDirectory: false, isSymlink: false } + ]) + expect(classify).toHaveBeenCalledTimes(4) + }) + + it('stops before classifying or retaining the first over-limit entry', async () => { + let enumerated = 0 + const classify = vi.fn(async () => false) + const source = { + async *[Symbol.asyncIterator]() { + while (enumerated < 100) { + enumerated += 1 + yield entry(`entry-${enumerated}`) + } + } + } + + await expect( + collectRelayFilesystemDirectoryEntries( + '/repo', + source, + { maxEntries: 3, maxRetainedBytes: 1024 }, + classify + ) + ).rejects.toThrow(FILESYSTEM_DIRECTORY_LIMIT_MESSAGE) + expect(enumerated).toBe(4) + expect(classify).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/relay/filesystem-directory-reader.ts b/src/relay/filesystem-directory-reader.ts new file mode 100644 index 00000000000..e6c6d6e46c4 --- /dev/null +++ b/src/relay/filesystem-directory-reader.ts @@ -0,0 +1,70 @@ +import { opendir, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { + createFilesystemDirectoryLimitState, + trackFilesystemDirectoryEntry, + type FilesystemDirectoryListingLimits +} from '../shared/filesystem-directory-listing-limit' + +type RelayDirectorySourceEntry = { + name: string + isDirectory(): boolean + isSymbolicLink(): boolean +} + +export type RelayFilesystemDirectoryEntry = { + name: string + isDirectory: boolean + isSymlink: boolean +} + +type DirectoryClassifier = (dirPath: string, entry: RelayDirectorySourceEntry) => Promise + +async function isRelayFilesystemDirectoryEntry( + dirPath: string, + entry: RelayDirectorySourceEntry +): Promise { + if (entry.isDirectory()) { + return true + } + if (!entry.isSymbolicLink()) { + return false + } + try { + // Why: linked directories must remain expandable remotely. + return (await stat(join(dirPath, entry.name))).isDirectory() + } catch { + return false + } +} + +export async function readRelayFilesystemDirectory( + dirPath: string, + requestedLimits?: Partial +): Promise { + return collectRelayFilesystemDirectoryEntries(dirPath, await opendir(dirPath), requestedLimits) +} + +export async function collectRelayFilesystemDirectoryEntries( + dirPath: string, + directory: AsyncIterable | Iterable, + requestedLimits?: Partial, + classifyDirectory: DirectoryClassifier = isRelayFilesystemDirectoryEntry +): Promise { + const entries: RelayFilesystemDirectoryEntry[] = [] + const limit = createFilesystemDirectoryLimitState(requestedLimits) + for await (const entry of directory) { + trackFilesystemDirectoryEntry(limit, entry) + entries.push({ + name: entry.name, + isDirectory: await classifyDirectory(dirPath, entry), + isSymlink: entry.isSymbolicLink() + }) + } + return entries.sort((left, right) => { + if (left.isDirectory !== right.isDirectory) { + return left.isDirectory ? -1 : 1 + } + return left.name.localeCompare(right.name) + }) +} diff --git a/src/relay/fs-handler-file-read.ts b/src/relay/fs-handler-file-read.ts index 003172c08fd..28670f1b26a 100644 --- a/src/relay/fs-handler-file-read.ts +++ b/src/relay/fs-handler-file-read.ts @@ -1,5 +1,12 @@ -import { open, readFile, stat } from 'node:fs/promises' +import { open, stat } from 'node:fs/promises' import { extname } from 'node:path' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' +import { + assertRasterImagePreviewWithinLimits, + isKnownRasterImageMimeType, + RASTER_IMAGE_PREVIEW_HEADER_MAX_BYTES +} from '../shared/raster-image-preview-limits' +import type { RasterImageDimensions } from '../shared/raster-image-dimensions' import type { RelayDispatcher, RequestContext } from './dispatcher' import { STREAM_ACK_WINDOW_CHUNKS, STREAM_CHUNK_SIZE, RelayErrorCode } from './protocol' import type { RelayStreamRegistry, TooManyStreamsError } from './fs-stream-registry' @@ -23,15 +30,22 @@ export async function readRelayFileContent(filePath: string) { } if (mimeType) { - const buffer = await readFile(filePath) - return { content: buffer.toString('base64'), isBinary: true, isImage: true, mimeType } + const { buffer } = await readNodeFileWithinLimit(filePath, sizeLimit) + const imageDimensions = assertRasterImagePreviewWithinLimits(buffer, mimeType) + return { + content: buffer.toString('base64'), + isBinary: true, + isImage: true, + mimeType, + ...(imageDimensions ? { imageDimensions } : {}) + } } if (stats.size > BINARY_PROBE_BYTES && (await isBinaryFilePrefix(filePath))) { return { content: '', isBinary: true } } - const buffer = await readFile(filePath) + const { buffer } = await readNodeFileWithinLimit(filePath, sizeLimit) if (isBinaryBuffer(buffer)) { return { content: '', isBinary: true } } @@ -44,6 +58,7 @@ export type StreamMetadata = { isBinary: boolean isImage?: boolean mimeType?: string + imageDimensions?: RasterImageDimensions /** On-the-wire encoding of each chunk's `data` field. Always 'base64'. */ chunkEncoding?: 'base64' /** Encoding of the assembled FileReadResult.content. */ @@ -105,31 +120,38 @@ export async function readRelayFileStreamMetadata( const handle = await open(filePath, 'r') let streamId: number try { - streamId = registry.register(handle) + let imageDimensions: RasterImageDimensions | undefined + if (isKnownRasterImageMimeType(mimeType)) { + const header = Buffer.alloc(Math.min(stats.size, RASTER_IMAGE_PREVIEW_HEADER_MAX_BYTES)) + const headerBytesRead = await readFullStreamChunk(handle, header, header.length, 0) + imageDimensions = assertRasterImagePreviewWithinLimits( + header.subarray(0, headerBytesRead), + mimeType + ) + } + streamId = registry.register(handle, context.clientId) + process.stderr.write(`[relay] stream start id=${streamId} size=${stats.size}\n`) + + // Why: start after metadata returns so the subscribed client cannot miss the first chunk. + const resolvedPumpOptions = pumpOptions ?? { paceWithAcks: false } + setImmediate(() => { + void pumpChunks(streamId, stats.size, dispatcher, registry, context, resolvedPumpOptions) + }) + + return { + streamId, + totalSize: stats.size, + isBinary: !!mimeType, + isImage: mimeType ? true : undefined, + mimeType, + ...(imageDimensions ? { imageDimensions } : {}), + chunkEncoding: 'base64', + resultEncoding: mimeType ? 'base64' : 'utf-8' + } } catch (err) { await handle.close() throw err } - - process.stderr.write(`[relay] stream start id=${streamId} size=${stats.size}\n`) - - // Why: pumpChunks owns its own try/finally for handle release; the outer - // setImmediate kicks the pump off the metadata-response task so the client - // sees the response before the first chunk frame. - const resolvedPumpOptions = pumpOptions ?? { paceWithAcks: false } - setImmediate(() => { - void pumpChunks(streamId, stats.size, dispatcher, registry, context, resolvedPumpOptions) - }) - - return { - streamId, - totalSize: stats.size, - isBinary: !!mimeType, - isImage: mimeType ? true : undefined, - mimeType, - chunkEncoding: 'base64', - resultEncoding: mimeType ? 'base64' : 'utf-8' - } } async function pumpChunks( @@ -203,6 +225,7 @@ async function pumpChunks( // Why: the bulk lane waits out sink saturation, so a flood of chunk // frames cannot pile up in the outbound pipe ahead of interactive // pty.data frames written via plain notify(). + registry.recordSent(streamId, seq) await dispatcher.notifyBulk( 'fs.streamChunk', { streamId, seq, data }, @@ -227,14 +250,23 @@ async function pumpChunks( try { if (endReason === 'end') { - dispatcher.notify('fs.streamEnd', { streamId }) + if (pumpOptions.clientId !== undefined) { + dispatcher.notifyClient(pumpOptions.clientId, 'fs.streamEnd', { streamId }) + } else { + dispatcher.notify('fs.streamEnd', { streamId }) + } process.stderr.write(`[relay] stream end id=${streamId}\n`) } else if (endReason === 'error') { - dispatcher.notify('fs.streamError', { + const params = { streamId, code: errorCode ?? 'ESTREAMERROR', message: errorMessage ?? 'stream error' - }) + } + if (pumpOptions.clientId !== undefined) { + dispatcher.notifyClient(pumpOptions.clientId, 'fs.streamError', params) + } else { + dispatcher.notify('fs.streamError', params) + } process.stderr.write(`[relay] stream error id=${streamId} code=${errorCode}\n`) } else if (endReason === 'aborted') { process.stderr.write(`[relay] stream cancel id=${streamId}\n`) diff --git a/src/relay/fs-handler-git-fallback.ts b/src/relay/fs-handler-git-fallback.ts index 8de2ad3394c..66f0e48d2ad 100644 --- a/src/relay/fs-handler-git-fallback.ts +++ b/src/relay/fs-handler-git-fallback.ts @@ -27,6 +27,14 @@ import { SEARCH_TIMEOUT_MS } from '../shared/text-search' import { buildRelayGitEnv } from './relay-command-env' +import { SearchSubprocessLineAccumulator } from '../shared/search-subprocess-lines' +import { + createQuickOpenListingBudget, + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QuickOpenSubprocessPathAccumulator, + resolveQuickOpenResultLimit, + retainQuickOpenPath +} from '../shared/quick-open-listing-limits' /** * List files using `git ls-files`. Fallback when rg is not installed. @@ -46,9 +54,14 @@ export function listFilesWithGit( if (signal?.aborted) { return Promise.reject(fileListingCancellationError(signal)) } + const resultLimit = resolveQuickOpenResultLimit(maxResults) + if (resultLimit === 0) { + return Promise.resolve([]) + } const gitPaths = new Set() const directoryPaths = new Set() const directFileCandidates = new Set() + const listingBudget = createQuickOpenListingBudget() const { primary, ignoredPass } = buildGitLsFilesArgsForQuickOpen(excludePathPrefixes) const children: { child: ReturnType @@ -59,7 +72,7 @@ export function listFilesWithGit( const runGitLsFiles = (args: string[]): Promise => { return new Promise((resolve, reject) => { - let buf = '' + const paths = new QuickOpenSubprocessPathAccumulator(0) let done = false const processPath = (path: string): boolean => { @@ -67,27 +80,23 @@ export function listFilesWithGit( return false } if (path.endsWith('/')) { - directoryPaths.add(path) + retainQuickOpenPath(directoryPaths, path, listingBudget) } else { - gitPaths.add(path) - if (maxResults !== undefined) { - // Why: this duplicate classification exists only to stop bounded - // scans; unbounded SSH scans must not retain another full listing. - const parsed = parseQuickOpenGitLsFilesEntry(path) - const relPath = parsed.path.replace(/\/+$/, '') - if ( - !parsed.isGitlink && - !parsed.isUntrackedDir && - shouldIncludeQuickOpenPath(relPath) && - !shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes) - ) { - directFileCandidates.add(relPath) - } + retainQuickOpenPath(gitPaths, path, listingBudget) + const parsed = parseQuickOpenGitLsFilesEntry(path) + const relPath = parsed.path.replace(/\/+$/, '') + if ( + !parsed.isGitlink && + !parsed.isUntrackedDir && + shouldIncludeQuickOpenPath(relPath) && + !shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes) + ) { + retainQuickOpenPath(directFileCandidates, relPath, listingBudget) } } // Why: placeholders need IO classification and can disappear; only // guaranteed final files are allowed to stop the remote Git processes. - return maxResults !== undefined && directFileCandidates.size >= maxResults + return directFileCandidates.size >= resultLimit } const child = spawn('git', ['ls-files', ...args], { @@ -111,7 +120,7 @@ export function listFilesWithGit( return } done = true - buf = '' + paths.clear() cleanup() reject(error) } @@ -120,6 +129,7 @@ export function listFilesWithGit( return } done = true + paths.clear() cleanup() resolve() } @@ -130,20 +140,23 @@ export function listFilesWithGit( resolve: resolvePass }) - function handleStdoutData(chunk: string): void { - buf += chunk - let start = 0 - let idx = buf.indexOf('\0', start) - while (idx !== -1) { - if (processPath(buf.substring(start, idx))) { - buf = '' + function failForOutput(error: unknown): void { + child.kill() + rejectPass(error instanceof Error ? error : new Error(String(error))) + } + function handleStdoutData(chunk: Buffer | string): void { + try { + const outcome = paths.push(chunk, (path) => !processPath(path)) + if (outcome === 'stopped') { finishAtLimit() - return + } else if (outcome === 'path-too-large') { + failForOutput( + new Error(`Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes`) + ) } - start = idx + 1 - idx = buf.indexOf('\0', start) + } catch (error) { + failForOutput(error) } - buf = start < buf.length ? buf.substring(start) : '' } function handleStderrData(): void { /* drain */ @@ -162,9 +175,14 @@ export function listFilesWithGit( rejectPass(new Error(`git ls-files killed by ${signal}`)) return } - if (buf && processPath(buf)) { - buf = '' - finishAtLimit() + try { + const trailingPath = paths.finish() + if (trailingPath && processPath(trailingPath)) { + finishAtLimit() + return + } + } catch (error) { + failForOutput(error) return } if (code === 0) { @@ -177,7 +195,6 @@ export function listFilesWithGit( rejectPass(new Error(`git ls-files exited with code ${code}`)) } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) @@ -232,12 +249,9 @@ export function listFilesWithGit( ) } }) - const passes = - maxResults === undefined - ? Promise.all([runGitLsFiles(primary), runIgnoredPass()]) - : runGitLsFiles(primary).then(() => - directFileCandidates.size < maxResults ? runIgnoredPass() : Promise.resolve() - ) + const passes = runGitLsFiles(primary).then(() => + directFileCandidates.size < resultLimit ? runIgnoredPass() : Promise.resolve() + ) return passes .then(async () => { @@ -247,11 +261,11 @@ export function listFilesWithGit( directoryPaths, excludePathPrefixes, signal, - maxResults + maxResults: resultLimit }) // Why: directory placeholders are expanded after Git exits; restore // Git's path order for empty queries and fuzzy-score ties over SSH. - return files.sort().slice(0, maxResults) + return files.sort().slice(0, resultLimit) }) .catch((err) => { killSurvivors('git ls-files canceled after sibling failure') @@ -277,7 +291,7 @@ export function searchWithGitGrep( const gitArgs = buildGitGrepArgs(query, opts) const matchRegex = buildSubmatchRegex(query, opts) const acc = createAccumulator() - let stdoutBuffer = '' + const stdoutLines = new SearchSubprocessLineAccumulator() let done = false const child = spawn('git', gitArgs, { @@ -309,12 +323,11 @@ export function searchWithGitGrep( } } - function handleStdoutData(chunk: string): void { - stdoutBuffer += chunk - const lines = stdoutBuffer.split('\n') - stdoutBuffer = lines.pop() ?? '' - for (const l of lines) { - processLine(l) + function handleStdoutData(chunk: Buffer): void { + if (!stdoutLines.push(chunk, processLine)) { + acc.truncated = true + child.kill() + resolveOnce() } } @@ -327,13 +340,13 @@ export function searchWithGitGrep( } function handleClose(): void { - if (stdoutBuffer) { - processLine(stdoutBuffer) + const trailingLine = stdoutLines.finish() + if (trailingLine !== null) { + processLine(trailingLine) } resolveOnce() } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) diff --git a/src/relay/fs-handler-install-rg.ts b/src/relay/fs-handler-install-rg.ts index 9b0cd21b065..4f4be970aee 100644 --- a/src/relay/fs-handler-install-rg.ts +++ b/src/relay/fs-handler-install-rg.ts @@ -1,85 +1,11 @@ -import { readFile } from 'node:fs/promises' import { - getProcessOutputFields, - iterateProcessOutputLines -} from '../shared/process-output-field-scanner' + buildInstallRgMessage as buildSharedInstallRgMessage, + detectInstallCommand, + detectLinuxInstallCommandFromOsRelease +} from '../shared/quick-open-install-rg' -const GENERIC_LINUX_RIPGREP_INSTALL = - 'install ripgrep via your package manager (e.g. apt/dnf/pacman)' -const OS_RELEASE_ID_LIKE_MAX_FIELDS = 16 +export { detectInstallCommand, detectLinuxInstallCommandFromOsRelease } -export async function detectInstallCommand(): Promise { - if (process.platform === 'darwin') { - return 'brew install ripgrep' - } - if (process.platform === 'linux') { - try { - const osRelease = await readFile('/etc/os-release', 'utf-8') - return detectLinuxInstallCommandFromOsRelease(osRelease) - } catch { - /* fall through to generic guidance */ - } - return GENERIC_LINUX_RIPGREP_INSTALL - } - return 'install ripgrep (https://github.com/BurntSushi/ripgrep#installation)' -} - -export function detectLinuxInstallCommandFromOsRelease(osRelease: string): string { - for (const id of getOsReleasePackageFamilyIds(osRelease)) { - if (id === 'debian' || id === 'ubuntu') { - return 'sudo apt install ripgrep' - } - if (id === 'fedora' || id === 'rhel' || id === 'centos') { - return 'sudo dnf install ripgrep' - } - if (id === 'arch') { - return 'sudo pacman -S ripgrep' - } - if (id === 'alpine') { - return 'sudo apk add ripgrep' - } - } - - return GENERIC_LINUX_RIPGREP_INSTALL -} - -function getOsReleasePackageFamilyIds(osRelease: string): string[] { - const ids: string[] = [] - - for (const line of iterateProcessOutputLines(osRelease)) { - const separatorIndex = line.indexOf('=') - if (separatorIndex <= 0) { - continue - } - - const key = line.slice(0, separatorIndex) - const value = readOsReleaseValue(line.slice(separatorIndex + 1)) - if (key === 'ID') { - const id = getProcessOutputFields(value, 1)[0] - if (id) { - ids.push(id) - } - } else if (key === 'ID_LIKE') { - ids.push(...getProcessOutputFields(value, OS_RELEASE_ID_LIKE_MAX_FIELDS)) - } - } - - return ids -} - -function readOsReleaseValue(rawValue: string): string { - const trimmed = rawValue.trim() - const quote = trimmed[0] - return (quote === '"' || quote === "'") && trimmed.at(-1) === quote - ? trimmed.slice(1, -1) - : trimmed -} - -export async function buildInstallRgMessage(cause: unknown): Promise { - const reason = cause instanceof Error ? cause.message : String(cause) - const cmd = await detectInstallCommand() - return ( - `Quick Open scan too large (${reason}). ` + - `Install ripgrep on the remote to enable fast, gitignore-aware listing: ${cmd}` - ) +export function buildInstallRgMessage(cause: unknown): Promise { + return buildSharedInstallRgMessage(cause, 'remote') } diff --git a/src/relay/fs-handler-list-files-cancel.test.ts b/src/relay/fs-handler-list-files-cancel.test.ts index 02cd8da6afc..e30bf3cc498 100644 --- a/src/relay/fs-handler-list-files-cancel.test.ts +++ b/src/relay/fs-handler-list-files-cancel.test.ts @@ -43,7 +43,7 @@ describe('relay list-files cancellation', () => { vi.useRealTimers() }) - it('listFilesWithRg kills both rg passes and rejects when aborted mid-flight', async () => { + it('listFilesWithRg kills the active rg pass and rejects when aborted mid-flight', async () => { const primaryProc = createMockProcess() const ignoredProc = createMockProcess() spawnMock.mockImplementation((_cmd: string, args: string[]) => @@ -59,7 +59,8 @@ describe('relay list-files cancellation', () => { await expect(promise).rejects.toSatisfy(isFileListingCancellation) expect(primaryProc.kill).toHaveBeenCalled() - expect(ignoredProc.kill).toHaveBeenCalled() + expect(ignoredProc.kill).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledTimes(1) // Late close events after cancellation must not fire anything. primaryProc.emit('close', null, 'SIGTERM') @@ -89,14 +90,16 @@ describe('relay list-files cancellation', () => { setTimeout(() => { ;(primaryProc.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\n') primaryProc.emit('close', 0, null) - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/out.js\n') - ignoredProc.emit('close', 0, null) + queueMicrotask(() => { + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/out.js\n') + ignoredProc.emit('close', 0, null) + }) }, 5) await expect(promise).resolves.toEqual(['src/index.ts', 'dist/out.js']) }) - it('listFilesWithGit kills both git passes and rejects when aborted mid-flight', async () => { + it('listFilesWithGit kills the active git pass and rejects when aborted mid-flight', async () => { const procs: ChildProcess[] = [] spawnMock.mockImplementation(() => { const proc = createMockProcess() @@ -107,12 +110,11 @@ describe('relay list-files cancellation', () => { const controller = new AbortController() const promise = listFilesWithGit('/remote/root', [], { signal: controller.signal }) - expect(procs).toHaveLength(2) + expect(procs).toHaveLength(1) controller.abort() await expect(promise).rejects.toSatisfy(isFileListingCancellation) expect(procs[0].kill).toHaveBeenCalled() - expect(procs[1].kill).toHaveBeenCalled() }) it('listFilesWithGit rejects without spawning when the signal is already aborted', async () => { diff --git a/src/relay/fs-handler-list-files-ignored.test.ts b/src/relay/fs-handler-list-files-ignored.test.ts index 1ca5c2f5327..acc1450cdc0 100644 --- a/src/relay/fs-handler-list-files-ignored.test.ts +++ b/src/relay/fs-handler-list-files-ignored.test.ts @@ -16,6 +16,10 @@ import { tmpdir } from 'node:os' import { listFilesWithGit } from './fs-handler-git-fallback' import { listFilesWithRg } from './fs-handler-list-files' import { searchWithRg } from './fs-handler-utils' +import { + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QUICK_OPEN_LISTING_MAX_RESULTS +} from '../shared/quick-open-listing-limits' const tempDirs: string[] = [] const SHA1 = '0123456789abcdef0123456789abcdef01234567' @@ -77,10 +81,12 @@ describe('relay quick open ignored file listing', () => { ;(primaryProc.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\n') primaryProc.emit('close', 0, null) - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\n') - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'node_modules/pkg/index.js\n') - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'packages/other/src/x.ts\n') - ignoredProc.emit('close', 0, null) + queueMicrotask(() => { + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\n') + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'node_modules/pkg/index.js\n') + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'packages/other/src/x.ts\n') + ignoredProc.emit('close', 0, null) + }) }, 10) await expect(promise).resolves.toEqual(['src/index.ts', 'dist/generated.js']) @@ -115,6 +121,46 @@ describe('relay quick open ignored file listing', () => { expect(callIndex).toBe(1) }) + it('bounds an omitted relay rg result limit without spawning the ignored pass', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + let callIndex = 0 + spawnMock.mockImplementation(() => (++callIndex === 1 ? primary : ignored)) + const promise = listFilesWithRg('/remote/root') + const paths = Array.from( + { length: QUICK_OPEN_LISTING_MAX_RESULTS + 1 }, + (_value, index) => `src/file-${index}.ts` + ) + + ;(primary.stdout as unknown as EventEmitter).emit('data', paths.join('\n')) + + const result = await promise + expect(result).toHaveLength(QUICK_OPEN_LISTING_MAX_RESULTS) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) + }) + + it('kills a relay rg scan whose residual path exceeds the field limit', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + let callIndex = 0 + spawnMock.mockImplementation(() => (++callIndex === 1 ? primary : ignored)) + const promise = listFilesWithRg('/remote/root') + + ;(primary.stdout as unknown as EventEmitter).emit( + 'data', + Buffer.alloc(QUICK_OPEN_LISTING_MAX_PATH_BYTES + 1, 0x61) + ) + + await expect(promise).rejects.toThrow( + `Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes` + ) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) + }) + it('git fallback ignored pass includes ignored non-env files', async () => { const root = await makeTempRoot() await writeRel(root, 'dist/generated.js') @@ -140,9 +186,11 @@ describe('relay quick open ignored file listing', () => { ) primaryProc.emit('close', 0, null) - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/\0') - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'packages/other/src/x.ts\0') - ignoredProc.emit('close', 0, null) + queueMicrotask(() => { + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/\0') + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'packages/other/src/x.ts\0') + ignoredProc.emit('close', 0, null) + }) }, 10) await expect(promise).resolves.toEqual(['dist/generated.js', 'src/index.ts', 'tab\tfile.txt']) @@ -179,6 +227,26 @@ describe('relay quick open ignored file listing', () => { expect(callIndex).toBe(1) }) + it('bounds an omitted relay Git result limit before spawning the ignored pass', async () => { + const primary = createMockProcess() + const ignored = createMockProcess() + let callIndex = 0 + spawnMock.mockImplementation(() => (++callIndex === 1 ? primary : ignored)) + const promise = listFilesWithGit('/remote/root') + const paths = Array.from( + { length: QUICK_OPEN_LISTING_MAX_RESULTS + 1 }, + (_value, index) => `src/file-${index}.ts` + ) + + ;(primary.stdout as unknown as EventEmitter).emit('data', paths.join('\0')) + + const result = await promise + expect(result).toHaveLength(QUICK_OPEN_LISTING_MAX_RESULTS) + expect(primary.kill).toHaveBeenCalled() + expect(ignored.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) + }) + it('does not let a discarded relay Git placeholder consume the result budget', async () => { const primaryProc = createMockProcess() spawnMock.mockReturnValue(primaryProc) @@ -219,7 +287,7 @@ describe('relay quick open ignored file listing', () => { `${staged('100644', 'README.md')}\0${staged('160000', 'packages/app')}\0packages/lib/\0` ) primaryProc.emit('close', 0, null) - ignoredProc.emit('close', 0, null) + queueMicrotask(() => ignoredProc.emit('close', 0, null)) }, 10) await expect(promise).resolves.toEqual([ @@ -247,9 +315,11 @@ describe('relay quick open ignored file listing', () => { ;(primaryProc.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\0') primaryProc.emit('close', 0, null) - // Entries streamed before the kill are kept alongside the primary pass. - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') - ignoredProc.emit('close', null, 'SIGTERM') + queueMicrotask(() => { + // Entries streamed before the kill are kept alongside the primary pass. + ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') + ignoredProc.emit('close', null, 'SIGTERM') + }) }, 10) await expect(promise).resolves.toEqual(['dist/generated.js', 'src/index.ts']) @@ -277,7 +347,7 @@ describe('relay quick open ignored file listing', () => { ;(primaryProc.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\0') primaryProc.emit('close', 0, null) - ignoredProc.emit('close', 128, null) + queueMicrotask(() => ignoredProc.emit('close', 128, null)) }, 10) await expect(promise).resolves.toEqual(['src/index.ts']) @@ -302,12 +372,10 @@ describe('relay quick open ignored file listing', () => { setTimeout(() => { ;(primaryProc.stdout as unknown as EventEmitter).emit('data', 'src/index.ts\0') primaryProc.emit('close', null, 'SIGTERM') - - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') - ignoredProc.emit('close', 0, null) }, 10) await expect(promise).rejects.toThrow('git ls-files killed by SIGTERM') + expect(callIndex).toBe(1) }) it('git fallback rejects when the primary pass exits non-zero', async () => { @@ -324,12 +392,10 @@ describe('relay quick open ignored file listing', () => { setTimeout(() => { primaryProc.emit('close', 128, null) - - ;(ignoredProc.stdout as unknown as EventEmitter).emit('data', 'dist/generated.js\0') - ignoredProc.emit('close', 0, null) }, 10) await expect(promise).rejects.toThrow('git ls-files exited with code 128') + expect(callIndex).toBe(1) }) it('git fallback rejects when a timed-out child does not emit close', async () => { @@ -355,15 +421,12 @@ describe('relay quick open ignored file listing', () => { expect(outcome).toContain('git ls-files timed out') expect(primaryProc.kill).toHaveBeenCalled() - expect(ignoredProc.kill).toHaveBeenCalled() + expect(ignoredProc.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) expect((primaryProc.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) expect((primaryProc.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) expect(primaryProc.listenerCount('error')).toBe(0) expect(primaryProc.listenerCount('close')).toBe(0) - expect((ignoredProc.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) - expect((ignoredProc.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) - expect(ignoredProc.listenerCount('error')).toBe(0) - expect(ignoredProc.listenerCount('close')).toBe(0) } finally { vi.useRealTimers() } @@ -392,15 +455,12 @@ describe('relay quick open ignored file listing', () => { expect(outcome).toBe('rejected:rg list timed out') expect(primaryProc.kill).toHaveBeenCalled() - expect(ignoredProc.kill).toHaveBeenCalled() + expect(ignoredProc.kill).not.toHaveBeenCalled() + expect(callIndex).toBe(1) expect((primaryProc.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) expect((primaryProc.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) expect(primaryProc.listenerCount('error')).toBe(0) expect(primaryProc.listenerCount('close')).toBe(0) - expect((ignoredProc.stdout as unknown as EventEmitter).listenerCount('data')).toBe(0) - expect((ignoredProc.stderr as unknown as EventEmitter).listenerCount('data')).toBe(0) - expect(ignoredProc.listenerCount('error')).toBe(0) - expect(ignoredProc.listenerCount('close')).toBe(0) } finally { vi.useRealTimers() } diff --git a/src/relay/fs-handler-list-files.ts b/src/relay/fs-handler-list-files.ts index c63a352786e..b6586c1af7e 100644 --- a/src/relay/fs-handler-list-files.ts +++ b/src/relay/fs-handler-list-files.ts @@ -23,6 +23,13 @@ import { shouldExcludeQuickOpenRelPath, shouldIncludeQuickOpenPath } from '../shared/quick-open-filter' +import { + createQuickOpenListingBudget, + QUICK_OPEN_LISTING_MAX_PATH_BYTES, + QuickOpenSubprocessPathAccumulator, + resolveQuickOpenResultLimit, + retainQuickOpenPath +} from '../shared/quick-open-listing-limits' export const LIST_FILES_TIMEOUT_MS = 25_000 @@ -35,8 +42,13 @@ export function listFilesWithRg( if (signal?.aborted) { return Promise.reject(fileListingCancellationError(signal)) } + const resultLimit = resolveQuickOpenResultLimit(maxResults) + if (resultLimit === 0) { + return Promise.resolve([]) + } return new Promise((resolve, reject) => { const files = new Set() + const listingBudget = createQuickOpenListingBudget() let done = false const children: { child: ChildProcess @@ -66,16 +78,15 @@ export function listFilesWithRg( if (shouldExcludeQuickOpenRelPath(relPath, excludePathPrefixes)) { return true } - files.add(relPath) - if (maxResults !== undefined && files.size >= maxResults) { - finishAtLimit() + if (files.size < resultLimit) { + retainQuickOpenPath(files, relPath, listingBudget) } return true } const runPass = (args: string[]): Promise => new Promise((passResolve, passReject) => { - let passBuf = '' + const paths = new QuickOpenSubprocessPathAccumulator(0x0a) let passDone = false let passFileCount = 0 // --no-messages: permission-denied noise on the remote (e.g. .ssh, @@ -104,7 +115,7 @@ export function listFilesWithRg( return } passDone = true - passBuf = '' + paths.clear() cleanup() passReject(error) } @@ -113,6 +124,7 @@ export function listFilesWithRg( return } passDone = true + paths.clear() cleanup() passResolve() } @@ -129,21 +141,30 @@ export function listFilesWithRg( rejectPass(new Error('rg list timed out')) }, LIST_FILES_TIMEOUT_MS) - function handleStdoutData(chunk: string): void { - passBuf += chunk - let start = 0 - let idx = passBuf.indexOf('\n', start) - while (idx !== -1) { - if (processLine(passBuf.substring(start, idx))) { - passFileCount++ + function failForOutput(error: unknown): void { + child.kill() + rejectPass(error instanceof Error ? error : new Error(String(error))) + } + function handleStdoutData(chunk: Buffer | string): void { + try { + const outcome = paths.push(chunk, (path) => { + if (processLine(path)) { + passFileCount++ + } + return files.size < resultLimit + }) + if (outcome === 'stopped') { + finishAtLimit() + } else if (outcome === 'path-too-large') { + failForOutput( + new Error( + `Quick Open file path exceeded ${QUICK_OPEN_LISTING_MAX_PATH_BYTES} bytes` + ) + ) } - if (done) { - return - } - start = idx + 1 - idx = passBuf.indexOf('\n', start) + } catch (error) { + failForOutput(error) } - passBuf = start < passBuf.length ? passBuf.substring(start) : '' } function handleStderrData(): void { /* drain to prevent backpressure stalls */ @@ -163,10 +184,18 @@ export function listFilesWithRg( return } // Flush residual line only on clean exit. - if (passBuf) { - if (processLine(passBuf)) { + try { + const trailingPath = paths.finish() + if (trailingPath && processLine(trailingPath)) { passFileCount++ } + if (files.size >= resultLimit) { + finishAtLimit() + return + } + } catch (error) { + failForOutput(error) + return } // exit 0 = matches found, 1 = no files (still success for --files). // exit 2 is documented as "a subdirectory could not be searched" @@ -182,7 +211,6 @@ export function listFilesWithRg( } } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) @@ -212,7 +240,7 @@ export function listFilesWithRg( done = true signal?.removeEventListener('abort', onAbort) killSurvivors('rg list reached bounded result limit') - resolve(Array.from(files).slice(0, maxResults)) + resolve(Array.from(files).slice(0, resultLimit)) } // Why: a cancelled scan (workspace switch, superseded request) must stop @@ -228,14 +256,11 @@ export function listFilesWithRg( } signal?.addEventListener('abort', onAbort, { once: true }) - const passes = - maxResults === undefined - ? Promise.all([runPass(primary), runPass(ignoredPass)]) - : // Why: deterministic primary-first budgeting prevents a large ignored - // tree from starving ordinary source paths on a remote host. - runPass(primary).then(() => - files.size < maxResults ? runPass(ignoredPass) : Promise.resolve() - ) + // Why: deterministic primary-first budgeting prevents a large ignored + // tree from starving ordinary source paths on a remote host. + const passes = runPass(primary).then(() => + files.size < resultLimit ? runPass(ignoredPass) : Promise.resolve() + ) passes .then(() => { @@ -244,7 +269,7 @@ export function listFilesWithRg( } done = true signal?.removeEventListener('abort', onAbort) - resolve(Array.from(files)) + resolve(Array.from(files).slice(0, resultLimit)) }) .catch((err) => { if (done) { diff --git a/src/relay/fs-handler-stream.test.ts b/src/relay/fs-handler-stream.test.ts index c7624afce65..7490fa7d2f9 100644 --- a/src/relay/fs-handler-stream.test.ts +++ b/src/relay/fs-handler-stream.test.ts @@ -5,19 +5,25 @@ import type { RelayDispatcher } from './dispatcher' import { STREAM_CHUNK_SIZE } from './protocol' import * as fs from 'node:fs/promises' import * as path from 'node:path' -import { mkdtempSync, writeFileSync } from 'node:fs' +import { mkdtempSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' vi.mock('@parcel/watcher', () => ({ subscribe: vi.fn() })) -type Notification = { method: string; params?: Record } +type Notification = { method: string; params?: Record; clientId?: number } function createMockDispatcher() { const requestHandlers = new Map< string, - (params: Record, context?: { isStale: () => boolean }) => Promise + ( + params: Record, + context?: { clientId: number; isStale: () => boolean } + ) => Promise + >() + const notificationHandlers = new Map< + string, + (params: Record, context: { clientId: number; isStale: () => boolean }) => void >() - const notificationHandlers = new Map) => void>() const notifications: Notification[] = [] return { onRequest: vi.fn( @@ -28,33 +34,55 @@ function createMockDispatcher() { requestHandlers.set(method, handler as never) } ), - onNotification: vi.fn((method: string, handler: (params: Record) => void) => { - notificationHandlers.set(method, handler) - }), + onNotification: vi.fn( + ( + method: string, + handler: ( + params: Record, + context: { clientId: number; isStale: () => boolean } + ) => void + ) => { + notificationHandlers.set(method, handler) + } + ), notify: vi.fn((method: string, params?: Record) => { notifications.push({ method, params }) }), - notifyBulk: vi.fn(async (method: string, params?: Record): Promise => { - notifications.push({ method, params }) - }), + notifyClient: vi.fn( + (clientId: number, method: string, params?: Record): void => { + notifications.push({ method, params, clientId }) + } + ), + notifyBulk: vi.fn( + async ( + method: string, + params?: Record, + options?: { clientId?: number } + ): Promise => { + notifications.push({ method, params, clientId: options?.clientId }) + } + ), _notifications: notifications, callRequest( method: string, params: Record = {}, - context?: { isStale: () => boolean } + context?: { clientId?: number; isStale: () => boolean } ) { const handler = requestHandlers.get(method) if (!handler) { throw new Error(`No handler for ${method}`) } - return handler(params, context) + return handler( + params, + context ? { clientId: context.clientId ?? 1, isStale: context.isStale } : undefined + ) }, - callNotification(method: string, params: Record = {}) { + callNotification(method: string, params: Record = {}, clientId = 1) { const handler = notificationHandlers.get(method) if (!handler) { throw new Error(`No handler for ${method}`) } - handler(params) + handler(params, { clientId, isStale: () => false }) } } } @@ -100,6 +128,16 @@ async function waitFor(predicate: () => boolean, timeoutMs = 5000): Promise { let dispatcher: ReturnType let handler: FsHandler @@ -116,19 +154,39 @@ describe('FsHandler readFileStream', () => { await fs.rm(tmpDir, { recursive: true, force: true }) }) + it('rejects a terminal-artifact raster dimension bomb', async () => { + const filePath = path.join(tmpDir, 'artifact-bomb.png') + writeFileSync(filePath, pngContent(24, 32_769, 1)) + + await expect( + dispatcher.callRequest('fs.readTerminalArtifact', { + filePath, + expectedRealPath: await fs.realpath(filePath), + maxBytes: 512 * 1024 + }) + ).rejects.toThrow('Image dimensions exceed the preview safety limit') + }) + it('streams a binary file in chunked notifications', async () => { const filePath = path.join(tmpDir, 'image.png') - const content = Buffer.alloc(300 * 1024, 0x42) + const content = pngContent(300 * 1024) + const ownerClientId = 7 writeFileSync(filePath, content) const meta = (await dispatcher.callRequest( 'fs.readFileStream', { filePath }, - { isStale: () => false } - )) as { streamId: number; totalSize: number; resultEncoding: string } + { clientId: ownerClientId, isStale: () => false } + )) as { + streamId: number + totalSize: number + resultEncoding: string + imageDimensions: { width: number; height: number } + } expect(meta.streamId).toBeDefined() expect(meta.totalSize).toBe(content.length) expect(meta.resultEncoding).toBe('base64') + expect(meta.imageDimensions).toEqual({ width: 1, height: 1 }) await waitFor(() => collectStream(dispatcher).end !== null) const { chunks, end, err } = collectStream(dispatcher) @@ -136,11 +194,48 @@ describe('FsHandler readFileStream', () => { expect(end).toEqual({ streamId: meta.streamId }) const reassembled = Buffer.concat(chunks.map((c) => Buffer.from(c.data, 'base64'))) expect(reassembled.equals(content)).toBe(true) + expect( + dispatcher._notifications.filter((notification) => notification.method === 'fs.streamEnd') + ).toEqual([ + { + method: 'fs.streamEnd', + params: { streamId: meta.streamId }, + clientId: ownerClientId + } + ]) + }) + + it('sends a stream error only to the requesting relay client', async () => { + const filePath = path.join(tmpDir, 'truncated.png') + const ownerClientId = 9 + writeFileSync(filePath, pngContent(STREAM_CHUNK_SIZE)) + + const meta = (await dispatcher.callRequest( + 'fs.readFileStream', + { filePath }, + { clientId: ownerClientId, isStale: () => false } + )) as { streamId: number } + truncateSync(filePath, 0) + + await waitFor(() => collectStream(dispatcher).err !== null) + expect( + dispatcher._notifications.filter((notification) => notification.method === 'fs.streamError') + ).toEqual([ + { + method: 'fs.streamError', + params: { + streamId: meta.streamId, + code: 'ESTREAMTRUNCATED', + message: `File truncated mid-stream: expected ${STREAM_CHUNK_SIZE}, got 0` + }, + clientId: ownerClientId + } + ]) }) it('fills protocol chunks when fs.read returns short before EOF', async () => { const filePath = path.join(tmpDir, 'short-read.png') - const content = Buffer.alloc(STREAM_CHUNK_SIZE + 17, 0x42) + const content = pngContent(STREAM_CHUNK_SIZE + 17) writeFileSync(filePath, content) const sampleHandle = await fs.open(filePath, 'r') @@ -242,7 +337,7 @@ describe('FsHandler readFileStream', () => { it('exits the pump and emits no further chunks when isStale flips', async () => { const filePath = path.join(tmpDir, 'big.png') - const content = Buffer.alloc(800 * 1024, 0x42) + const content = pngContent(800 * 1024) writeFileSync(filePath, content) let stale = false @@ -264,7 +359,7 @@ describe('FsHandler readFileStream', () => { it('honors fs.cancelStream by stopping the pump and emitting no end frame', async () => { const filePath = path.join(tmpDir, 'cancel.png') - writeFileSync(filePath, Buffer.alloc(2 * 1024 * 1024, 0x42)) + writeFileSync(filePath, pngContent(2 * 1024 * 1024)) const meta = (await dispatcher.callRequest( 'fs.readFileStream', @@ -283,7 +378,7 @@ describe('FsHandler readFileStream', () => { it('parks the pump at the ack credit window and resumes on fs.streamAck', async () => { const filePath = path.join(tmpDir, 'paced.png') - const content = Buffer.alloc(1536 * 1024, 0x42) // 6 chunks + const content = pngContent(1536 * 1024) // 6 chunks writeFileSync(filePath, content) const meta = (await dispatcher.callRequest( @@ -298,6 +393,11 @@ describe('FsHandler readFileStream', () => { expect(collectStream(dispatcher).chunks.length).toBe(4) expect(collectStream(dispatcher).end).toBeNull() + // A future ACK must not mint credit for chunks the relay has not sent. + dispatcher.callNotification('fs.streamAck', { streamId: meta.streamId, seq: 10_000 }) + await flush(10) + expect(collectStream(dispatcher).chunks.length).toBe(4) + // Ack one chunk → exactly one more is admitted. dispatcher.callNotification('fs.streamAck', { streamId: meta.streamId, seq: 0 }) await waitFor(() => collectStream(dispatcher).chunks.length === 5) @@ -315,9 +415,38 @@ describe('FsHandler readFileStream', () => { expect(reassembled.equals(content)).toBe(true) }) + it('ignores stream credit and cancellation from a different relay client', async () => { + const filePath = path.join(tmpDir, 'owned-paced.png') + writeFileSync(filePath, pngContent(4 * 1024 * 1024)) + const ownerClientId = 7 + const meta = (await dispatcher.callRequest( + 'fs.readFileStream', + { filePath, flowControl: 'ack' }, + { clientId: ownerClientId, isStale: () => false } + )) as { streamId: number } + + await waitFor(() => collectStream(dispatcher).chunks.length === 4) + dispatcher.callNotification('fs.streamAck', { streamId: meta.streamId, seq: 10_000 }, 8) + dispatcher.callNotification('fs.cancelStream', { streamId: meta.streamId }, 8) + await flush(10) + expect(collectStream(dispatcher).chunks).toHaveLength(4) + + dispatcher.callNotification('fs.streamAck', { streamId: meta.streamId, seq: 0 }, ownerClientId) + await waitFor(() => collectStream(dispatcher).chunks.length === 5) + dispatcher.callNotification('fs.cancelStream', { streamId: meta.streamId }, ownerClientId) + + const registry = (handler as unknown as { streamRegistry: { size(): number } }).streamRegistry + await waitFor(() => registry.size() === 0) + expect( + dispatcher._notifications + .filter((notification) => notification.method.startsWith('fs.stream')) + .every((notification) => notification.clientId === ownerClientId) + ).toBe(true) + }) + it('releases a pump parked on the ack window when the stream is cancelled', async () => { const filePath = path.join(tmpDir, 'parked-cancel.png') - writeFileSync(filePath, Buffer.alloc(4 * 1024 * 1024, 0x42)) // 16 chunks + writeFileSync(filePath, pngContent(4 * 1024 * 1024)) // 16 chunks const meta = (await dispatcher.callRequest( 'fs.readFileStream', @@ -340,7 +469,7 @@ describe('FsHandler readFileStream', () => { const paths: string[] = [] for (let i = 0; i < 17; i++) { const p = path.join(tmpDir, `s${i}.png`) - writeFileSync(p, Buffer.alloc(8 * 1024 * 1024, 0x42)) + writeFileSync(p, pngContent(8 * 1024 * 1024)) paths.push(p) } @@ -370,4 +499,15 @@ describe('FsHandler readFileStream', () => { } await flush(50) }, 20_000) + + it('rejects an oversized raster before registering or pumping a stream', async () => { + const filePath = path.join(tmpDir, 'bomb.png') + writeFileSync(filePath, pngContent(24, 32_769, 1)) + + await expect( + dispatcher.callRequest('fs.readFileStream', { filePath }, { isStale: () => false }) + ).rejects.toThrow('Image dimensions exceed the preview safety limit') + await flush() + expect(collectStream(dispatcher).chunks).toHaveLength(0) + }) }) diff --git a/src/relay/fs-handler-terminal-artifact.ts b/src/relay/fs-handler-terminal-artifact.ts index 02af753f134..2f5829ad0b7 100644 --- a/src/relay/fs-handler-terminal-artifact.ts +++ b/src/relay/fs-handler-terminal-artifact.ts @@ -10,6 +10,7 @@ import { MAX_PREVIEWABLE_BINARY_SIZE, MAX_TEXT_FILE_SIZE } from './fs-handler-utils' +import { assertRasterImagePreviewWithinLimits } from '../shared/raster-image-preview-limits' type TerminalArtifactStat = { size: number @@ -43,7 +44,14 @@ export async function readVerifiedTerminalArtifact(params: Record { const rgArgs = buildRgArgs(query, rootPath, opts) const acc = createAccumulator() - let buffer = '' + const stdoutLines = new SearchSubprocessLineAccumulator() let resolved = false // Why: spawn can throw synchronously on invalid options (e.g. bad cwd), @@ -134,12 +135,11 @@ export function searchWithRg( } } - function handleStdoutData(chunk: string): void { - buffer += chunk - const lines = buffer.split('\n') - buffer = lines.pop() ?? '' - for (const line of lines) { - processLine(line) + function handleStdoutData(chunk: Buffer): void { + if (!stdoutLines.push(chunk, processLine)) { + acc.truncated = true + child.kill() + resolveOnce() } } @@ -152,13 +152,13 @@ export function searchWithRg( } function handleClose(): void { - if (buffer) { - processLine(buffer) + const trailingLine = stdoutLines.finish() + if (trailingLine !== null) { + processLine(trailingLine) } resolveOnce() } - child.stdout!.setEncoding('utf-8') child.stdout!.on('data', handleStdoutData) child.stderr!.on('data', handleStderrData) child.once('error', handleError) diff --git a/src/relay/fs-handler.test.ts b/src/relay/fs-handler.test.ts index 23dd28c35cb..3c8bbd847b9 100644 --- a/src/relay/fs-handler.test.ts +++ b/src/relay/fs-handler.test.ts @@ -17,6 +17,8 @@ vi.mock('@parcel/watcher', () => ({ subscribe: mockSubscribe })) +const PNG_1X1 = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB', 'base64') + function createMockDispatcher() { const requestHandlers = new Map< string, @@ -138,6 +140,7 @@ describe('FsHandler', () => { it('registers all expected handlers', () => { const methods = Array.from(dispatcher._requestHandlers.keys()) expect(methods).toContain('fs.readDir') + expect(methods).toContain('fs.readDirBounded') expect(methods).toContain('fs.readFile') expect(methods).toContain('fs.tempDir') expect(methods).toContain('fs.writeFile') @@ -152,6 +155,7 @@ describe('FsHandler', () => { expect(methods).toContain('fs.realpath') expect(methods).toContain('fs.search') expect(methods).toContain('fs.listFiles') + expect(methods).toContain('fs.listMarkdownDocuments') expect(methods).toContain('fs.workspaceSpaceScan') expect(methods).toContain('fs.watch') expect(methods).toContain('fs.unwatchAndWait') @@ -164,12 +168,12 @@ describe('FsHandler', () => { await expect(dispatcher.callRequest('fs.tempDir')).resolves.toBe(tmpdir()) }) - it('readDir returns sorted entries with directories first', async () => { + it('bounded readDir returns sorted entries with directories first', async () => { mkdirSync(path.join(tmpDir, 'subdir')) writeFileSync(path.join(tmpDir, 'file.txt'), 'hello') writeFileSync(path.join(tmpDir, 'aaa.txt'), 'world') - const result = (await dispatcher.callRequest('fs.readDir', { dirPath: tmpDir })) as { + const result = (await dispatcher.callRequest('fs.readDirBounded', { dirPath: tmpDir })) as { name: string isDirectory: boolean }[] @@ -239,17 +243,19 @@ describe('FsHandler', () => { it('readFile returns base64 for image files', async () => { const filePath = path.join(tmpDir, 'test.png') - writeFileSync(filePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) + writeFileSync(filePath, PNG_1X1) const result = (await dispatcher.callRequest('fs.readFile', { filePath })) as { content: string isBinary: boolean isImage: boolean mimeType: string + imageDimensions: { width: number; height: number } } expect(result.isBinary).toBe(true) expect(result.isImage).toBe(true) expect(result.mimeType).toBe('image/png') + expect(result.imageDimensions).toEqual({ width: 1, height: 1 }) expect(result.content).toBeTruthy() }) diff --git a/src/relay/fs-handler.ts b/src/relay/fs-handler.ts index 9c146af1450..c741d8f9e99 100644 --- a/src/relay/fs-handler.ts +++ b/src/relay/fs-handler.ts @@ -1,9 +1,8 @@ /* eslint-disable max-lines -- Why: relay filesystem request handling shares path expansion, file IO, search, streaming reads, and Space scans. */ -import { readdir, writeFile, stat, lstat, mkdir, rename, cp, rm, realpath } from 'node:fs/promises' +import { writeFile, stat, lstat, mkdir, rename, cp, rm, realpath } from 'node:fs/promises' import { execFile } from 'node:child_process' import { tmpdir } from 'node:os' -import { join } from 'node:path' import type { RelayDispatcher, RequestContext } from './dispatcher' import type { RelayContext } from './context' // Why: RelayContext is accepted in the constructor for protocol back-compat @@ -36,24 +35,27 @@ import { buildRelayCommandEnv } from './relay-command-env' import { assertNoClobberRenameDestinationAvailable } from '../shared/filesystem-rename-collision' import { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' import type { RelayWatcherProcessPool } from './relay-watcher-process-pool' +import { + MOBILE_FILE_DIRECTORY_MAX_ENTRIES, + MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES +} from '../shared/mobile-file-directory-limit' +import { readMobileRelayDirectory } from './mobile-file-directory-reader' +import { readRelayFilesystemDirectory } from './filesystem-directory-reader' +import type { FilesystemDirectoryListingLimits } from '../shared/filesystem-directory-listing-limit' +import { listRelayMarkdownDocumentPaths } from './markdown-document-listing' +import { resolveQuickOpenResultLimit } from '../shared/quick-open-listing-limits' -async function isDirectoryEntry( - dirPath: string, - entry: { name: string; isDirectory(): boolean; isSymbolicLink(): boolean } -): Promise { - if (entry.isDirectory()) { - return true +function readDirectoryLimitsFromParams( + params: Record +): Partial { + const limits: Partial = {} + if (typeof params.maxEntries === 'number') { + limits.maxEntries = params.maxEntries } - if (!entry.isSymbolicLink()) { - return false - } - try { - // Why: the file explorer needs target type for symlinked directories so a - // workspace link to an external folder expands instead of opening as a file. - return (await stat(join(dirPath, entry.name))).isDirectory() - } catch { - return false + if (typeof params.maxRetainedBytes === 'number') { + limits.maxRetainedBytes = params.maxRetainedBytes } + return limits } function fileStatFromLstat(stats: Awaited>) { @@ -102,6 +104,7 @@ export class FsHandler { private registerHandlers(): void { this.dispatcher.onRequest('fs.readDir', (p) => this.readDir(p)) + this.dispatcher.onRequest('fs.readDirBounded', (p) => this.readDir(p)) this.dispatcher.onRequest('fs.readFile', (p) => this.readFile(p)) this.dispatcher.onRequest('fs.readFileStream', (p, c) => this.readFileStream(p, c)) this.dispatcher.onRequest('fs.readTerminalArtifact', (p) => this.readTerminalArtifact(p)) @@ -120,6 +123,9 @@ export class FsHandler { this.dispatcher.onRequest('fs.realpath', (p) => this.realpath(p)) this.dispatcher.onRequest('fs.search', (p) => this.search(p)) this.dispatcher.onRequest('fs.listFiles', (p, c) => this.listFiles(p, c)) + this.dispatcher.onRequest('fs.listMarkdownDocuments', (p, c) => + listRelayMarkdownDocumentPaths(expandTilde(p.rootPath as string), c.signal) + ) this.dispatcher.onRequest('fs.workspaceSpaceScan', (p, c) => this.workspaceSpaceScan(p, c)) this.dispatcher.onRequest('fs.watch', (p, context) => this.watchRegistry.watch( @@ -134,21 +140,25 @@ export class FsHandler { this.dispatcher.onNotification('fs.unwatch', (p, context) => this.watchRegistry.unwatch(expandTilde(p.rootPath as string), context) ) - this.dispatcher.onNotification('fs.cancelStream', (p) => this.cancelStream(p)) - this.dispatcher.onNotification('fs.streamAck', (p) => this.streamAck(p)) + this.dispatcher.onNotification('fs.cancelStream', (p, context) => this.cancelStream(p, context)) + this.dispatcher.onNotification('fs.streamAck', (p, context) => this.streamAck(p, context)) } private async readDir(params: Record) { const dirPath = expandTilde(params.dirPath as string) - const entries = await readdir(dirPath, { withFileTypes: true }) - const mapped = await Promise.all( - entries.map(async (entry) => ({ - name: entry.name, - isDirectory: await isDirectoryEntry(dirPath, entry), - isSymlink: entry.isSymbolicLink() - })) - ) - return mapped.sort((a, b) => { + if ( + params.maxEntries === MOBILE_FILE_DIRECTORY_MAX_ENTRIES && + params.maxRetainedBytes === MOBILE_FILE_DIRECTORY_MAX_RETAINED_BYTES + ) { + return this.sortDirectoryEntries(await readMobileRelayDirectory(dirPath)) + } + return readRelayFilesystemDirectory(dirPath, readDirectoryLimitsFromParams(params)) + } + + private sortDirectoryEntries( + entries: T[] + ): T[] { + return entries.sort((a, b) => { if (a.isDirectory !== b.isDirectory) { return a.isDirectory ? -1 : 1 } @@ -184,18 +194,18 @@ export class FsHandler { return tmpdir() } - private cancelStream(params: Record): void { + private cancelStream(params: Record, context: RequestContext): void { const streamId = params.streamId as number | undefined if (typeof streamId === 'number') { - this.streamRegistry.abort(streamId) + this.streamRegistry.abort(streamId, context.clientId) } } - private streamAck(params: Record): void { + private streamAck(params: Record, context: RequestContext): void { const streamId = params.streamId as number | undefined const seq = params.seq as number | undefined if (typeof streamId === 'number' && typeof seq === 'number') { - this.streamRegistry.recordAck(streamId, seq) + this.streamRegistry.recordAck(streamId, seq, context.clientId) } } @@ -356,12 +366,13 @@ export class FsHandler { private listFiles(params: Record, context?: RequestContext): Promise { const rootPath = expandTilde(params.rootPath as string) - const maxResults = + const requestedMaxResults = typeof params.maxResults === 'number' && Number.isInteger(params.maxResults) && params.maxResults > 0 - ? Math.min(params.maxResults, 20_001) + ? params.maxResults : undefined + const maxResults = resolveQuickOpenResultLimit(requestedMaxResults) // Why: the main-to-relay RPC adds excludePaths so nested linked worktrees // don't get double-scanned. The shared helper validates the shape and // normalizes into root-relative prefixes; malformed input yields [] so @@ -382,7 +393,7 @@ export class FsHandler { rootPath: string, excludePathPrefixes: string[], signal: AbortSignal, - maxResults?: number + maxResults: number ): Promise { const rgAvailable = await checkRgAvailable() throwIfFileListingCancelled(signal) diff --git a/src/relay/fs-list-files-cancel.integration.test.ts b/src/relay/fs-list-files-cancel.integration.test.ts index 588ddb5fe51..4f9118e608d 100644 --- a/src/relay/fs-list-files-cancel.integration.test.ts +++ b/src/relay/fs-list-files-cancel.integration.test.ts @@ -27,7 +27,7 @@ const { fakeListFiles } = vi.hoisted(() => { ( rootPath: string, _excludes: readonly string[] = [], - options: { signal?: AbortSignal } = {} + options: { signal?: AbortSignal; maxResults?: number } = {} ) => new Promise((resolve, reject) => { scans.push({ rootPath, signal: options.signal, resolve }) @@ -67,6 +67,7 @@ import { RelayDispatcher } from './dispatcher' import { RelayContext } from './context' import { FsHandler } from './fs-handler' import { LIST_FILES_SUPERSEDED_MESSAGE } from './fs-list-files-scan-coordinator' +import { QUICK_OPEN_LISTING_MAX_RESULTS } from '../shared/quick-open-listing-limits' async function flushPipe(): Promise { // The in-memory pipe defers each hop with setImmediate; a few macrotask @@ -123,6 +124,7 @@ describe('Integration: cancellable fs.listFiles (#7721)', () => { const scanPromise = mux.request('fs.listFiles', { rootPath: '/big/workspace' }) await flushPipe() expect(fakeListFiles.scans).toHaveLength(1) + expect(fakeListFiles.mock.calls[0][2]?.maxResults).toBe(QUICK_OPEN_LISTING_MAX_RESULTS) // The interactive request must complete while the scan is still pending. const entries = (await mux.request('fs.readDir', { dirPath: tmpDir })) as { name: string }[] diff --git a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts index 8cca2787891..3875d22a282 100644 --- a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts +++ b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts @@ -37,6 +37,16 @@ const FRAMED_CHUNK_BYTES = Math.ceil((STREAM_CHUNK_SIZE * 4) / 3) + 512 // Node pipe/socket sinks report saturation via write() === false past the HWM. const SINK_HIGH_WATER_MARK = 64 * 1024 +function randomPngBytes(size: number): Buffer { + const content = randomBytes(size) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(content) + content.writeUInt32BE(13, 8) + content.write('IHDR', 12, 'ascii') + content.writeUInt32BE(1, 16) + content.writeUInt32BE(1, 20) + return content +} + async function waitUntil( predicate: () => boolean, what: string, @@ -182,7 +192,7 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { const harness = createHarness({ congested: true }) try { const filePath = path.join(tmpDir, 'big.png') - const original = randomBytes(3 * 1024 * 1024) // 12 chunks + const original = randomPngBytes(3 * 1024 * 1024) // 12 chunks writeFileSync(filePath, original) // Relay-side fake PTY: echoes input back immediately, mirroring @@ -222,11 +232,17 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { const harness = createHarness({ congested: false }) try { const filePath = path.join(tmpDir, 'big.png') - writeFileSync(filePath, randomBytes(3 * 1024 * 1024)) // 12 chunks + writeFileSync(filePath, randomPngBytes(3 * 1024 * 1024)) // 12 chunks const receivedSeqs: number[] = [] + let acknowledgeChunks = false + let streamId: number | null = null harness.mux.onNotificationByMethod('fs.streamChunk', (params) => { - receivedSeqs.push(params.seq as number) + const seq = params.seq as number + receivedSeqs.push(seq) + if (acknowledgeChunks && streamId !== null) { + harness.mux.notify('fs.streamAck', { streamId, seq }) + } }) let streamEnded = false harness.mux.onNotificationByMethod('fs.streamEnd', () => { @@ -240,6 +256,7 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { filePath, flowControl: 'ack' })) as { streamId: number } + streamId = metadata.streamId await waitUntil(() => receivedSeqs.length > 0, 'first chunk received') await waitUntilSettled(() => receivedSeqs.length) @@ -250,8 +267,9 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { expect(streamEnded).toBe(false) // Acking releases the window and the stream completes. + acknowledgeChunks = true const totalChunks = 12 - for (let seq = 0; seq < totalChunks; seq += 1) { + for (const seq of receivedSeqs) { harness.mux.notify('fs.streamAck', { streamId: metadata.streamId, seq }) } await waitUntil(() => streamEnded, 'stream completed after acks') @@ -265,7 +283,7 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { const harness = createHarness({ congested: false }) try { const filePath = path.join(tmpDir, 'legacy.png') - const original = randomBytes(1024 * 1024 + 12345) + const original = randomPngBytes(1024 * 1024 + 12345) writeFileSync(filePath, original) const chunks = new Map() diff --git a/src/relay/fs-stream-registry.ts b/src/relay/fs-stream-registry.ts index 2bca76c9495..09b687aa714 100644 --- a/src/relay/fs-stream-registry.ts +++ b/src/relay/fs-stream-registry.ts @@ -3,7 +3,10 @@ import { MAX_CONCURRENT_STREAMS, RelayErrorCode, STREAM_ACK_STALL_RECHECK_MS } f type StreamEntry = { handle: FileHandle + ownerClientId: number aborted: boolean + /** Highest chunk seq admitted to the outbound bulk lane. */ + sentThroughSeq: number /** Highest chunk seq the client acknowledged (in-order; -1 = none yet). */ ackedThroughSeq: number /** Pumps parked on the ack credit window. Woken by acks, abort, release, @@ -22,23 +25,25 @@ export class RelayStreamRegistry { private streams = new Map() private nextId = 1 - register(handle: FileHandle): number { + register(handle: FileHandle, ownerClientId: number): number { if (this.streams.size >= MAX_CONCURRENT_STREAMS) { throw new TooManyStreamsError() } const streamId = this.nextId++ this.streams.set(streamId, { handle, + ownerClientId, aborted: false, + sentThroughSeq: -1, ackedThroughSeq: -1, ackWaiters: new Set() }) return streamId } - abort(streamId: number): void { + abort(streamId: number, clientId: number): void { const entry = this.streams.get(streamId) - if (entry) { + if (entry?.ownerClientId === clientId) { entry.aborted = true this.wakeAckWaiters(entry) } @@ -52,9 +57,22 @@ export class RelayStreamRegistry { return this.streams.get(streamId) } - recordAck(streamId: number, seq: number): void { + recordSent(streamId: number, seq: number): void { const entry = this.streams.get(streamId) - if (!entry || typeof seq !== 'number' || !Number.isFinite(seq)) { + if (entry && Number.isSafeInteger(seq) && seq === entry.sentThroughSeq + 1) { + entry.sentThroughSeq = seq + } + } + + recordAck(streamId: number, seq: number, clientId: number): void { + const entry = this.streams.get(streamId) + if ( + !entry || + entry.ownerClientId !== clientId || + !Number.isSafeInteger(seq) || + seq < 0 || + seq > entry.sentThroughSeq + ) { return } if (seq > entry.ackedThroughSeq) { @@ -129,7 +147,11 @@ export class RelayStreamRegistry { // cleanly on the next iteration boundary instead of seeing EBADF when // release closes the handle out from under an in-flight read. for (const id of this.streams.keys()) { - this.abort(id) + const entry = this.streams.get(id) + if (entry) { + entry.aborted = true + this.wakeAckWaiters(entry) + } } const ids = Array.from(this.streams.keys()) await Promise.all(ids.map((id) => this.release(id))) diff --git a/src/relay/git-handler-status-ops-bounds.test.ts b/src/relay/git-handler-status-ops-bounds.test.ts new file mode 100644 index 00000000000..8849ad9b334 --- /dev/null +++ b/src/relay/git-handler-status-ops-bounds.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { mkdir, mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { resolveGitDir } from './git-handler-status-ops' + +describe('relay resolveGitDir metadata bounds', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + async function makeWorktree(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-relay-resolve-git-dir-')) + roots.push(root) + const worktreePath = join(root, 'checkout') + await mkdir(worktreePath) + return worktreePath + } + + it('preserves a normal linked-worktree pointer', async () => { + const worktreePath = await makeWorktree() + await writeFile(join(worktreePath, '.git'), 'gitdir: ../common/.git/worktrees/checkout\n') + + await expect(resolveGitDir(worktreePath)).resolves.toBe( + join(worktreePath, '..', 'common', '.git', 'worktrees', 'checkout') + ) + }) + + it('falls back to the .git path for an oversized sparse pointer', async () => { + const worktreePath = await makeWorktree() + const dotGitPath = join(worktreePath, '.git') + await writeFile(dotGitPath, 'x') + await truncate(dotGitPath, 64 * 1024 + 1) + + await expect(resolveGitDir(worktreePath)).resolves.toBe(dotGitPath) + }) +}) diff --git a/src/relay/git-handler-status-ops.ts b/src/relay/git-handler-status-ops.ts index 1bd05821f50..e802b31677f 100644 --- a/src/relay/git-handler-status-ops.ts +++ b/src/relay/git-handler-status-ops.ts @@ -4,7 +4,6 @@ */ import * as path from 'node:path' import { existsSync } from 'node:fs' -import { readFile } from 'node:fs/promises' import { parseUnmergedEntry } from './git-handler-utils' import type { GitExec } from './git-handler-ops' import type { RelayGitStreamExec } from './git-stdout-stream' @@ -24,11 +23,16 @@ import { clearGitStatusLineStatsCacheKey, reuseOrRecomputeGitStatusLineStats } from '../shared/git-status-line-stats-cache' +import { readNodeFileWithinLimit } from '../shared/node-bounded-file-reader' + +const MAX_GIT_POINTER_FILE_BYTES = 64 * 1024 export async function resolveGitDir(worktreePath: string): Promise { const dotGitPath = path.join(worktreePath, '.git') try { - const contents = await readFile(dotGitPath, 'utf-8') + const contents = ( + await readNodeFileWithinLimit(dotGitPath, MAX_GIT_POINTER_FILE_BYTES) + ).buffer.toString('utf-8') const match = contents.match(/^gitdir:\s*(.+)\s*$/m) if (match) { return path.resolve(worktreePath, match[1]) diff --git a/src/relay/git-handler-stream-serialization-memory.test.ts b/src/relay/git-handler-stream-serialization-memory.test.ts new file mode 100644 index 00000000000..0fae5409820 --- /dev/null +++ b/src/relay/git-handler-stream-serialization-memory.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from 'vitest' +import { RelayContext } from './context' +import { RelayDispatcher, type RequestContext } from './dispatcher' +import { GitHandler } from './git-handler' +import { GIT_RESPONSE_STREAM_THRESHOLD, type GitResponseStreamMarker } from './protocol' + +type StreamableGitHandler = { + maybeStreamResponse( + result: unknown, + params: Record, + context: RequestContext + ): GitResponseStreamMarker +} + +describe('GitHandler streamed response serialization', () => { + it('registers a large serialized string without allocating a full payload Buffer', () => { + const dispatcher = new RelayDispatcher(() => true) + const handler = new GitHandler(dispatcher, new RelayContext()) + const streamable = handler as unknown as StreamableGitHandler + const result = { text: 'x'.repeat(GIT_RESPONSE_STREAM_THRESHOLD + 1) } + const fromSpy = vi.spyOn(Buffer, 'from') + + try { + const marker = streamable.maybeStreamResponse( + result, + { __streamResponse: true }, + { clientId: 1, isStale: () => false } + ) + + expect(marker.__orcaGitResponseStream.totalBytes).toBe( + Buffer.byteLength(JSON.stringify(result), 'utf8') + ) + expect(fromSpy).not.toHaveBeenCalled() + } finally { + handler.dispose() + dispatcher.dispose() + fromSpy.mockRestore() + } + }) +}) diff --git a/src/relay/git-handler-submodule-ops.test.ts b/src/relay/git-handler-submodule-ops.test.ts index 15481097688..937d5d19850 100644 --- a/src/relay/git-handler-submodule-ops.test.ts +++ b/src/relay/git-handler-submodule-ops.test.ts @@ -2,11 +2,19 @@ import { describe, expect, it } from 'vitest' import * as path from 'node:path' import type { GitExec } from './git-handler-ops' import { + MAX_SUBMODULE_PATH_CODE_UNITS, + MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS, MAX_SUBMODULE_PATHS_CACHE_ENTRIES, + MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES, + MAX_SUBMODULE_PATHS_OUTPUT_BYTES, + MAX_SUBMODULE_PATHS_PER_REPO, + MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS, SUBMODULE_PATHS_CACHE_TTL_MS, clearSubmodulePathsCache, createSubmodulePathsCache, + getSubmodulePathsCacheCodeUnits, getSubmodulePathsCacheCount, + listSubmodulePaths, listSubmodulePathsCached, resolveSubmoduleWorktreePath } from './git-handler-submodule-ops' @@ -26,6 +34,19 @@ function gitmodulesExec(paths: string[]): { git: GitExec; calls: () => number } return { git, calls: () => calls } } +function pathsUsingCodeUnits(codeUnits: number): string[] { + const fullPath = 'x'.repeat(MAX_SUBMODULE_PATH_CODE_UNITS) + const paths = Array.from( + { length: Math.floor(codeUnits / MAX_SUBMODULE_PATH_CODE_UNITS) }, + () => fullPath + ) + const remainder = codeUnits % MAX_SUBMODULE_PATH_CODE_UNITS + if (remainder > 0) { + paths.push('x'.repeat(remainder)) + } + return paths +} + describe('listSubmodulePathsCached', () => { it('reads .gitmodules once for repeated diffs on the same worktree within TTL', async () => { const { git, calls } = gitmodulesExec(['vendor/lib']) @@ -39,6 +60,19 @@ describe('listSubmodulePathsCached', () => { expect(calls()).toBe(1) }) + it('keeps the existing 10 MiB command-output ceiling explicit', async () => { + let receivedOptions: Parameters[2] + const git: GitExec = async (_args, _cwd, options) => { + receivedOptions = options + return { stdout: 'submodule.lib.path vendor/lib\n', stderr: '' } + } + + await listSubmodulePaths(git, '/repo') + + expect(receivedOptions).toEqual({ maxBuffer: MAX_SUBMODULE_PATHS_OUTPUT_BYTES }) + expect(MAX_SUBMODULE_PATHS_OUTPUT_BYTES).toBe(10 * 1024 * 1024) + }) + it('re-reads after the TTL expires', async () => { const { git, calls } = gitmodulesExec(['vendor/lib']) const cache = createSubmodulePathsCache() @@ -69,6 +103,7 @@ describe('listSubmodulePathsCached', () => { await listSubmodulePathsCached(git, '/repo-c', cache, SUBMODULE_PATHS_CACHE_TTL_MS + 1) expect(getSubmodulePathsCacheCount(cache)).toBe(1) + expect(getSubmodulePathsCacheCodeUnits(cache)).toBe('/repo-c'.length + 'vendor/lib'.length) }) it('caches an empty result so a submodule-free repo is not re-read', async () => { @@ -121,6 +156,69 @@ describe('listSubmodulePathsCached', () => { expect(calls()).toBe(callsBeforeReads + 1) }) + it('does not retain worktree keys above the UTF-8 byte ceiling', async () => { + const exact = gitmodulesExec(['vendor/lib']) + const exactCache = createSubmodulePathsCache() + const exactKey = 'r'.repeat(MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES) + + await listSubmodulePathsCached(exact.git, exactKey, exactCache, 1_000) + await listSubmodulePathsCached(exact.git, exactKey, exactCache, 1_001) + + expect(exact.calls()).toBe(1) + expect(getSubmodulePathsCacheCount(exactCache)).toBe(1) + + const overflow = gitmodulesExec(['vendor/lib']) + const overflowCache = createSubmodulePathsCache() + const overflowKey = '界'.repeat(Math.floor(MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES / 3) + 1) + expect(Buffer.byteLength(overflowKey, 'utf8')).toBeGreaterThan( + MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES + ) + + await listSubmodulePathsCached(overflow.git, overflowKey, overflowCache, 1_000) + await listSubmodulePathsCached(overflow.git, overflowKey, overflowCache, 1_001) + + expect(overflow.calls()).toBe(2) + expect(getSubmodulePathsCacheCount(overflowCache)).toBe(0) + expect(getSubmodulePathsCacheCodeUnits(overflowCache)).toBe(0) + }) + + it('evicts LRU payloads above the global retained-code-unit ceiling and recovers', async () => { + const calls = new Map() + const git: GitExec = async (_args, cwd) => { + calls.set(cwd, (calls.get(cwd) ?? 0) + 1) + const paths = + cwd === '/overflow' + ? ['vendor/lib'] + : pathsUsingCodeUnits(MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS - cwd.length) + return { + stdout: paths + .map((submodulePath, i) => `submodule.sub${i}.path ${submodulePath}`) + .join('\n'), + stderr: '' + } + } + const cache = createSubmodulePathsCache() + + for (let i = 0; i < 4; i += 1) { + await listSubmodulePathsCached(git, `/repo-${i}`, cache, 1_000) + } + expect(getSubmodulePathsCacheCodeUnits(cache)).toBe(MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS) + + await listSubmodulePathsCached(git, '/overflow', cache, 1_000) + + expect(getSubmodulePathsCacheCount(cache)).toBe(4) + expect(getSubmodulePathsCacheCodeUnits(cache)).toBeLessThan( + MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS + ) + await listSubmodulePathsCached(git, '/repo-1', cache, 1_001) + expect(calls.get('/repo-1')).toBe(1) + await listSubmodulePathsCached(git, '/repo-0', cache, 1_001) + expect(calls.get('/repo-0')).toBe(2) + expect(getSubmodulePathsCacheCodeUnits(cache)).toBeLessThanOrEqual( + MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS + ) + }) + it('does not let a pre-mutation SSH read repopulate the cache', async () => { let resolveOldRead: ((value: { stdout: string; stderr: string }) => void) | undefined let calls = 0 @@ -138,6 +236,7 @@ describe('listSubmodulePathsCached', () => { const oldRead = listSubmodulePathsCached(git, '/repo', cache, 1_000) expect(resolveOldRead).toBeTypeOf('function') clearSubmodulePathsCache(cache) + expect(getSubmodulePathsCacheCodeUnits(cache)).toBe(0) resolveOldRead?.({ stdout: 'submodule.lib.path old-lib\n', stderr: '' }) await expect(oldRead).resolves.toEqual(['old-lib']) @@ -149,6 +248,46 @@ describe('listSubmodulePathsCached', () => { }) }) +describe('listSubmodulePaths', () => { + it('preserves the exact path-count boundary and rejects one more', async () => { + const exactPaths = Array.from( + { length: MAX_SUBMODULE_PATHS_PER_REPO }, + (_, index) => `vendor/lib-${index}` + ) + const exact = await listSubmodulePaths(gitmodulesExec(exactPaths).git, '/repo') + const overflow = await listSubmodulePaths( + gitmodulesExec([...exactPaths, 'vendor/overflow']).git, + '/repo' + ) + + expect(exact).toHaveLength(MAX_SUBMODULE_PATHS_PER_REPO) + expect(exact.at(-1)).toBe(`vendor/lib-${MAX_SUBMODULE_PATHS_PER_REPO - 1}`) + expect(overflow).toEqual([]) + }) + + it('preserves the exact per-path boundary and rejects one more code unit', async () => { + const exactPath = 'x'.repeat(MAX_SUBMODULE_PATH_CODE_UNITS) + + await expect(listSubmodulePaths(gitmodulesExec([exactPath]).git, '/repo')).resolves.toEqual([ + exactPath + ]) + await expect( + listSubmodulePaths(gitmodulesExec([`${exactPath}x`]).git, '/repo') + ).resolves.toEqual([]) + }) + + it('preserves the exact per-repo payload boundary and rejects one more code unit', async () => { + const exactPaths = pathsUsingCodeUnits(MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS) + const exact = await listSubmodulePaths(gitmodulesExec(exactPaths).git, '/repo') + const overflow = await listSubmodulePaths(gitmodulesExec([...exactPaths, 'x']).git, '/repo') + + expect(exact.reduce((total, submodulePath) => total + submodulePath.length, 0)).toBe( + MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS + ) + expect(overflow).toEqual([]) + }) +}) + describe('resolveSubmoduleWorktreePath', () => { it('resolves relative submodule paths inside the selected worktree', () => { expect(resolveSubmoduleWorktreePath('/repo', 'vendor/lib')).toBe( diff --git a/src/relay/git-handler-submodule-ops.ts b/src/relay/git-handler-submodule-ops.ts index 0409ba61177..d0f5ed595f8 100644 --- a/src/relay/git-handler-submodule-ops.ts +++ b/src/relay/git-handler-submodule-ops.ts @@ -11,6 +11,7 @@ import * as path from 'node:path' import { buildDiffResult } from './git-diff-result' import { parseBranchDiff } from './git-handler-utils' import { parseNumstat } from '../shared/git-uncommitted-line-stats' +import { iterateProcessOutputLines } from '../shared/process-output-field-scanner' import { readBlobAtOid, type GitBufferExec, type GitExec } from './git-handler-ops' /** @@ -20,18 +21,26 @@ import { readBlobAtOid, type GitBufferExec, type GitExec } from './git-handler-o */ export const SUBMODULE_PATHS_CACHE_TTL_MS = 5_000 export const MAX_SUBMODULE_PATHS_CACHE_ENTRIES = 512 -type SubmodulePathsCacheEntry = { paths: string[]; expiresAt: number } +export const MAX_SUBMODULE_PATHS_OUTPUT_BYTES = 10 * 1024 * 1024 +export const MAX_SUBMODULE_PATHS_PER_REPO = 10_000 +export const MAX_SUBMODULE_PATH_CODE_UNITS = 64 * 1024 +export const MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS = 4 * 1024 * 1024 +export const MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS = 16 * 1024 * 1024 +export const MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES = 64 * 1024 +type SubmodulePathsCacheEntry = { paths: string[]; expiresAt: number; retainedCodeUnits: number } export type SubmodulePathsCache = { entries: Map generation: number + retainedCodeUnits: number } export function createSubmodulePathsCache(): SubmodulePathsCache { - return { entries: new Map(), generation: 0 } + return { entries: new Map(), generation: 0, retainedCodeUnits: 0 } } export function clearSubmodulePathsCache(cache: SubmodulePathsCache): void { cache.entries.clear() + cache.retainedCodeUnits = 0 // Why: a pre-mutation SSH read must not restore stale .gitmodules paths // after the mutation invalidated them. cache.generation += 1 @@ -41,6 +50,19 @@ export function getSubmodulePathsCacheCount(cache: SubmodulePathsCache): number return cache.entries.size } +export function getSubmodulePathsCacheCodeUnits(cache: SubmodulePathsCache): number { + return cache.retainedCodeUnits +} + +function deleteSubmodulePathsCacheEntry(cache: SubmodulePathsCache, worktreePath: string): void { + const entry = cache.entries.get(worktreePath) + if (!entry) { + return + } + cache.entries.delete(worktreePath) + cache.retainedCodeUnits -= entry.retainedCodeUnits +} + function getCachedSubmodulePaths( cache: SubmodulePathsCache, worktreePath: string, @@ -51,7 +73,7 @@ function getCachedSubmodulePaths( return null } if (cached.expiresAt <= now) { - cache.entries.delete(worktreePath) + deleteSubmodulePathsCacheEntry(cache, worktreePath) return null } cache.entries.delete(worktreePath) @@ -62,7 +84,7 @@ function getCachedSubmodulePaths( function pruneExpiredSubmodulePaths(cache: SubmodulePathsCache, now: number): void { for (const [worktreePath, entry] of cache.entries) { if (entry.expiresAt <= now) { - cache.entries.delete(worktreePath) + deleteSubmodulePathsCacheEntry(cache, worktreePath) } } } @@ -73,14 +95,31 @@ function rememberSubmodulePaths( paths: string[], now: number ): void { - cache.entries.delete(worktreePath) - cache.entries.set(worktreePath, { paths, expiresAt: now + SUBMODULE_PATHS_CACHE_TTL_MS }) - while (cache.entries.size > MAX_SUBMODULE_PATHS_CACHE_ENTRIES) { + deleteSubmodulePathsCacheEntry(cache, worktreePath) + if (Buffer.byteLength(worktreePath, 'utf8') > MAX_SUBMODULE_PATHS_CACHE_KEY_BYTES) { + return + } + // Why: JS strings retain UTF-16 code units, so byte counts understate non-ASCII heap use. + const retainedCodeUnits = + worktreePath.length + paths.reduce((total, submodulePath) => total + submodulePath.length, 0) + if (retainedCodeUnits > MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS) { + return + } + cache.entries.set(worktreePath, { + paths, + expiresAt: now + SUBMODULE_PATHS_CACHE_TTL_MS, + retainedCodeUnits + }) + cache.retainedCodeUnits += retainedCodeUnits + while ( + cache.entries.size > MAX_SUBMODULE_PATHS_CACHE_ENTRIES || + cache.retainedCodeUnits > MAX_SUBMODULE_PATHS_CACHE_CODE_UNITS + ) { const oldestPath = cache.entries.keys().next().value if (oldestPath === undefined) { break } - cache.entries.delete(oldestPath) + deleteSubmodulePathsCacheEntry(cache, oldestPath) } } @@ -119,20 +158,37 @@ export async function listSubmodulePaths(git: GitExec, worktreePath: string): Pr try { const { stdout } = await git( ['config', '--file', '.gitmodules', '--get-regexp', '^submodule\\..*\\.path$'], - worktreePath + worktreePath, + { maxBuffer: MAX_SUBMODULE_PATHS_OUTPUT_BYTES } ) - return stdout - .split(/\r?\n/) - .map((line) => { - const spaceIndex = line.indexOf(' ') - return spaceIndex === -1 + const paths: string[] = [] + let retainedCodeUnits = 0 + for (const line of iterateProcessOutputLines(stdout)) { + if (line.length > MAX_SUBMODULE_PATH_CODE_UNITS + 4_096) { + return [] + } + const spaceIndex = line.indexOf(' ') + const submodulePath = + spaceIndex === -1 ? '' : line .slice(spaceIndex + 1) .trim() .replace(/\/+$/, '') - }) - .filter((value) => value.length > 0) + if (!submodulePath) { + continue + } + if ( + paths.length >= MAX_SUBMODULE_PATHS_PER_REPO || + submodulePath.length > MAX_SUBMODULE_PATH_CODE_UNITS || + submodulePath.length > MAX_SUBMODULE_PATHS_PER_REPO_CODE_UNITS - retainedCodeUnits + ) { + return [] + } + paths.push(submodulePath) + retainedCodeUnits += submodulePath.length + } + return paths } catch { return [] } diff --git a/src/relay/git-handler-utils.ts b/src/relay/git-handler-utils.ts index ab3bb61faba..329797b687e 100644 --- a/src/relay/git-handler-utils.ts +++ b/src/relay/git-handler-utils.ts @@ -10,6 +10,7 @@ import * as path from 'node:path' import { decodeGitCQuotedPath } from '../shared/git-cquoted-path' import { isBinaryBuffer } from '../shared/binary-buffer' import type { GitLineStats } from '../shared/git-uncommitted-line-stats' +import { iterateNulDelimitedFields } from '../shared/nul-delimited-fields' export { isUnsupportedWorktreeListZError } from '../shared/git-worktree-command-capabilities' export function parseBranchStatusChar(char: string): string { @@ -209,7 +210,7 @@ function splitNulWorktreeList(output: string): string[][] { const blocks: string[][] = [] let currentBlock: string[] = [] - for (const field of output.split('\0')) { + for (const field of iterateNulDelimitedFields(output)) { if (field) { currentBlock.push(field) continue diff --git a/src/relay/git-handler.ts b/src/relay/git-handler.ts index 54dfa86a674..61f1728f79c 100644 --- a/src/relay/git-handler.ts +++ b/src/relay/git-handler.ts @@ -11,6 +11,7 @@ import { parseWorktreeList } from './git-handler-utils' import { parseNumstat } from '../shared/git-uncommitted-line-stats' +import { iterateNulDelimitedFields } from '../shared/nul-delimited-fields' import { computeDiff, branchCompare as branchCompareOp, @@ -77,7 +78,15 @@ import { isUnsupportedRevParsePathFormatError } from '../shared/git-worktree-command-capabilities' import { GitResponseStreamRegistry } from './git-response-stream' -import { GIT_RESPONSE_STREAM_THRESHOLD } from './protocol' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../shared/node-bounded-json-stringify' +import { + GIT_RESPONSE_STREAM_THRESHOLD, + MAX_GIT_RESPONSE_STREAM_BYTES, + RelayErrorCode +} from './protocol' import { endSubprocessStdin } from '../shared/subprocess-stdin-write' import { clearGitStatusLineStatsCache } from '../shared/git-status-line-stats-cache' import { streamRelayGitStdout } from './git-stdout-stream' @@ -269,11 +278,26 @@ export class GitHandler { if (params.__streamResponse !== true || !context) { return result } - const payload = Buffer.from(JSON.stringify(result ?? null), 'utf-8') - if (payload.length <= GIT_RESPONSE_STREAM_THRESHOLD) { + let serialized: string + let payloadBytes: number + try { + const bounded = stringifyJsonWithinByteLimit(result ?? null, MAX_GIT_RESPONSE_STREAM_BYTES) + serialized = bounded.serialized + payloadBytes = bounded.byteLength + } catch (error) { + if (!(error instanceof JsonStringifyByteLimitError)) { + throw error + } + const oversized = new Error( + `Git response is too large to transfer safely (more than ${MAX_GIT_RESPONSE_STREAM_BYTES} bytes)` + ) as Error & { code: number } + oversized.code = RelayErrorCode.StreamProtocolError + throw oversized + } + if (payloadBytes <= GIT_RESPONSE_STREAM_THRESHOLD) { return result } - return this.responseStreams.startStream(payload, this.dispatcher, context) + return this.responseStreams.startStream(serialized, this.dispatcher, context) } private clearGitMutationReadCaches(): void { @@ -691,7 +715,7 @@ export class GitHandler { worktreePath ) // Why: a selected tracked directory can make `ls-files -z` return enough descendants for push(...split) to exceed the argument limit. - for (const trackedPathSpec of stdout.split('\0')) { + for (const trackedPathSpec of iterateNulDelimitedFields(stdout)) { if (trackedPathSpec) { trackedPathSpecs.push(trackedPathSpec) } diff --git a/src/relay/git-response-stream-ownership.test.ts b/src/relay/git-response-stream-ownership.test.ts index c6340265238..758a898d8ee 100644 --- a/src/relay/git-response-stream-ownership.test.ts +++ b/src/relay/git-response-stream-ownership.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { RelayDispatcher, RequestContext } from './dispatcher' -import { GitResponseStreamRegistry } from './git-response-stream' +import { + GitResponseStreamRegistry, + MAX_CONCURRENT_GIT_RESPONSE_STREAMS +} from './git-response-stream' import { GIT_RESPONSE_CHUNK_SIZE, STREAM_ACK_WINDOW_CHUNKS } from './protocol' async function flushPump(): Promise { @@ -37,10 +40,20 @@ describe('GitResponseStreamRegistry client ownership', () => { await flushPump() expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS) + registry.recordAck(streamId, 10_000, ownerClientId) + await flushPump() + expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS) + registry.recordAck(streamId, 10_000, ownerClientId + 1) await flushPump() expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS) + for (const invalidSeq of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + registry.recordAck(streamId, invalidSeq, ownerClientId) + } + await flushPump() + expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS) + registry.abort(streamId, ownerClientId + 1) registry.recordAck(streamId, STREAM_ACK_WINDOW_CHUNKS - 1, ownerClientId) await flushPump() @@ -65,4 +78,115 @@ describe('GitResponseStreamRegistry client ownership', () => { expect(notifyBulk.mock.calls[0]?.[0]).toBe('git.responseChunk') expect(notifyBulk.mock.calls[1]?.[0]).toBe('git.responseError') }) + + it('caps parked streams and defers base64 expansion until the pump runs', () => { + const dispatcher = { + notifyBulk: vi.fn().mockResolvedValue(undefined), + notify: vi.fn() + } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry() + registries.push(registry) + const payload = Buffer.from('payload') + const toStringSpy = vi.spyOn(Buffer.prototype, 'toString') + + try { + for (let index = 0; index < MAX_CONCURRENT_GIT_RESPONSE_STREAMS; index += 1) { + registry.startStream(payload, dispatcher, { + clientId: 7, + isStale: () => false + }) + } + + expect(toStringSpy.mock.calls.some(([encoding]) => encoding === 'base64')).toBe(false) + expect(() => + registry.startStream(payload, dispatcher, { + clientId: 7, + isStale: () => false + }) + ).toThrow(`Too many concurrent git response streams`) + } finally { + toStringSpy.mockRestore() + } + }) + + it('caps aggregate bytes retained by concurrent parked responses', () => { + const dispatcher = { + notifyBulk: vi.fn().mockResolvedValue(undefined), + notify: vi.fn() + } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry(10) + registries.push(registry) + const context = { clientId: 7, isStale: () => false } + + registry.startStream(Buffer.alloc(6), dispatcher, context) + registry.startStream(Buffer.alloc(4), dispatcher, context) + + expect(() => registry.startStream(Buffer.alloc(1), dispatcher, context)).toThrow( + 'Concurrent git responses exceed retained-byte limit (10 bytes)' + ) + }) + + it('rejects a serialized string before allocating any encoded chunk', () => { + const dispatcher = { + notifyBulk: vi.fn().mockResolvedValue(undefined), + notify: vi.fn() + } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry(0) + registries.push(registry) + const fromSpy = vi.spyOn(Buffer, 'from') + + try { + expect(() => + registry.startStream('serialized response', dispatcher, { + clientId: 7, + isStale: () => false + }) + ).toThrow('Concurrent git responses exceed retained-byte limit (0 bytes)') + expect(fromSpy).not.toHaveBeenCalled() + } finally { + fromSpy.mockRestore() + } + }) + + it('streams a serialized string without changing its UTF-8 bytes', async () => { + const chunks: Buffer[] = [] + const notifyBulk = vi.fn().mockImplementation((method, params) => { + if (method === 'git.responseChunk') { + chunks.push(Buffer.from(params.data, 'base64')) + } + return Promise.resolve() + }) + const dispatcher = { notifyBulk, notify: vi.fn() } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry() + registries.push(registry) + const serialized = JSON.stringify({ text: `boundary-${'🐋'.repeat(40_000)}` }) + + const marker = registry.startStream(serialized, dispatcher, { + clientId: 7, + isStale: () => false + }) + await flushPump() + await flushPump() + + expect(Buffer.concat(chunks)).toEqual(Buffer.from(serialized, 'utf8')) + expect(chunks).toHaveLength(marker.__orcaGitResponseStream.chunkCount) + expect(marker.__orcaGitResponseStream.totalBytes).toBe(Buffer.byteLength(serialized, 'utf8')) + }) + + it('releases aggregate bytes after completion and disposal', async () => { + const dispatcher = { + notifyBulk: vi.fn().mockResolvedValue(undefined), + notify: vi.fn() + } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry(10) + registries.push(registry) + const context = { clientId: 7, isStale: () => false } + + registry.startStream(Buffer.alloc(10), dispatcher, context) + await flushPump() + expect(() => registry.startStream(Buffer.alloc(10), dispatcher, context)).not.toThrow() + + registry.disposeAll() + expect(() => registry.startStream(Buffer.alloc(10), dispatcher, context)).not.toThrow() + }) }) diff --git a/src/relay/git-response-stream.ts b/src/relay/git-response-stream.ts index bced3b47e46..2cdb6a73944 100644 --- a/src/relay/git-response-stream.ts +++ b/src/relay/git-response-stream.ts @@ -6,42 +6,95 @@ import type { RelayDispatcher, RequestContext } from './dispatcher' import { GIT_RESPONSE_CHUNK_SIZE, + MAX_GIT_RESPONSE_STREAM_BYTES, + MAX_GIT_RESPONSE_STREAM_CHUNKS, + MAX_CONCURRENT_STREAMS, + RelayErrorCode, STREAM_ACK_WINDOW_CHUNKS, STREAM_ACK_STALL_RECHECK_MS, type GitResponseStreamMarker } from './protocol' +import { + encodeUtf8StringChunk, + planUtf8StringChunks, + type Utf8StringChunk +} from './git-response-utf8-chunks' type GitResponseStreamEntry = { ownerClientId: number + retainedBytes: number aborted: boolean + /** Highest chunk seq admitted to the outbound bulk lane. */ + sentThroughSeq: number /** Highest chunk seq the client acknowledged (in-order; -1 = none yet). */ ackedThroughSeq: number ackWaiters: Set<() => void> } -/** Serialized git responses are chunked as base64 so multi-byte UTF-8 - * sequences never split across a chunk boundary (the client concatenates the - * decoded bytes and parses once). */ -function encodeChunks(payload: Buffer): string[] { - const chunks: string[] = [] - for (let offset = 0; offset < payload.length; offset += GIT_RESPONSE_CHUNK_SIZE) { - chunks.push(payload.subarray(offset, offset + GIT_RESPONSE_CHUNK_SIZE).toString('base64')) - } - return chunks -} +type PreparedGitResponse = + | { kind: 'buffer'; value: Buffer; byteLength: number; chunkCount: number } + | { + kind: 'string' + value: string + byteLength: number + chunks: Utf8StringChunk[] + chunkCount: number + } + +export const MAX_CONCURRENT_GIT_RESPONSE_STREAMS = MAX_CONCURRENT_STREAMS +export const MAX_ACTIVE_GIT_RESPONSE_STREAM_BYTES = 128 * 1024 * 1024 export class GitResponseStreamRegistry { private streams = new Map() private nextId = 1 + private retainedBytes = 0 - private register(ownerClientId: number): number { + constructor(private readonly maxRetainedBytes: number = MAX_ACTIVE_GIT_RESPONSE_STREAM_BYTES) {} + + private preparePayload(payload: Buffer | string): PreparedGitResponse { + if (Buffer.isBuffer(payload)) { + return { + kind: 'buffer', + value: payload, + byteLength: payload.length, + chunkCount: Math.ceil(payload.length / GIT_RESPONSE_CHUNK_SIZE) + } + } + const chunks = planUtf8StringChunks(payload, GIT_RESPONSE_CHUNK_SIZE) + return { + kind: 'string', + value: payload, + byteLength: Buffer.byteLength(payload, 'utf8'), + chunks, + chunkCount: chunks.length + } + } + + private register(ownerClientId: number, retainedBytes: number): number { + if (this.streams.size >= MAX_CONCURRENT_GIT_RESPONSE_STREAMS) { + const error = new Error( + `Too many concurrent git response streams (max ${MAX_CONCURRENT_GIT_RESPONSE_STREAMS})` + ) as Error & { code: number } + error.code = RelayErrorCode.TooManyStreams + throw error + } + if (retainedBytes > this.maxRetainedBytes - this.retainedBytes) { + const error = new Error( + `Concurrent git responses exceed retained-byte limit (${this.maxRetainedBytes} bytes)` + ) as Error & { code: number } + error.code = RelayErrorCode.TooManyStreams + throw error + } const streamId = this.nextId++ this.streams.set(streamId, { ownerClientId, + retainedBytes, aborted: false, + sentThroughSeq: -1, ackedThroughSeq: -1, ackWaiters: new Set() }) + this.retainedBytes += retainedBytes return streamId } @@ -50,8 +103,9 @@ export class GitResponseStreamRegistry { if ( !entry || entry.ownerClientId !== clientId || - typeof seq !== 'number' || - !Number.isFinite(seq) + !Number.isSafeInteger(seq) || + seq < 0 || + seq > entry.sentThroughSeq ) { return } @@ -111,25 +165,39 @@ export class GitResponseStreamRegistry { * sentinel marker to send as the RPC result. */ startStream( - payload: Buffer, + payload: Buffer | string, dispatcher: RelayDispatcher, context: RequestContext ): GitResponseStreamMarker { - const streamId = this.register(context.clientId) - const chunks = encodeChunks(payload) + const prepared = this.preparePayload(payload) + if ( + prepared.byteLength > MAX_GIT_RESPONSE_STREAM_BYTES || + prepared.chunkCount > MAX_GIT_RESPONSE_STREAM_CHUNKS + ) { + const error = new Error( + `Git response exceeds stream limit (${prepared.byteLength} bytes, ${prepared.chunkCount} chunks)` + ) as Error & { code: number } + error.code = RelayErrorCode.StreamProtocolError + throw error + } + const streamId = this.register(context.clientId, prepared.byteLength) // Why: kick the pump off the response task so the client sees the sentinel // (and can subscribe/reassemble) before the first chunk frame arrives. setImmediate(() => { - void this.pump(streamId, chunks, dispatcher, context) + void this.pump(streamId, prepared, dispatcher, context) }) return { - __orcaGitResponseStream: { streamId, totalBytes: payload.length, chunkCount: chunks.length } + __orcaGitResponseStream: { + streamId, + totalBytes: prepared.byteLength, + chunkCount: prepared.chunkCount + } } } private async pump( streamId: number, - chunks: string[], + payload: PreparedGitResponse, dispatcher: RelayDispatcher, context: RequestContext ): Promise { @@ -141,7 +209,7 @@ export class GitResponseStreamRegistry { let seq = 0 let endReason: 'end' | 'aborted' | 'stale' = 'end' try { - for (seq = 0; seq < chunks.length; seq += 1) { + for (seq = 0; seq < payload.chunkCount; seq += 1) { if (context.isStale()) { endReason = 'stale' break @@ -167,11 +235,20 @@ export class GitResponseStreamRegistry { endReason = 'aborted' break } - // Why: notifyBulk waits out sink saturation so chunk frames never pile - // up in the outbound pipe ahead of interactive pty.data frames. + const chunk = + payload.kind === 'buffer' + ? payload.value.subarray( + seq * GIT_RESPONSE_CHUNK_SIZE, + (seq + 1) * GIT_RESPONSE_CHUNK_SIZE + ) + : encodeUtf8StringChunk(payload.value, payload.chunks[seq]) + // Why: encode only the current chunk; eager base64 retained a second, + // 4/3-expanded copy of every parked response until its final ACK. + const data = chunk.toString('base64') + entry.sentThroughSeq = seq await dispatcher.notifyBulk( 'git.responseChunk', - { streamId, seq, data: chunks[seq] }, + { streamId, seq, data }, { clientId } @@ -197,15 +274,25 @@ export class GitResponseStreamRegistry { } } } finally { - this.streams.delete(streamId) + this.deleteStream(streamId) } } + private deleteStream(streamId: number): void { + const entry = this.streams.get(streamId) + if (!entry) { + return + } + this.streams.delete(streamId) + this.retainedBytes -= entry.retainedBytes + } + disposeAll(): void { for (const entry of this.streams.values()) { entry.aborted = true this.wake(entry) } this.streams.clear() + this.retainedBytes = 0 } } diff --git a/src/relay/git-response-utf8-chunks.test.ts b/src/relay/git-response-utf8-chunks.test.ts new file mode 100644 index 00000000000..1b2239dcb90 --- /dev/null +++ b/src/relay/git-response-utf8-chunks.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { encodeUtf8StringChunk, planUtf8StringChunks } from './git-response-utf8-chunks' + +describe('Git response UTF-8 chunk planning', () => { + it('round-trips mixed-width text without splitting code points', () => { + const value = 'ascii-é-€-🐋-end'.repeat(20) + const plan = planUtf8StringChunks(value, 17) + const encoded = plan.map((chunk) => encodeUtf8StringChunk(value, chunk)) + + expect(Buffer.concat(encoded)).toEqual(Buffer.from(value, 'utf8')) + expect(plan.every((chunk) => chunk.byteLength <= 17)).toBe(true) + expect(plan.reduce((total, chunk) => total + chunk.byteLength, 0)).toBe( + Buffer.byteLength(value, 'utf8') + ) + }) + + it('matches Node replacement encoding for unmatched surrogate code units', () => { + const value = 'before-\ud800-after-\udc00' + const plan = planUtf8StringChunks(value, 8) + + expect(Buffer.concat(plan.map((chunk) => encodeUtf8StringChunk(value, chunk)))).toEqual( + Buffer.from(value, 'utf8') + ) + }) + + it('returns no chunks for empty text and rejects unsafe chunk sizes', () => { + expect(planUtf8StringChunks('', 4)).toEqual([]) + expect(() => planUtf8StringChunks('value', 3)).toThrow(RangeError) + }) +}) diff --git a/src/relay/git-response-utf8-chunks.ts b/src/relay/git-response-utf8-chunks.ts new file mode 100644 index 00000000000..1c55dbeed5b --- /dev/null +++ b/src/relay/git-response-utf8-chunks.ts @@ -0,0 +1,55 @@ +export type Utf8StringChunk = { + start: number + end: number + byteLength: number +} + +function encodedCodePointSize(value: string, index: number): { bytes: number; codeUnits: number } { + const code = value.charCodeAt(index) + if (code <= 0x7f) { + return { bytes: 1, codeUnits: 1 } + } + if (code <= 0x7ff) { + return { bytes: 2, codeUnits: 1 } + } + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + return { bytes: 4, codeUnits: 2 } + } + } + return { bytes: 3, codeUnits: 1 } +} + +export function planUtf8StringChunks(value: string, maxChunkBytes: number): Utf8StringChunk[] { + if (!Number.isSafeInteger(maxChunkBytes) || maxChunkBytes < 4) { + throw new RangeError('UTF-8 chunk limit must be a safe integer of at least 4 bytes') + } + + const chunks: Utf8StringChunk[] = [] + let chunkStart = 0 + let chunkBytes = 0 + let index = 0 + while (index < value.length) { + const encoded = encodedCodePointSize(value, index) + if (chunkBytes > 0 && chunkBytes + encoded.bytes > maxChunkBytes) { + chunks.push({ start: chunkStart, end: index, byteLength: chunkBytes }) + chunkStart = index + chunkBytes = 0 + } + chunkBytes += encoded.bytes + index += encoded.codeUnits + } + if (chunkBytes > 0) { + chunks.push({ start: chunkStart, end: value.length, byteLength: chunkBytes }) + } + return chunks +} + +export function encodeUtf8StringChunk(value: string, chunk: Utf8StringChunk): Buffer { + const encoded = Buffer.from(value.slice(chunk.start, chunk.end), 'utf8') + if (encoded.length !== chunk.byteLength) { + throw new Error('UTF-8 chunk plan did not match encoded byte length') + } + return encoded +} diff --git a/src/relay/git-status-upstream-negative-cache.test.ts b/src/relay/git-status-upstream-negative-cache.test.ts index c69fa11e4e8..e6085e3b2ce 100644 --- a/src/relay/git-status-upstream-negative-cache.test.ts +++ b/src/relay/git-status-upstream-negative-cache.test.ts @@ -4,6 +4,7 @@ import { clearNoEffectiveUpstreamStatusCacheEntry, getNoEffectiveUpstreamStatusCacheCountForTests, getNoEffectiveUpstreamStatusGenerationCountForTests, + MAX_NO_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES, readOrProbeNoEffectiveUpstreamStatus } from './git-status-upstream-negative-cache' @@ -327,6 +328,59 @@ describe('relay upstream negative cache', () => { expect(getNoEffectiveUpstreamStatusGenerationCountForTests()).toBe(512) }) + it('retains the exact key-byte boundary but not oversized identities', async () => { + const worktreePath = '/r' + const keyOverheadBytes = Buffer.byteLength(`${worktreePath}\0\0`, 'utf8') + const exactBranch = 'x'.repeat(MAX_NO_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES - keyOverheadBytes) + const oversizedBranch = `${exactBranch}x` + const createRunner = (branchName: string) => + vi.fn(async (args: string[]): Promise<{ stdout: string }> => { + if (args[0] === 'symbolic-ref') { + return { stdout: `${branchName}\n` } + } + if (args[0] === 'rev-parse' && args.includes('HEAD@{u}')) { + throw new Error(`fatal: no upstream configured for branch ${branchName}`) + } + if (isConfigListSnapshotCommand(args)) { + return emptyGitConfigSnapshot() + } + if (args[0] === 'rev-parse') { + throw new Error('missing remote branch') + } + throw new Error(`No upstream fixture for git ${args.join(' ')}`) + }) + const exactRunner = createRunner(exactBranch) + const oversizedRunner = createRunner(oversizedBranch) + + await readOrProbeNoEffectiveUpstreamStatus( + { worktreePath, branchName: exactBranch }, + exactRunner + ) + await readOrProbeNoEffectiveUpstreamStatus( + { worktreePath, branchName: exactBranch }, + exactRunner + ) + await readOrProbeNoEffectiveUpstreamStatus( + { worktreePath, branchName: oversizedBranch }, + oversizedRunner + ) + await readOrProbeNoEffectiveUpstreamStatus( + { worktreePath, branchName: oversizedBranch }, + oversizedRunner + ) + clearNoEffectiveUpstreamStatusCacheEntry({ + worktreePath, + branchName: oversizedBranch + }) + + expect(exactRunner.mock.calls.filter(([args]) => args[0] === 'symbolic-ref')).toHaveLength(1) + expect(oversizedRunner.mock.calls.filter(([args]) => args[0] === 'symbolic-ref')).toHaveLength( + 2 + ) + expect(getNoEffectiveUpstreamStatusCacheCountForTests()).toBe(1) + expect(getNoEffectiveUpstreamStatusGenerationCountForTests()).toBe(0) + }) + it('coalesces no-upstream config reads into one snapshot subprocess', async () => { const runGit = vi.fn(async (args: string[]): Promise<{ stdout: string }> => { if (args[0] === 'symbolic-ref') { diff --git a/src/relay/git-status-upstream-negative-cache.ts b/src/relay/git-status-upstream-negative-cache.ts index e3ad9239cc9..387c2a8b060 100644 --- a/src/relay/git-status-upstream-negative-cache.ts +++ b/src/relay/git-status-upstream-negative-cache.ts @@ -5,6 +5,7 @@ import type { GitUpstreamStatus } from '../shared/types' const NO_EFFECTIVE_UPSTREAM_CACHE_TTL_MS = 5 * 60_000 const MAX_NO_EFFECTIVE_UPSTREAM_CACHE_ENTRIES = 512 +export const MAX_NO_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES = 64 * 1024 type NoEffectiveUpstreamCacheIdentity = { worktreePath: string @@ -26,6 +27,10 @@ function noEffectiveUpstreamCacheKey(identity: NoEffectiveUpstreamCacheIdentity) return [identity.worktreePath, identity.branchName, identity.upstreamName ?? ''].join('\0') } +function canRetainNoEffectiveUpstreamCacheKey(cacheKey: string): boolean { + return Buffer.byteLength(cacheKey, 'utf8') <= MAX_NO_EFFECTIVE_UPSTREAM_CACHE_KEY_BYTES +} + function readCachedNoEffectiveUpstreamStatus( cacheKey: string, nowMs = Date.now() @@ -66,6 +71,11 @@ function cacheNoEffectiveUpstreamStatus( writeGeneration: number, nowMs = Date.now() ): void { + if (!canRetainNoEffectiveUpstreamCacheKey(cacheKey)) { + noEffectiveUpstreamByIdentity.delete(cacheKey) + noEffectiveUpstreamWriteGeneration.delete(cacheKey) + return + } // Why: hasConfiguredPushTarget controls publish behavior; keep that signal // fresh rather than serving a stale positive from status polling. if (status.hasUpstream || status.hasConfiguredPushTarget) { @@ -154,6 +164,10 @@ export function clearNoEffectiveUpstreamStatusCacheEntry( retireNoEffectiveUpstreamProbe(cacheKey) noEffectiveUpstreamByIdentity.delete(cacheKey) noEffectiveUpstreamInFlight.delete(cacheKey) + if (!canRetainNoEffectiveUpstreamCacheKey(cacheKey)) { + noEffectiveUpstreamWriteGeneration.delete(cacheKey) + return + } noEffectiveUpstreamWriteGeneration.set( cacheKey, (noEffectiveUpstreamWriteGeneration.get(cacheKey) ?? 0) + 1 diff --git a/src/relay/git-working-file-read.test.ts b/src/relay/git-working-file-read.test.ts index 7f9824ead94..4f52b30196c 100644 --- a/src/relay/git-working-file-read.test.ts +++ b/src/relay/git-working-file-read.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import * as path from 'node:path' import { afterEach, describe, expect, it } from 'vitest' @@ -29,7 +29,8 @@ describe('readWorkingDiffFile', () => { it('marks oversized working-tree files as binary before diffing', async () => { tmpDir = await mkdtemp(path.join(tmpdir(), 'relay-working-file-')) const filePath = path.join(tmpDir, 'large.log') - await writeFile(filePath, Buffer.alloc(10 * 1024 * 1024 + 1, 'a')) + await writeFile(filePath, 'a') + await truncate(filePath, 10 * 1024 * 1024 + 1) await expect(readWorkingDiffFile(filePath)).resolves.toEqual({ content: '', diff --git a/src/relay/git-working-file-read.ts b/src/relay/git-working-file-read.ts index 05cf1ebcdce..8f639189863 100644 --- a/src/relay/git-working-file-read.ts +++ b/src/relay/git-working-file-read.ts @@ -1,5 +1,9 @@ -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import { bufferToBlob } from './git-handler-utils' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../shared/node-bounded-file-reader' const MAX_RELAY_DIFF_WORKING_FILE_BYTES = 10 * 1024 * 1024 @@ -19,16 +23,16 @@ export async function readWorkingDiffFile( if (!fileStat.isFile()) { return { content: '', isBinary: false, missing: true } } - if (fileStat.size > MAX_RELAY_DIFF_WORKING_FILE_BYTES) { - // Why: mirror local git diff reads, which cap blob transfer at 10MB. - return { content: '', isBinary: true, missing: false } - } try { - const buffer = await readFile(absPath) + const { buffer } = await readNodeFileWithinLimit(absPath, MAX_RELAY_DIFF_WORKING_FILE_BYTES) // Why: bufferToBlob needs the path's extension to know an image is // previewable; omitting it made every relay-side binary diff empty. return { ...bufferToBlob(buffer, absPath), missing: false } - } catch { + } catch (error) { + if (error instanceof NodeFileReadTooLargeError) { + // Why: mirror local git diff reads, which classify blobs over 10MB as binary. + return { content: '', isBinary: true, missing: false } + } // Why: the file exists but could not be read — a read failure, not a deletion. return { content: '', isBinary: false, missing: false } } diff --git a/src/relay/integration.test.ts b/src/relay/integration.test.ts index 71d09c78de6..a3fcfd75adb 100644 --- a/src/relay/integration.test.ts +++ b/src/relay/integration.test.ts @@ -23,6 +23,7 @@ import { RelayDispatcher } from './dispatcher' import { RelayContext } from './context' import { FsHandler } from './fs-handler' import { GitHandler } from './git-handler' +import { SshFilesystemProvider } from '../main/providers/ssh-filesystem-provider' function gitInit(dir: string): void { execFileSync('git', ['init'], { cwd: dir, stdio: 'pipe' }) @@ -108,6 +109,17 @@ describe('Integration: Client Mux ↔ Relay Dispatcher', () => { expect(names).toEqual(['hello.txt', 'readme.md']) }) + it('routes SSH provider directory reads through the bounded relay handler', async () => { + writeFileSync(path.join(tmpDir, 'hello.txt'), 'world') + const provider = new SshFilesystemProvider('test-connection', mux) + + await expect(provider.readDir(tmpDir)).resolves.toEqual([ + { name: 'hello.txt', isDirectory: false, isSymlink: false } + ]) + + provider.dispose() + }) + it('readFile returns text content', async () => { writeFileSync(path.join(tmpDir, 'data.txt'), 'some content') @@ -122,6 +134,11 @@ describe('Integration: Client Mux ↔ Relay Dispatcher', () => { it('readFileStream round-trip preserves a 12 MB binary file', async () => { const filePath = path.join(tmpDir, 'big.png') const original = randomBytes(12 * 1024 * 1024) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(original) + original.writeUInt32BE(13, 8) + original.write('IHDR', 12, 'ascii') + original.writeUInt32BE(1, 16) + original.writeUInt32BE(1, 20) writeFileSync(filePath, original) const { readFileViaStream } = await import('../main/ssh/ssh-filesystem-stream-reader') const { content } = await readFileViaStream(mux, filePath) diff --git a/src/relay/markdown-document-listing.test.ts b/src/relay/markdown-document-listing.test.ts new file mode 100644 index 00000000000..38f93db2ee6 --- /dev/null +++ b/src/relay/markdown-document-listing.test.ts @@ -0,0 +1,93 @@ +import { EventEmitter } from 'node:events' +import type { ChildProcess } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { describe, expect, it, vi } from 'vitest' +import { + MARKDOWN_DOCUMENT_LISTING_MAX_DOCUMENTS, + MARKDOWN_DOCUMENT_LISTING_MAX_PATH_BYTES, + MarkdownDocumentListingCapacityError +} from '../shared/markdown-document-listing-limits' +import { + listMarkdownPathsWithRg, + listRelayMarkdownDocumentPaths +} from './markdown-document-listing' + +function successfulChild(output: Buffer): ChildProcess { + const child = Object.assign(new EventEmitter(), { + exitCode: null, + signalCode: null, + stdout: new PassThrough(), + stderr: new PassThrough(), + kill: vi.fn(() => true) + }) as unknown as ChildProcess + queueMicrotask(() => { + child.stdout!.emit('data', output) + child.emit('close', 0, null) + }) + return child +} + +describe('relay Markdown document producer', () => { + it('returns only Markdown paths from a real under-limit workspace', async () => { + const rootPath = await mkdtemp(join(tmpdir(), 'orca-markdown-listing-')) + try { + await mkdir(join(rootPath, 'docs')) + await writeFile(join(rootPath, 'README.md'), 'readme') + await writeFile(join(rootPath, 'docs', 'Guide.MDX'), 'guide') + await writeFile(join(rootPath, 'docs', 'app.ts'), 'code') + + const result = await listRelayMarkdownDocumentPaths(rootPath) + expect(result.sort()).toEqual(['README.md', 'docs/Guide.MDX']) + } finally { + await rm(rootPath, { recursive: true, force: true }) + } + }) + + it('filters a large non-Markdown repository before applying document retention limits', async () => { + const nonMarkdown = Array.from({ length: 100_001 }, (_value, index) => `src/${index}.ts\0`) + const output = Buffer.from(`${nonMarkdown.join('')}README.md\0docs/GUIDE.MDX\0`) + const spawnProcess = vi.fn((_args: string[]) => successfulChild(output)) + + await expect(listMarkdownPathsWithRg('/repo', undefined, spawnProcess)).resolves.toEqual([ + 'README.md', + 'docs/GUIDE.MDX' + ]) + expect(spawnProcess).toHaveBeenCalledTimes(2) + expect(spawnProcess.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(['--iglob', '*.md', '--null']) + ) + }) + + it('rejects rather than truncating the first document beyond the count limit', async () => { + const output = Buffer.from( + Array.from( + { length: MARKDOWN_DOCUMENT_LISTING_MAX_DOCUMENTS + 1 }, + (_value, index) => `${index}.md\0` + ).join('') + ) + const children: ChildProcess[] = [] + const spawnProcess = vi.fn(() => { + const child = successfulChild(output) + children.push(child) + return child + }) + + await expect(listMarkdownPathsWithRg('/repo', undefined, spawnProcess)).rejects.toBeInstanceOf( + MarkdownDocumentListingCapacityError + ) + expect(children[0]?.kill).toHaveBeenCalled() + expect(spawnProcess).toHaveBeenCalledTimes(1) + }) + + it('bounds an unterminated subprocess path before decoding or retaining it', async () => { + const child = successfulChild(Buffer.alloc(MARKDOWN_DOCUMENT_LISTING_MAX_PATH_BYTES + 1, 0x61)) + + await expect(listMarkdownPathsWithRg('/repo', undefined, () => child)).rejects.toBeInstanceOf( + MarkdownDocumentListingCapacityError + ) + expect(child.kill).toHaveBeenCalled() + }) +}) diff --git a/src/relay/markdown-document-listing.ts b/src/relay/markdown-document-listing.ts new file mode 100644 index 00000000000..a4330f3442b --- /dev/null +++ b/src/relay/markdown-document-listing.ts @@ -0,0 +1,238 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { checkRgAvailable } from './fs-handler-utils' +import { + buildRgArgsForQuickOpen, + normalizeQuickOpenRgLine, + shouldIncludeQuickOpenPath +} from '../shared/quick-open-filter' +import { + assertMarkdownDocumentPathWithinLimit, + createMarkdownDocumentListingBudget, + isMarkdownDocumentListingCapacityError, + MARKDOWN_DOCUMENT_LISTING_ERROR_MESSAGE, + MARKDOWN_DOCUMENT_LISTING_MAX_PATH_BYTES, + MarkdownDocumentListingCapacityError, + retainMarkdownRelativePath, + visitMarkdownDocumentListingEntry +} from '../shared/markdown-document-listing-limits' +import { + discoverMarkdownRelativePaths, + isMarkdownDocumentPath +} from '../shared/node-markdown-document-discovery' +import { fileListingCancellationError } from '../shared/file-listing-cancellation' +import { RelayErrorCode } from './protocol' +import { GrowingByteBuffer } from '../shared/growing-byte-buffer' + +const MARKDOWN_LISTING_TIMEOUT_MS = 25_000 +const MARKDOWN_GLOBS = ['*.md', '*.mdx', '*.markdown'] + +class MarkdownPathFieldAccumulator { + private readonly field = new GrowingByteBuffer() + + push(chunk: Buffer, onPath: (path: string) => void): boolean { + let cursor = 0 + while (cursor < chunk.length) { + const delimiter = chunk.indexOf(0, cursor) + const end = delimiter === -1 ? chunk.length : delimiter + const segmentBytes = end - cursor + if (this.field.byteLength + segmentBytes > MARKDOWN_DOCUMENT_LISTING_MAX_PATH_BYTES) { + this.clear() + return false + } + if (delimiter !== -1 && this.field.byteLength === 0) { + onPath(chunk.toString('utf8', cursor, end)) + } else if (segmentBytes > 0) { + this.field.append(chunk.subarray(cursor, end)) + if (delimiter !== -1) { + onPath(this.take()) + } + } else if (delimiter !== -1) { + onPath(this.take()) + } + if (delimiter === -1) { + return true + } + cursor = delimiter + 1 + } + return true + } + + finish(): string | null { + return this.field.byteLength > 0 ? this.take() : null + } + + private take(): string { + return this.field.takeString() + } + + private clear(): void { + this.field.clear() + } +} + +function markdownRgArgs(args: string[]): string[] { + const target = args.at(-1) + if (!target) { + throw new Error('Markdown rg scan is missing its search root') + } + const result = [...args.slice(0, -1), '--null'] + for (const glob of MARKDOWN_GLOBS) { + result.push('--iglob', glob) + } + result.push(target) + return result +} + +function relativePathDepth(path: string): number { + let depth = 0 + for (let index = 0; index < path.length; index += 1) { + if (path[index] === '/') { + depth += 1 + } + } + return depth +} + +export async function listMarkdownPathsWithRg( + rootPath: string, + signal?: AbortSignal, + spawnProcess: (args: string[]) => ChildProcess = (args) => + spawn('rg', args, { cwd: rootPath, stdio: ['ignore', 'pipe', 'pipe'] }) +): Promise { + const paths = new Set() + const budget = createMarkdownDocumentListingBudget() + assertMarkdownDocumentPathWithinLimit(rootPath, budget.limits.maxPathBytes) + const passes = buildRgArgsForQuickOpen({ + searchRoot: '.', + excludePathPrefixes: [], + forceSlashSeparator: true + }) + + const runPass = (args: string[]): Promise => + new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(fileListingCancellationError(signal)) + return + } + const child = spawnProcess(markdownRgArgs(args)) + const fields = new MarkdownPathFieldAccumulator() + let done = false + let parseablePathCount = 0 + let timer: ReturnType | null = null + + const cleanup = (): void => { + if (timer) { + clearTimeout(timer) + timer = null + } + child.stdout?.off('data', onStdout) + child.stderr?.off('data', onStderr) + child.off('error', onError) + child.off('close', onClose) + signal?.removeEventListener('abort', onAbort) + } + const finish = (error?: Error, kill = false): void => { + if (done) { + return + } + done = true + cleanup() + if (kill && child.exitCode === null && child.signalCode === null) { + child.kill() + } + if (error) { + reject(error) + } else { + resolve() + } + } + const processPath = (rawPath: string): void => { + const relativePath = normalizeQuickOpenRgLine(rawPath, { kind: 'cwd-relative' }) + if (relativePath === null) { + return + } + parseablePathCount += 1 + if ( + paths.has(relativePath) || + !shouldIncludeQuickOpenPath(relativePath) || + !isMarkdownDocumentPath(relativePath) + ) { + return + } + visitMarkdownDocumentListingEntry(budget, relativePath, relativePathDepth(relativePath)) + retainMarkdownRelativePath(budget, rootPath, relativePath) + paths.add(relativePath) + } + const onStdout = (chunk: Buffer): void => { + try { + if (!fields.push(chunk, processPath)) { + finish(new MarkdownDocumentListingCapacityError(), true) + } + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error)), true) + } + } + const onStderr = (): void => {} + const onError = (error: Error): void => finish(error) + const onClose = (code: number | null, exitSignal: NodeJS.Signals | null): void => { + if (exitSignal) { + finish(new Error(`rg killed by ${exitSignal}`)) + return + } + try { + const trailingPath = fields.finish() + if (trailingPath !== null) { + processPath(trailingPath) + } + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))) + return + } + if (code === 0 || code === 1 || (code === 2 && parseablePathCount > 0)) { + finish() + } else { + finish(new Error(`rg exited with code ${code}`)) + } + } + const onAbort = (): void => finish(fileListingCancellationError(signal), true) + + child.stdout?.on('data', onStdout) + child.stderr?.on('data', onStderr) + child.once('error', onError) + child.once('close', onClose) + signal?.addEventListener('abort', onAbort, { once: true }) + timer = setTimeout( + () => finish(new Error('Markdown document listing timed out'), true), + MARKDOWN_LISTING_TIMEOUT_MS + ) + }) + + await runPass(passes.primary) + await runPass(passes.ignoredPass) + return Array.from(paths) +} + +export async function listRelayMarkdownDocumentPaths( + rootPath: string, + signal?: AbortSignal +): Promise { + try { + if (await checkRgAvailable()) { + return await listMarkdownPathsWithRg(rootPath, signal) + } + return await discoverMarkdownRelativePaths(rootPath, { + ignoreNestedDirectoryErrors: true, + shouldDescend: (relativePath) => shouldIncludeQuickOpenPath(relativePath), + signal + }) + } catch (error) { + if (!isMarkdownDocumentListingCapacityError(error)) { + throw error + } + const relayError = new Error(MARKDOWN_DOCUMENT_LISTING_ERROR_MESSAGE) as Error & { + code: number + } + relayError.code = RelayErrorCode.MarkdownDocumentListingCapacity + throw relayError + } +} diff --git a/src/relay/mobile-file-directory-reader.test.ts b/src/relay/mobile-file-directory-reader.test.ts new file mode 100644 index 00000000000..8000dad5c00 --- /dev/null +++ b/src/relay/mobile-file-directory-reader.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from 'vitest' + +const { statMock } = vi.hoisted(() => ({ statMock: vi.fn() })) + +vi.mock('node:fs/promises', () => ({ opendir: vi.fn(), stat: statMock })) + +import { collectMobileRelayDirectoryEntries } from './mobile-file-directory-reader' + +describe('mobile relay directory reader', () => { + it('stops enumeration at the mobile entry limit', async () => { + let enumerated = 0 + const directory = { + async *[Symbol.asyncIterator]() { + while (enumerated < 20_000) { + enumerated += 1 + yield { + name: 'entry', + isDirectory: () => false, + isSymbolicLink: () => false + } + } + } + } + + await expect(collectMobileRelayDirectoryEntries('/repo', directory)).rejects.toThrow( + 'This folder is too large to show safely on mobile' + ) + expect(enumerated).toBe(10_001) + }) + + it('bounds concurrent symlink stat work while preserving entry order', async () => { + let active = 0 + let maxActive = 0 + statMock.mockImplementation(async () => { + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setImmediate(resolve)) + active -= 1 + return { isDirectory: () => true } + }) + const directory = { + async *[Symbol.asyncIterator]() { + for (let index = 0; index < 100; index += 1) { + yield { + name: `link-${index}`, + isDirectory: () => false, + isSymbolicLink: () => true + } + } + } + } + + const entries = await collectMobileRelayDirectoryEntries('/repo', directory) + + expect(maxActive).toBe(32) + expect(entries.map((entry) => entry.name)).toEqual( + Array.from({ length: 100 }, (_, index) => `link-${index}`) + ) + expect(entries.every((entry) => entry.isDirectory && entry.isSymlink)).toBe(true) + }) +}) diff --git a/src/relay/mobile-file-directory-reader.ts b/src/relay/mobile-file-directory-reader.ts new file mode 100644 index 00000000000..7db3cdf7bb6 --- /dev/null +++ b/src/relay/mobile-file-directory-reader.ts @@ -0,0 +1,76 @@ +import { opendir, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { + createMobileFileDirectoryLimitState, + trackMobileFileDirectoryEntry +} from '../shared/mobile-file-directory-limit' + +type RelayDirectorySourceEntry = { + name: string + isDirectory(): boolean + isSymbolicLink(): boolean +} + +export type RelayDirectoryEntry = { + name: string + isDirectory: boolean + isSymlink: boolean +} + +const MOBILE_DIRECTORY_STAT_CONCURRENCY = 32 + +export async function isRelayDirectoryEntry( + dirPath: string, + entry: RelayDirectorySourceEntry +): Promise { + if (entry.isDirectory()) { + return true + } + if (!entry.isSymbolicLink()) { + return false + } + try { + // Why: linked workspace directories must remain expandable in the file explorer. + return (await stat(join(dirPath, entry.name))).isDirectory() + } catch { + return false + } +} + +export async function readMobileRelayDirectory(dirPath: string): Promise { + return collectMobileRelayDirectoryEntries(dirPath, await opendir(dirPath)) +} + +export async function collectMobileRelayDirectoryEntries( + dirPath: string, + directory: AsyncIterable +): Promise { + const entries: RelayDirectoryEntry[] = [] + let batch: RelayDirectorySourceEntry[] = [] + const limit = createMobileFileDirectoryLimitState() + + const flushBatch = async (): Promise => { + entries.push( + ...(await Promise.all( + batch.map(async (entry) => ({ + name: entry.name, + isDirectory: await isRelayDirectoryEntry(dirPath, entry), + isSymlink: entry.isSymbolicLink() + })) + )) + ) + batch = [] + } + + for await (const entry of directory) { + trackMobileFileDirectoryEntry(limit, entry) + batch.push(entry) + if (batch.length === MOBILE_DIRECTORY_STAT_CONCURRENCY) { + await flushBatch() + } + } + if (batch.length > 0) { + await flushBatch() + } + return entries +} diff --git a/src/relay/plugin-overlay.test.ts b/src/relay/plugin-overlay.test.ts index 9b90692cd28..11348f99b2b 100644 --- a/src/relay/plugin-overlay.test.ts +++ b/src/relay/plugin-overlay.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { existsSync, mkdirSync, @@ -11,6 +11,7 @@ import { } from 'node:fs' import { tmpdir } from 'node:os' import { basename, join } from 'node:path' +import * as configOverlayMirroring from '../main/pty/config-overlay-mirroring' import { PluginOverlayManager } from './plugin-overlay' import { resolvePiSourceAgentDir } from './plugin-overlay-env' @@ -70,6 +71,30 @@ describe('PluginOverlayManager', () => { expect(manager.materializeOpenCode('tab-missing:0', join(homeDir, 'missing'))).toBeNull() }) + it('rejects an over-capacity remote config before replacing the last good overlay', () => { + manager.setSources({ opencodePluginSource: 'first plugin' }) + const overlayDir = manager.materializeOpenCode('tab-capacity:0')! + const pluginPath = join(overlayDir, 'plugins', 'orca-opencode-status.js') + const userConfigDir = join(homeDir, 'large-opencode-config') + mkdirSync(userConfigDir) + const planSpy = vi + .spyOn(configOverlayMirroring, 'createConfigOverlayPlan') + .mockImplementation(() => { + throw new configOverlayMirroring.ConfigOverlayCapacityError('entries', 4_097, 4_096) + }) + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + try { + expect(manager.materializeOpenCode('tab-capacity:0', userConfigDir)).toBeNull() + expect(readFileSync(pluginPath, 'utf8')).toBe('first plugin') + expect(stderrSpy).toHaveBeenCalledWith( + expect.stringContaining('Agent config overlay entries exceeded its 4096 limit') + ) + } finally { + planSpy.mockRestore() + stderrSpy.mockRestore() + } + }) + it('installs Pi extension into the real agent extensions dir', () => { manager.setSources({ piExtensionSource: '// pi extension' }) const dir = manager.materializePi('tab-2:0') diff --git a/src/relay/plugin-overlay.ts b/src/relay/plugin-overlay.ts index 15adf7004e5..e9c6dd56845 100644 --- a/src/relay/plugin-overlay.ts +++ b/src/relay/plugin-overlay.ts @@ -17,19 +17,19 @@ // Pi/OMP homes for those agents. import { createHash } from 'node:crypto' -import { - existsSync, - mkdirSync, - readFileSync, - readdirSync, - realpathSync, - statSync, - unlinkSync, - writeFileSync -} from 'node:fs' +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' -import { mirrorEntry, safeRemoveOverlay } from '../main/pty/overlay-mirror' +import { + applyConfigOverlayPlan, + createConfigOverlayPlan, + type ConfigOverlayPlan +} from '../main/pty/config-overlay-mirroring' +import { safeRemoveOverlay } from '../main/pty/overlay-mirror' +import { + isManagedPiExtensionFile, + withOrcaManagedPiExtensionMarker +} from '../main/pi/managed-extension-ownership' import type { PiAgentKind } from '../shared/pi-agent-kind' const RELAY_HOOKS_DIR = '.orca-relay' @@ -41,13 +41,6 @@ const PI_OVERLAY_SUBDIR_BY_KIND: Record = { const OPENCODE_PLUGIN_FILE = 'orca-opencode-status.js' const PI_EXTENSION_FILE = 'orca-agent-status.ts' const PI_AGENT_SUBDIR = 'agent' -const ORCA_MANAGED_EXTENSION_MARKER = '@orca-managed-pi-extension' - -function withOrcaManagedPiExtensionMarker(source: string): string { - return source.includes(ORCA_MANAGED_EXTENSION_MARKER) - ? source - : `// ${ORCA_MANAGED_EXTENSION_MARKER}\n${source}` -} // Why: source-dir resolution is keyed off the launching agent (Pi or OMP). // Both consume `PI_CODING_AGENT_DIR` but default to different `~/./agent` // paths on the remote disk. The renderer-chosen launch command flows in via @@ -137,50 +130,16 @@ export class PluginOverlayManager { return this.piExtensionSources[kind] ?? this.piExtensionSources.pi } - private mirrorOpenCodeConfig(sourceDir: string, overlayDir: string): void { - for (const entry of readdirSync(sourceDir, { withFileTypes: true })) { - const sourcePath = join(sourceDir, entry.name) - - if (entry.name === 'plugins') { - const isSymlink = entry.isSymbolicLink() - let isLinkPointingToDir = false - if (isSymlink) { - try { - isLinkPointingToDir = statSync(sourcePath).isDirectory() - } catch { - isLinkPointingToDir = false - } - } - - if ((!isSymlink && entry.isDirectory()) || isLinkPointingToDir) { - const resolvedSource = isLinkPointingToDir ? realpathSync(sourcePath) : sourcePath - const overlayPluginsDir = join(overlayDir, 'plugins') - mkdirSync(overlayPluginsDir, { recursive: true }) - for (const pluginEntry of readdirSync(resolvedSource, { withFileTypes: true })) { - if (pluginEntry.name === OPENCODE_PLUGIN_FILE) { - continue - } - mirrorEntry( - join(resolvedSource, pluginEntry.name), - join(overlayPluginsDir, pluginEntry.name) - ) - } - continue - } - } - - mirrorEntry(sourcePath, join(overlayDir, entry.name)) - } - } - private writeOpenCodePlugin(overlayDir: string): void { const pluginsDir = join(overlayDir, 'plugins') mkdirSync(pluginsDir, { recursive: true }) const pluginPath = join(pluginsDir, OPENCODE_PLUGIN_FILE) try { unlinkSync(pluginPath) - } catch { - // Fresh overlay or no same-named stale symlink. + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } } writeFileSync(pluginPath, this.opencodePluginSource!) } @@ -197,8 +156,7 @@ export class PluginOverlayManager { } const dir = join(this.opencodeRoot, safeDirName(id)) try { - safeRemoveOverlay(dir, this.opencodeRoot) - mkdirSync(dir, { recursive: true }) + let plan: ConfigOverlayPlan | null = null if (existingConfigDir) { if (!existsSync(existingConfigDir)) { return null @@ -206,7 +164,16 @@ export class PluginOverlayManager { // Why: OPENCODE_CONFIG_DIR is a single config root. Mirror the user's // remote root into the overlay before adding Orca's plugin so status // reporting does not hide their auth, models, keybinds, or plugins. - this.mirrorOpenCodeConfig(existingConfigDir, dir) + plan = createConfigOverlayPlan(existingConfigDir, { + reservedPluginFile: OPENCODE_PLUGIN_FILE + }) + } + if (!safeRemoveOverlay(dir, this.opencodeRoot)) { + throw new Error('OpenCode overlay cleanup exceeded its safety limits') + } + mkdirSync(dir, { recursive: true }) + if (plan) { + applyConfigOverlayPlan(plan, dir) } this.writeOpenCodePlugin(dir) return dir @@ -222,14 +189,6 @@ export class PluginOverlayManager { return join(this.homeDir, PI_AGENT_HOME_DIR_NAME[kind], PI_AGENT_SUBDIR) } - private canOverwritePiExtension(path: string): boolean { - try { - return readFileSync(path, 'utf8').includes(ORCA_MANAGED_EXTENSION_MARKER) - } catch { - return true - } - } - /** Install the Pi/OMP status extension into the remote real agent dir and * return that directory. `kind` selects which Pi-compatible agent's default * dir to use when `existingAgentDir` is not supplied. */ @@ -246,7 +205,7 @@ export class PluginOverlayManager { const extensionsDir = join(sourceAgentDir, 'extensions') mkdirSync(extensionsDir, { recursive: true }) const extensionPath = join(extensionsDir, PI_EXTENSION_FILE) - if (!this.canOverwritePiExtension(extensionPath)) { + if (existsSync(extensionPath) && !isManagedPiExtensionFile(extensionPath)) { return null } writeFileSync(extensionPath, extensionSource) @@ -259,10 +218,8 @@ export class PluginOverlayManager { } } - /** Drop a paneKey's overlay dirs on PTY exit. Best-effort; cleanup over a - * recursive tree may fail on exotic filesystems but the worst-case - * outcome is unbounded growth on a long-lived relay, which the per-pane - * caches alone do not bound. */ + /** Drop a paneKey's overlay dirs on PTY exit. Best-effort and bounded; an + * exotic or pathological tree is logged and left for a later cleanup. */ clearOverlay(id: string): void { if (!isUsableId(id)) { return @@ -273,7 +230,11 @@ export class PluginOverlayManager { // safeRemoveOverlay keeps each call bounded to its own tree. for (const root of [this.opencodeRoot, ...Object.values(this.piRoots)]) { try { - safeRemoveOverlay(join(root, safe), root) + if (!safeRemoveOverlay(join(root, safe), root)) { + process.stderr.write( + `[plugin-overlay] overlay cleanup stopped at its safety limit: ${join(root, safe)}\n` + ) + } } catch (err) { // Why: log the failed cleanup so a permission/IO error is observable. // The leak is the failure mode the per-pane cache eviction exists to diff --git a/src/relay/port-scan-handler.test.ts b/src/relay/port-scan-handler.test.ts index 9df947b1aad..01f4ad7300f 100644 --- a/src/relay/port-scan-handler.test.ts +++ b/src/relay/port-scan-handler.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' -import { parseHexAddress } from './port-scan-handler' +import { LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES } from '../shared/linux-proc-port-scan-limits' +import { parseHexAddress, parseLinuxProcListeningSockets } from './port-scan-handler' import { parseWindowsNetstatOutput, parseWindowsPowerShellPortRows } from './windows-port-scan' describe('parseHexAddress', () => { @@ -68,6 +69,20 @@ describe('parseHexAddress', () => { }) }) +describe('parseLinuxProcListeningSockets', () => { + it('caps retained listeners before inode and process scans', () => { + const rows = Array.from( + { length: LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES + 5 }, + (_, index) => + ` ${index}: 0100007F:${(index + 1).toString(16).padStart(4, '0')} 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 ${index + 1}` + ) + + expect(parseLinuxProcListeningSockets(['header', ...rows].join('\n'))).toHaveLength( + LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES + ) + }) +}) + describe('parseWindowsPowerShellPortRows', () => { it('parses PowerShell JSON arrays', () => { expect( diff --git a/src/relay/port-scan-handler.ts b/src/relay/port-scan-handler.ts index a576d8921fa..f3ead301c7a 100644 --- a/src/relay/port-scan-handler.ts +++ b/src/relay/port-scan-handler.ts @@ -1,5 +1,12 @@ -import { readFile, readdir, readlink } from 'node:fs/promises' import { getProcessOutputFields } from '../shared/process-output-field-scanner' +import { mapLinuxSocketInodesToPids } from '../shared/linux-proc-socket-owner-scanner' +import { + createLinuxProcTextReadBudget, + LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES, + readLinuxProcNetworkTable, + readLinuxProcTextWithinBudget, + type LinuxProcTextReadBudget +} from '../shared/linux-proc-port-scan-limits' import type { RelayDispatcher, RequestContext } from './dispatcher' import { scanWindowsListeningPorts } from './windows-port-scan' @@ -43,16 +50,17 @@ export class PortScanHandler { this.readProcNet('/proc/net/tcp6') ]) - const listeningSockets = [...tcp4, ...tcp6] + const listeningSockets = [...tcp4, ...tcp6].slice(0, LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES) if (listeningSockets.length === 0) { return [] } const inodeSet = new Set(listeningSockets.map((s) => s.inode)) - const inodeToPid = await this.mapInodesToPids(inodeSet) + const inodeToPid = await mapLinuxSocketInodesToPids(inodeSet) const seen = new Set() const results: DetectedPort[] = [] + const metadataBudget = createLinuxProcTextReadBudget() const relayPid = process.pid const relayParentPid = process.ppid @@ -72,7 +80,7 @@ export class PortScanHandler { continue } - const processName = pid != null ? await this.getProcessName(pid) : undefined + const processName = pid != null ? await this.getProcessName(pid, metadataBudget) : undefined if (processName === 'sshd') { continue @@ -95,111 +103,49 @@ export class PortScanHandler { private async readProcNet( path: string ): Promise<{ port: number; host: string; inode: number }[]> { - let content: string - try { - content = await readFile(path, 'utf-8') - } catch { - return [] - } - - const lines = content.split('\n') - const results: { port: number; host: string; inode: number }[] = [] - - for (let i = 1; i < lines.length; i++) { - const fields = getProcessOutputFields(lines[i], 10) - if (fields.length < 10) { - continue - } - - // State field (index 3): 0A = TCP_LISTEN - if (fields[3] !== '0A') { - continue - } - - const localAddress = fields[1] - const parsed = parseHexAddress(localAddress) - if (!parsed) { - continue - } - - const inode = Number.parseInt(fields[9], 10) - if (Number.isNaN(inode) || inode === 0) { - continue - } - - results.push({ port: parsed.port, host: parsed.host, inode }) - } - - return results + const content = await readLinuxProcNetworkTable(path) + return content === null ? [] : parseLinuxProcListeningSockets(content) } - private async mapInodesToPids(inodes: Set): Promise> { - const result = new Map() - if (inodes.size === 0) { - return result - } - - let pids: string[] - try { - pids = (await readdir('/proc')).filter((name) => /^\d+$/.test(name)) - } catch { - return result - } - - for (const pidStr of pids) { - const fdDir = `/proc/${pidStr}/fd` - let fds: string[] - try { - fds = await readdir(fdDir) - } catch { - continue - } - - const pid = Number.parseInt(pidStr, 10) - - for (const fd of fds) { - let link: string - try { - link = await readlink(`${fdDir}/${fd}`) - } catch { - continue - } - - const match = link.match(/^socket:\[(\d+)\]$/) - if (!match) { - continue - } - - const inode = Number.parseInt(match[1], 10) - if (inodes.has(inode)) { - result.set(inode, pid) - } - } - } - - return result - } - - private async getProcessName(pid: number): Promise { - try { - const cmdline = await readFile(`/proc/${pid}/cmdline`, 'utf-8') - if (!cmdline) { - return undefined - } - - const exe = cmdline.split('\0')[0] - if (!exe) { - return undefined - } - - const parts = exe.split('/') - return parts.at(-1) - } catch { + private async getProcessName( + pid: number, + budget: LinuxProcTextReadBudget + ): Promise { + const cmdline = await readLinuxProcTextWithinBudget(`/proc/${pid}/cmdline`, budget) + if (!cmdline) { return undefined } + + const exe = cmdline.split('\0')[0] + return exe ? exe.split('/').at(-1) : undefined } } +export function parseLinuxProcListeningSockets( + content: string +): { port: number; host: string; inode: number }[] { + const lines = content.split('\n') + const results: { port: number; host: string; inode: number }[] = [] + + for (let i = 1; i < lines.length; i++) { + const fields = getProcessOutputFields(lines[i], 10) + if (fields.length < 10 || fields[3] !== '0A') { + continue + } + const parsed = parseHexAddress(fields[1]) + const inode = Number.parseInt(fields[9], 10) + if (!parsed || !Number.isFinite(inode) || inode === 0) { + continue + } + results.push({ port: parsed.port, host: parsed.host, inode }) + if (results.length >= LINUX_PROC_LISTENING_SOCKET_MAX_ENTRIES) { + break + } + } + + return results +} + // Why: /proc/net/tcp encodes addresses as hex pairs in host-byte-order. // IPv4: 8 hex chars for address + ':' + 4 hex chars for port. // IPv6: 32 hex chars for address + ':' + 4 hex chars for port. diff --git a/src/relay/preflight-handler.test.ts b/src/relay/preflight-handler.test.ts index 80bc2ba490a..9729ea14192 100644 --- a/src/relay/preflight-handler.test.ts +++ b/src/relay/preflight-handler.test.ts @@ -32,7 +32,8 @@ import { buildCommandLookupSpecs, hasAbsoluteCommandPath, isCommandOnPathForRelay, - PreflightHandler + PreflightHandler, + RELAY_AGENT_PATH_PROBE_CONCURRENCY } from './preflight-handler' function lookupArgs(command: string, mode: '-lc' | '-ilc' = '-lc'): string[] { @@ -233,6 +234,54 @@ describe('hasAbsoluteCommandPath', () => { }) describe('PreflightHandler', () => { + it.each([ + ['at the limit', RELAY_AGENT_PATH_PROBE_CONCURRENCY], + ['above the limit', RELAY_AGENT_PATH_PROBE_CONCURRENCY + 1] + ])('bounds relay-supplied command probes %s', async (_, count) => { + let active = 0 + let peak = 0 + const releases: (() => void)[] = [] + execFileAsyncMock.mockImplementation(async (_file, args) => { + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + const command = String(args[1]).match(/command -v '([^']+)'/)?.[1] ?? 'agent' + return { stdout: `__ORCA_AGENT_PATH__/relay/path/${command}\n` } + }) + const requestHandlers = new Map) => Promise>() + const dispatcher = { + onRequest: vi.fn( + (method: string, handler: (params: Record) => Promise) => { + requestHandlers.set(method, handler) + } + ) + } + new PreflightHandler(dispatcher as never) + + const detection = requestHandlers.get('preflight.detectAgents')!({ + commands: Array.from({ length: count }, (_, index) => ({ + id: `agent-${index}`, + cmd: `agent-${index}` + })) + }) + await vi.waitFor(() => + expect(execFileAsyncMock).toHaveBeenCalledTimes( + Math.min(count, RELAY_AGENT_PATH_PROBE_CONCURRENCY) + ) + ) + if (count > RELAY_AGENT_PATH_PROBE_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(execFileAsyncMock).toHaveBeenCalledTimes(count)) + } + releases.splice(0).forEach((release) => release()) + + await expect(detection).resolves.toMatchObject({ + agents: expect.arrayContaining(Array.from({ length: count }, (_, index) => `agent-${index}`)) + }) + expect(peak).toBe(Math.min(count, RELAY_AGENT_PATH_PROBE_CONCURRENCY)) + }) + it('honors required commands when reporting detected agents', async () => { execFileAsyncMock.mockImplementation(async (_file, args) => { const script = String(args[1]) diff --git a/src/relay/preflight-handler.ts b/src/relay/preflight-handler.ts index 101c796137c..d3505614f0d 100644 --- a/src/relay/preflight-handler.ts +++ b/src/relay/preflight-handler.ts @@ -8,6 +8,7 @@ import { isPwshAvailable } from '../main/pwsh' import { isWslAvailable, listWslDistros } from '../main/wsl' import { isGitBashAvailable } from '../main/git-bash' import { buildPosixCommandPathLookupScript } from '../shared/posix-command-path-lookup' +import { mapWithConcurrency } from '../shared/map-with-concurrency' const execFileAsync = promisify(execFile) @@ -35,6 +36,7 @@ type AgentDetectionCommand = { const SUPPORTED_POSIX_SHELLS = new Set(['sh', 'dash', 'bash', 'zsh', 'fish']) const CONSERVATIVE_SYSTEM_SHELL_DIRS = new Set(['/bin', '/usr/bin']) const AGENT_PATH_PREFIX = '__ORCA_AGENT_PATH__' +export const RELAY_AGENT_PATH_PROBE_CONCURRENCY = 8 export class PreflightHandler { private dispatcher: RelayDispatcher @@ -67,11 +69,13 @@ export class PreflightHandler { ) ] - const results = await Promise.all( - probeCommands.map(async (cmd) => ({ + const results = await mapWithConcurrency( + probeCommands, + RELAY_AGENT_PATH_PROBE_CONCURRENCY, + async (cmd) => ({ cmd, installed: await this.isCommandOnPath(cmd) - })) + }) ) const foundCommands = new Set( results.filter((result) => result.installed).map(({ cmd }) => cmd) diff --git a/src/relay/protocol-handshake.test.ts b/src/relay/protocol-handshake.test.ts index 822fea3e20b..8cfaaea92c5 100644 --- a/src/relay/protocol-handshake.test.ts +++ b/src/relay/protocol-handshake.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { MessageType, HEADER_LENGTH, + MAX_BUFFERED_FRAME_CHUNKS, FrameDecoder, encodeHandshakeFrame, parseHandshakeMessage, @@ -9,6 +10,45 @@ import { } from './protocol' describe('handshake framing', () => { + it('bounds retained chunk wrappers for a byte-fragmented frame', () => { + const frames: DecodedFrame[] = [] + const decoder = new FrameDecoder((frame) => frames.push(frame)) + const payloadLength = MAX_BUFFERED_FRAME_CHUNKS + 512 + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(payloadLength, 9) + decoder.feed(header) + + const byte = Buffer.from('x') + const state = decoder as unknown as { chunks: Buffer[] } + for (let index = 0; index < payloadLength; index += 1) { + decoder.feed(byte) + expect(state.chunks.length).toBeLessThanOrEqual(MAX_BUFFERED_FRAME_CHUNKS) + } + + expect(frames).toHaveLength(1) + expect(frames[0].payload.equals(Buffer.alloc(payloadLength, 0x78))).toBe(true) + }) + + it('does not coalesce ordinary transport chunks while a frame is incomplete', () => { + const decoder = new FrameDecoder(() => {}) + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(1024 * 1024, 9) + decoder.feed(header) + const ordinaryChunks = Array.from({ length: 16 }, () => Buffer.alloc(32 * 1024)) + + for (const chunk of ordinaryChunks) { + decoder.feed(chunk) + } + + const state = decoder as unknown as { chunks: Buffer[] } + expect(state.chunks).toHaveLength(ordinaryChunks.length + 1) + for (const [index, chunk] of ordinaryChunks.entries()) { + expect(state.chunks[index + 1]).toBe(chunk) + } + }) + it('round-trips an orca-relay-handshake envelope through the existing framing', () => { const sent = encodeHandshakeFrame({ type: 'orca-relay-handshake', @@ -65,4 +105,31 @@ describe('handshake framing', () => { expect(MessageType.Handshake).not.toBe(MessageType.Regular) expect(MessageType.Handshake).not.toBe(MessageType.KeepAlive) }) + + it('stream-discards an oversized payload instead of buffering toward its advertised size', () => { + const errors: Error[] = [] + const decoder = new FrameDecoder( + () => {}, + (error) => errors.push(error) + ) + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(0xffffffff, 9) + + decoder.feed(header) + const state = decoder as unknown as { + bufferedLength: number + oversizedPayloadBytesRemaining: number + } + expect(errors).toHaveLength(1) + expect(state.bufferedLength).toBe(0) + + const payloadChunk = Buffer.alloc(64 * 1024) + for (let index = 0; index < 128; index += 1) { + decoder.feed(payloadChunk) + expect(state.bufferedLength).toBe(0) + } + expect(state.oversizedPayloadBytesRemaining).toBe(0xffffffff - 128 * payloadChunk.length) + expect(errors).toHaveLength(1) + }) }) diff --git a/src/relay/protocol-json-encoding-memory.test.ts b/src/relay/protocol-json-encoding-memory.test.ts new file mode 100644 index 00000000000..aed0fca223b --- /dev/null +++ b/src/relay/protocol-json-encoding-memory.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from 'vitest' +import { + encodeJsonRpcFrame, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + parseJsonRpcMessage +} from './protocol' +import { RELAY_JSON_MAX_STRUCTURAL_TOKENS } from '../shared/relay-json-admission' + +describe('relay JSON frame encoding memory', () => { + it('preserves accepted JSON bytes', () => { + const message = { + jsonrpc: '2.0' as const, + id: 1, + result: { escaped: 'line\n🐋', omitted: undefined } + } + + const frame = encodeJsonRpcFrame(message, 7, 3) + + expect(frame.subarray(HEADER_LENGTH).toString('utf8')).toBe(JSON.stringify(message)) + }) + + it('rejects an oversized message before allocating its payload Buffer', () => { + const message = { + jsonrpc: '2.0' as const, + id: 1, + result: '\n'.repeat(MAX_MESSAGE_SIZE) + } + const fromSpy = vi.spyOn(Buffer, 'from') + + try { + expect(() => encodeJsonRpcFrame(message, 1, 0)).toThrow('Message too large') + expect(fromSpy).not.toHaveBeenCalled() + } finally { + fromSpy.mockRestore() + } + }) + + it('rejects structurally amplified inbound JSON before parsing', () => { + const payload = Buffer.from( + `{"jsonrpc":"2.0","id":1,"method":"x","params":{"values":[${'0,'.repeat( + RELAY_JSON_MAX_STRUCTURAL_TOKENS + )}0]}}` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + try { + expect(() => parseJsonRpcMessage(payload)).toThrow(/JSON structure exceeds/) + expect(parseSpy).not.toHaveBeenCalled() + } finally { + parseSpy.mockRestore() + } + }) +}) diff --git a/src/relay/protocol.ts b/src/relay/protocol.ts index 4df0f7bd9b5..9adda8942f0 100644 --- a/src/relay/protocol.ts +++ b/src/relay/protocol.ts @@ -1,11 +1,19 @@ // Self-contained relay protocol — mirrors src/main/ssh/relay-protocol.ts // but has no Electron dependencies. Deployed standalone to remote hosts. +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from '../shared/node-bounded-json-stringify' +import { parseRelayJsonText } from '../shared/relay-json-admission' + export const RELAY_VERSION = '0.1.0' export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n` export const HEADER_LENGTH = 13 export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 +export const MAX_BUFFERED_FRAME_CHUNKS = 4_096 +const MAX_COALESCED_TINY_FRAME_CHUNK_BYTES = 8 * 1024 export const MessageType = { Regular: 1, @@ -28,7 +36,7 @@ export function encodeHandshakeFrame(msg: HandshakeMessage): Buffer { } export function parseHandshakeMessage(payload: Buffer): HandshakeMessage { - const msg = JSON.parse(payload.toString('utf-8')) as HandshakeMessage + const msg = parseRelayJsonText(payload.toString('utf-8')) const t = (msg as { type?: string }).type if ( t !== 'orca-relay-handshake' && @@ -73,6 +81,11 @@ export const GIT_RESPONSE_STREAM_THRESHOLD = 256 * 1024 * the client reassembles by concatenation (it does not depend on chunk size). */ export const GIT_RESPONSE_CHUNK_SIZE = 128 * 1024 +/** Bounds retained serialized Git results while preserving responses far above + * the per-file render limit and git.exec's output ceiling. */ +export const MAX_GIT_RESPONSE_STREAM_BYTES = 64 * 1024 * 1024 +export const MAX_GIT_RESPONSE_STREAM_CHUNKS = 4_096 + /** Sentinel result returned in place of a large git response: the real payload * follows as git.responseChunk frames on the bulk lane. Old relays never emit * this, so a new client falls back to the plain result they return. */ @@ -82,7 +95,8 @@ export type GitResponseStreamMarker = { export const RelayErrorCode = { TooManyStreams: -33006, - StreamProtocolError: -33007 + StreamProtocolError: -33007, + MarkdownDocumentListingCapacity: -33008 } as const export type JsonRpcRequest = { @@ -129,10 +143,16 @@ export function encodeFrame( } export function encodeJsonRpcFrame(msg: JsonRpcMessage, id: number, ack: number): Buffer { - const payload = Buffer.from(JSON.stringify(msg), 'utf-8') - if (payload.length > MAX_MESSAGE_SIZE) { - throw new Error(`Message too large: ${payload.length} bytes`) + let serialized: string + try { + serialized = stringifyJsonWithinByteLimit(msg, MAX_MESSAGE_SIZE).serialized + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new Error(`Message too large: more than ${MAX_MESSAGE_SIZE} bytes`) + } + throw error } + const payload = Buffer.from(serialized, 'utf-8') return encodeFrame(MessageType.Regular, id, ack, payload) } @@ -147,6 +167,7 @@ export class FrameDecoder { // frame exactly once instead. private chunks: Buffer[] = [] private bufferedLength = 0 + private oversizedPayloadBytesRemaining = 0 private onFrame: (frame: DecodedFrame) => void private onError: ((err: Error) => void) | null @@ -156,12 +177,18 @@ export class FrameDecoder { } feed(chunk: Buffer | Uint8Array): void { - const buf = Buffer.isBuffer(chunk) + let buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) + if (this.oversizedPayloadBytesRemaining > 0 && buf.length > 0) { + const discarded = Math.min(this.oversizedPayloadBytesRemaining, buf.length) + this.oversizedPayloadBytesRemaining -= discarded + buf = buf.subarray(discarded) + } if (buf.length > 0) { this.chunks.push(buf) this.bufferedLength += buf.length + this.coalesceBufferedChunks() } while (this.bufferedLength >= HEADER_LENGTH) { @@ -169,28 +196,28 @@ export class FrameDecoder { const length = header.readUInt32BE(9) const totalLength = HEADER_LENGTH + length - if (this.bufferedLength < totalLength) { - // Not fully received yet (also holds oversized frames until they can - // be skipped whole, keeping the decoder synchronized). - break - } - if (length > MAX_MESSAGE_SIZE) { - // Why: Throwing here would leave the buffer in a partially consumed - // state — subsequent feed() calls would try to parse the leftover - // payload bytes as a new header, corrupting every future frame. - // Instead we skip the entire oversized frame so the decoder stays - // synchronized with the stream. - this.discardBytes(totalLength) + // Why: retain only the header, then stream-discard the advertised + // payload so a 13-byte malicious header cannot pin up to 4 GiB. + this.discardBytes(HEADER_LENGTH) + this.oversizedPayloadBytesRemaining = length + this.discardBufferedOversizedPayload() const err = new Error(`Frame payload too large: ${length} bytes — discarded`) if (this.onError) { this.onError(err) } else { process.stderr.write(`[relay] ${err.message}\n`) } + if (this.oversizedPayloadBytesRemaining > 0) { + break + } continue } + if (this.bufferedLength < totalLength) { + break + } + const framed = this.takeBytes(totalLength) const frame: DecodedFrame = { type: framed[0], @@ -205,6 +232,7 @@ export class FrameDecoder { reset(): void { this.chunks = [] this.bufferedLength = 0 + this.oversizedPayloadBytesRemaining = 0 } // Why: at the handshake → dispatcher transition, the next consumer must @@ -280,11 +308,37 @@ export class FrameDecoder { } this.bufferedLength -= count } + + private discardBufferedOversizedPayload(): void { + const discarded = Math.min(this.oversizedPayloadBytesRemaining, this.bufferedLength) + if (discarded === 0) { + return + } + this.discardBytes(discarded) + this.oversizedPayloadBytesRemaining -= discarded + } + + private coalesceBufferedChunks(): void { + // Why: merge adversarial tiny fragments without copying ordinary transport chunks. + while (this.chunks.length >= 2) { + const right = this.chunks.at(-1)! + const left = this.chunks.at(-2)! + if ( + left.length > right.length || + left.length + right.length > MAX_COALESCED_TINY_FRAME_CHUNK_BYTES + ) { + break + } + this.chunks.splice(-2, 2, Buffer.concat([left, right], left.length + right.length)) + } + if (this.chunks.length > MAX_BUFFERED_FRAME_CHUNKS) { + this.chunks = [Buffer.concat(this.chunks, this.bufferedLength)] + } + } } export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage { - const text = payload.toString('utf-8') - const msg = JSON.parse(text) as JsonRpcMessage + const msg = parseRelayJsonText(payload.toString('utf-8')) if (msg.jsonrpc !== '2.0') { throw new Error(`Invalid JSON-RPC version: ${(msg as Record).jsonrpc}`) } diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 60f944456d4..5369a1e80b0 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -11,6 +11,7 @@ import { SETUP_AGENT_SEQUENCE_STARTUP_COMMAND_ENV } from '../shared/setup-agent-sequencing' import { PTY_STARTUP_INGRESS_VERSION } from '../shared/pty-startup-ingress' +import { MAX_TERMINAL_COLS } from '../shared/terminal-size-limits' const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ mockPtySpawn: vi.fn(), @@ -24,7 +25,9 @@ const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ write: vi.fn(), resize: vi.fn(), kill: vi.fn(), - clear: vi.fn() + clear: vi.fn(), + pause: vi.fn(), + resume: vi.fn() } })) @@ -38,9 +41,11 @@ import { PtyHandler, attachIdentityMismatches } from './pty-handler' -import type { RelayDispatcher } from './dispatcher' +import { MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES } from './pty-persistence-envelope' +import type { RelayDispatcher, RelayNotificationWriteResult } from './dispatcher' type TestRequestContext = { + clientId?: number isStale: () => boolean signal?: AbortSignal } @@ -50,8 +55,18 @@ function createMockDispatcher() { string, (params: Record, context?: TestRequestContext) => Promise >() - const notificationHandlers = new Map) => void>() + const notificationHandlers = new Map< + string, + (params: Record, context: TestRequestContext) => void + >() const notifications: { method: string; params?: Record }[] = [] + const targetedNotifications: { + clientId: number + method: string + params?: Record + }[] = [] + const clientDetachListeners = new Set<(clientId: number) => void>() + const connectedClientIds = new Set([1]) const dispatcher = { onRequest: vi.fn( @@ -62,33 +77,75 @@ function createMockDispatcher() { requestHandlers.set(method, handler) } ), - onNotification: vi.fn((method: string, handler: (params: Record) => void) => { - notificationHandlers.set(method, handler) + onNotification: vi.fn( + ( + method: string, + handler: (params: Record, context: TestRequestContext) => void + ) => { + notificationHandlers.set(method, handler) + } + ), + onClientDetached: vi.fn((listener: (clientId: number) => void) => { + clientDetachListeners.add(listener) + return () => clientDetachListeners.delete(listener) + }), + connectedClientIds: vi.fn(() => Array.from(connectedClientIds)), + evictClient: vi.fn((clientId: number) => { + connectedClientIds.delete(clientId) + for (const listener of clientDetachListeners) { + listener(clientId) + } }), notify: vi.fn((method: string, params?: Record) => { notifications.push({ method, params }) }), + notifyClientWithBackpressure: vi.fn( + ( + clientId: number, + method: string, + params?: Record + ): RelayNotificationWriteResult => { + targetedNotifications.push({ clientId, method, params }) + const { deliveryToken: _deliveryToken, ...legacyParams } = params ?? {} + dispatcher.notify(method, legacyParams) + return { delivered: true, saturated: false, drained: Promise.resolve() } + } + ), // Helpers for tests _requestHandlers: requestHandlers, _notificationHandlers: notificationHandlers, _notifications: notifications, + _targetedNotifications: targetedNotifications, async callRequest( method: string, params: Record = {}, - context?: TestRequestContext + context: TestRequestContext = { clientId: 1, isStale: () => false } ) { const handler = requestHandlers.get(method) if (!handler) { throw new Error(`No handler for ${method}`) } + if (typeof context?.clientId === 'number') { + connectedClientIds.add(context.clientId) + } return handler(params, context) }, - callNotification(method: string, params: Record = {}) { + callNotification( + method: string, + params: Record = {}, + context: TestRequestContext = { clientId: 1, isStale: () => false } + ) { const handler = notificationHandlers.get(method) if (!handler) { throw new Error(`No handler for ${method}`) } - handler(params) + handler(params, context) + }, + emitClientDetached(clientId: number) { + connectedClientIds.delete(clientId) + for (const listener of clientDetachListeners) { + listener(clientId) + } } } @@ -126,6 +183,8 @@ describe('PtyHandler', () => { mockPtyInstance.resize.mockReset() mockPtyInstance.kill.mockReset() mockPtyInstance.clear.mockReset() + mockPtyInstance.pause.mockReset() + mockPtyInstance.resume.mockReset() mockPtySpawn.mockReturnValue({ ...mockPtyInstance }) @@ -151,6 +210,7 @@ describe('PtyHandler', () => { expect(methods).toContain('pty.clearBuffer') expect(methods).toContain('pty.hasChildProcesses') expect(methods).toContain('pty.getForegroundProcess') + expect(methods).toContain('pty.inspectProcess') expect(methods).toContain('pty.listProcesses') expect(methods).toContain('pty.getDefaultShell') @@ -160,6 +220,12 @@ describe('PtyHandler', () => { expect(notifMethods).toContain('pty.ackData') }) + it('rejects strict process inspection for a missing relay PTY', async () => { + await expect(dispatcher.callRequest('pty.inspectProcess', { id: 'missing' })).rejects.toThrow( + 'terminal_gone' + ) + }) + it('allows callers to shorten a grace timer for empty startup relays', () => { const onExpire = vi.fn() handler.startGraceTimer(onExpire, 100) @@ -202,6 +268,13 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(1) }) + it('rejects oversized spawn dimensions before native allocation', async () => { + await expect( + dispatcher.callRequest('pty.spawn', { cols: MAX_TERMINAL_COLS + 1, rows: 24 }) + ).rejects.toThrow(`1 through ${MAX_TERMINAL_COLS}`) + expect(mockPtySpawn).not.toHaveBeenCalled() + }) + it("does not forward Orca's own NODE_ENV into the spawned shell", async () => { // Why: NODE_ENV in the relay host process is a build-mode flag, not the // user's; leaking it breaks `next build` and Vitest in the terminal. @@ -407,6 +480,25 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(MAX_RELAY_PTY_SESSIONS) }) + it('rejects oversized retained spawn fields before creating a native PTY', async () => { + await expect( + dispatcher.callRequest('pty.spawn', { + cwd: 'x'.repeat(MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES + 1) + }) + ).rejects.toThrow(`exceeds ${MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES} bytes`) + + expect(mockPtySpawn).not.toHaveBeenCalled() + expect(handler.activePtyCount).toBe(0) + }) + + it('bounds the requested PTY serialization id list', async () => { + await expect( + dispatcher.callRequest('pty.serialize', { + ids: Array.from({ length: MAX_RELAY_PTY_SESSIONS + 1 }, (_, index) => `pty-${index}`) + }) + ).rejects.toThrow(`PTY persistence request exceeds ${MAX_RELAY_PTY_SESSIONS} entries`) + }) + it('spawns a PTY without post-Node-18 array copy methods', async () => { const descriptor = Object.getOwnPropertyDescriptor(Array.prototype, 'toReversed') Reflect.deleteProperty(Array.prototype, 'toReversed') @@ -2331,7 +2423,7 @@ describe('PtyHandler', () => { expect(mockPtySpawn).not.toHaveBeenCalled() }) - it('applies the physical PTY cap to untrusted revive state', async () => { + it('rejects untrusted revive state above the physical PTY cap transactionally', async () => { const state = JSON.stringify( Array.from({ length: MAX_RELAY_PTY_SESSIONS + 1 }, (_, index) => ({ id: `pty-${index + 1}`, @@ -2342,17 +2434,11 @@ describe('PtyHandler', () => { worktreeId: 'repo-id::/repo' })) ) - const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) - try { - await expect(dispatcher.callRequest('pty.revive', { state })).rejects.toThrow( - 'Maximum number of PTY sessions reached (50)' - ) - } finally { - killSpy.mockRestore() - } - - expect(mockPtySpawn).toHaveBeenCalledTimes(MAX_RELAY_PTY_SESSIONS) - expect(handler.activePtyCount).toBe(MAX_RELAY_PTY_SESSIONS) + await expect(dispatcher.callRequest('pty.revive', { state })).rejects.toThrow( + `PTY persistence state exceeds ${MAX_RELAY_PTY_SESSIONS} entries` + ) + expect(mockPtySpawn).not.toHaveBeenCalled() + expect(handler.activePtyCount).toBe(0) }) it('deduplicates concurrent revive requests for the same physical PTY id', async () => { @@ -2874,6 +2960,40 @@ describe('PtyHandler', () => { expect(vi.getTimerCount()).toBe(0) }) + it('keeps output delivery registered when relay disposal cannot kill a live PTY', async () => { + let onDataCb: ((data: string) => void) | undefined + let rejectKill = true + const mockKill = vi.fn(() => { + if (rejectKill) { + throw new Error('persistent dispose kill failure') + } + }) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn((cb: (data: string) => void) => { + onDataCb = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', {}) + const dispose = handler.dispose({ waitForPhysicalExit: false }) + const rejected = expect(dispose).rejects.toThrow('persistent dispose kill failure') + await vi.advanceTimersByTimeAsync(250) + await rejected + + onDataCb?.('still alive') + await vi.advanceTimersByTimeAsync(8) + expect(dispatcher._notifications).toContainEqual({ + method: 'pty.data', + params: { id: 'pty-1', data: 'still alive' } + }) + + rejectKill = false + await handler.dispose({ waitForPhysicalExit: false }) + expect(handler.activePtyCount).toBe(0) + }) + it('takes ownership when dispose overlaps a queued graceful force-kill retry', async () => { let onExitCb: ((evt: { exitCode: number }) => void) | undefined let forceAttempts = 0 diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 2bc9ead9cfe..361cc3eecca 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -59,6 +59,25 @@ import { isAgentSessionSurfaceBinding, type AgentSessionOwnerBinding } from '../shared/agent-session-host-authority' +import { PtyOutputBroadcast } from './pty-output-broadcast' +import { + assertRelayPtyPersistenceFieldWithinLimit, + assertRelayPtyRetainedFieldsWithinLimits, + parseRelayPtyPersistenceEnvelope, + parseRelayPtyPersistenceIds, + sanitizeRelayPtyEnvToDelete, + serializeRelayPtyPersistenceEnvelope, + type RelayPtyIdentity, + type RelayPtyPersistenceEntry +} from './pty-persistence-envelope' +import { + MAX_TERMINAL_COLS, + MAX_TERMINAL_ROWS, + normalizeTerminalSize, + terminalSizeAdmissionError +} from '../shared/terminal-size-limits' + +export { PTY_OUTPUT_HIGH_WATER_CHARS, PTY_OUTPUT_LOW_WATER_CHARS } from './pty-output-broadcast' function isMissingNodePtyNativeBinding(error: unknown): boolean { return ( @@ -110,13 +129,6 @@ const AGENT_SESSION_CREATE_OPERATION_ID_PATTERN = /^[A-Za-z0-9_-]{43}$/ const AGENT_SESSION_CREATE_OPERATION_RETENTION_MS = 24 * 60 * 60 * 1000 const AGENT_SESSION_CREATE_OPERATION_LIMIT = 4_096 -type PendingPtyOutput = { - data: string - rawLength?: number - transformed?: boolean - seq?: number -} - type ManagedStartupCommand = { command: string delivered: boolean @@ -173,13 +185,9 @@ const DEFAULT_GRACE_TIME_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 export const IMMEDIATE_PTY_EXIT_TIMEOUT_MS = 8_000 export const MAX_RELAY_PTY_SESSIONS = 50 export const REPLAY_BUFFER_MAX = 100 * 1024 -const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 -const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 -const PTY_OUTPUT_FLUSH_CHUNK_CHARS = 16 * 1024 -const PTY_OUTPUT_FLUSH_MAX_WRITES = 2 const INTERACTIVE_OUTPUT_WINDOW_MS = 100 const INTERACTIVE_OUTPUT_MAX_CHARS = 1024 -const INTERACTIVE_REDRAW_MAX_CHARS = PTY_OUTPUT_FLUSH_CHUNK_CHARS +const INTERACTIVE_REDRAW_MAX_CHARS = 16 * 1024 const INTERACTIVE_OUTPUT_BUDGET_CHARS = 32 * 1024 const STARTUP_COMMAND_WRITE_DELAY_MS = 50 const STARTUP_COMMAND_SHELL_READY_FALLBACK_MS = 1500 @@ -233,35 +241,9 @@ type PtyProcessSummary = { agentSessionOwners?: AgentSessionOwnerBinding[] } -type SerializedPtyEntry = { - id: string - pid: number - cols: number - rows: number - cwd: string - paneKey?: string - tabId?: string - attachIdentity?: PtyIdentity - worktreeId?: string - terminalHandle?: string - explicitTerm?: string - envToDelete?: string[] - /** Optional for state serialized by relays predating the credential guard. */ - gitCredentialPromptGuarded?: boolean - agentSessionOwners?: AgentSessionOwnerBinding[] -} - -function sanitizeEnvToDelete(value: unknown): string[] { - return Array.isArray(value) - ? value - .filter((key): key is string => typeof key === 'string' && key.length > 0) - .slice(0, 1_024) - : [] -} - export type PtyExitListener = (event: { id: string; paneKey?: string }) => void -type PtyIdentity = { paneKey?: string; tabId?: string } +type PtyIdentity = RelayPtyIdentity /** * True when a reattach's expected pane identity contradicts the target PTY's own. @@ -292,10 +274,9 @@ export class PtyHandler { private ptys = new Map() private nextId = 1 private dispatcher: RelayDispatcher + private readonly outputBroadcast: PtyOutputBroadcast private graceTimeMs: number private graceTimer: ReturnType | null = null - private outputFlushTimer: ReturnType | null = null - private pendingOutputByPty = new Map() private lastInputAtByPty = new Map() private interactiveOutputCharsByPty = new Map() private pendingSpawnCount = 0 @@ -319,6 +300,7 @@ export class PtyHandler { constructor(dispatcher: RelayDispatcher, graceTimeMs = DEFAULT_GRACE_TIME_MS) { this.dispatcher = dispatcher + this.outputBroadcast = new PtyOutputBroadcast(dispatcher) this.graceTimeMs = graceTimeMs this.registerHandlers() } @@ -523,6 +505,7 @@ export class PtyHandler { private wireAndStore(managed: ManagedPty): void { managed.physicalExit = new PhysicalExitTracker() this.ptys.set(managed.id, managed) + this.outputBroadcast.register(managed.id, managed.pty) const emitIngressData = (emission: PtyIngressEmission): void => { const rawLength = emission.rawEndSeq - emission.rawStartSeq this.appendReplayBuffer(managed, emission.data) @@ -567,7 +550,7 @@ export class PtyHandler { } this.clearStartupCommandTimer(managed) this.releaseRelayIngress(managed) - this.flushPtyOutput(managed.id) + this.outputBroadcast.flushForExit(managed.id) this.dispatcher.notify('pty.exit', { id: managed.id, code: exitCode, @@ -612,7 +595,7 @@ export class PtyHandler { private registerHandlers(): void { this.dispatcher.onRequest('pty.spawn', (p, context) => this.spawn(p, context)) - this.dispatcher.onRequest('pty.attach', (p) => this.attach(p)) + this.dispatcher.onRequest('pty.attach', (p, context) => this.attach(p, context)) this.dispatcher.onRequest('pty.shutdown', (p) => this.shutdown(p)) this.dispatcher.onRequest('pty.sendSignal', (p) => this.sendSignal(p)) this.dispatcher.onRequest('pty.getCwd', (p) => this.getCwd(p)) @@ -621,6 +604,7 @@ export class PtyHandler { this.dispatcher.onRequest('pty.clearBuffer', (p) => this.clearBuffer(p)) this.dispatcher.onRequest('pty.hasChildProcesses', (p) => this.hasChildProcesses(p)) this.dispatcher.onRequest('pty.getForegroundProcess', (p) => this.getForegroundProcess(p)) + this.dispatcher.onRequest('pty.inspectProcess', (p) => this.inspectProcess(p)) this.dispatcher.onRequest('pty.getCapabilities', async () => ({ startupIngressVersion: PTY_STARTUP_INGRESS_VERSION, agentSessionClaimVersion: AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION, @@ -637,9 +621,9 @@ export class PtyHandler { this.dispatcher.onNotification('pty.data', (p) => this.writeData(p)) this.dispatcher.onNotification('pty.resize', (p) => this.resize(p)) - this.dispatcher.onNotification('pty.ackData', (_p) => { - /* flow control ack -- not yet enforced */ - }) + this.dispatcher.onNotification('pty.ackData', (p, context) => + this.outputBroadcast.acknowledge(p, context) + ) } private isLikelyInteractiveRedraw(data: string): boolean { @@ -685,105 +669,21 @@ export class PtyHandler { data: string, meta: { rawLength?: number; transformed?: boolean; seq?: number } = {} ): void { - const existing = this.pendingOutputByPty.get(id) - if (meta.transformed === true) { - // Why: transformed spans lack a raw-to-clean slice mapping, so they can't be folded into the output batch. - if (existing) { - this.flushPtyOutput(id) - } - this.dispatcher.notify('pty.data', { id, data, ...meta }) + const managed = this.ptys.get(id) + if (!managed || managed.disposed) { return } - const pending: PendingPtyOutput = { data: (existing?.data ?? '') + data } - if (existing?.rawLength !== undefined || meta.rawLength !== undefined) { - pending.rawLength = - (existing?.rawLength ?? existing?.data.length ?? 0) + (meta.rawLength ?? data.length) - } - if (meta.seq !== undefined) { - pending.seq = meta.seq - } - if (this.shouldSendInteractiveOutputNow(id, pending.data)) { - this.pendingOutputByPty.delete(id) - this.clearOutputFlushTimerIfIdle() - // Why: send interactive echo immediately — batching must not add visible input delay for TUIs. - this.dispatcher.notify('pty.data', { id, ...pending }) - return - } - this.pendingOutputByPty.set(id, pending) - this.scheduleOutputFlush(PTY_OUTPUT_BATCH_INTERVAL_MS) - } - - private scheduleOutputFlush(delayMs: number): void { - if (this.outputFlushTimer !== null) { - return - } - this.outputFlushTimer = setTimeout(() => this.flushPendingOutput(), delayMs) - } - - private flushPendingOutput(): void { - this.outputFlushTimer = null - let writes = 0 - for (const [id, pending] of Array.from(this.pendingOutputByPty.entries())) { - if (writes >= PTY_OUTPUT_FLUSH_MAX_WRITES) { - break - } - this.pendingOutputByPty.delete(id) - const chunk = pending.transformed - ? pending.data - : pending.data.slice(0, PTY_OUTPUT_FLUSH_CHUNK_CHARS) - const remaining = pending.transformed ? '' : pending.data.slice(PTY_OUTPUT_FLUSH_CHUNK_CHARS) - if (remaining) { - this.pendingOutputByPty.set(id, { - data: remaining, - ...(pending.rawLength === undefined ? {} : { rawLength: remaining.length }), - seq: pending.seq - }) - } - const chunkRawLength = pending.transformed - ? pending.rawLength - : pending.rawLength === undefined - ? undefined - : chunk.length - const chunkSeq = - pending.seq === undefined ? undefined : pending.seq - (pending.data.length - chunk.length) - this.dispatcher.notify('pty.data', { - id, - data: chunk, - ...(chunkSeq === undefined ? {} : { seq: chunkSeq }), - ...(chunkRawLength === undefined ? {} : { rawLength: chunkRawLength }), - ...(pending.transformed ? { transformed: true } : {}) - }) - writes++ - } - if (this.pendingOutputByPty.size > 0 && writes > 0) { - // Why: yield between slices of a large chunk so client input and control frames can interleave. - this.scheduleOutputFlush(PTY_OUTPUT_DRAIN_CONTINUE_MS) - } - } - - private flushPtyOutput(id: string): void { - const pending = this.pendingOutputByPty.get(id) - if (!pending) { - return - } - this.pendingOutputByPty.delete(id) - this.dispatcher.notify('pty.data', { id, ...pending }) - this.clearOutputFlushTimerIfIdle() - } - - private clearOutputFlushTimerIfIdle(): void { - if (this.pendingOutputByPty.size > 0 || this.outputFlushTimer === null) { - return - } - clearTimeout(this.outputFlushTimer) - this.outputFlushTimer = null + this.outputBroadcast.enqueue( + id, + { data, ...meta }, + meta.transformed === true || this.shouldSendInteractiveOutputNow(id, data) + ) } private clearPtyFlowState(id: string): void { - this.pendingOutputByPty.delete(id) + this.outputBroadcast.unregister(id) this.lastInputAtByPty.delete(id) this.interactiveOutputCharsByPty.delete(id) - this.clearOutputFlushTimerIfIdle() } private beginPtyCreation(operationPaths: readonly (string | undefined)[]): () => void { @@ -900,9 +800,13 @@ export class PtyHandler { context?: RequestContext ): Promise { const env = params.env as Record | undefined - const worktreeId = env?.ORCA_WORKTREE_ID + const worktreeId = typeof env?.ORCA_WORKTREE_ID === 'string' ? env.ORCA_WORKTREE_ID : undefined + if (worktreeId !== undefined) { + assertRelayPtyPersistenceFieldWithinLimit('worktreeId', worktreeId) + } const worktreePath = worktreeId ? splitWorktreeId(worktreeId)?.worktreePath : undefined const cwd = typeof params.cwd === 'string' ? params.cwd : resolveDefaultCwd() + assertRelayPtyPersistenceFieldWithinLimit('cwd', cwd) const finishCreation = this.beginPtyCreation([worktreePath, cwd]) let physicalSpawnCommitted = false const markPhysicalSpawnCommitted = (): void => { @@ -988,16 +892,21 @@ export class PtyHandler { context?: RequestContext, onPhysicalSpawnCommitted?: () => void ): Promise<{ id: string; incarnationId: string }> { + const sizeError = terminalSizeAdmissionError(params.cols, params.rows, 'pty.spawn', { + allowMissing: true + }) + if (sizeError) { + throw new Error(sizeError) + } const pty = await this.loadPty() if (!pty) { throw new Error('node-pty is not available on this remote host') } - const cols = (params.cols as number) || 80 - const rows = (params.rows as number) || 24 + const { cols, rows } = normalizeTerminalSize(params.cols, params.rows) const cwd = (params.cwd as string) || resolveDefaultCwd() const env = params.env as Record | undefined - const envToDelete = sanitizeEnvToDelete(params.envToDelete) + const envToDelete = sanitizeRelayPtyEnvToDelete(params.envToDelete) const explicitTerm = !envToDelete.includes('TERM') && env && @@ -1020,6 +929,23 @@ export class PtyHandler { // Why: kept so a restarted runtime can re-adopt this PTY under its original handle (survives revive). const terminalHandle = typeof env?.ORCA_TERMINAL_HANDLE === 'string' ? env.ORCA_TERMINAL_HANDLE : undefined + const tabId = typeof env?.ORCA_TAB_ID === 'string' ? env.ORCA_TAB_ID : undefined + const attachIdentity = { + paneKey: typeof params.paneKey === 'string' ? params.paneKey : paneKey, + tabId: typeof params.tabId === 'string' ? params.tabId : tabId + } + const worktreeId = typeof env?.ORCA_WORKTREE_ID === 'string' ? env.ORCA_WORKTREE_ID : undefined + assertRelayPtyRetainedFieldsWithinLimits({ + id, + cwd, + paneKey, + tabId, + attachIdentity, + worktreeId, + terminalHandle, + explicitTerm, + envToDelete + }) const command = typeof params.command === 'string' ? params.command : undefined const terminalWindowsWslDistro = typeof params.terminalWindowsWslDistro === 'string' ? params.terminalWindowsWslDistro : null @@ -1078,12 +1004,6 @@ export class PtyHandler { onPhysicalSpawnCommitted?.() // Why: capture paneKey so the exit listener can evict per-pane caches without a separate ptyId→paneKey map. - const tabId = typeof env?.ORCA_TAB_ID === 'string' ? env.ORCA_TAB_ID : undefined - const attachIdentity = { - paneKey: typeof params.paneKey === 'string' ? params.paneKey : paneKey, - tabId: typeof params.tabId === 'string' ? params.tabId : tabId - } - const worktreeId = typeof env?.ORCA_WORKTREE_ID === 'string' ? env.ORCA_WORKTREE_ID : undefined const startupIngressIntent = params.startupIngressVersion === PTY_STARTUP_INGRESS_VERSION ? parsePtyStartupIngressIntent(params.startupIngress) @@ -1142,7 +1062,8 @@ export class PtyHandler { } private async attach( - params: Record + params: Record, + context?: RequestContext ): Promise<{ incarnationId: string; replay?: string }> { const id = params.id as string const managed = this.ptys.get(id) @@ -1155,7 +1076,7 @@ export class PtyHandler { if (managed.pty.pid && !isProcessAlive(managed.pty.pid)) { managed.physicalExit?.markExited() this.releaseRelayIngress(managed) - this.flushPtyOutput(id) + this.outputBroadcast.flushForExit(id) this.notifyExitListener(managed) this.agentSessionOwners.release(managed.id) disposeManagedPty(managed) @@ -1176,14 +1097,14 @@ export class PtyHandler { throw new Error(`PTY "${id}" not found (identity mismatch)`) } + if (typeof context?.clientId === 'number') { + this.outputBroadcast.resetClient(id, context.clientId) + } managed.startupIngress?.snapshotBarrier() // Why: renderer hasn't registered replay handlers yet during spawn, so return to the caller instead of notifying too early. // Why: buffer intentionally NOT cleared after replay (client clears xterm first) so later restarts still replay full history. if (managed.buffered) { - // Why: drop pending batched bytes already in the replay buffer so attach doesn't render them twice. - this.pendingOutputByPty.delete(id) - this.clearOutputFlushTimerIfIdle() if (params.suppressReplayNotification) { return { incarnationId: managed.incarnationId, replay: managed.buffered } } @@ -1208,8 +1129,8 @@ export class PtyHandler { private resize(params: Record): void { const id = params.id as string - const cols = Math.max(1, Math.min(500, Math.floor(Number(params.cols) || 80))) - const rows = Math.max(1, Math.min(500, Math.floor(Number(params.rows) || 24))) + const cols = Math.max(1, Math.min(MAX_TERMINAL_COLS, Math.floor(Number(params.cols) || 80))) + const rows = Math.max(1, Math.min(MAX_TERMINAL_ROWS, Math.floor(Number(params.rows) || 24))) const managed = this.ptys.get(id) if (managed && !managed.disposed) { managed.pty.resize(cols, rows) @@ -1236,7 +1157,6 @@ export class PtyHandler { if (immediate) { this.releaseStartupCommand(managed) - this.flushPtyOutput(id) this.requestForceKill(managed) // Why: remote Git deletion must not race the child's native handles; on timeout keep the map entry so onExit/retry still owns it. await this.waitForPhysicalExit(managed, IMMEDIATE_PTY_EXIT_TIMEOUT_MS) @@ -1278,6 +1198,7 @@ export class PtyHandler { if (managed.gracefulKillSent) { return } + this.outputBroadcast.flushForExit(managed.id) managed.gracefulKillSent = true if (process.platform === 'win32') { // Why: ConPTY's bare kill is already force-final; block any later close of the handle. @@ -1332,6 +1253,7 @@ export class PtyHandler { if (managed.forceKillSent || (process.platform === 'win32' && managed.gracefulKillSent)) { return } + this.outputBroadcast.flushForExit(managed.id) managed.forceKillSent = true try { killPtyProcess(managed.pty, 'SIGKILL') @@ -1386,6 +1308,25 @@ export class PtyHandler { return await getForegroundProcessName(managed.pty.pid, managed.pty.process || null) } + private async inspectProcess(params: Record): Promise<{ + foregroundProcess: string | null + hasChildProcesses: boolean + }> { + const id = params.id as string + const managed = this.ptys.get(id) + if (!managed || managed.disposed) { + throw new Error('terminal_gone') + } + const foregroundProcess = await getForegroundProcessName( + managed.pty.pid, + managed.pty.process || null + ) + return { + foregroundProcess, + hasChildProcesses: await processHasChildren(managed.pty.pid) + } + } + private async listProcesses(): Promise { const results: PtyProcessSummary[] = [] for (const [id, managed] of this.ptys) { @@ -1407,8 +1348,8 @@ export class PtyHandler { } private async serialize(params: Record): Promise { - const ids = params.ids as string[] - const entries: SerializedPtyEntry[] = [] + const ids = parseRelayPtyPersistenceIds(params.ids, MAX_RELAY_PTY_SESSIONS) + const entries: RelayPtyPersistenceEntry[] = [] for (const id of ids) { const managed = this.ptys.get(id) if (!managed) { @@ -1431,12 +1372,11 @@ export class PtyHandler { ...(managed.terminalHandle ? { terminalHandle: managed.terminalHandle } : {}) }) } - return JSON.stringify(entries) + return serializeRelayPtyPersistenceEnvelope(entries, MAX_RELAY_PTY_SESSIONS) } private async revive(params: Record): Promise { - const state = params.state as string - const entries = JSON.parse(state) as SerializedPtyEntry[] + const entries = parseRelayPtyPersistenceEnvelope(params.state, MAX_RELAY_PTY_SESSIONS) for (const entry of entries) { if (this.ptys.has(entry.id) || this.pendingReviveIds.has(entry.id)) { @@ -1462,7 +1402,7 @@ export class PtyHandler { } } - private async reviveEntry(entry: SerializedPtyEntry): Promise { + private async reviveEntry(entry: RelayPtyPersistenceEntry): Promise { const ptyMod = await this.loadPty() if (!ptyMod) { return @@ -1489,7 +1429,7 @@ export class PtyHandler { revivedEnv.TERM = explicitTerm } // Why: serialized state may come from an older/untrusted client; reapply fresh-spawn bounds. - const envToDelete = sanitizeEnvToDelete(entry.envToDelete) + const envToDelete = sanitizeRelayPtyEnvToDelete(entry.envToDelete) const shell = resolveDefaultShell() const spawnEnv = this.buildSpawnEnv( revivedEnv, @@ -1576,16 +1516,8 @@ export class PtyHandler { this.cancelGraceTimer() await this.waitForPendingPtyCreations() for (const managed of this.ptys.values()) { - this.releaseRelayIngress(managed) - this.flushPtyOutput(managed.id) + this.outputBroadcast.flushForExit(managed.id) } - if (this.outputFlushTimer !== null) { - clearTimeout(this.outputFlushTimer) - this.outputFlushTimer = null - } - this.pendingOutputByPty.clear() - this.lastInputAtByPty.clear() - this.interactiveOutputCharsByPty.clear() const results = await Promise.allSettled( [...this.ptys.values()].map((managed) => this.disposePtyForRelayShutdown(managed, waitForPhysicalExit) @@ -1597,6 +1529,9 @@ export class PtyHandler { if (rejected) { throw rejected.reason } + this.outputBroadcast.dispose() + this.lastInputAtByPty.clear() + this.interactiveOutputCharsByPty.clear() } private async disposePtyForRelayShutdown( @@ -1608,9 +1543,9 @@ export class PtyHandler { managed.killTimer = undefined } this.clearStartupCommandTimer(managed) - this.releaseRelayIngress(managed) // Why: retain the native owner until SIGKILL is accepted (one bounded retry) or onExit proves it gone. await this.requestForceKillForRelayShutdown(managed) + this.releaseRelayIngress(managed) if (waitForPhysicalExit && this.ptys.get(managed.id) === managed && !managed.disposed) { try { await this.waitForPhysicalExit(managed, IMMEDIATE_PTY_EXIT_TIMEOUT_MS) diff --git a/src/relay/pty-output-broadcast.test.ts b/src/relay/pty-output-broadcast.test.ts new file mode 100644 index 00000000000..79f3947d37d --- /dev/null +++ b/src/relay/pty-output-broadcast.test.ts @@ -0,0 +1,245 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RelayDispatcher, RelayNotificationWriteResult } from './dispatcher' +import { PTY_OUTPUT_HIGH_WATER_CHARS, PtyOutputBroadcast } from './pty-output-broadcast' + +type Delivery = { + clientId: number + data: string + deliveryToken: string +} + +function createDispatcher(clientIds: number[]) { + const connected = new Set(clientIds) + const detachListeners = new Set<(clientId: number) => void>() + const deliveries: Delivery[] = [] + const evicted: number[] = [] + const writers = new Map< + number, + (delivery: Delivery) => RelayNotificationWriteResult | undefined + >() + const dispatcher = { + connectedClientIds: () => Array.from(connected), + onClientDetached: (listener: (clientId: number) => void) => { + detachListeners.add(listener) + return () => detachListeners.delete(listener) + }, + notifyClientWithBackpressure: ( + clientId: number, + _method: string, + params: Record = {} + ): RelayNotificationWriteResult => { + const delivery = { + clientId, + data: String(params.data ?? ''), + deliveryToken: String(params.deliveryToken ?? '') + } + deliveries.push(delivery) + return ( + writers.get(clientId)?.(delivery) ?? { + delivered: true, + saturated: false, + drained: Promise.resolve() + } + ) + }, + evictClient: (clientId: number) => { + if (!connected.delete(clientId)) { + return + } + evicted.push(clientId) + for (const listener of detachListeners) { + listener(clientId) + } + } + } as unknown as RelayDispatcher + return { dispatcher, deliveries, evicted, writers } +} + +function deliveryData(deliveries: Delivery[], clientId: number): string { + return deliveries + .filter((delivery) => delivery.clientId === clientId) + .map((delivery) => delivery.data) + .join('') +} + +describe('PtyOutputBroadcast', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('broadcasts identical ordered output to every healthy client', async () => { + const { dispatcher, deliveries } = createDispatcher([1, 2]) + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause: vi.fn(), resume: vi.fn() }) + + output.enqueue('pty-1', { data: 'hello ' }) + output.enqueue('pty-1', { data: 'world' }) + await vi.advanceTimersByTimeAsync(20) + + expect(deliveryData(deliveries, 1)).toBe('hello world') + expect(deliveryData(deliveries, 2)).toBe('hello world') + }) + + it('lets a transiently saturated client drain its exact backlog without stalling peers', async () => { + const { dispatcher, deliveries, writers } = createDispatcher([1, 2]) + let signalDrain!: () => void + let firstWrite = true + writers.set(1, () => { + if (!firstWrite) { + return undefined + } + firstWrite = false + return { + delivered: true, + saturated: true, + drained: new Promise((resolve) => { + signalDrain = resolve + }) + } + }) + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause: vi.fn(), resume: vi.fn() }) + + output.enqueue('pty-1', { data: 'first' }) + await vi.advanceTimersByTimeAsync(8) + output.enqueue('pty-1', { data: ' second' }) + await vi.advanceTimersByTimeAsync(8) + + expect(deliveryData(deliveries, 1)).toBe('first') + expect(deliveryData(deliveries, 2)).toBe('first second') + + signalDrain() + await Promise.resolve() + + expect(deliveryData(deliveries, 1)).toBe('first second') + expect(deliveryData(deliveries, 2)).toBe('first second') + }) + + it('evicts only a permanently slow observer at the shared backlog bound', async () => { + const { dispatcher, deliveries, evicted, writers } = createDispatcher([1, 2]) + writers.set(1, () => ({ + delivered: true, + saturated: true, + drained: new Promise(() => {}) + })) + const pause = vi.fn() + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause, resume: vi.fn() }) + const chunk = 'x'.repeat(16 * 1024) + + for (let index = 0; index <= PTY_OUTPUT_HIGH_WATER_CHARS / chunk.length; index++) { + output.enqueue('pty-1', { data: chunk }) + await vi.advanceTimersByTimeAsync(8) + const healthyDelivery = deliveries.findLast((delivery) => delivery.clientId === 2) + output.acknowledge( + { + id: 'pty-1', + charCount: chunk.length, + deliveryToken: healthyDelivery?.deliveryToken + }, + { clientId: 2, isStale: () => false } + ) + } + + expect(evicted).toEqual([1]) + expect(deliveryData(deliveries, 2)).toHaveLength(PTY_OUTPUT_HIGH_WATER_CHARS + chunk.length) + expect(pause).not.toHaveBeenCalled() + + output.enqueue('pty-1', { data: 'tail' }) + await vi.advanceTimersByTimeAsync(20) + expect(deliveryData(deliveries, 2).endsWith('tail')).toBe(true) + }) + + it('pauses all-blocked producers and rejects cross-client or stale credit', async () => { + const { dispatcher, deliveries } = createDispatcher([7]) + const pause = vi.fn() + const resume = vi.fn() + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause, resume }) + + output.enqueue('pty-1', { data: 'x'.repeat(PTY_OUTPUT_HIGH_WATER_CHARS) }) + const deliveryToken = deliveries[0]?.deliveryToken + + expect(pause).toHaveBeenCalledOnce() + output.acknowledge( + { id: 'pty-1', charCount: PTY_OUTPUT_HIGH_WATER_CHARS, deliveryToken }, + { clientId: 8, isStale: () => false } + ) + output.acknowledge( + { id: 'pty-1', charCount: PTY_OUTPUT_HIGH_WATER_CHARS, deliveryToken: 'stale' }, + { clientId: 7, isStale: () => false } + ) + expect(resume).not.toHaveBeenCalled() + + output.acknowledge( + { id: 'pty-1', charCount: PTY_OUTPUT_HIGH_WATER_CHARS, deliveryToken }, + { clientId: 7, isStale: () => false } + ) + expect(resume).toHaveBeenCalledOnce() + }) + + it('rotates only the reattaching client token and preserves other observers', async () => { + const { dispatcher, deliveries } = createDispatcher([1, 2]) + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause: vi.fn(), resume: vi.fn() }) + + output.enqueue('pty-1', { data: 'before' }) + await vi.advanceTimersByTimeAsync(20) + const oldOne = deliveries.find((delivery) => delivery.clientId === 1)?.deliveryToken + const oldTwo = deliveries.find((delivery) => delivery.clientId === 2)?.deliveryToken + + output.resetClient('pty-1', 1) + output.enqueue('pty-1', { data: 'after' }) + await vi.advanceTimersByTimeAsync(20) + const latestOne = deliveries.findLast((delivery) => delivery.clientId === 1)?.deliveryToken + const latestTwo = deliveries.findLast((delivery) => delivery.clientId === 2)?.deliveryToken + + expect(latestOne).not.toBe(oldOne) + expect(latestTwo).toBe(oldTwo) + expect(deliveryData(deliveries, 2)).toBe('beforeafter') + }) + + it('cancels drain waiters and releases a paused producer on cleanup', async () => { + const { dispatcher, writers } = createDispatcher([1]) + const cancelDrain = vi.fn() + writers.set(1, () => ({ + delivered: true, + saturated: true, + drained: new Promise(() => {}), + cancelDrain + })) + const pause = vi.fn() + const resume = vi.fn() + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause, resume }) + + output.enqueue('pty-1', { data: 'blocked' }) + await vi.advanceTimersByTimeAsync(8) + expect(pause).toHaveBeenCalledOnce() + + output.unregister('pty-1') + + expect(resume).toHaveBeenCalledOnce() + expect(cancelDrain).toHaveBeenCalledOnce() + }) + + it('queues the bounded final tail before exit even when normal credit is exhausted', () => { + const { dispatcher, deliveries } = createDispatcher([1, 2]) + const output = new PtyOutputBroadcast(dispatcher) + output.register('pty-1', { pause: vi.fn(), resume: vi.fn() }) + const data = `${'x'.repeat(PTY_OUTPUT_HIGH_WATER_CHARS)}final tail` + + output.enqueue('pty-1', { data }) + expect(deliveryData(deliveries, 1)).toHaveLength(PTY_OUTPUT_HIGH_WATER_CHARS) + expect(deliveryData(deliveries, 2)).toHaveLength(PTY_OUTPUT_HIGH_WATER_CHARS) + + output.flushForExit('pty-1') + + expect(deliveryData(deliveries, 1)).toBe(data) + expect(deliveryData(deliveries, 2)).toBe(data) + }) +}) diff --git a/src/relay/pty-output-broadcast.ts b/src/relay/pty-output-broadcast.ts new file mode 100644 index 00000000000..379539a4ee6 --- /dev/null +++ b/src/relay/pty-output-broadcast.ts @@ -0,0 +1,300 @@ +import type { IPty } from 'node-pty' +import type { RelayDispatcher, RequestContext } from './dispatcher' +import { + appendPtyOutput, + cancelPtyOutputDrains, + canSendPtyOutput, + createPtyOutputState, + ensurePtyOutputClient, + evictablePtyOutputClientIds, + hasSendablePtyOutput, + pendingPtyOutputCharCount, + PTY_OUTPUT_HIGH_WATER_CHARS, + relayPtyOutputCharCount, + removePtyOutputClient, + resumePtyOutputProducer, + updatePtyOutputProducer, + type PtyOutputClientFlow, + type PtyOutputState, + type RelayPtyOutput +} from './pty-output-flow-state' + +export { + PTY_OUTPUT_HIGH_WATER_CHARS, + PTY_OUTPUT_LOW_WATER_CHARS, + type RelayPtyOutput +} from './pty-output-flow-state' + +const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 +const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 +const PTY_OUTPUT_FLUSH_MAX_WRITES = 2 + +export class PtyOutputBroadcast { + private readonly states = new Map() + private flushTimer: ReturnType | null = null + private unsubscribeClientDetach: (() => void) | null + + constructor(private readonly dispatcher: RelayDispatcher) { + this.unsubscribeClientDetach = dispatcher.onClientDetached((clientId) => { + this.detachClient(clientId) + }) + } + + register(id: string, producer: Pick): void { + this.unregister(id) + this.states.set(id, createPtyOutputState(id, producer)) + } + + enqueue(id: string, output: RelayPtyOutput, immediate = false): void { + const state = this.states.get(id) + if (!state || (output.data.length === 0 && relayPtyOutputCharCount(output) === 0)) { + return + } + const clientIds = this.dispatcher.connectedClientIds() + if (clientIds.length === 0) { + return + } + for (const clientId of clientIds) { + ensurePtyOutputClient(state, clientId) + } + appendPtyOutput(state, output, new Set(clientIds)) + if (pendingPtyOutputCharCount(state) >= PTY_OUTPUT_HIGH_WATER_CHARS) { + this.enforceBacklogBound(state) + } else { + updatePtyOutputProducer(state) + } + if (immediate || output.transformed === true) { + this.flushState(state, PTY_OUTPUT_FLUSH_MAX_WRITES) + this.scheduleContinuationIfNeeded() + return + } + this.scheduleFlush(PTY_OUTPUT_BATCH_INTERVAL_MS) + } + + acknowledge(params: Record, context: RequestContext): void { + const id = params.id + const charCount = params.charCount + const deliveryToken = params.deliveryToken + if ( + typeof id !== 'string' || + typeof charCount !== 'number' || + !Number.isFinite(charCount) || + charCount <= 0 || + typeof deliveryToken !== 'string' + ) { + return + } + const state = this.states.get(id) + const flow = state?.clients.get(context.clientId) + if (!state || !flow || flow.deliveryToken !== deliveryToken) { + return + } + flow.unackedChars = Math.max(0, flow.unackedChars - Math.floor(charCount)) + this.flushState(state, PTY_OUTPUT_FLUSH_MAX_WRITES) + this.scheduleContinuationIfNeeded() + } + + resetClient(id: string, clientId: number): void { + const state = this.states.get(id) + if (!state) { + return + } + removePtyOutputClient(state, clientId) + if (this.dispatcher.connectedClientIds().includes(clientId)) { + ensurePtyOutputClient(state, clientId) + } + updatePtyOutputProducer(state) + this.clearFlushTimerIfIdle() + } + + flushForExit(id: string): void { + const state = this.states.get(id) + if (!state) { + return + } + // Why bypass normal credit: the bounded final tail must be queued before pty.exit tears down the client-side terminal. + this.flushState(state, Number.POSITIVE_INFINITY, true) + } + + unregister(id: string): void { + const state = this.states.get(id) + if (!state) { + return + } + resumePtyOutputProducer(state) + for (const flow of state.clients.values()) { + cancelPtyOutputDrains(flow) + } + state.clients.clear() + state.pending.length = 0 + this.states.delete(id) + this.clearFlushTimerIfIdle() + } + + dispose(): void { + this.unsubscribeClientDetach?.() + this.unsubscribeClientDetach = null + for (const id of Array.from(this.states.keys())) { + this.unregister(id) + } + if (this.flushTimer) { + clearTimeout(this.flushTimer) + this.flushTimer = null + } + } + + private flushPending(): void { + this.flushTimer = null + let writesRemaining = PTY_OUTPUT_FLUSH_MAX_WRITES + for (const state of this.states.values()) { + if (writesRemaining <= 0) { + break + } + writesRemaining -= this.flushState(state, 1) + } + if (this.hasSendablePendingOutput()) { + this.scheduleFlush(PTY_OUTPUT_DRAIN_CONTINUE_MS) + } + } + + private flushState(state: PtyOutputState, maxWrites: number, force = false): number { + let writes = 0 + const blockedClientIds = new Set() + for (let entryIndex = 0; entryIndex < state.pending.length; ) { + if (writes >= maxWrites) { + break + } + const pending = state.pending[entryIndex] + for (const clientId of Array.from(pending.pendingClientIds)) { + if (writes >= maxWrites) { + break + } + if (blockedClientIds.has(clientId)) { + continue + } + const flow = state.clients.get(clientId) + if (!flow) { + pending.pendingClientIds.delete(clientId) + continue + } + if (!force && !canSendPtyOutput(flow)) { + blockedClientIds.add(clientId) + continue + } + pending.pendingClientIds.delete(clientId) + this.sendToClient(state, flow, pending) + writes++ + if (state.pending[entryIndex] !== pending) { + break + } + if (!force && !canSendPtyOutput(flow)) { + blockedClientIds.add(clientId) + } + } + if (state.pending[entryIndex] !== pending) { + continue + } + if (pending.pendingClientIds.size === 0) { + state.pending.splice(entryIndex, 1) + } else { + entryIndex++ + } + } + if (pendingPtyOutputCharCount(state) >= PTY_OUTPUT_HIGH_WATER_CHARS) { + this.evictLaggingClientsWhenHealthy(state) + } + updatePtyOutputProducer(state) + this.clearFlushTimerIfIdle() + return writes + } + + private sendToClient( + state: PtyOutputState, + flow: PtyOutputClientFlow, + output: RelayPtyOutput + ): void { + const result = this.dispatcher.notifyClientWithBackpressure(flow.clientId, 'pty.data', { + id: state.id, + data: output.data, + ...(output.rawLength === undefined ? {} : { rawLength: output.rawLength }), + ...(output.transformed ? { transformed: true } : {}), + ...(output.seq === undefined ? {} : { seq: output.seq }), + deliveryToken: flow.deliveryToken + }) + if (!result.delivered || state.clients.get(flow.clientId) !== flow) { + removePtyOutputClient(state, flow.clientId) + return + } + flow.unackedChars += relayPtyOutputCharCount(output) + if (!result.saturated) { + return + } + flow.sinkBackpressured = true + if (result.cancelDrain) { + flow.drainCancellations.add(result.cancelDrain) + } + void result.drained.then(() => { + if (result.cancelDrain) { + flow.drainCancellations.delete(result.cancelDrain) + } + if (state.clients.get(flow.clientId) !== flow) { + return + } + flow.sinkBackpressured = false + this.flushState(state, PTY_OUTPUT_FLUSH_MAX_WRITES) + this.scheduleContinuationIfNeeded() + }) + } + + private detachClient(clientId: number): void { + for (const state of this.states.values()) { + removePtyOutputClient(state, clientId) + updatePtyOutputProducer(state) + } + this.clearFlushTimerIfIdle() + } + + private enforceBacklogBound(state: PtyOutputState): void { + this.flushState(state, Number.POSITIVE_INFINITY) + if (pendingPtyOutputCharCount(state) < PTY_OUTPUT_HIGH_WATER_CHARS) { + return + } + this.evictLaggingClientsWhenHealthy(state) + updatePtyOutputProducer(state) + } + + private evictLaggingClientsWhenHealthy(state: PtyOutputState): void { + for (const clientId of evictablePtyOutputClientIds(state)) { + this.dispatcher.evictClient(clientId) + this.detachClient(clientId) + } + } + + private hasSendablePendingOutput(): boolean { + return hasSendablePtyOutput(this.states.values()) + } + + private scheduleFlush(delayMs: number): void { + if (this.flushTimer) { + return + } + this.flushTimer = setTimeout(() => this.flushPending(), delayMs) + } + + private scheduleContinuationIfNeeded(): void { + if (this.hasSendablePendingOutput()) { + this.scheduleFlush(PTY_OUTPUT_DRAIN_CONTINUE_MS) + } + } + + private clearFlushTimerIfIdle(): void { + if ( + this.flushTimer === null || + Array.from(this.states.values()).some((s) => s.pending.length) + ) { + return + } + clearTimeout(this.flushTimer) + this.flushTimer = null + } +} diff --git a/src/relay/pty-output-flow-state.ts b/src/relay/pty-output-flow-state.ts new file mode 100644 index 00000000000..f0761985c03 --- /dev/null +++ b/src/relay/pty-output-flow-state.ts @@ -0,0 +1,240 @@ +import { randomUUID } from 'node:crypto' +import type { IPty } from 'node-pty' + +export const PTY_OUTPUT_HIGH_WATER_CHARS = 256 * 1024 +export const PTY_OUTPUT_LOW_WATER_CHARS = 32 * 1024 + +const PTY_OUTPUT_FLUSH_CHUNK_CHARS = 16 * 1024 + +export type RelayPtyOutput = { + data: string + rawLength?: number + transformed?: boolean + seq?: number +} + +export type PtyOutputClientFlow = { + clientId: number + deliveryToken: string + unackedChars: number + sinkBackpressured: boolean + drainCancellations: Set<() => void> +} + +export type PendingPtyOutput = RelayPtyOutput & { + pendingClientIds: Set +} + +export type PtyOutputState = { + id: string + producer: Pick + clients: Map + pending: PendingPtyOutput[] + producerPaused: boolean +} + +export function createPtyOutputState( + id: string, + producer: Pick +): PtyOutputState { + return { id, producer, clients: new Map(), pending: [], producerPaused: false } +} + +export function relayPtyOutputCharCount(output: RelayPtyOutput): number { + return typeof output.rawLength === 'number' && Number.isFinite(output.rawLength) + ? Math.max(0, Math.floor(output.rawLength)) + : output.data.length +} + +function setsEqual(left: ReadonlySet, right: ReadonlySet): boolean { + if (left.size !== right.size) { + return false + } + for (const value of left) { + if (!right.has(value)) { + return false + } + } + return true +} + +function appendChunk( + state: PtyOutputState, + output: RelayPtyOutput, + pendingClientIds: Set +): void { + const tail = state.pending.at(-1) + const tailChars = tail ? relayPtyOutputCharCount(tail) : 0 + const addedChars = relayPtyOutputCharCount(output) + if ( + tail && + tail.transformed === output.transformed && + setsEqual(tail.pendingClientIds, pendingClientIds) && + tailChars + addedChars <= PTY_OUTPUT_FLUSH_CHUNK_CHARS + ) { + const previousLength = tail.data.length + tail.data += output.data + if (tail.rawLength !== undefined || output.rawLength !== undefined) { + tail.rawLength = (tail.rawLength ?? previousLength) + addedChars + } + if (output.seq !== undefined) { + tail.seq = output.seq + } + return + } + state.pending.push({ ...output, pendingClientIds }) +} + +export function appendPtyOutput( + state: PtyOutputState, + output: RelayPtyOutput, + pendingClientIds: Set +): void { + if (output.transformed || output.data.length <= PTY_OUTPUT_FLUSH_CHUNK_CHARS) { + appendChunk(state, output, pendingClientIds) + return + } + for (let offset = 0; offset < output.data.length; offset += PTY_OUTPUT_FLUSH_CHUNK_CHARS) { + const data = output.data.slice(offset, offset + PTY_OUTPUT_FLUSH_CHUNK_CHARS) + const remainingChars = output.data.length - offset - data.length + appendChunk( + state, + { + data, + ...(output.rawLength === undefined ? {} : { rawLength: data.length }), + ...(output.seq === undefined ? {} : { seq: output.seq - remainingChars }) + }, + new Set(pendingClientIds) + ) + } +} + +export function ensurePtyOutputClient( + state: PtyOutputState, + clientId: number +): PtyOutputClientFlow { + const existing = state.clients.get(clientId) + if (existing) { + return existing + } + const flow: PtyOutputClientFlow = { + clientId, + deliveryToken: randomUUID(), + unackedChars: 0, + sinkBackpressured: false, + drainCancellations: new Set() + } + state.clients.set(clientId, flow) + return flow +} + +export function cancelPtyOutputDrains(flow: PtyOutputClientFlow): void { + for (const cancel of flow.drainCancellations) { + cancel() + } + flow.drainCancellations.clear() +} + +export function removePtyOutputClient(state: PtyOutputState, clientId: number): void { + const flow = state.clients.get(clientId) + if (flow) { + cancelPtyOutputDrains(flow) + state.clients.delete(clientId) + } + for (let index = state.pending.length - 1; index >= 0; index--) { + const pending = state.pending[index] + pending.pendingClientIds.delete(clientId) + if (pending.pendingClientIds.size === 0) { + state.pending.splice(index, 1) + } + } +} + +export function canSendPtyOutput(flow: PtyOutputClientFlow): boolean { + return !flow.sinkBackpressured && flow.unackedChars < PTY_OUTPUT_HIGH_WATER_CHARS +} + +export function pendingPtyOutputCharCount(state: PtyOutputState, clientId?: number): number { + return state.pending.reduce( + (total, pending) => + total + + (clientId === undefined || pending.pendingClientIds.has(clientId) + ? relayPtyOutputCharCount(pending) + : 0), + 0 + ) +} + +export function resumePtyOutputProducer(state: PtyOutputState): void { + if (!state.producerPaused) { + return + } + try { + state.producer.resume() + } catch { + /* PTY already destroyed */ + } + state.producerPaused = false +} + +export function updatePtyOutputProducer(state: PtyOutputState): void { + const flows = Array.from(state.clients.values()) + if (flows.length === 0) { + resumePtyOutputProducer(state) + return + } + if (flows.every((flow) => !canSendPtyOutput(flow))) { + if (!state.producerPaused) { + try { + state.producer.pause() + state.producerPaused = true + } catch { + /* PTY already destroyed */ + } + } + return + } + if ( + state.producerPaused && + flows.some( + (flow) => + !flow.sinkBackpressured && + flow.unackedChars + pendingPtyOutputCharCount(state, flow.clientId) <= + PTY_OUTPUT_LOW_WATER_CHARS + ) + ) { + resumePtyOutputProducer(state) + } +} + +export function hasSendablePtyOutput(states: Iterable): boolean { + for (const state of states) { + for (const pending of state.pending) { + for (const clientId of pending.pendingClientIds) { + const flow = state.clients.get(clientId) + if (flow && canSendPtyOutput(flow)) { + return true + } + } + } + } + return false +} + +export function evictablePtyOutputClientIds(state: PtyOutputState): number[] { + const laggingClientIds = new Set() + for (const pending of state.pending) { + for (const clientId of pending.pendingClientIds) { + laggingClientIds.add(clientId) + } + } + if ( + laggingClientIds.size === 0 || + !Array.from(state.clients.values()).some( + (flow) => !laggingClientIds.has(flow.clientId) && canSendPtyOutput(flow) + ) + ) { + return [] + } + return Array.from(laggingClientIds) +} diff --git a/src/relay/pty-persistence-envelope.test.ts b/src/relay/pty-persistence-envelope.test.ts new file mode 100644 index 00000000000..3e01c635641 --- /dev/null +++ b/src/relay/pty-persistence-envelope.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES, + MAX_RELAY_PTY_PERSISTENCE_RETAINED_BYTES, + MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES, + parseRelayPtyPersistenceEnvelope, + serializeRelayPtyPersistenceEnvelope, + type RelayPtyPersistenceEntry +} from './pty-persistence-envelope' +import { MAX_TERMINAL_COLS } from '../shared/terminal-size-limits' + +function entry(index: number, overrides: Partial = {}) { + return { + id: `pty-${index}`, + pid: 100 + index, + cols: 80, + rows: 24, + cwd: '/repo', + envToDelete: [], + gitCredentialPromptGuarded: false, + ...overrides + } +} + +describe('relay PTY persistence envelope', () => { + it('round-trips normal state without changing retained fields', () => { + const entries = [ + entry(1, { + paneKey: 'tab-1:leaf-1', + attachIdentity: { paneKey: 'tab-1:leaf-1', tabId: 'tab-1' }, + worktreeId: 'repo::/repo', + terminalHandle: 'terminal-1', + explicitTerm: 'screen-256color', + envToDelete: ['ORCA_ATTRIBUTION_SHIM_DIR'], + gitCredentialPromptGuarded: true + }) + ] + + const serialized = serializeRelayPtyPersistenceEnvelope(entries, 50) + + expect(parseRelayPtyPersistenceEnvelope(serialized, 50)).toEqual(entries) + }) + + it('accepts the field limit and rejects limit plus one', () => { + expect(() => + serializeRelayPtyPersistenceEnvelope( + [entry(1, { cwd: 'x'.repeat(MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES) })], + 50 + ) + ).not.toThrow() + expect(() => + serializeRelayPtyPersistenceEnvelope( + [entry(1, { cwd: 'x'.repeat(MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES + 1) })], + 50 + ) + ).toThrow(`exceeds ${MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES} bytes`) + }) + + it('rejects aggregate retained fields before serialization allocates the output', () => { + const field = 'x'.repeat(63 * 1024) + const entries = Array.from({ length: 49 }, (_, index) => + entry(index, { cwd: field, paneKey: field }) + ) + + expect(() => serializeRelayPtyPersistenceEnvelope(entries, 50)).toThrow( + `exceeds ${MAX_RELAY_PTY_PERSISTENCE_RETAINED_BYTES} retained bytes` + ) + }) + + it('rejects oversized and deeply nested input before JSON.parse', () => { + expect(() => + parseRelayPtyPersistenceEnvelope(' '.repeat(MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES + 1), 50) + ).toThrow(`exceeds ${MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES} bytes`) + + const deeplyNested = JSON.stringify([ + { + ...entry(1), + ignored: [[[[[[[[]]]]]]]] + } + ]) + expect(() => parseRelayPtyPersistenceEnvelope(deeplyNested, 50)).toThrow( + 'JSON nesting exceeds 8 levels' + ) + }) + + it('rejects entry-count overflow transactionally', () => { + const serialized = JSON.stringify([entry(1), entry(2)]) + expect(() => parseRelayPtyPersistenceEnvelope(serialized, 1)).toThrow( + 'PTY persistence state exceeds 1 entries' + ) + }) + + it('rejects oversized terminal dimensions on writes and reads', () => { + const oversized = entry(1, { cols: MAX_TERMINAL_COLS + 1 }) + expect(() => serializeRelayPtyPersistenceEnvelope([oversized], 50)).toThrow( + `1 through ${MAX_TERMINAL_COLS}` + ) + expect(() => parseRelayPtyPersistenceEnvelope(JSON.stringify([oversized]), 50)).toThrow( + `1 through ${MAX_TERMINAL_COLS}` + ) + }) +}) diff --git a/src/relay/pty-persistence-envelope.ts b/src/relay/pty-persistence-envelope.ts new file mode 100644 index 00000000000..65380233190 --- /dev/null +++ b/src/relay/pty-persistence-envelope.ts @@ -0,0 +1,243 @@ +import { assertJsonTextStructureWithinLimits } from '../shared/json-text-structure-limit' +import { stringifyJsonWithinByteLimit } from '../shared/node-bounded-json-stringify' +import { terminalSizeAdmissionError } from '../shared/terminal-size-limits' + +export const MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES = 8 * 1024 * 1024 +export const MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES = 64 * 1024 +export const MAX_RELAY_PTY_PERSISTENCE_ENTRY_BYTES = 128 * 1024 +export const MAX_RELAY_PTY_PERSISTENCE_RETAINED_BYTES = 6 * 1024 * 1024 +export const MAX_RELAY_PTY_ENV_DELETE_KEYS = 1_024 + +const PTY_PERSISTENCE_JSON_STRUCTURE_LIMITS = { + structuralTokens: 131_072, + nestingDepth: 8 +} as const + +export type RelayPtyIdentity = { + paneKey?: string + tabId?: string +} + +export type RelayPtyPersistenceEntry = { + id: string + pid: number + cols: number + rows: number + cwd: string + paneKey?: string + tabId?: string + attachIdentity?: RelayPtyIdentity + worktreeId?: string + terminalHandle?: string + explicitTerm?: string + envToDelete?: string[] + gitCredentialPromptGuarded?: boolean +} + +export type RelayPtyRetainedFields = Pick< + RelayPtyPersistenceEntry, + | 'id' + | 'cwd' + | 'paneKey' + | 'tabId' + | 'attachIdentity' + | 'worktreeId' + | 'terminalHandle' + | 'explicitTerm' + | 'envToDelete' +> + +export function sanitizeRelayPtyEnvToDelete(value: unknown): string[] { + return Array.isArray(value) + ? value + .filter((key): key is string => typeof key === 'string' && key.length > 0) + .slice(0, MAX_RELAY_PTY_ENV_DELETE_KEYS) + : [] +} + +export function assertRelayPtyRetainedFieldsWithinLimits(fields: RelayPtyRetainedFields): number { + let retainedBytes = 0 + const add = (field: string, value: string | undefined): void => { + if (value === undefined) { + return + } + const bytes = Buffer.byteLength(value, 'utf8') + if (bytes > MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES) { + throw new Error( + `PTY persistence field "${field}" exceeds ${MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES} bytes` + ) + } + retainedBytes += bytes + } + + add('id', fields.id) + add('cwd', fields.cwd) + add('paneKey', fields.paneKey) + add('tabId', fields.tabId) + add('attachIdentity.paneKey', fields.attachIdentity?.paneKey) + add('attachIdentity.tabId', fields.attachIdentity?.tabId) + add('worktreeId', fields.worktreeId) + add('terminalHandle', fields.terminalHandle) + add('explicitTerm', fields.explicitTerm) + for (const key of fields.envToDelete ?? []) { + add('envToDelete', key) + } + if (retainedBytes > MAX_RELAY_PTY_PERSISTENCE_ENTRY_BYTES) { + throw new Error( + `PTY persistence entry exceeds ${MAX_RELAY_PTY_PERSISTENCE_ENTRY_BYTES} retained bytes` + ) + } + return retainedBytes +} + +export function serializeRelayPtyPersistenceEnvelope( + entries: readonly RelayPtyPersistenceEntry[], + maxEntries: number +): string { + assertEnvelopeEntryCount(entries, maxEntries) + assertEnvelopeRetainedBytes(entries) + return stringifyJsonWithinByteLimit(entries, MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES).serialized +} + +export function parseRelayPtyPersistenceEnvelope( + state: unknown, + maxEntries: number +): RelayPtyPersistenceEntry[] { + if (typeof state !== 'string') { + throw new Error('PTY persistence state must be JSON text') + } + if (Buffer.byteLength(state, 'utf8') > MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES) { + throw new Error(`PTY persistence state exceeds ${MAX_RELAY_PTY_PERSISTENCE_STATE_BYTES} bytes`) + } + assertJsonTextStructureWithinLimits(state, PTY_PERSISTENCE_JSON_STRUCTURE_LIMITS) + return normalizeEnvelope(JSON.parse(state) as unknown, maxEntries) +} + +export function parseRelayPtyPersistenceIds(value: unknown, maxEntries: number): string[] { + if (!Array.isArray(value) || value.length > maxEntries) { + throw new Error(`PTY persistence request exceeds ${maxEntries} entries`) + } + return value.map((id) => requiredString(id, 'id')) +} + +function normalizeEnvelope(value: unknown, maxEntries: number): RelayPtyPersistenceEntry[] { + if (!Array.isArray(value)) { + throw new Error(`PTY persistence state exceeds ${maxEntries} entries`) + } + assertEnvelopeEntryCount(value, maxEntries) + + const entries = value.map((entry, index) => normalizeEntry(entry, index)) + assertEnvelopeRetainedBytes(entries) + return entries +} + +function assertEnvelopeEntryCount(value: readonly unknown[], maxEntries: number): void { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 0) { + throw new RangeError('PTY persistence entry limit must be a non-negative safe integer') + } + if (value.length > maxEntries) { + throw new Error(`PTY persistence state exceeds ${maxEntries} entries`) + } +} + +function assertEnvelopeRetainedBytes(entries: readonly RelayPtyRetainedFields[]): void { + let retainedBytes = 0 + for (const entry of entries) { + const dimensions = entry as Partial + const sizeError = terminalSizeAdmissionError( + dimensions.cols, + dimensions.rows, + 'PTY persistence entry', + { allowMissing: true } + ) + if (sizeError) { + throw new Error(sizeError) + } + retainedBytes += assertRelayPtyRetainedFieldsWithinLimits(entry) + if (retainedBytes > MAX_RELAY_PTY_PERSISTENCE_RETAINED_BYTES) { + throw new Error( + `PTY persistence state exceeds ${MAX_RELAY_PTY_PERSISTENCE_RETAINED_BYTES} retained bytes` + ) + } + } +} + +function normalizeEntry(value: unknown, index: number): RelayPtyPersistenceEntry { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`PTY persistence entry ${index} must be an object`) + } + const entry = value as Record + const attachIdentity = normalizeIdentity(entry.attachIdentity) + const paneKey = optionalString(entry.paneKey, 'paneKey') + const tabId = optionalString(entry.tabId, 'tabId') + const worktreeId = optionalString(entry.worktreeId, 'worktreeId') + const terminalHandle = optionalString(entry.terminalHandle, 'terminalHandle') + const explicitTerm = optionalString(entry.explicitTerm, 'explicitTerm') + const envToDelete = sanitizeRelayPtyEnvToDelete(entry.envToDelete) + const cols = positiveSafeIntegerOrDefault(entry.cols, 'cols', 80) + const rows = positiveSafeIntegerOrDefault(entry.rows, 'rows', 24) + const sizeError = terminalSizeAdmissionError(cols, rows, 'PTY persistence entry') + if (sizeError) { + throw new Error(sizeError) + } + + return { + id: requiredString(entry.id, 'id'), + pid: positiveSafeInteger(entry.pid, 'pid'), + cols, + rows, + cwd: requiredString(entry.cwd, 'cwd'), + ...(paneKey === undefined ? {} : { paneKey }), + ...(tabId === undefined ? {} : { tabId }), + ...(attachIdentity === undefined ? {} : { attachIdentity }), + ...(worktreeId === undefined ? {} : { worktreeId }), + ...(terminalHandle === undefined ? {} : { terminalHandle }), + ...(explicitTerm === undefined ? {} : { explicitTerm }), + ...(Array.isArray(entry.envToDelete) ? { envToDelete } : {}), + gitCredentialPromptGuarded: entry.gitCredentialPromptGuarded === true + } +} + +function normalizeIdentity(value: unknown): RelayPtyIdentity | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return undefined + } + const identity = value as Record + const paneKey = optionalString(identity.paneKey, 'attachIdentity.paneKey') + const tabId = optionalString(identity.tabId, 'attachIdentity.tabId') + return paneKey === undefined && tabId === undefined ? undefined : { paneKey, tabId } +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string') { + throw new Error(`PTY persistence field "${field}" must be a string`) + } + assertRelayPtyPersistenceFieldWithinLimit(field, value) + return value +} + +function optionalString(value: unknown, field: string): string | undefined { + if (value === undefined) { + return undefined + } + return requiredString(value, field) +} + +export function assertRelayPtyPersistenceFieldWithinLimit(field: string, value: string): void { + if (Buffer.byteLength(value, 'utf8') > MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES) { + throw new Error( + `PTY persistence field "${field}" exceeds ${MAX_RELAY_PTY_PERSISTENCE_FIELD_BYTES} bytes` + ) + } +} + +function positiveSafeInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new Error(`PTY persistence field "${field}" must be a positive safe integer`) + } + return value as number +} + +function positiveSafeIntegerOrDefault(value: unknown, field: string, fallback: number): number { + return value === undefined ? fallback : positiveSafeInteger(value, field) +} diff --git a/src/relay/pty-shell-launch.test.ts b/src/relay/pty-shell-launch.test.ts index c4c5b5f11ad..fb8fc4dc693 100644 --- a/src/relay/pty-shell-launch.test.ts +++ b/src/relay/pty-shell-launch.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, mkdirSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { spawnSync } from 'node:child_process' @@ -137,6 +137,24 @@ describe('getRelayShellLaunchConfig', () => { ) }) + it.skipIf(process.platform === 'win32')( + 'replaces a sparse oversized wrapper without reading its payload', + () => { + const zshRoot = join(homeDir, '.orca-relay', 'shell-ready', 'zsh') + const wrapperPath = join(zshRoot, '.zshenv') + mkdirSync(zshRoot, { recursive: true }) + writeFileSync(wrapperPath, '') + truncateSync(wrapperPath, 1024 * 1024 * 1024) + + getRelayShellLaunchConfig('/bin/zsh', { + HOME: homeDir, + ORCA_OPENCODE_CONFIG_DIR: '/tmp/orca-opencode-overlay' + }) + + expect(readFileSync(wrapperPath, 'utf8')).toContain('export ORCA_USER_ZDOTDIR=') + } + ) + it.skipIf(process.platform === 'win32')( 'wraps zsh when MiMo home must survive shell startup', () => { diff --git a/src/relay/pty-shell-launch.ts b/src/relay/pty-shell-launch.ts index 5f58b89702c..34ba6883044 100644 --- a/src/relay/pty-shell-launch.ts +++ b/src/relay/pty-shell-launch.ts @@ -1,4 +1,4 @@ -import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' import { getPosixOmpShellWrapper } from '../main/pty/omp-shell-wrapper' @@ -7,6 +7,7 @@ import { getZshShellReadyMarkerRegistrationBlock, getZshStartupFileSourceBlock } from '../main/shell-templates' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' const RELAY_SHELL_READY_DIR = '.orca-relay/shell-ready' const POSIX_LOGIN_ARGS = ['-l'] @@ -246,7 +247,10 @@ trap '__orca_osc133_preexec' DEBUG mkdirSync(dirname(path), { recursive: true }) let existing: string | null = null try { - existing = readFileSync(path, 'utf8') + existing = readNodeFileSyncWithinLimit( + path, + Buffer.byteLength(content, 'utf8') + ).buffer.toString('utf8') } catch { existing = null } diff --git a/src/relay/pty-shell-utils.ts b/src/relay/pty-shell-utils.ts index 6d589b8c6a0..23ce31500fb 100644 --- a/src/relay/pty-shell-utils.ts +++ b/src/relay/pty-shell-utils.ts @@ -1,8 +1,9 @@ import { execFile as execFileCb } from 'node:child_process' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { homedir } from 'node:os' import { win32 as pathWin32 } from 'node:path' import { promisify } from 'node:util' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' import { isAgentForegroundWrapperProcess, isExpectedAgentProcess, @@ -22,6 +23,7 @@ import { } from '../main/providers/windows-agent-foreground-process' const execFile = promisify(execFileCb) +const MAX_ETC_SHELLS_BYTES = 64 * 1024 export function resolveWindowsDefaultShell( env: NodeJS.ProcessEnv = process.env, @@ -311,7 +313,10 @@ export function listShellProfiles(): { name: string; path: string }[] { const seen = new Set() try { - const content = readFileSync('/etc/shells', 'utf-8') + const content = readNodeFileSyncWithinLimit( + '/etc/shells', + MAX_ETC_SHELLS_BYTES + ).buffer.toString('utf8') for (const line of content.split('\n')) { const trimmed = line.trim() if (!trimmed || trimmed.startsWith('#')) { diff --git a/src/relay/relay-filesystem-watch-registry.test.ts b/src/relay/relay-filesystem-watch-registry.test.ts index 9a21fb9eace..800cb43fd86 100644 --- a/src/relay/relay-filesystem-watch-registry.test.ts +++ b/src/relay/relay-filesystem-watch-registry.test.ts @@ -11,6 +11,7 @@ import type { import type { RelayDispatcher, RequestContext } from './dispatcher' import { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' import { createRelayWatcherProcessPool } from './relay-watcher-process-pool' +import { MAX_RELAY_WATCH_ROOT_KEY_BYTES } from './relay-watcher-root-capacity' type InstalledWatch = { rootPath: string @@ -86,6 +87,21 @@ describe('RelayFilesystemWatchRegistry', () => { registry = new RelayFilesystemWatchRegistry(dispatcher as unknown as RelayDispatcher, pool) }) + it('rejects an oversized raw root before retaining a pending setup', async () => { + await expect( + registry.watch('/'.repeat(MAX_RELAY_WATCH_ROOT_KEY_BYTES + 1), context(1)) + ).rejects.toThrow('File watcher root path is too long') + + expect(pool.installed).toHaveLength(0) + expect( + ( + registry as unknown as { + pendingSetups: Map + } + ).pendingSetups.size + ).toBe(0) + }) + it('emits overflow around child replacement and resumes ordered event delivery', async () => { await registry.watch('/repo', context(1)) const first = pool.installed[0] diff --git a/src/relay/relay-filesystem-watch-registry.ts b/src/relay/relay-filesystem-watch-registry.ts index 4acb786fa23..94977c01ab2 100644 --- a/src/relay/relay-filesystem-watch-registry.ts +++ b/src/relay/relay-filesystem-watch-registry.ts @@ -16,7 +16,10 @@ import { type RelayWatcherTeardownState } from './relay-watcher-teardown-tracker' import { emitRelayWatcherTerminalFailure } from './relay-watcher-terminal-notifier' -import { assertRelayWatcherRootCapacity } from './relay-watcher-root-capacity' +import { + assertRelayWatcherRootCapacity, + assertRelayWatcherRootKeyCapacity +} from './relay-watcher-root-capacity' import { normalizeRuntimePathForComparison } from '../shared/cross-platform-path' import { trackRelayWatcherSetup, @@ -85,6 +88,7 @@ export class RelayFilesystemWatchRegistry { context?: RequestContext, watchId?: number ): Promise { + assertRelayWatcherRootKeyCapacity(rootKey, rootPath) const staleTeardown = releaseStaleRelayWatches(this.watches.values(), (state) => this.closeWatch(state) ) diff --git a/src/relay/relay-handshake.ts b/src/relay/relay-handshake.ts index e021c392561..b1a7da5b5dc 100644 --- a/src/relay/relay-handshake.ts +++ b/src/relay/relay-handshake.ts @@ -1,8 +1,9 @@ // Wire-level handshake helpers for the Orca relay. import { dirname, join } from 'node:path' -import { existsSync, readFileSync, realpathSync } from 'node:fs' +import { existsSync, realpathSync } from 'node:fs' import type { Socket } from 'node:net' +import { readRelayVersionMarkerSync } from '../shared/relay-version-marker' import { RELAY_VERSION, MessageType, @@ -34,7 +35,7 @@ export function readLaunchVersion(): string { } const versionFile = join(dir, '.version') if (existsSync(versionFile)) { - const v = readFileSync(versionFile, 'utf-8').trim() + const v = readRelayVersionMarkerSync(versionFile) if (v) { return v } diff --git a/src/relay/relay-watcher-removal-fence.ts b/src/relay/relay-watcher-removal-fence.ts index 9b0b62ab780..fdda69abc67 100644 --- a/src/relay/relay-watcher-removal-fence.ts +++ b/src/relay/relay-watcher-removal-fence.ts @@ -6,6 +6,9 @@ import type { RelayDispatcher } from './dispatcher' import { emitRelayWatcherTerminalFailure } from './relay-watcher-terminal-notifier' import { isPathInsideOrEqual } from '../shared/cross-platform-path' import type { RelayWatcherPendingSetup } from './relay-watcher-setup-tracking' +import { forEachWithConcurrency, mapSettledWithConcurrency } from '../shared/map-with-concurrency' + +const RELAY_WATCH_CLOSE_CONCURRENCY = 8 export class RelayWatcherRemovalFence { private readonly roots = new Set() @@ -102,7 +105,9 @@ export class RelayWatcherRemovalFence { const pending = [...this.pendingSetups.entries()].filter(([setupRoot]) => isPathInsideOrEqual(rootKey, setupRoot) ) - await Promise.all(pending.map(([, setup]) => setup.promise.catch(() => undefined))) + await forEachWithConcurrency(pending, RELAY_WATCH_CLOSE_CONCURRENCY, async ([, setup]) => { + await setup.promise.catch(() => undefined) + }) const states = [...this.watches.entries()] .filter(([watchRoot]) => isPathInsideOrEqual(rootKey, watchRoot)) .map(([, state]) => state) @@ -111,7 +116,15 @@ export class RelayWatcherRemovalFence { state.clients.clear() state.clientWatchIds.clear() } - await Promise.all(states.map((state) => this.closeWatch(state))) + const closes = await mapSettledWithConcurrency(states, RELAY_WATCH_CLOSE_CONCURRENCY, (state) => + this.closeWatch(state) + ) + const failedClose = closes.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failedClose) { + throw failedClose.reason + } const trackedRoots = this.teardownTracker .rootPaths() .filter((trackedRoot) => isPathInsideOrEqual(rootKey, trackedRoot)) diff --git a/src/relay/relay-watcher-root-capacity.test.ts b/src/relay/relay-watcher-root-capacity.test.ts new file mode 100644 index 00000000000..e8434a82011 --- /dev/null +++ b/src/relay/relay-watcher-root-capacity.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_RELAY_WATCH_ROOT_KEY_BYTES, + MAX_RELAY_WATCH_ROOT_KEYS_BYTES, + MAX_RELAY_WATCH_ROOTS, + assertRelayWatcherRootCapacity +} from './relay-watcher-root-capacity' + +function keyWithBytes(label: string, bytes: number): string { + return `${label}${'x'.repeat(bytes - Buffer.byteLength(label))}` +} + +describe('relay watcher root capacity', () => { + it('accepts a root key at the exact UTF-8 byte boundary', () => { + const root = keyWithBytes('/', MAX_RELAY_WATCH_ROOT_KEY_BYTES) + + expect(() => assertRelayWatcherRootCapacity([], [], [], root)).not.toThrow() + }) + + it('rejects a root key one byte over the boundary', () => { + const root = keyWithBytes('/', MAX_RELAY_WATCH_ROOT_KEY_BYTES + 1) + + expect(() => assertRelayWatcherRootCapacity([], [], [], root)).toThrow( + 'File watcher root path is too long' + ) + }) + + it('rejects an oversized raw path even when normalization made its key small', () => { + const rawPath = '/'.repeat(MAX_RELAY_WATCH_ROOT_KEY_BYTES + 1) + + expect(() => assertRelayWatcherRootCapacity([], [], [], '/', rawPath)).toThrow( + 'File watcher root path is too long' + ) + }) + + it('caps aggregate keys retained across active, pending, and failed teardowns', () => { + const retained = Array.from({ length: 4 }, (_, index) => + keyWithBytes(`/${index}/`, MAX_RELAY_WATCH_ROOT_KEY_BYTES - 1) + ) + const exactProspective = keyWithBytes('/p', 4) + expect( + retained.reduce((total, root) => total + Buffer.byteLength(root), 0) + + Buffer.byteLength(exactProspective) + ).toBe(MAX_RELAY_WATCH_ROOT_KEYS_BYTES) + + expect(() => + assertRelayWatcherRootCapacity( + [retained[0]!], + [retained[1]!], + [retained[2]!, retained[3]!], + exactProspective + ) + ).not.toThrow() + expect(() => + assertRelayWatcherRootCapacity( + [retained[0]!], + [retained[1]!], + [retained[2]!, retained[3]!], + `${exactProspective}x` + ) + ).toThrow('Maximum file watcher root path memory reached') + }) + + it('keeps the existing physical watcher count boundary', () => { + const retained = Array.from({ length: MAX_RELAY_WATCH_ROOTS }, (_, index) => `/${index}`) + + expect(() => assertRelayWatcherRootCapacity(retained, [], [], retained[0]!)).not.toThrow() + expect(() => assertRelayWatcherRootCapacity(retained, [], [], '/overflow')).toThrow( + 'Maximum number of file watchers reached' + ) + }) +}) diff --git a/src/relay/relay-watcher-root-capacity.ts b/src/relay/relay-watcher-root-capacity.ts index 025462e7bd3..4f51b1eeafe 100644 --- a/src/relay/relay-watcher-root-capacity.ts +++ b/src/relay/relay-watcher-root-capacity.ts @@ -1,13 +1,37 @@ -const MAX_RELAY_WATCH_ROOTS = 20 +export const MAX_RELAY_WATCH_ROOTS = 20 +export const MAX_RELAY_WATCH_ROOT_KEY_BYTES = 64 * 1024 +export const MAX_RELAY_WATCH_ROOT_KEYS_BYTES = 256 * 1024 + +export function assertRelayWatcherRootKeyCapacity(rootKey: string, rootPath = rootKey): void { + if ( + Buffer.byteLength(rootKey, 'utf8') > MAX_RELAY_WATCH_ROOT_KEY_BYTES || + Buffer.byteLength(rootPath, 'utf8') > MAX_RELAY_WATCH_ROOT_KEY_BYTES + ) { + throw new Error('File watcher root path is too long') + } +} export function assertRelayWatcherRootCapacity( activeRoots: Iterable, pendingRoots: Iterable, teardownRoots: Iterable, - prospectiveRoot: string + prospectiveRoot: string, + prospectiveRootPath = prospectiveRoot ): void { + assertRelayWatcherRootKeyCapacity(prospectiveRoot, prospectiveRootPath) const physicalRoots = new Set([...activeRoots, ...pendingRoots, ...teardownRoots]) physicalRoots.add(prospectiveRoot) + let retainedKeyBytes = 0 + for (const root of physicalRoots) { + const keyBytes = Buffer.byteLength(root, 'utf8') + if (keyBytes > MAX_RELAY_WATCH_ROOT_KEY_BYTES) { + throw new Error('File watcher root path is too long') + } + retainedKeyBytes += keyBytes + if (retainedKeyBytes > MAX_RELAY_WATCH_ROOT_KEYS_BYTES) { + throw new Error('Maximum file watcher root path memory reached') + } + } if (physicalRoots.size > MAX_RELAY_WATCH_ROOTS) { throw new Error('Maximum number of file watchers reached') } diff --git a/src/relay/relay-watcher-teardown-tracker.ts b/src/relay/relay-watcher-teardown-tracker.ts index 91b1460054d..6e250dbbd52 100644 --- a/src/relay/relay-watcher-teardown-tracker.ts +++ b/src/relay/relay-watcher-teardown-tracker.ts @@ -15,6 +15,7 @@ export type RelayWatcherTeardownState = { } export class RelayWatcherTeardownTracker { + // Why: these entries own native handles and cannot be evicted; watcher admission bounds their count and aggregate root-key bytes. private readonly pending = new Map>() private readonly failed = new Map() diff --git a/src/relay/relay.ts b/src/relay/relay.ts index e279bec49c7..08bbb9f9aad 100644 --- a/src/relay/relay.ts +++ b/src/relay/relay.ts @@ -14,6 +14,7 @@ import { resolve, join } from 'node:path' import { unlinkSync, existsSync, statSync } from 'node:fs' import { RELAY_SENTINEL, + MAX_MESSAGE_SIZE, FrameDecoder, MessageType, encodeJsonRpcFrame, @@ -22,7 +23,7 @@ import { type JsonRpcResponse } from './protocol' import { readLaunchVersion, runConnectHandshake, setupDaemonHandshake } from './relay-handshake' -import { RelayDispatcher } from './dispatcher' +import { MAX_RELAY_SOCKET_CONNECTIONS, RelayDispatcher } from './dispatcher' import { RelayContext } from './context' import { PtyHandler } from './pty-handler' import { FsHandler } from './fs-handler' @@ -53,6 +54,10 @@ import { relayLogLine } from './relay-diagnostic-log' import { remoteCliRequestTimeoutMs } from './remote-cli-timeout' import { shouldReadRemoteCliStdin } from './remote-cli-stdin' import { registerManagedHookInstaller } from './managed-hook-installer' +import { + NodeReadableTextTooLargeError, + readNodeReadableTextWithinLimit +} from '../shared/node-readable-text' const DEFAULT_GRACE_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 const SOCK_NAME = 'relay.sock' @@ -283,11 +288,14 @@ async function readOrcaCliStdin(): Promise { if (process.stdin.isTTY) { return undefined } - const chunks: Buffer[] = [] - for await (const chunk of process.stdin) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))) + try { + return await readNodeReadableTextWithinLimit(process.stdin, MAX_MESSAGE_SIZE) + } catch (error) { + if (error instanceof NodeReadableTextTooLargeError) { + throw new Error(`Remote CLI stdin exceeds ${MAX_MESSAGE_SIZE} byte relay frame limit`) + } + throw error } - return Buffer.concat(chunks).toString('utf8') } // ── Normal mode ────────────────────────────────────────────────────── @@ -377,6 +385,12 @@ async function main(): Promise { return } stdoutDrainWaiters.add(cb) + }, + disconnect: () => { + stdoutAlive = false + flushStdoutDrainWaiters() + process.stdin.destroy() + process.stdout.destroy() } } ) @@ -612,17 +626,6 @@ async function main(): Promise { } function attachAcceptedSocket(sock: Socket, leftover: Buffer): void { - // Why: remove the initial stdin data listener once a socket client is accepted, so stale SSH-channel bytes can't interleave. - process.stdin.pause() - process.stdin.removeAllListeners('data') - - hasAcceptedSocketClient = true - acceptedSocketConnections++ - relayLogLine( - `[relay] Socket client accepted (clients=${socketClients.size + 1}, accepted=${acceptedSocketConnections})` - ) - cancelGrace('socket client accepted') - // Why: same backpressure surface as stdout — bulk frames wait for socket drain so they can't bury interactive PTY frames. const sockDrainWaiters = new Set<() => void>() const flushSockDrainWaiters = (): void => { @@ -634,23 +637,44 @@ async function main(): Promise { sock.on('drain', flushSockDrainWaiters) sock.on('close', flushSockDrainWaiters) sock.on('error', flushSockDrainWaiters) - const clientId = dispatcher.attachClient( - (data) => { - if (!sock.destroyed) { - return sock.write(data) - } - return undefined - }, - { - waitWriteDrain: (cb) => { + let clientId: number + try { + clientId = dispatcher.attachClient( + (data) => { if (sock.destroyed) { - cb() - return + return undefined } - sockDrainWaiters.add(cb) + return sock.write(data) + }, + { + waitWriteDrain: (cb) => { + if (sock.destroyed) { + cb() + return + } + sockDrainWaiters.add(cb) + }, + disconnect: () => sock.destroy() } - } + ) + } catch (error) { + relayLogLine( + `[relay] Socket client rejected: ${error instanceof Error ? error.message : String(error)}` + ) + sock.destroy() + return + } + + // Why: remove the initial stdin data listener once a socket client is accepted, so stale SSH-channel bytes can't interleave. + process.stdin.pause() + process.stdin.removeAllListeners('data') + + hasAcceptedSocketClient = true + acceptedSocketConnections++ + relayLogLine( + `[relay] Socket client accepted (clients=${socketClients.size + 1}, accepted=${acceptedSocketConnections})` ) + cancelGrace('socket client accepted') socketClients.set(sock, clientId) // Why: feed handshake-buffered leftover bytes before wiring sock.on('data') so frame ordering is preserved. @@ -692,6 +716,7 @@ async function main(): Promise { } }) }) + server.maxConnections = MAX_RELAY_SOCKET_CONNECTIONS // Why: umask 0o177 before listen makes the socket 0o600 atomically, closing the chmod-after-listen TOCTOU window. const shouldSetSocketUmask = !isWindowsNamedPipePath(sockPath) @@ -861,6 +886,7 @@ async function main(): Promise { if (detached) { // Why: detached stdin is /dev/null, so listening would EOF → grace → shutdown before --connect arrives; use the socket instead. stdoutAlive = false + dispatcher.invalidateClient() startGrace('detached startup') } else { process.stdin.on('data', (chunk: Buffer) => { diff --git a/src/relay/workspace-session-handler.test.ts b/src/relay/workspace-session-handler.test.ts index 7c3917e9c2a..40baa7a9155 100644 --- a/src/relay/workspace-session-handler.test.ts +++ b/src/relay/workspace-session-handler.test.ts @@ -1,9 +1,15 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { closeSync, ftruncateSync, mkdtempSync, openSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { RelayDispatcher } from './dispatcher' -import { WorkspaceSessionHandler } from './workspace-session-handler' +import { + MAX_PRESENCE_CLIENTS_PER_NAMESPACE, + MAX_PRESENCE_NAMESPACES, + MAX_WORKSPACE_SESSION_SNAPSHOT_BYTES, + MAX_WORKSPACE_SESSION_SNAPSHOT_STRUCTURAL_TOKENS, + WorkspaceSessionHandler +} from './workspace-session-handler' import { encodeJsonRpcFrame, MessageType, type JsonRpcRequest } from './protocol' function decodeJsonFrames(written: Buffer[]): unknown[] { @@ -34,6 +40,7 @@ async function sendRequest( describe('WorkspaceSessionHandler', () => { let baseDir: string let dispatcher: RelayDispatcher + let handler: WorkspaceSessionHandler let written: Buffer[] beforeEach(() => { @@ -42,11 +49,12 @@ describe('WorkspaceSessionHandler', () => { dispatcher = new RelayDispatcher((data) => { written.push(Buffer.from(data)) }) - new WorkspaceSessionHandler(dispatcher, baseDir) + handler = new WorkspaceSessionHandler(dispatcher, baseDir) }) afterEach(() => { dispatcher.dispose() + vi.useRealTimers() rmSync(baseDir, { recursive: true, force: true }) }) @@ -104,6 +112,40 @@ describe('WorkspaceSessionHandler', () => { expect(staleResponse.result.snapshot.revision).toBe(1) }) + it('starts fresh when a persisted snapshot exceeds the file cap', async () => { + const snapshotPath = join(baseDir, 'oversized.json') + const file = openSync(snapshotPath, 'w') + ftruncateSync(file, MAX_WORKSPACE_SESSION_SNAPSHOT_BYTES + 1) + closeSync(file) + + await sendRequest(dispatcher, 'workspace.get', { namespace: 'oversized' }, 1) + + const response = decodeJsonFrames(written).find( + (frame) => (frame as { id?: number }).id === 1 + ) as { result: { revision: number; session: { activeRepoId: unknown } } } + expect(response.result.revision).toBe(0) + expect(response.result.session.activeRepoId).toBeNull() + }) + + it('starts fresh when a bounded snapshot amplifies structure before parsing', () => { + writeFileSync( + join(baseDir, 'amplified.json'), + `[${'0,'.repeat(MAX_WORKSPACE_SESSION_SNAPSHOT_STRUCTURAL_TOKENS)}0]` + ) + const parseSpy = vi.spyOn(JSON, 'parse') + + const snapshot = ( + handler as unknown as { + read(namespace: string): { revision: number; session: { activeRepoId: unknown } } + } + ).read('amplified') + + expect(snapshot.revision).toBe(0) + expect(snapshot.session.activeRepoId).toBeNull() + expect(parseSpy).not.toHaveBeenCalled() + parseSpy.mockRestore() + }) + it('tracks presence per namespace', async () => { await sendRequest( dispatcher, @@ -139,4 +181,83 @@ describe('WorkspaceSessionHandler', () => { 'Laptop A' ) }) + + it('does not retain namespaces for presence queries without a client id', async () => { + for (let index = 0; index < MAX_PRESENCE_NAMESPACES + 10; index += 1) { + await sendRequest( + dispatcher, + 'workspace.presence', + { namespace: `query-${index}` }, + index + 1 + ) + } + + const namespaces = (handler as unknown as { clientsByNamespace: Map }) + .clientsByNamespace + expect(namespaces.size).toBe(0) + }) + + it('sweeps expired clients from every namespace on any heartbeat', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + await sendRequest(dispatcher, 'workspace.presence', { namespace: 'a', clientId: 'a' }, 1) + await sendRequest(dispatcher, 'workspace.presence', { namespace: 'b', clientId: 'b' }, 2) + + vi.setSystemTime(new Date('2026-01-01T00:01:00Z')) + await sendRequest(dispatcher, 'workspace.presence', { namespace: 'c', clientId: 'c' }, 3) + + const namespaces = (handler as unknown as { clientsByNamespace: Map }) + .clientsByNamespace + expect(Array.from(namespaces.keys())).toEqual(['c']) + }) + + it('bounds clients per namespace by evicting the oldest heartbeat', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + for (let index = 0; index <= MAX_PRESENCE_CLIENTS_PER_NAMESPACE; index += 1) { + vi.setSystemTime(Date.now() + 1) + await sendRequest( + dispatcher, + 'workspace.presence', + { namespace: 'team', clientId: `client-${String(index).padStart(3, '0')}` }, + index + 1 + ) + } + + const clients = ( + handler as unknown as { + clientsByNamespace: Map> + } + ).clientsByNamespace.get('team') + expect(clients?.size).toBe(MAX_PRESENCE_CLIENTS_PER_NAMESPACE) + expect(clients?.has('client-000')).toBe(false) + expect( + clients?.has(`client-${String(MAX_PRESENCE_CLIENTS_PER_NAMESPACE).padStart(3, '0')}`) + ).toBe(true) + }) + + it('bounds namespaces by evicting the least recently active one', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + for (let index = 0; index <= MAX_PRESENCE_NAMESPACES; index += 1) { + vi.setSystemTime(Date.now() + 1) + await sendRequest( + dispatcher, + 'workspace.presence', + { + namespace: `space-${String(index).padStart(3, '0')}`, + clientId: `client-${index}` + }, + index + 1 + ) + } + + const namespaces = (handler as unknown as { clientsByNamespace: Map }) + .clientsByNamespace + expect(namespaces.size).toBe(MAX_PRESENCE_NAMESPACES) + expect(namespaces.has('space-000')).toBe(false) + expect(namespaces.has(`space-${MAX_PRESENCE_NAMESPACES}`)).toBe(true) + }) }) + +type ConnectedClientForTest = { clientId: string; name: string; lastSeenAt: number } diff --git a/src/relay/workspace-session-handler.ts b/src/relay/workspace-session-handler.ts index c87a8b6076f..f42e9e80de8 100644 --- a/src/relay/workspace-session-handler.ts +++ b/src/relay/workspace-session-handler.ts @@ -1,6 +1,9 @@ -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' +import { assertJsonTextStructureWithinLimits } from '../shared/json-text-structure-limit' +import { readNodeFileSyncWithinLimit } from '../shared/node-bounded-file-reader' +import { stringifyJsonWithinByteLimit } from '../shared/node-bounded-json-stringify' import type { RelayDispatcher } from './dispatcher' type RemoteWorkspaceSnapshot = { @@ -28,6 +31,11 @@ type PatchResult = const SNAPSHOT_SCHEMA_VERSION = 1 const PRESENCE_TTL_MS = 45_000 +export const MAX_PRESENCE_NAMESPACES = 256 +export const MAX_PRESENCE_CLIENTS_PER_NAMESPACE = 64 +export const MAX_WORKSPACE_SESSION_SNAPSHOT_BYTES = 16 * 1024 * 1024 +export const MAX_WORKSPACE_SESSION_SNAPSHOT_STRUCTURAL_TOKENS = 1_000_000 +export const MAX_WORKSPACE_SESSION_SNAPSHOT_NESTING_DEPTH = 128 function emptySession(): Record { return { @@ -81,7 +89,15 @@ export class WorkspaceSessionHandler { } try { - const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial + const content = readNodeFileSyncWithinLimit( + path, + MAX_WORKSPACE_SESSION_SNAPSHOT_BYTES + ).buffer.toString('utf8') + assertJsonTextStructureWithinLimits(content, { + structuralTokens: MAX_WORKSPACE_SESSION_SNAPSHOT_STRUCTURAL_TOKENS, + nestingDepth: MAX_WORKSPACE_SESSION_SNAPSHOT_NESTING_DEPTH + }) + const parsed = JSON.parse(content) as Partial return { namespace, revision: @@ -116,7 +132,11 @@ export class WorkspaceSessionHandler { const path = this.snapshotPath(snapshot.namespace) mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) const tmpPath = `${path}.tmp` - writeFileSync(tmpPath, JSON.stringify(snapshot, null, 2), { mode: 0o600 }) + const serialized = stringifyJsonWithinByteLimit( + snapshot, + MAX_WORKSPACE_SESSION_SNAPSHOT_BYTES + ).serialized + writeFileSync(tmpPath, serialized, { mode: 0o600 }) renameSync(tmpPath, path) } @@ -163,25 +183,72 @@ export class WorkspaceSessionHandler { const namespace = sanitizeNamespace(params.namespace) const clientId = typeof params.clientId === 'string' ? sanitizeClientId(params.clientId) : '' const name = typeof params.clientName === 'string' ? sanitizeClientName(params.clientName) : '' - const clients = this.clientsByNamespace.get(namespace) ?? new Map() - this.clientsByNamespace.set(namespace, clients) - const now = Date.now() - for (const [id, client] of clients) { - if (now - client.lastSeenAt > PRESENCE_TTL_MS) { - clients.delete(id) + this.sweepPresence(now) + + let clients = this.clientsByNamespace.get(namespace) + if (!clientId) { + return { clients: this.sortedClients(clients) } + } + if (!clients) { + if (this.clientsByNamespace.size >= MAX_PRESENCE_NAMESPACES) { + this.evictOldestNamespace() + } + clients = new Map() + this.clientsByNamespace.set(namespace, clients) + } + if (!clients.has(clientId) && clients.size >= MAX_PRESENCE_CLIENTS_PER_NAMESPACE) { + this.evictOldestClient(clients) + } + clients.set(clientId, { + clientId, + name: name || 'Unknown device', + lastSeenAt: now + }) + + return { clients: this.sortedClients(clients) } + } + + private sweepPresence(now: number): void { + for (const [namespace, clients] of this.clientsByNamespace) { + for (const [id, client] of clients) { + if (now - client.lastSeenAt > PRESENCE_TTL_MS) { + clients.delete(id) + } + } + if (clients.size === 0) { + this.clientsByNamespace.delete(namespace) } } - if (clientId) { - clients.set(clientId, { - clientId, - name: name || 'Unknown device', - lastSeenAt: now - }) - } + } - return { - clients: Array.from(clients.values()).sort((a, b) => b.lastSeenAt - a.lastSeenAt) + private evictOldestNamespace(): void { + let oldest: { namespace: string; lastSeenAt: number } | undefined + for (const [namespace, clients] of this.clientsByNamespace) { + const lastSeenAt = Math.max(...Array.from(clients.values(), (client) => client.lastSeenAt)) + if ( + !oldest || + lastSeenAt < oldest.lastSeenAt || + (lastSeenAt === oldest.lastSeenAt && namespace < oldest.namespace) + ) { + oldest = { namespace, lastSeenAt } + } + } + if (oldest) { + this.clientsByNamespace.delete(oldest.namespace) } } + + private evictOldestClient(clients: Map): void { + const oldest = Array.from(clients.values()).sort( + (a, b) => a.lastSeenAt - b.lastSeenAt || a.clientId.localeCompare(b.clientId) + )[0] + if (oldest) { + clients.delete(oldest.clientId) + } + } + + private sortedClients(clients: Map | undefined): ConnectedClient[] { + return Array.from(clients?.values() ?? []).sort((a, b) => b.lastSeenAt - a.lastSeenAt) + } } diff --git a/src/relay/workspace-space-scan.ts b/src/relay/workspace-space-scan.ts index cc823457142..d2ffb8af378 100644 --- a/src/relay/workspace-space-scan.ts +++ b/src/relay/workspace-space-scan.ts @@ -2,16 +2,25 @@ cancellation, symlink, and top-level compaction semantics in one scanner. */ import { execFile } from 'node:child_process' import type { Dirent } from 'node:fs' -import { lstat, readdir } from 'node:fs/promises' +import { lstat, opendir } from 'node:fs/promises' import { basename, join } from 'node:path' import { platform } from 'node:process' import { promisify } from 'node:util' import type { WorkspaceSpaceDirectoryScanResult, - WorkspaceSpaceItem, - WorkspaceSpaceItemKind + WorkspaceSpaceItem } from '../shared/workspace-space-types' import { compactWorkspaceSpaceItems } from '../shared/workspace-space-compaction' +import { mapWithConcurrency } from '../shared/map-with-concurrency' +import { + scanWorkspaceSpaceEntryTree, + type WorkspaceSpaceEntryScan +} from '../shared/workspace-space-entry-traversal' +import { + collectWorkspaceSpaceDirectoryEntries, + createWorkspaceSpaceScanBudget, + WorkspaceSpaceScanCapacityError +} from '../shared/workspace-space-scan-budget' import type { RequestContext } from './dispatcher' const RELAY_FS_CONCURRENCY = 48 @@ -19,15 +28,7 @@ const DU_TIMEOUT_MS = 120_000 const DU_MAX_BUFFER_BYTES = 16 * 1024 * 1024 const execFileAsync = promisify(execFile) -type AsyncLimiter = (task: () => Promise) => Promise - -type ScanStats = { - name: string - path: string - kind: WorkspaceSpaceItemKind - sizeBytes: number - skippedEntryCount: number -} +type ScanStats = WorkspaceSpaceEntryScan class RelayWorkspaceSpaceScanCancelledError extends Error { constructor() { @@ -42,57 +43,6 @@ function throwIfCancelled(context: RequestContext): void { } } -function createAsyncLimiter(maxConcurrent: number, context: RequestContext): AsyncLimiter { - let active = 0 - const queue: { resolve: () => void }[] = [] - - const acquire = async (): Promise => { - throwIfCancelled(context) - if (active < maxConcurrent) { - active += 1 - return - } - await new Promise((resolve, reject) => { - let onAbort: (() => void) | null = null - const waiter = { - resolve: () => { - if (onAbort) { - context.signal?.removeEventListener('abort', onAbort) - } - resolve() - } - } - onAbort = () => { - const index = queue.indexOf(waiter) - if (index !== -1) { - queue.splice(index, 1) - } - reject(new RelayWorkspaceSpaceScanCancelledError()) - } - queue.push(waiter) - if (context.signal) { - context.signal.addEventListener('abort', onAbort, { once: true }) - if (context.signal.aborted) { - onAbort() - } - } - }) - throwIfCancelled(context) - active += 1 - } - - return async (task: () => Promise): Promise => { - await acquire() - try { - return await task() - } finally { - active -= 1 - const next = queue.shift() - next?.resolve() - } - } -} - function normalizeDuPath(pathValue: string): string { const trimmed = pathValue.replace(/\/+$/, '') return trimmed.length > 0 ? trimmed : pathValue @@ -142,11 +92,10 @@ async function scanTopLevelEntryWithDu( entryPath: string, name: string, duSizes: Map, - limit: AsyncLimiter, context: RequestContext ): Promise { throwIfCancelled(context) - const stats = await limit(() => lstat(entryPath)) + const stats = await lstat(entryPath) throwIfCancelled(context) if (stats.isSymbolicLink()) { @@ -181,77 +130,30 @@ async function scanTopLevelEntryWithDu( async function scanEntryAggregate( entryPath: string, name: string, - limit: AsyncLimiter, context: RequestContext ): Promise { - throwIfCancelled(context) - const stats = await limit(() => lstat(entryPath)) - throwIfCancelled(context) - - if (stats.isSymbolicLink()) { - return { - name, - path: entryPath, - kind: 'symlink', - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - if (!stats.isDirectory()) { - return { - name, - path: entryPath, - kind: 'file', - sizeBytes: stats.size, - skippedEntryCount: 0 - } - } - - let entries: Dirent[] - try { - entries = await limit(() => readdir(entryPath, { withFileTypes: true })) - } catch { - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes: stats.size, - skippedEntryCount: 1 - } - } - - const childStats = await Promise.all( - entries.map(async (entry): Promise => { - try { - return await scanEntryAggregate(join(entryPath, entry.name), entry.name, limit, context) - } catch (error) { - if (error instanceof RelayWorkspaceSpaceScanCancelledError) { - throw error - } - return null + return scanWorkspaceSpaceEntryTree({ + rootPath: entryPath, + rootName: name, + concurrency: RELAY_FS_CONCURRENCY, + signal: context.signal, + entryName: (entry) => entry.name, + joinPath: join, + classifyEntry: async (path) => { + const stats = await lstat(path) + throwIfCancelled(context) + if (stats.isSymbolicLink()) { + return { kind: 'symlink', sizeBytes: stats.size } } - }) - ) - - let sizeBytes = stats.size - let skippedEntryCount = 0 - for (const child of childStats) { - if (!child) { - skippedEntryCount += 1 - continue - } - sizeBytes += child.sizeBytes - skippedEntryCount += child.skippedEntryCount - } - - return { - name, - path: entryPath, - kind: 'directory', - sizeBytes, - skippedEntryCount - } + return stats.isDirectory() + ? { kind: 'directory', sizeBytes: stats.size } + : { kind: 'file', sizeBytes: stats.size } + }, + readDirectory: (path) => opendir(path), + checkCancelled: () => throwIfCancelled(context), + createCancellationError: () => new RelayWorkspaceSpaceScanCancelledError(), + isCancellationError: (error) => error instanceof RelayWorkspaceSpaceScanCancelledError + }) } async function scanDirectoryWithDu( @@ -266,19 +168,27 @@ async function scanDirectoryWithDu( } const [entries, duSizes] = await Promise.all([ - readdir(rootPath, { withFileTypes: true }), + opendir(rootPath).then((directory) => + collectWorkspaceSpaceDirectoryEntries( + directory, + rootPath, + (entry) => entry.name, + createWorkspaceSpaceScanBudget(), + () => throwIfCancelled(context) + ) + ), readDuDepthOne(rootPath, context) ]) throwIfCancelled(context) - const limit = createAsyncLimiter(RELAY_FS_CONCURRENCY, context) - const childStats = await Promise.all( - entries.map(async (entry): Promise => { + const childStats = await mapWithConcurrency( + entries, + RELAY_FS_CONCURRENCY, + async (entry): Promise => { try { return await scanTopLevelEntryWithDu( join(rootPath, entry.name), entry.name, duSizes, - limit, context ) } catch (error) { @@ -287,7 +197,7 @@ async function scanDirectoryWithDu( } return null } - }) + } ) const children = childStats.filter((child): child is ScanStats => child !== null) const compact = compactWorkspaceSpaceItems(children.map(toWorkspaceSpaceItem)) @@ -305,55 +215,13 @@ async function scanDirectoryWithNode( rootPath: string, context: RequestContext ): Promise { - throwIfCancelled(context) - const limit = createAsyncLimiter(RELAY_FS_CONCURRENCY, context) - const rootStats = await lstat(rootPath) - throwIfCancelled(context) - if (!rootStats.isDirectory() || rootStats.isSymbolicLink()) { - const root = await scanEntryAggregate(rootPath, basename(rootPath), limit, context) - return { - sizeBytes: root.sizeBytes, - skippedEntryCount: root.skippedEntryCount, - topLevelItems: [], - omittedTopLevelItemCount: 0, - omittedTopLevelSizeBytes: 0 - } - } - - let entries: Dirent[] - try { - entries = await readdir(rootPath, { withFileTypes: true }) - } catch { - return { - sizeBytes: rootStats.size, - skippedEntryCount: 1, - topLevelItems: [], - omittedTopLevelItemCount: 0, - omittedTopLevelSizeBytes: 0 - } - } - - const childStats = await Promise.all( - entries.map(async (entry): Promise => { - try { - return await scanEntryAggregate(join(rootPath, entry.name), entry.name, limit, context) - } catch (error) { - if (error instanceof RelayWorkspaceSpaceScanCancelledError) { - throw error - } - return null - } - }) - ) - const children = childStats.filter((child): child is ScanStats => child !== null) + const root = await scanEntryAggregate(rootPath, basename(rootPath), context) + const children = root.children ?? [] const compact = compactWorkspaceSpaceItems(children.map(toWorkspaceSpaceItem)) return { - sizeBytes: rootStats.size + children.reduce((sum, child) => sum + child.sizeBytes, 0), - skippedEntryCount: - children.reduce((sum, child) => sum + child.skippedEntryCount, 0) + - childStats.length - - children.length, + sizeBytes: root.sizeBytes, + skippedEntryCount: root.skippedEntryCount, ...compact } } @@ -366,7 +234,10 @@ export async function scanWorkspaceSpaceDirectory( try { return await scanDirectoryWithDu(rootPath, context) } catch (error) { - if (error instanceof RelayWorkspaceSpaceScanCancelledError) { + if ( + error instanceof RelayWorkspaceSpaceScanCancelledError || + error instanceof WorkspaceSpaceScanCapacityError + ) { throw error } } diff --git a/src/relay/wsl-hook-fs-bridge.test.ts b/src/relay/wsl-hook-fs-bridge.test.ts index f656b4aa3bd..8ca16770b41 100644 --- a/src/relay/wsl-hook-fs-bridge.test.ts +++ b/src/relay/wsl-hook-fs-bridge.test.ts @@ -2,13 +2,14 @@ // `posix.resolve` against a POSIX guest home. On win32 `posix.resolve` of a // Windows tmpdir yields an invalid path, so the whole suite is skipped there // (the bridge only ever runs inside a Linux WSL guest). -import { mkdtempSync, rmSync, statSync } from 'node:fs' +import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' import { posix } from 'node:path' import { tmpdir } from 'node:os' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { registerWslHookFsHandlers } from './wsl-hook-fs-bridge' import type { MethodHandler, RelayDispatcher, RequestContext } from './dispatcher' +import { estimateFilesystemDirectoryEntryBytes } from '../shared/filesystem-directory-listing-limit' import { WSL_HOOK_FS_METHODS, type WslFsResult } from '../shared/wsl-hook-relay-contract' describe.skipIf(process.platform === 'win32')('registerWslHookFsHandlers (WSL fs bridge)', () => { @@ -57,6 +58,22 @@ describe.skipIf(process.platform === 'win32')('registerWslHookFsHandlers (WSL fs expect(read).toEqual({ ok: true, content: 'hello guest' }) }) + it('accepts an exact byte-limit read and rejects an oversized file before materializing it', async () => { + const path = posix.join(home, 'bounded.txt') + writeFileSync(path, '🐋') + + await expect( + call<{ content: string }>(WSL_HOOK_FS_METHODS.readFile, { path, maxBytes: 4 }) + ).resolves.toEqual({ ok: true, content: '🐋' }) + await expect( + call<{ content: string }>(WSL_HOOK_FS_METHODS.readFile, { path, maxBytes: 3 }) + ).resolves.toMatchObject({ + ok: false, + errno: 'EFBIG', + fileCapacity: { observedBytes: 4, maxBytes: 3 } + }) + }) + it('refuses writeFile to an absolute path outside home', async () => { const result = await call(WSL_HOOK_FS_METHODS.writeFile, { path: '/etc/orca-evil.txt', @@ -100,6 +117,48 @@ describe.skipIf(process.platform === 'win32')('registerWslHookFsHandlers (WSL fs expect(root.ok).toBe(true) }) + it('streams directory entries through requested entry and retained-byte limits', async () => { + const directory = posix.join(home, 'bounded-dir') + mkdirSync(directory) + writeFileSync(posix.join(directory, 'alpha'), '') + writeFileSync(posix.join(directory, 'beta'), '') + + const exactEntries = await call<{ entries: { filename: string }[] }>( + WSL_HOOK_FS_METHODS.readdir, + { path: directory, maxEntries: 2, maxRetainedBytes: 4096 } + ) + expect(exactEntries).toMatchObject({ ok: true }) + if (exactEntries.ok) { + expect(exactEntries.entries).toHaveLength(2) + } + await expect( + call(WSL_HOOK_FS_METHODS.readdir, { + path: directory, + maxEntries: 1, + maxRetainedBytes: 4096 + }) + ).resolves.toMatchObject({ ok: false }) + + const single = posix.join(home, 'single-entry-dir') + mkdirSync(single) + writeFileSync(posix.join(single, 'one'), '') + const exactRetainedBytes = estimateFilesystemDirectoryEntryBytes({ name: 'one' }) + await expect( + call(WSL_HOOK_FS_METHODS.readdir, { + path: single, + maxEntries: 1, + maxRetainedBytes: exactRetainedBytes + }) + ).resolves.toMatchObject({ ok: true }) + await expect( + call(WSL_HOOK_FS_METHODS.readdir, { + path: single, + maxEntries: 1, + maxRetainedBytes: exactRetainedBytes - 1 + }) + ).resolves.toMatchObject({ ok: false }) + }) + it('refuses readdir on a non-ancestor dir outside home', async () => { const result = await call(WSL_HOOK_FS_METHODS.readdir, { path: '/etc' }) expect(result).toMatchObject({ ok: false, errno: 'EACCES' }) diff --git a/src/relay/wsl-hook-fs-bridge.ts b/src/relay/wsl-hook-fs-bridge.ts index 3ebb6047b1b..482554451f3 100644 --- a/src/relay/wsl-hook-fs-bridge.ts +++ b/src/relay/wsl-hook-fs-bridge.ts @@ -7,17 +7,54 @@ import { promises as fs } from 'node:fs' import { posix } from 'node:path' import type { RelayDispatcher } from './dispatcher' +import { + createFilesystemDirectoryLimitState, + trackFilesystemDirectoryEntry +} from '../shared/filesystem-directory-listing-limit' +import { + NodeFileReadTooLargeError, + readNodeFileWithinLimit +} from '../shared/node-bounded-file-reader' import { WSL_HOOK_FS_METHODS, + WSL_HOOK_FS_MAX_DIRECTORY_ENTRIES, + WSL_HOOK_FS_MAX_DIRECTORY_RETAINED_BYTES, + WSL_HOOK_FS_MAX_READ_BYTES, type WslFsFailure, type WslFsResult } from '../shared/wsl-hook-relay-contract' function failure(err: unknown): WslFsFailure { + if (err instanceof NodeFileReadTooLargeError) { + return { + ok: false, + errno: 'EFBIG', + message: err.message, + fileCapacity: { observedBytes: err.observedBytes, maxBytes: err.maxBytes } + } + } const e = err as NodeJS.ErrnoException return { ok: false, errno: e?.code ?? 'EUNKNOWN', message: e?.message ?? String(err) } } +function requestedLimit(value: unknown, maximum: number): number { + if (value === undefined) { + return maximum + } + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw Object.assign(new Error('invalid capacity limit'), { code: 'EINVAL' }) + } + return Math.min(value, maximum) +} + +function requestedDirectoryLimit(value: unknown, maximum: number): number { + const limit = requestedLimit(value, maximum) + if (limit === 0) { + throw Object.assign(new Error('directory capacity limit must be positive'), { code: 'EINVAL' }) + } + return limit +} + export function registerWslHookFsHandlers( dispatcher: RelayDispatcher, home: string, @@ -46,7 +83,7 @@ export function registerWslHookFsHandlers( return resolved } // Why: the installers' mkdir-p walks top-down from `/`, probing every - // ancestor of home with readdir before it ever creates a dir. Allow + // ancestor of home before it ever creates a dir. Allow // read-only existence probes on those ancestors; everything else stays // home-scoped. const scopedProbe = (rawPath: unknown): string => { @@ -68,7 +105,10 @@ export function registerWslHookFsHandlers( WSL_HOOK_FS_METHODS.readFile, async (params): Promise> => { try { - const content = await fs.readFile(scoped(params.path), 'utf8') + const maxBytes = requestedLimit(params.maxBytes, WSL_HOOK_FS_MAX_READ_BYTES) + const content = ( + await readNodeFileWithinLimit(scoped(params.path), maxBytes) + ).buffer.toString('utf8') return { ok: true, content } } catch (err) { return failure(err) @@ -93,7 +133,7 @@ export function registerWslHookFsHandlers( WSL_HOOK_FS_METHODS.stat, async (params): Promise> => { try { - const stats = await fs.stat(scoped(params.path)) + const stats = await fs.stat(scopedProbe(params.path)) return { ok: true, mode: stats.mode } } catch (err) { return failure(err) @@ -133,11 +173,39 @@ export function registerWslHookFsHandlers( dispatcher.onRequest( WSL_HOOK_FS_METHODS.readdir, async (params): Promise> => { + let directory: Awaited> | undefined try { - const names = await fs.readdir(scopedProbe(params.path)) - return { ok: true, entries: names.map((filename) => ({ filename })) } + const limits = { + maxEntries: requestedDirectoryLimit(params.maxEntries, WSL_HOOK_FS_MAX_DIRECTORY_ENTRIES), + maxRetainedBytes: requestedDirectoryLimit( + params.maxRetainedBytes, + WSL_HOOK_FS_MAX_DIRECTORY_RETAINED_BYTES + ) + } + const limit = createFilesystemDirectoryLimitState(limits) + const entries: { filename: string }[] = [] + const directoryPath = scopedProbe(params.path) + directory = await fs.opendir(directoryPath, { bufferSize: 32 }) + // Ancestors outside home are authorized only as existence probes. + if ( + directoryPath !== homeRoot && + (directoryPath === '/' || homeRoot.startsWith(`${directoryPath}/`)) + ) { + return { ok: true, entries } + } + for (;;) { + const entry = await directory.read() + if (entry === null) { + break + } + trackFilesystemDirectoryEntry(limit, { name: entry.name }) + entries.push({ filename: entry.name }) + } + return { ok: true, entries } } catch (err) { return failure(err) + } finally { + await directory?.close().catch(() => undefined) } } ) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index d958e1fbf12..869ebda72e8 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -141,7 +141,7 @@ import { timeRendererStartupStep, timeRendererStartupSyncStep } from './startup/startup-diagnostics' -import { reconnectSshTargetForRendererStartup } from './startup/ssh-startup-reconnect' +import { reconnectSshTargetsForRendererStartup } from './startup/ssh-startup-reconnect' import { shouldRenderPetOverlay } from './components/pet/pet-overlay-visibility' import { applyDocumentTheme } from './lib/document-theme' import { getSystemPrefersDark } from './lib/terminal-theme' @@ -978,26 +978,18 @@ function App(): React.JSX.Element { } // Why: treat timed-out eager targets as deferred so their PTYs reattach on tab focus (ssh.connect keeps running in main and likely finishes by then). - const timedOutTargets: string[] = [] - await timeRendererStartupStep( + const timedOutTargets = await timeRendererStartupStep( 'ssh-reconnect', () => - Promise.all( - eagerTargets.map(async ({ targetId }) => { - const result = await reconnectSshTargetForRendererStartup({ - targetId, - timeoutMs: SSH_RECONNECT_TIMEOUT_MS, - connect: (id) => window.api.ssh.connect({ targetId: id }), - publishState: actions.setSshConnectionState, - onFailure: (id, error) => { - console.warn(`SSH auto-reconnect failed for ${id}:`, error) - } - }) - if (result.timedOut) { - timedOutTargets.push(targetId) - } - }) - ), + reconnectSshTargetsForRendererStartup({ + targetIds: eagerTargets.map((target) => target.targetId), + timeoutMs: SSH_RECONNECT_TIMEOUT_MS, + connect: (id) => window.api.ssh.connect({ targetId: id }), + publishState: actions.setSshConnectionState, + onFailure: (id, error) => { + console.warn(`SSH auto-reconnect failed for ${id}:`, error) + } + }), { eagerTargets: eagerTargets.length, deferredTargets: deferredTargets.length diff --git a/src/renderer/src/components/DetachedHeadBadge.tsx b/src/renderer/src/components/DetachedHeadBadge.tsx index 58a12e6395e..bc03ce2ab52 100644 --- a/src/renderer/src/components/DetachedHeadBadge.tsx +++ b/src/renderer/src/components/DetachedHeadBadge.tsx @@ -12,13 +12,15 @@ type DetachedHeadBadgeProps = { label?: 'sidebar' | 'source-control' side?: React.ComponentProps['side'] className?: string + tabIndex?: number } export function DetachedHeadBadge({ display, label = 'source-control', side = 'right', - className + className, + tabIndex }: DetachedHeadBadgeProps): React.JSX.Element { const visibleLabel = label === 'sidebar' ? display.sidebarLabel : display.sourceControlLabel @@ -27,6 +29,8 @@ export function DetachedHeadBadge({ - {visibleLabel} + {visibleLabel} diff --git a/src/renderer/src/components/GitHubItemDialog.tsx b/src/renderer/src/components/GitHubItemDialog.tsx index 815c8dac7e4..a10c688a1ac 100644 --- a/src/renderer/src/components/GitHubItemDialog.tsx +++ b/src/renderer/src/components/GitHubItemDialog.tsx @@ -6,8 +6,7 @@ import React, { useLayoutEffect, useMemo, useRef, - useState, - useSyncExternalStore + useState } from 'react' import { lazyWithRetry as lazy } from '@/lib/lazy-with-retry' import { useVirtualizer } from '@tanstack/react-virtual' @@ -90,6 +89,8 @@ import { } from '@/components/editor/diff-section-layout' import type { DiffSection } from '@/components/editor/diff-section-types' import { removeDiffSectionMeasuredHeight } from '@/components/editor/diff-section-height-cache' +import { createCombinedDiffLoadScheduler } from '@/components/editor/combined-diff-load-scheduler' +import { retainCombinedDiffSectionText } from '@/components/editor/combined-diff-text-retention' import { MAX_RENDERED_DIFF_COMBINED_CHARACTERS, MAX_RENDERED_DIFF_LINES_PER_SIDE, @@ -160,10 +161,18 @@ import { getCommentBodySubmitState, hasBoundedCommentBodyText } from '@/lib/comment-body-submit-state' +import { emitGitHubWorkItemDetailsCacheMutation } from '@/lib/github-work-item-details-cache-events' import { - emitGitHubWorkItemDetailsCacheMutation, - onGitHubWorkItemDetailsCacheMutation -} from '@/lib/github-work-item-details-cache-events' + WORK_ITEM_DETAILS_FRESH_MS, + WORK_ITEM_DETAILS_UNAVAILABLE_MESSAGE, + getWorkItemDetailsCacheEntry, + getWorkItemDetailsCacheGeneration, + getWorkItemDetailsCacheKey, + invalidateWorkItemDetailsCacheByMatch, + invalidateWorkItemDetailsCacheForKey, + touchWorkItemDetailsCache, + useWorkItemDetailsCacheEntry +} from '@/lib/github-work-item-details-cache' import { lookupGitHubWorkItemDetailsForSource } from '@/lib/github-work-item-source-lookup' import { canUseGitHubRepoContext, @@ -1409,102 +1418,12 @@ function isPRFileViewed(file: GitHubPRFile): boolean { return file.viewerViewedState === 'VIEWED' } -// Why: SWR cache for work-item details so reopening paints instantly instead of paying IPC + `gh` startup; keyed to avoid source/type collisions, LRU-bounded, FRESH_MS refetch on open. See docs/gh-work-item-drawer-cache.md. -const WORK_ITEM_DETAILS_CACHE_MAX = 50 -const WORK_ITEM_DETAILS_FRESH_MS = 30_000 -const WORK_ITEM_DETAILS_UNAVAILABLE_MESSAGE = 'Unable to load details for this GitHub item.' -type WorkItemDetailsCacheEntry = { - details: GitHubWorkItemDetails | null - fetchedAt: number - pending?: Promise - error?: string -} -const workItemDetailsCache = new Map() - -// Why: drawers subscribe via useSyncExternalStore so a cached item paints synchronously; snapshot stability relies on every write replacing entry identity (delete+set). -const workItemDetailsCacheListeners = new Set<() => void>() -function subscribeWorkItemDetailsCache(listener: () => void): () => void { - workItemDetailsCacheListeners.add(listener) - return () => { - workItemDetailsCacheListeners.delete(listener) - } -} -function notifyWorkItemDetailsCache(): void { - for (const listener of workItemDetailsCacheListeners) { - listener() - } -} - -function getWorkItemDetailsCacheKey(args: { - repoPath: string - repoId: string - issueSourcePreference: string | undefined - sourceCacheScope?: string | null - type: 'issue' | 'pr' - number: number -}): string { - // Why: key on every axis that changes which (repo, item) the IPC resolves to; `\0` separator avoids ambiguity with fields containing `:` or `/`. - const keyParts = args.sourceCacheScope - ? [args.repoId, args.sourceCacheScope, args.issueSourcePreference ?? 'auto', args.type] - : [args.repoId, args.issueSourcePreference ?? 'auto', args.type] - return [...keyParts, args.number].join('\0') -} - -function touchWorkItemDetailsCache(key: string, entry: WorkItemDetailsCacheEntry): void { - // Why: re-insert to move to MRU position; Map insertion order keeps the oldest key first for eviction. - workItemDetailsCache.delete(key) - workItemDetailsCache.set(key, entry) - while (workItemDetailsCache.size > WORK_ITEM_DETAILS_CACHE_MAX) { - const oldest = workItemDetailsCache.keys().next().value - if (oldest === undefined) { - break - } - workItemDetailsCache.delete(oldest) - } - notifyWorkItemDetailsCache() -} - -// Why: exposed so mutation handlers can drop a stale entry after a local mutation; cross-window invalidation arrives via the gh:workItemMutated listener below. -export function invalidateWorkItemDetailsCacheForKey(key: string): void { - // Why: bump generation so a fetch launched before this invalidation won't write its stale result back. - workItemDetailsCacheGeneration += 1 - const existed = workItemDetailsCache.delete(key) - if (existed) { - notifyWorkItemDetailsCache() - } -} - -// Why: bumped on every invalidation so an in-flight refetch started before a mutation can detect its result is stale and skip writing it back. -let workItemDetailsCacheGeneration = 0 - -// Why: without the exact key (e.g. a cross-window event carries only repoPath+number+type), drop every entry matching that tuple regardless of source preference. -function invalidateWorkItemDetailsCacheByMatch(args: { - repoPath: string - repoId?: string - type: 'issue' | 'pr' - number: number -}): void { - const suffix = `\0${args.type}\0${args.number}` - const prefix = `${args.repoId ?? args.repoPath}\0` - let removed = false - for (const key of Array.from(workItemDetailsCache.keys())) { - if (key.startsWith(prefix) && key.endsWith(suffix)) { - workItemDetailsCache.delete(key) - removed = true - } - } - if (removed) { - workItemDetailsCacheGeneration += 1 - notifyWorkItemDetailsCache() - } -} - function patchCachedPRFileViewedState( cacheKey: string, path: string, viewerViewedState: GitHubPRFileViewedState ): GitHubPRFileViewedState | undefined { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) const files = prev?.details?.files if (!prev?.details || !files) { return undefined @@ -1529,7 +1448,7 @@ function patchCachedPRFileViewedState( } function patchCachedPRChecks(cacheKey: string, checks: PRCheckDetail[]): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1545,7 +1464,7 @@ function patchCachedPRReviewRequests( cacheKey: string, reviewRequests: GitHubAssignableUser[] ): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1561,7 +1480,7 @@ function patchCachedPRReviewRequests( } function patchCachedWorkItemBody(cacheKey: string, body: string): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1573,29 +1492,6 @@ function patchCachedWorkItemBody(cacheKey: string, body: string): void { }) } -// Why: install once — all dialogs share the cache; track unsubscribe so Vite HMR doesn't accumulate listeners across dev reloads. -let workItemMutatedUnsub: (() => void) | undefined -let workItemDetailsCacheEventUnsub: (() => void) | undefined -if (typeof window !== 'undefined' && window.api?.gh?.onWorkItemMutated) { - workItemMutatedUnsub = window.api.gh.onWorkItemMutated((payload) => { - invalidateWorkItemDetailsCacheByMatch({ - repoPath: payload.repoPath, - repoId: payload.repoId, - type: payload.type, - number: payload.number - }) - }) - workItemDetailsCacheEventUnsub = onGitHubWorkItemDetailsCacheMutation((payload) => { - invalidateWorkItemDetailsCacheByMatch(payload) - }) -} -if (typeof import.meta !== 'undefined' && import.meta.hot) { - import.meta.hot.dispose(() => { - workItemMutatedUnsub?.() - workItemDetailsCacheEventUnsub?.() - }) -} - // Why: bounded LRU so opening many PRs with many files doesn't grow this module-level map unboundedly until reload. const PR_FILE_CONTENT_CACHE_MAX = 64 // Why: overflow is a sentinel; force reported size past the render budget so downstream checks reliably pick fallback mode. @@ -2277,13 +2173,47 @@ function PRFilesCombinedDiffViewer({ const loadedIndicesRef = useRef>(new Set()) const loadingIndicesRef = useRef>(new Set()) const sectionsRef = useRef([]) + const protectedSectionKeysRef = useRef>(new Set()) const generationRef = useRef(0) const modifiedEditorsRef = useRef>(new Map()) const handleSectionSaveRef = useRef<(index: number) => Promise>(async () => {}) + const loadSectionRef = useRef<(index: number) => Promise>(async () => {}) + const loadSchedulerRef = useRef( + createCombinedDiffLoadScheduler({ + loadSection: (index) => loadSectionRef.current(index), + // Why: keep hosted fetch latency responsive without stacking a PR worth of response bodies. + maxConcurrent: 2 + }) + ) + const applySectionTextRetention = useCallback( + (nextSections: DiffSection[], additionallyProtectedKey?: string): DiffSection[] => { + const protectedSectionKeys = new Set(protectedSectionKeysRef.current) + if (additionallyProtectedKey) { + protectedSectionKeys.add(additionallyProtectedKey) + } + const retained = retainCombinedDiffSectionText({ + sections: nextSections, + loadedIndices: loadedIndicesRef.current, + protectedSectionKeys + }) + for (const index of retained.evictedIndices) { + loadedIndicesRef.current.delete(index) + } + return retained.sections + }, + [] + ) sectionsRef.current = sections + useEffect(() => { + const scheduler = loadSchedulerRef.current + scheduler.reset() + return () => scheduler.dispose() + }, []) + useEffect(() => { generationRef.current += 1 + loadSchedulerRef.current.reset() loadedIndicesRef.current.clear() loadingIndicesRef.current.clear() setSectionHeights({}) @@ -2308,10 +2238,10 @@ function PRFilesCombinedDiffViewer({ ) }, [entries, entrySignature]) - const loadSection = useCallback( - (index: number) => { + const loadSectionNow = useCallback( + async (index: number) => { const section = sectionsRef.current[index] - if (!section || section.collapsed) { + if (!section || section.collapsed || !protectedSectionKeysRef.current.has(section.key)) { return } if (loadedIndicesRef.current.has(index) || loadingIndicesRef.current.has(index)) { @@ -2368,74 +2298,86 @@ function PRFilesCombinedDiffViewer({ return { result: getPRFileDiffResult(contents), resultContents: contents } } - load() - .catch((error) => ({ - result: { - kind: 'text', - originalContent: '', - modifiedContent: '', - originalIsBinary: false, - modifiedIsBinary: false - } as GitDiffResult, - resultContents: undefined, - error: error instanceof Error ? error.message : 'Failed to load diff.' - })) - .then(({ result, resultContents, error }) => { - loadingIndicesRef.current.delete(index) - if (generationRef.current !== generation) { - return - } - const largeDiffRenderLimit = - !error && result.kind === 'text' && resultContents - ? getPRFileContentsRenderLimit(resultContents) - : null - const storedContent = getStoredTextDiffContent(result, largeDiffRenderLimit) - const storedResult = getStoredTextDiffResult(result, largeDiffRenderLimit) - loadedIndicesRef.current.add(index) - setSections((prev) => - prev.map((current, currentIndex) => - currentIndex === index - ? { - ...current, - diffResult: storedResult, - originalContent: storedContent.originalContent, - modifiedContent: storedContent.modifiedContent, - loading: false, - error, - largeDiffRenderLimit - } - : current - ) - ) - }) - }, - [baseSha, fileByPath, headSha, prNumber, prRepo, repoId, repoPath, sourceContext] - ) - - const retrySection = useCallback( - (index: number) => { - loadedIndicesRef.current.delete(index) + const { result, resultContents, error } = await load().catch((error) => ({ + result: { + kind: 'text', + originalContent: '', + modifiedContent: '', + originalIsBinary: false, + modifiedIsBinary: false + } as GitDiffResult, + resultContents: undefined, + error: error instanceof Error ? error.message : 'Failed to load diff.' + })) loadingIndicesRef.current.delete(index) - setSectionHeights((prev) => removeDiffSectionMeasuredHeight(prev, index)) - setSections((prev) => - prev.map((section, sectionIndex) => - sectionIndex === index + if (generationRef.current !== generation) { + return + } + const largeDiffRenderLimit = + !error && result.kind === 'text' && resultContents + ? getPRFileContentsRenderLimit(resultContents) + : null + const storedContent = getStoredTextDiffContent(result, largeDiffRenderLimit) + const storedResult = getStoredTextDiffResult(result, largeDiffRenderLimit) + loadedIndicesRef.current.add(index) + setSections((prev) => { + const nextSections = prev.map((current, currentIndex) => + currentIndex === index ? { - ...section, - diffResult: null, - originalContent: '', - modifiedContent: '', - loading: true, - error: undefined, - largeDiffRenderLimit: null + ...current, + diffResult: storedResult, + originalContent: storedContent.originalContent, + modifiedContent: storedContent.modifiedContent, + loading: false, + error, + largeDiffRenderLimit } - : section + : current ) - ) - loadSection(index) + return applySectionTextRetention(nextSections, nextSections[index]?.key) + }) }, - [loadSection] + [ + applySectionTextRetention, + baseSha, + fileByPath, + headSha, + prNumber, + prRepo, + repoId, + repoPath, + sourceContext + ] ) + loadSectionRef.current = loadSectionNow + + const loadSection = useCallback((index: number) => { + if (!sectionsRef.current[index]?.collapsed) { + loadSchedulerRef.current.request(index) + } + }, []) + + const retrySection = useCallback((index: number) => { + loadedIndicesRef.current.delete(index) + loadingIndicesRef.current.delete(index) + setSectionHeights((prev) => removeDiffSectionMeasuredHeight(prev, index)) + setSections((prev) => + prev.map((section, sectionIndex) => + sectionIndex === index + ? { + ...section, + diffResult: null, + originalContent: '', + modifiedContent: '', + loading: true, + error: undefined, + largeDiffRenderLimit: null + } + : section + ) + ) + loadSchedulerRef.current.rerequest(index) + }, []) const toggleSection = useCallback( (index: number) => { @@ -2452,17 +2394,11 @@ function PRFilesCombinedDiffViewer({ [loadSection] ) - const setAllSectionsCollapsed = useCallback( - (collapsed: boolean) => { - setSections((prev) => prev.map((section) => ({ ...section, collapsed }))) - if (!collapsed) { - window.requestAnimationFrame(() => { - sectionsRef.current.forEach((_, index) => loadSection(index)) - }) - } - }, - [loadSection] - ) + const setAllSectionsCollapsed = useCallback((collapsed: boolean) => { + // Why: expanded virtual rows load on mount; prefetching every offscreen file + // can queue an entire large PR's response bodies before retention runs. + setSections((prev) => prev.map((section) => ({ ...section, collapsed }))) + }, []) const allSectionsCollapsed = sections.length > 0 && sections.every((section) => section.collapsed) const sectionIndexByKey = useMemo(() => createCombinedDiffSectionIndexMap(sections), [sections]) @@ -2501,6 +2437,18 @@ function PRFilesCombinedDiffViewer({ : `${index}:${entrySignature}` } }) + const protectedSectionKeys = [ + ...virtualizer + .getVirtualItems() + .map((item) => sections[item.index]?.key) + .filter((key): key is string => key !== undefined), + ...(activeTreeSectionKey ? [activeTreeSectionKey] : []) + ] + protectedSectionKeysRef.current = new Set(protectedSectionKeys) + const protectedSectionSignature = protectedSectionKeys.join('\0') + useLayoutEffect(() => { + setSections((current) => applySectionTextRetention(current)) + }, [applySectionTextRetention, protectedSectionSignature, sections]) useLayoutEffect(() => { virtualizer.measure() @@ -6874,14 +6822,8 @@ export default function GitHubItemDialog({ } }, [workItem]) - // Why: subscribe to the module-level cache so reopening a cached item paints synchronously on first render. - const cachedEntry = useSyncExternalStore( - subscribeWorkItemDetailsCache, - useCallback( - () => (detailsCacheKey ? workItemDetailsCache.get(detailsCacheKey) : undefined), - [detailsCacheKey] - ) - ) + // Why: rejected oversized results stay local to the mounted drawer without re-entering retention. + const cachedEntry = useWorkItemDetailsCacheEntry(detailsCacheKey) // Why: bumped on cold open (no cached details yet) so the details memo re-runs and surfaces the optimistic comment before the fetch lands. const [optimisticTick, setOptimisticTick] = useState(0) @@ -6948,7 +6890,7 @@ export default function GitHubItemDialog({ prevItemIdRef.current = workItem.id setTab(normalizeItemDialogTab(workItem, initialTab)) - const cached = workItemDetailsCache.get(detailsCacheKey) + const cached = getWorkItemDetailsCacheEntry(detailsCacheKey) const now = Date.now() const hasFreshData = cached?.details && now - cached.fetchedAt <= WORK_ITEM_DETAILS_FRESH_MS @@ -6968,7 +6910,7 @@ export default function GitHubItemDialog({ }) // Why: snapshot the invalidation generation; if it advances before resolve, a mid-flight mutation invalidated the entry — don't write back. - const launchedAtGeneration = workItemDetailsCacheGeneration + const launchedAtGeneration = getWorkItemDetailsCacheGeneration() if (!cached?.pending) { touchWorkItemDetailsCache(detailsCacheKey, { @@ -6981,8 +6923,8 @@ export default function GitHubItemDialog({ inflight .then((result) => { - const invalidatedMidFlight = workItemDetailsCacheGeneration !== launchedAtGeneration - const prev = workItemDetailsCache.get(detailsCacheKey) + const invalidatedMidFlight = getWorkItemDetailsCacheGeneration() !== launchedAtGeneration + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (invalidatedMidFlight && prev?.pending !== inflight) { // Why: entry was deliberately dropped (or later repopulated) — don't recreate or touch it. return @@ -7010,8 +6952,8 @@ export default function GitHubItemDialog({ }) .catch((err) => { const message = err instanceof Error ? err.message : 'Failed to load details' - const invalidatedMidFlight = workItemDetailsCacheGeneration !== launchedAtGeneration - const prev = workItemDetailsCache.get(detailsCacheKey) + const invalidatedMidFlight = getWorkItemDetailsCacheGeneration() !== launchedAtGeneration + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (invalidatedMidFlight && prev?.pending !== inflight) { return } @@ -7115,7 +7057,7 @@ export default function GitHubItemDialog({ optimisticCommentsRef.current.push(comment) // Why: write through the module cache so concurrent drawers re-render; mark fetchedAt stale (0) so next open refetches server fields. if (detailsCacheKey) { - const prev = workItemDetailsCache.get(detailsCacheKey) + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (prev?.details) { const ids = new Set(prev.details.comments.map((c) => c.id)) if (!ids.has(comment.id)) { diff --git a/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx b/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx index e3af7f426d8..ddb11435c4d 100644 --- a/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx +++ b/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx @@ -287,9 +287,10 @@ function openRunTargetPicker(container: HTMLElement): void { } function findRunTargetItem(label: string): HTMLElement | undefined { - return [...document.body.querySelectorAll('[cmdk-item]')].find((item) => - item.textContent?.includes(label) - ) + // Why: "Add host" is a pinned footer button (mirrors the Project combobox), not a cmdk row. + return [ + ...document.body.querySelectorAll('[cmdk-item], [data-run-target-add-host]') + ].find((item) => item.textContent?.includes(label)) } let current: { container: HTMLDivElement; root: Root } | null = null diff --git a/src/renderer/src/components/NewWorkspaceComposerCard.tsx b/src/renderer/src/components/NewWorkspaceComposerCard.tsx index 02542787fd3..062714ff678 100644 --- a/src/renderer/src/components/NewWorkspaceComposerCard.tsx +++ b/src/renderer/src/components/NewWorkspaceComposerCard.tsx @@ -13,6 +13,7 @@ import { CornerDownLeft, FolderPlus, LoaderCircle, + Monitor, PlugZap, Plus, Settings2, @@ -26,7 +27,7 @@ import { CommandList, CommandSeparator } from '@/components/ui/command' -import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { Popover, PopoverAnchor, PopoverContent, PopoverTrigger } from '@/components/ui/popover' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { SettingsSwitch } from '@/components/settings/SettingsFormControls' import type RepoCombobox from '@/components/repo/RepoCombobox' @@ -71,6 +72,7 @@ import type { ReadyProjectHostSetupOption } from '@/lib/project-host-setup-options' import type { WorkspaceCreateErrorDisplay } from '@/lib/workspace-create-error-format' +import { LOCAL_EXECUTION_HOST_ID, type ExecutionHostId } from '../../../shared/execution-host' import type { SshConnectionStatus } from '../../../shared/ssh-types' import type { TaskSourceContext } from '../../../shared/task-source-context' import type { RuntimeStatus } from '../../../shared/runtime-types' @@ -355,6 +357,12 @@ function HostPathTooltip({ path }: { path: string }): React.JSX.Element { ) } +// Why: the local machine isn't a server — give it a monitor glyph so it reads as "this computer". +function HostRowIcon({ hostId }: { hostId: ExecutionHostId }): React.JSX.Element { + const Icon = hostId === LOCAL_EXECUTION_HOST_ID ? Monitor : Server + return +} + function WorkspaceRunTargetCombobox({ hostOptions, hostValue, @@ -501,7 +509,7 @@ function WorkspaceRunTargetCombobox({ ) : selectedHost ? ( - + {selectedHost.label} ) : ( @@ -534,7 +542,7 @@ function WorkspaceRunTargetCombobox({ onSelect={() => handleHostSelect(option.id)} onPointerEnter={closeSubmenus} onFocus={closeSubmenus} - className="items-center gap-2 px-3 py-2" + className="items-center gap-2 px-3 py-1.5" > - +
{option.label}
@@ -564,7 +572,7 @@ function WorkspaceRunTargetCombobox({ onSelect={() => {}} onPointerEnter={closeSubmenus} onFocus={closeSubmenus} - className="items-center gap-2 px-3 py-2" + className="items-center gap-2 px-3 py-1.5" >
@@ -576,7 +584,7 @@ function WorkspaceRunTargetCombobox({ ) : option.attention ? ( ) : ( - + )}
{option.label}
@@ -626,10 +634,10 @@ function WorkspaceRunTargetCombobox({ ))} ) : null} - {/* Why: separate the host list from the environment/add-host actions below, mirroring - the divider above the not-connected group so the two action rows read as their own - section rather than trailing the hosts. */} - {readyHostOptions.length > 0 || needsSetupHostOptions.length > 0 ? ( + {/* Why: separate the host list from the per-workspace-env row; the "Add host" action + is pinned below the list with its own border, so it needs no separator here. */} + {recipes.length > 0 && + (readyHostOptions.length > 0 || needsSetupHostOptions.length > 0) ? ( ) : null} {recipes.length > 0 ? ( @@ -641,7 +649,7 @@ function WorkspaceRunTargetCombobox({ onSelect={openVmRecipesSubmenu} onPointerEnter={openVmRecipesSubmenu} onFocus={openVmRecipesSubmenu} - className="items-center gap-2 px-3 py-2" + className="items-center gap-2 px-3 py-1.5" > handleRecipeSelect(recipe.id)} - className="items-center gap-2 px-3 py-2" + className="items-center gap-2 px-3 py-1.5" > ) : null} + + {/* Why: pin "Add host" below the scrollable list — mirrors the Project combobox's + "Add a new project" footer so it keeps a compact single-row height and one clean + divider instead of a taller in-list row above the popover edge. */} +
- - + + {/* Why: pin an empty value so cmdk doesn't auto-highlight the first row on open — matches the recipes submenu, which leaves nothing highlighted by default. */} @@ -731,7 +739,7 @@ function WorkspaceRunTargetCombobox({
@@ -752,7 +760,7 @@ function WorkspaceRunTargetCombobox({
@@ -774,7 +782,7 @@ function WorkspaceRunTargetCombobox({ - +
diff --git a/src/renderer/src/components/PullRequestPage.tsx b/src/renderer/src/components/PullRequestPage.tsx index 5ae19bf9303..8c8bdf1e4f5 100644 --- a/src/renderer/src/components/PullRequestPage.tsx +++ b/src/renderer/src/components/PullRequestPage.tsx @@ -6,8 +6,7 @@ import React, { useLayoutEffect, useMemo, useRef, - useState, - useSyncExternalStore + useState } from 'react' import { lazyWithRetry as lazy } from '@/lib/lazy-with-retry' import { useVirtualizer } from '@tanstack/react-virtual' @@ -85,6 +84,12 @@ import { } from '@/components/editor/diff-section-layout' import type { DiffSection } from '@/components/editor/diff-section-types' import { removeDiffSectionMeasuredHeight } from '@/components/editor/diff-section-height-cache' +import { createCombinedDiffLoadScheduler } from '@/components/editor/combined-diff-load-scheduler' +import { + COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES, + retainCombinedDiffSectionText, + retainCombinedDiffViewStateText +} from '@/components/editor/combined-diff-text-retention' import { MAX_RENDERED_DIFF_COMBINED_CHARACTERS, MAX_RENDERED_DIFF_LINES_PER_SIDE, @@ -163,10 +168,18 @@ import { getCommentBodySubmitState, hasBoundedCommentBodyText } from '@/lib/comment-body-submit-state' +import { emitGitHubWorkItemDetailsCacheMutation } from '@/lib/github-work-item-details-cache-events' import { - emitGitHubWorkItemDetailsCacheMutation, - onGitHubWorkItemDetailsCacheMutation -} from '@/lib/github-work-item-details-cache-events' + WORK_ITEM_DETAILS_FRESH_MS, + WORK_ITEM_DETAILS_UNAVAILABLE_MESSAGE, + getWorkItemDetailsCacheEntry, + getWorkItemDetailsCacheGeneration, + getWorkItemDetailsCacheKey, + invalidateWorkItemDetailsCacheByMatch, + invalidateWorkItemDetailsCacheForKey, + touchWorkItemDetailsCache, + useWorkItemDetailsCacheEntry +} from '@/lib/github-work-item-details-cache' import { lookupGitHubWorkItemDetailsForSource } from '@/lib/github-work-item-source-lookup' import { canUseGitHubRepoContext, @@ -1452,102 +1465,12 @@ function isPRFileViewed(file: GitHubPRFile): boolean { return file.viewerViewedState === 'VIEWED' } -// SWR cache: reopening a drawer paints cached data instantly while a background refetch reconciles. See docs/gh-work-item-drawer-cache.md. -const WORK_ITEM_DETAILS_CACHE_MAX = 50 -const WORK_ITEM_DETAILS_FRESH_MS = 30_000 -const WORK_ITEM_DETAILS_UNAVAILABLE_MESSAGE = 'Unable to load details for this GitHub item.' -type WorkItemDetailsCacheEntry = { - details: GitHubWorkItemDetails | null - fetchedAt: number - pending?: Promise - error?: string -} -const workItemDetailsCache = new Map() - -// Why: useSyncExternalStore snapshot stability relies on every cache write replacing the entry object identity (delete+set). -const workItemDetailsCacheListeners = new Set<() => void>() -function subscribeWorkItemDetailsCache(listener: () => void): () => void { - workItemDetailsCacheListeners.add(listener) - return () => { - workItemDetailsCacheListeners.delete(listener) - } -} -function notifyWorkItemDetailsCache(): void { - for (const listener of workItemDetailsCacheListeners) { - listener() - } -} - -function getWorkItemDetailsCacheKey(args: { - repoPath: string - repoId: string - issueSourcePreference: string | undefined - sourceCacheScope?: string | null - type: 'issue' | 'pr' - number: number -}): string { - // Why: `\0` separator avoids collisions between key fields that may contain `:` or `/`. - const keyParts = args.sourceCacheScope - ? [args.repoId, args.sourceCacheScope, args.issueSourcePreference ?? 'auto', args.type] - : [args.repoId, args.issueSourcePreference ?? 'auto', args.type] - return [...keyParts, args.number].join('\0') -} - -function touchWorkItemDetailsCache(key: string, entry: WorkItemDetailsCacheEntry): void { - // Why: re-insert moves the key to MRU; Map insertion order keeps the oldest key first when evicting. - workItemDetailsCache.delete(key) - workItemDetailsCache.set(key, entry) - while (workItemDetailsCache.size > WORK_ITEM_DETAILS_CACHE_MAX) { - const oldest = workItemDetailsCache.keys().next().value - if (oldest === undefined) { - break - } - workItemDetailsCache.delete(oldest) - } - notifyWorkItemDetailsCache() -} - -// Exposed so mutation handlers can drop a stale entry after a local mutation (cross-window invalidation arrives via the `gh:workItemMutated` listener below). -export function invalidateWorkItemDetailsCacheForKey(key: string): void { - // Why: bump generation so an in-flight fetch launched before this invalidation won't write its stale result back. - workItemDetailsCacheGeneration += 1 - const existed = workItemDetailsCache.delete(key) - if (existed) { - notifyWorkItemDetailsCache() - } -} - -// Why: monotonic counter so an in-flight refetch that started before a mutation detects its result is stale and skips the write-back. -let workItemDetailsCacheGeneration = 0 - -// Why: without the exact key (cross-window events carry only repoPath+number+type), drop every entry matching that tuple regardless of source preference. -function invalidateWorkItemDetailsCacheByMatch(args: { - repoPath: string - repoId?: string - type: 'issue' | 'pr' - number: number -}): void { - const suffix = `\0${args.type}\0${args.number}` - const prefix = `${args.repoId ?? args.repoPath}\0` - let removed = false - for (const key of Array.from(workItemDetailsCache.keys())) { - if (key.startsWith(prefix) && key.endsWith(suffix)) { - workItemDetailsCache.delete(key) - removed = true - } - } - if (removed) { - workItemDetailsCacheGeneration += 1 - notifyWorkItemDetailsCache() - } -} - function patchCachedPRFileViewedState( cacheKey: string, path: string, viewerViewedState: GitHubPRFileViewedState ): GitHubPRFileViewedState | undefined { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) const files = prev?.details?.files if (!prev?.details || !files) { return undefined @@ -1572,7 +1495,7 @@ function patchCachedPRFileViewedState( } function patchCachedPRChecks(cacheKey: string, checks: PRCheckDetail[]): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1588,7 +1511,7 @@ function patchCachedPRReviewRequests( cacheKey: string, reviewRequests: GitHubAssignableUser[] ): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1604,7 +1527,7 @@ function patchCachedPRReviewRequests( } function patchCachedWorkItemBody(cacheKey: string, body: string): void { - const prev = workItemDetailsCache.get(cacheKey) + const prev = getWorkItemDetailsCacheEntry(cacheKey) if (!prev?.details) { return } @@ -1616,29 +1539,6 @@ function patchCachedWorkItemBody(cacheKey: string, body: string): void { }) } -// Install once at module load (all dialogs share the cache); track the unsubscribe so Vite HMR doesn't accumulate listeners across reloads. -let workItemMutatedUnsub: (() => void) | undefined -let workItemDetailsCacheEventUnsub: (() => void) | undefined -if (typeof window !== 'undefined' && window.api?.gh?.onWorkItemMutated) { - workItemMutatedUnsub = window.api.gh.onWorkItemMutated((payload) => { - invalidateWorkItemDetailsCacheByMatch({ - repoPath: payload.repoPath, - repoId: payload.repoId, - type: payload.type, - number: payload.number - }) - }) - workItemDetailsCacheEventUnsub = onGitHubWorkItemDetailsCacheMutation((payload) => { - invalidateWorkItemDetailsCacheByMatch(payload) - }) -} -if (typeof import.meta !== 'undefined' && import.meta.hot) { - import.meta.hot.dispose(() => { - workItemMutatedUnsub?.() - workItemDetailsCacheEventUnsub?.() - }) -} - // Why: bounded LRU so a session of opening many PR files can't grow this module map without bound. const PR_FILE_CONTENT_CACHE_MAX = 64 // Why: overflow sentinel — force reported size past the render budget so downstream reliably picks fallback mode. @@ -2339,14 +2239,48 @@ function PRFilesCombinedDiffViewer({ const loadedIndicesRef = useRef>(new Set()) const loadingIndicesRef = useRef>(new Set()) const sectionsRef = useRef([]) + const protectedSectionKeysRef = useRef>(new Set()) const generationRef = useRef(0) const modifiedEditorsRef = useRef>(new Map()) const handleSectionSaveRef = useRef<(index: number) => Promise>(async () => {}) + const loadSectionRef = useRef<(index: number) => Promise>(async () => {}) + const loadSchedulerRef = useRef( + createCombinedDiffLoadScheduler({ + loadSection: (index) => loadSectionRef.current(index), + // Why: keep hosted fetch latency responsive without stacking a PR worth of response bodies. + maxConcurrent: 2 + }) + ) + const applySectionTextRetention = useCallback( + (nextSections: DiffSection[], additionallyProtectedKey?: string): DiffSection[] => { + const protectedSectionKeys = new Set(protectedSectionKeysRef.current) + if (additionallyProtectedKey) { + protectedSectionKeys.add(additionallyProtectedKey) + } + const retained = retainCombinedDiffSectionText({ + sections: nextSections, + loadedIndices: loadedIndicesRef.current, + protectedSectionKeys + }) + for (const index of retained.evictedIndices) { + loadedIndicesRef.current.delete(index) + } + return retained.sections + }, + [] + ) sectionsRef.current = sections + useEffect(() => { + const scheduler = loadSchedulerRef.current + scheduler.reset() + return () => scheduler.dispose() + }, []) + useEffect(() => { // Why: bump generation so stale async diff loads from the previous view can't patch the restored sections. generationRef.current += 1 + loadSchedulerRef.current.reset() const cached = prFilesDiffViewStateCache.get(viewStateKey) if (cached && cached.entrySignature === entrySignature) { const restoredSections = cached.sections @@ -2389,10 +2323,10 @@ function PRFilesCombinedDiffViewer({ ) }, [entries, entrySignature, viewStateKey]) - const loadSection = useCallback( - (index: number) => { + const loadSectionNow = useCallback( + async (index: number) => { const section = sectionsRef.current[index] - if (!section || section.collapsed) { + if (!section || section.collapsed || !protectedSectionKeysRef.current.has(section.key)) { return } if (loadedIndicesRef.current.has(index) || loadingIndicesRef.current.has(index)) { @@ -2449,74 +2383,86 @@ function PRFilesCombinedDiffViewer({ return { result: getPRFileDiffResult(contents), resultContents: contents } } - load() - .catch((error) => ({ - result: { - kind: 'text', - originalContent: '', - modifiedContent: '', - originalIsBinary: false, - modifiedIsBinary: false - } as GitDiffResult, - resultContents: undefined, - error: error instanceof Error ? error.message : 'Failed to load diff.' - })) - .then(({ result, resultContents, error }) => { - loadingIndicesRef.current.delete(index) - if (generationRef.current !== generation) { - return - } - const largeDiffRenderLimit = - !error && result.kind === 'text' && resultContents - ? getPRFileContentsRenderLimit(resultContents) - : null - const storedContent = getStoredTextDiffContent(result, largeDiffRenderLimit) - const storedResult = getStoredTextDiffResult(result, largeDiffRenderLimit) - loadedIndicesRef.current.add(index) - setSections((prev) => - prev.map((current, currentIndex) => - currentIndex === index - ? { - ...current, - diffResult: storedResult, - originalContent: storedContent.originalContent, - modifiedContent: storedContent.modifiedContent, - loading: false, - error, - largeDiffRenderLimit - } - : current - ) - ) - }) - }, - [baseSha, fileByPath, headSha, prNumber, prRepo, repoId, repoPath, sourceContext] - ) - - const retrySection = useCallback( - (index: number) => { - loadedIndicesRef.current.delete(index) + const { result, resultContents, error } = await load().catch((error) => ({ + result: { + kind: 'text', + originalContent: '', + modifiedContent: '', + originalIsBinary: false, + modifiedIsBinary: false + } as GitDiffResult, + resultContents: undefined, + error: error instanceof Error ? error.message : 'Failed to load diff.' + })) loadingIndicesRef.current.delete(index) - setSectionHeights((prev) => removeDiffSectionMeasuredHeight(prev, index)) - setSections((prev) => - prev.map((section, sectionIndex) => - sectionIndex === index + if (generationRef.current !== generation) { + return + } + const largeDiffRenderLimit = + !error && result.kind === 'text' && resultContents + ? getPRFileContentsRenderLimit(resultContents) + : null + const storedContent = getStoredTextDiffContent(result, largeDiffRenderLimit) + const storedResult = getStoredTextDiffResult(result, largeDiffRenderLimit) + loadedIndicesRef.current.add(index) + setSections((prev) => { + const nextSections = prev.map((current, currentIndex) => + currentIndex === index ? { - ...section, - diffResult: null, - originalContent: '', - modifiedContent: '', - loading: true, - error: undefined, - largeDiffRenderLimit: null + ...current, + diffResult: storedResult, + originalContent: storedContent.originalContent, + modifiedContent: storedContent.modifiedContent, + loading: false, + error, + largeDiffRenderLimit } - : section + : current ) - ) - loadSection(index) + return applySectionTextRetention(nextSections, nextSections[index]?.key) + }) }, - [loadSection] + [ + applySectionTextRetention, + baseSha, + fileByPath, + headSha, + prNumber, + prRepo, + repoId, + repoPath, + sourceContext + ] ) + loadSectionRef.current = loadSectionNow + + const loadSection = useCallback((index: number) => { + if (!sectionsRef.current[index]?.collapsed) { + loadSchedulerRef.current.request(index) + } + }, []) + + const retrySection = useCallback((index: number) => { + loadedIndicesRef.current.delete(index) + loadingIndicesRef.current.delete(index) + setSectionHeights((prev) => removeDiffSectionMeasuredHeight(prev, index)) + setSections((prev) => + prev.map((section, sectionIndex) => + sectionIndex === index + ? { + ...section, + diffResult: null, + originalContent: '', + modifiedContent: '', + loading: true, + error: undefined, + largeDiffRenderLimit: null + } + : section + ) + ) + loadSchedulerRef.current.rerequest(index) + }, []) const toggleSection = useCallback( (index: number) => { @@ -2533,17 +2479,11 @@ function PRFilesCombinedDiffViewer({ [loadSection] ) - const setAllSectionsCollapsed = useCallback( - (collapsed: boolean) => { - setSections((prev) => prev.map((section) => ({ ...section, collapsed }))) - if (!collapsed) { - window.requestAnimationFrame(() => { - sectionsRef.current.forEach((_, index) => loadSection(index)) - }) - } - }, - [loadSection] - ) + const setAllSectionsCollapsed = useCallback((collapsed: boolean) => { + // Why: expanded virtual rows load on mount; prefetching every offscreen file + // can queue an entire large PR's response bodies before retention runs. + setSections((prev) => prev.map((section) => ({ ...section, collapsed }))) + }, []) const allSectionsCollapsed = sections.length > 0 && sections.every((section) => section.collapsed) const sectionIndexByKey = useMemo(() => createCombinedDiffSectionIndexMap(sections), [sections]) @@ -2582,6 +2522,18 @@ function PRFilesCombinedDiffViewer({ : `${index}:${entrySignature}` } }) + const protectedSectionKeys = [ + ...virtualizer + .getVirtualItems() + .map((item) => sections[item.index]?.key) + .filter((key): key is string => key !== undefined), + ...(activeTreeSectionKey ? [activeTreeSectionKey] : []) + ] + protectedSectionKeysRef.current = new Set(protectedSectionKeys) + const protectedSectionSignature = protectedSectionKeys.join('\0') + useLayoutEffect(() => { + setSections((current) => applySectionTextRetention(current)) + }, [applySectionTextRetention, protectedSectionSignature, sections]) useLayoutEffect(() => { virtualizer.measure() @@ -2593,18 +2545,24 @@ function PRFilesCombinedDiffViewer({ } const preservedScrollTop = prFilesDiffScrollTopCache.get(viewStateKey) ?? scrollContainerRef.current?.scrollTop ?? 0 - setWithLRU(prFilesDiffViewStateCache, viewStateKey, { - entrySignature, - sections, - sectionHeights, - loadedIndices: Array.from(loadedIndicesRef.current).filter( - (index) => !sections[index]?.loading - ), - scrollTop: preservedScrollTop, - sideBySide, - fileTreeCollapsed, - activeTreeSectionKey - }) + setWithLRU( + prFilesDiffViewStateCache, + viewStateKey, + { + entrySignature, + sections, + sectionHeights, + loadedIndices: Array.from(loadedIndicesRef.current).filter( + (index) => !sections[index]?.loading + ), + scrollTop: preservedScrollTop, + sideBySide, + fileTreeCollapsed, + activeTreeSectionKey + }, + COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES + ) + retainCombinedDiffViewStateText(prFilesDiffViewStateCache) }, [ activeTreeSectionKey, entries.length, @@ -6619,14 +6577,8 @@ export default function PullRequestPage({ } }, [workItem]) - // Why: subscribe to the module cache so reopening a cached item paints synchronously; writes replace entry identity (delete+set), so Map.get is a stable snapshot. - const cachedEntry = useSyncExternalStore( - subscribeWorkItemDetailsCache, - useCallback( - () => (detailsCacheKey ? workItemDetailsCache.get(detailsCacheKey) : undefined), - [detailsCacheKey] - ) - ) + // Why: rejected oversized results stay local to the mounted page without re-entering retention. + const cachedEntry = useWorkItemDetailsCacheEntry(detailsCacheKey) // Why: bumped on cold open (no cached details) so the details memo re-runs and surfaces the optimistic comment via the loading shell; cache-notify handles the warm case. const [optimisticTick, setOptimisticTick] = useState(0) @@ -6680,7 +6632,7 @@ export default function PullRequestPage({ } prevItemIdRef.current = workItem.id - const cached = workItemDetailsCache.get(detailsCacheKey) + const cached = getWorkItemDetailsCacheEntry(detailsCacheKey) const now = Date.now() const hasFreshData = cached?.details && now - cached.fetchedAt <= WORK_ITEM_DETAILS_FRESH_MS @@ -6700,7 +6652,7 @@ export default function PullRequestPage({ }) // Why: snapshot the generation so a mid-flight invalidation (generation advance) blocks writing a stale result back. - const launchedAtGeneration = workItemDetailsCacheGeneration + const launchedAtGeneration = getWorkItemDetailsCacheGeneration() if (!cached?.pending) { touchWorkItemDetailsCache(detailsCacheKey, { @@ -6713,8 +6665,8 @@ export default function PullRequestPage({ inflight .then((result) => { - const invalidatedMidFlight = workItemDetailsCacheGeneration !== launchedAtGeneration - const prev = workItemDetailsCache.get(detailsCacheKey) + const invalidatedMidFlight = getWorkItemDetailsCacheGeneration() !== launchedAtGeneration + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (invalidatedMidFlight && prev?.pending !== inflight) { // Why: entry was deliberately dropped (or later repopulated) — don't recreate or clobber it. return @@ -6742,8 +6694,8 @@ export default function PullRequestPage({ }) .catch((err) => { const message = err instanceof Error ? err.message : 'Failed to load details' - const invalidatedMidFlight = workItemDetailsCacheGeneration !== launchedAtGeneration - const prev = workItemDetailsCache.get(detailsCacheKey) + const invalidatedMidFlight = getWorkItemDetailsCacheGeneration() !== launchedAtGeneration + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (invalidatedMidFlight && prev?.pending !== inflight) { return } @@ -6854,7 +6806,7 @@ export default function PullRequestPage({ optimisticCommentsRef.current.push(comment) // Why: write through the shared cache so subscribers re-render; fetchedAt=0 forces a background refresh next open for server-side fields. if (detailsCacheKey) { - const prev = workItemDetailsCache.get(detailsCacheKey) + const prev = getWorkItemDetailsCacheEntry(detailsCacheKey) if (prev?.details) { const ids = new Set(prev.details.comments.map((c) => c.id)) if (!ids.has(comment.id)) { diff --git a/src/renderer/src/components/QuickOpen.tsx b/src/renderer/src/components/QuickOpen.tsx index 395628c0b3f..a2f2d48f3e9 100644 --- a/src/renderer/src/components/QuickOpen.tsx +++ b/src/renderer/src/components/QuickOpen.tsx @@ -130,6 +130,7 @@ export default function QuickOpen(): React.JSX.Element | null { return guidance ? ( diff --git a/src/renderer/src/components/TaskPage.tsx b/src/renderer/src/components/TaskPage.tsx index 2430505d6cc..0eafd37caae 100644 --- a/src/renderer/src/components/TaskPage.tsx +++ b/src/renderer/src/components/TaskPage.tsx @@ -124,6 +124,7 @@ import { getLinearStatePillStyle } from '@/components/linear-state-pill-style' import { parseTaskQuery, stripRepoQualifiers, withQualifier } from '../../../shared/task-query' +import { mapSettledWithConcurrency } from '../../../shared/map-with-concurrency' import { githubProjectHost } from '../../../shared/github-project-identity' import { buildLinearTeamUrl, @@ -384,6 +385,7 @@ const TASK_SEARCH_DEBOUNCE_MS = 300 const LINEAR_ITEM_LIMIT = 36 const JIRA_ITEM_LIMIT = 50 const PR_CHECKS_EAGER_PREFETCH_LIMIT = 20 +const GITLAB_REPO_FETCH_CONCURRENCY = 8 const GITHUB_TASK_GRID_CLASS = 'min-w-[790px] grid-cols-[72px_minmax(320px,1fr)_84px_100px_92px_122px]' @@ -4909,7 +4911,12 @@ export default function TaskPage(): React.JSX.Element { return { repoId: repo.id, items: typed.items, error } }) - void Promise.allSettled(eligibleRepos.map(fetchItems)) + void mapSettledWithConcurrency(eligibleRepos, GITLAB_REPO_FETCH_CONCURRENCY, async (repo) => { + if (stale) { + return { repoId: repo.id, items: [] as GitLabWorkItem[], error: undefined } + } + return fetchItems(repo) + }) .then((results) => { if (stale) { return diff --git a/src/renderer/src/components/Terminal.tsx b/src/renderer/src/components/Terminal.tsx index 410b3bc8243..6724c05ef45 100644 --- a/src/renderer/src/components/Terminal.tsx +++ b/src/renderer/src/components/Terminal.tsx @@ -12,6 +12,7 @@ import { } from '@/constants/terminal' import { useAppStore } from '../store' import { folderWorkspaceKey } from '../../../shared/workspace-scope' +import { mapWithConcurrency } from '../../../shared/map-with-concurrency' import { useAllWorktrees } from '../store/selectors' import { getConnectionId } from '../lib/connection-context' import { basename } from '../lib/path' @@ -141,6 +142,7 @@ const EditorPanel = lazy(() => import('./editor/EditorPanel')) // Why: gate handler runs after a dialog advances so a stray carry-over click can't act on the next dialog; ~200ms absorbs a physical double-click while staying responsive. const CLOSE_DIALOG_DEBOUNCE_MS = 200 +const CHILD_PROCESS_CHECK_CONCURRENCY = 8 const EDITOR_TAB_CONTENT_TYPES = new Set([ 'editor', 'diff', @@ -447,15 +449,15 @@ function Terminal(): React.JSX.Element | null { } ) if (localPtyIds.length > 0) { - void Promise.all(localPtyIds.map((id) => window.api.pty.hasChildProcesses(id))).then( - (results) => { - if (results.some(Boolean)) { - setWindowCloseDialogOpen(true) - } else { - confirmNativeWindowClose() - } + void mapWithConcurrency(localPtyIds, CHILD_PROCESS_CHECK_CONCURRENCY, (id) => + window.api.pty.hasChildProcesses(id) + ).then((results) => { + if (results.some(Boolean)) { + setWindowCloseDialogOpen(true) + } else { + confirmNativeWindowClose() } - ) + }) return } } diff --git a/src/renderer/src/components/automations/AutomationsPage.tsx b/src/renderer/src/components/automations/AutomationsPage.tsx index 538cec979e3..34a8a66eedb 100644 --- a/src/renderer/src/components/automations/AutomationsPage.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.tsx @@ -1913,7 +1913,8 @@ export default function AutomationsPage(): React.JSX.Element { async ({ manager, job, page, pageSize }) => { const fallbackRunsPage = { runs: job.runs.slice(page * pageSize, page * pageSize + pageSize), - totalCount: job.runCount + totalCount: job.runCount, + totalCountSaturated: job.runCountSaturated === true } const listExternalRuns = ( window.api.automations as Partial> @@ -1932,7 +1933,8 @@ export default function AutomationsPage(): React.JSX.Element { }) return { runs: result.runs, - totalCount: result.total + totalCount: result.total, + totalCountSaturated: result.totalSaturated === true } } catch (error) { if (isMissingExternalRunsApiError(error)) { @@ -2653,7 +2655,7 @@ export default function AutomationsPage(): React.JSX.Element { · {entry.manager.provider === 'hermes' - ? `${entry.job.runCount} ${entry.job.runCount === 1 ? 'run' : 'runs'}` + ? `${entry.job.runCount}${entry.job.runCountSaturated ? '+' : ''} ${entry.job.runCount === 1 ? 'run' : 'runs'}` : entry.manager.canManage ? translate( 'auto.components.automations.AutomationsPage.aecdc3681f', diff --git a/src/renderer/src/components/automations/ExternalAutomationManagers.test.tsx b/src/renderer/src/components/automations/ExternalAutomationManagers.test.tsx index 8d21b62da6f..3b53d84d514 100644 --- a/src/renderer/src/components/automations/ExternalAutomationManagers.test.tsx +++ b/src/renderer/src/components/automations/ExternalAutomationManagers.test.tsx @@ -151,6 +151,13 @@ describe('ExternalAutomationManagers toggle', () => { expect(getSwitch().getAttribute('aria-checked')).toBe('false') }) + it('marks a capped run count as a lower bound', () => { + renderManagers([ + makeManager({ jobs: [makeJob({ runCount: 10_000, runCountSaturated: true })] }) + ]) + expect(container.textContent).toContain('10000+ runs found') + }) + it('dispatches pause when toggling an enabled job', () => { const manager = makeManager({ jobs: [makeJob({ enabled: true })] }) const { onAction } = renderManagers([manager]) diff --git a/src/renderer/src/components/automations/ExternalAutomationManagers.tsx b/src/renderer/src/components/automations/ExternalAutomationManagers.tsx index bd6e1b67549..8e993b69de6 100644 --- a/src/renderer/src/components/automations/ExternalAutomationManagers.tsx +++ b/src/renderer/src/components/automations/ExternalAutomationManagers.tsx @@ -241,7 +241,8 @@ export function ExternalAutomationManagers({
{manager.provider === 'hermes' ? (
- {job.runCount}{' '} + {job.runCount} + {job.runCountSaturated ? '+' : ''}{' '} {job.runCount === 1 ? translate( 'auto.components.automations.ExternalAutomationManagers.8e9165af08', diff --git a/src/renderer/src/components/automations/ExternalAutomationRunTable.tsx b/src/renderer/src/components/automations/ExternalAutomationRunTable.tsx index f6e5f693371..38200c6bbd2 100644 --- a/src/renderer/src/components/automations/ExternalAutomationRunTable.tsx +++ b/src/renderer/src/components/automations/ExternalAutomationRunTable.tsx @@ -24,6 +24,7 @@ const PAGE_SIZE = 8 export type ExternalAutomationRunPage = { runs: ExternalAutomationRun[] totalCount?: number + totalCountSaturated?: boolean } export type FetchExternalAutomationRuns = (input: { @@ -110,7 +111,14 @@ export function ExternalAutomationRunTable({ // before paint so stale fetched rows/selection never flash for the new job. setTableState(resolvedTableState) } - const { page, selectedRunId, fetchedRuns, fetchedTotalCount, fetchError } = resolvedTableState + const { + page, + selectedRunId, + fetchedRuns, + fetchedTotalCount, + fetchedTotalCountSaturated, + fetchError + } = resolvedTableState useEffect(() => { if (!onFetchRuns) { @@ -143,6 +151,7 @@ export function ExternalAutomationRunTable({ ...resolveExternalAutomationRunTableState(current, jobRef.current), fetchedRuns: null, fetchedTotalCount: null, + fetchedTotalCountSaturated: false, fetchError: error instanceof Error ? error.message : 'Failed to load runs.' })) } @@ -162,6 +171,11 @@ export function ExternalAutomationRunTable({ ? (fetchedRuns ?? fallbackRuns.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE)) : fallbackRuns.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE) const totalCount = onFetchRuns ? (fetchedTotalCount ?? job.runCount) : job.runCount + const totalCountSaturated = onFetchRuns + ? fetchedTotalCount === null + ? job.runCountSaturated === true + : fetchedTotalCountSaturated + : job.runCountSaturated === true const totalPages = Math.max(1, Math.ceil(totalCount / PAGE_SIZE)) const selectedRun = useMemo( () => @@ -199,7 +213,8 @@ export function ExternalAutomationRunTable({ ) : null}
- {totalCount}{' '} + {totalCount} + {totalCountSaturated ? '+' : ''}{' '} {totalCount === 1 ? translate('auto.components.automations.ExternalAutomationRunTable.872d032d05', 'run') : translate( @@ -290,6 +305,7 @@ export function ExternalAutomationRunTable({ {pageStart}-{pageEnd}{' '} {translate('auto.components.automations.ExternalAutomationRunTable.7475c0ce96', 'of')}{' '} {totalCount} + {totalCountSaturated ? '+' : ''}
diff --git a/src/renderer/src/components/automations/automation-run-output-snapshot.test.ts b/src/renderer/src/components/automations/automation-run-output-snapshot.test.ts index 32f3b4cac75..68f1bf0926b 100644 --- a/src/renderer/src/components/automations/automation-run-output-snapshot.test.ts +++ b/src/renderer/src/components/automations/automation-run-output-snapshot.test.ts @@ -86,6 +86,18 @@ describe('automation run output snapshot buffer', () => { expect(snapshot?.truncated).toBe(true) }) + it('preserves output delivered as 100,000 one-character chunks', () => { + const buffer = createAutomationRunOutputSnapshotBuffer() + + for (let index = 0; index < 100_000; index += 1) { + buffer.append(String.fromCharCode(97 + (index % 26))) + } + + const snapshot = buffer.snapshot() + expect(snapshot?.content).toHaveLength(100_000) + expect(snapshot?.truncated).toBe(false) + }) + it('creates a saved snapshot from agent transcript text', () => { expect(createAutomationRunOutputSnapshotFromText('\nFinal summary.\n')).toEqual({ format: 'plain_text', diff --git a/src/renderer/src/components/automations/automation-run-output-snapshot.ts b/src/renderer/src/components/automations/automation-run-output-snapshot.ts index a7b9151fbfc..c29dd2fc329 100644 --- a/src/renderer/src/components/automations/automation-run-output-snapshot.ts +++ b/src/renderer/src/components/automations/automation-run-output-snapshot.ts @@ -1,5 +1,6 @@ /* eslint-disable no-control-regex -- terminal snapshots normalize ANSI/control output. */ import type { AutomationRunOutputSnapshot } from '../../../../shared/automations-types' +import { appendCompactedStringChunk } from '../../../../shared/string-chunk-compaction' const MAX_OUTPUT_SNAPSHOT_CHARS = 256 * 1024 @@ -63,7 +64,7 @@ export function createAutomationRunOutputSnapshotBuffer(): AutomationRunOutputSn if (!chunk) { return } - chunks.push(chunk) + appendCompactedStringChunk(chunks, chunk) totalChars += chunk.length let overflowChars = totalChars - MAX_OUTPUT_SNAPSHOT_CHARS while (overflowChars > 0 && chunks.length > 0) { diff --git a/src/renderer/src/components/automations/external-automation-run-table-state.test.ts b/src/renderer/src/components/automations/external-automation-run-table-state.test.ts index 10da77d348f..6d6bcc05485 100644 --- a/src/renderer/src/components/automations/external-automation-run-table-state.test.ts +++ b/src/renderer/src/components/automations/external-automation-run-table-state.test.ts @@ -53,6 +53,7 @@ describe('external automation run table state', () => { selectedRunId: 'run-2', fetchedRuns: null, fetchedTotalCount: null, + fetchedTotalCountSaturated: false, fetchError: null }) }) @@ -83,12 +84,14 @@ describe('external automation run table state', () => { expect( resolveExternalAutomationFetchedRuns(current, job('job-1', current.sourceRuns), { runs: [run('run-1'), run('run-2')], - totalCount: 12 + totalCount: 12, + totalCountSaturated: true }) ).toMatchObject({ selectedRunId: 'run-2', fetchedRuns: [run('run-1'), run('run-2')], - fetchedTotalCount: 12 + fetchedTotalCount: 12, + fetchedTotalCountSaturated: true }) }) diff --git a/src/renderer/src/components/automations/external-automation-run-table-state.ts b/src/renderer/src/components/automations/external-automation-run-table-state.ts index ef25c085049..99f13a2c973 100644 --- a/src/renderer/src/components/automations/external-automation-run-table-state.ts +++ b/src/renderer/src/components/automations/external-automation-run-table-state.ts @@ -8,6 +8,7 @@ type ExternalAutomationRunTableJob = { type ExternalAutomationRunPageResult = { runs: ExternalAutomationRun[] totalCount?: number + totalCountSaturated?: boolean } export type ExternalAutomationRunTableState = { @@ -17,6 +18,7 @@ export type ExternalAutomationRunTableState = { selectedRunId: string | null fetchedRuns: ExternalAutomationRun[] | null fetchedTotalCount: number | null + fetchedTotalCountSaturated: boolean fetchError: string | null } @@ -30,6 +32,7 @@ export function createExternalAutomationRunTableState( selectedRunId: job.runs[0]?.id ?? null, fetchedRuns: null, fetchedTotalCount: null, + fetchedTotalCountSaturated: false, fetchError: null } } @@ -70,6 +73,7 @@ export function resolveExternalAutomationFetchedRuns( ...resolved, fetchedRuns: result.runs, fetchedTotalCount: result.totalCount ?? null, + fetchedTotalCountSaturated: result.totalCountSaturated === true, selectedRunId } } diff --git a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx index b6b82f0d133..4d01ce480c8 100644 --- a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx +++ b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx @@ -20,6 +20,7 @@ import { } from '@/components/terminal-pane/terminal-ime-native-text-forwarder' import { getMacNativeTextInputSourceTracker } from '@/components/terminal-pane/terminal-ime-input-source' import { composeActiveTerminalTheme } from '@/components/terminal-pane/terminal-appearance' +import { resolveTerminalMinimumContrastRatio } from '@/lib/terminal-contrast-correction' import { useSystemPrefersDark } from '@/components/terminal-pane/use-system-prefers-dark' import { translate } from '@/i18n/i18n' import { getBuiltinTheme, resolveEffectiveTerminalAppearance } from '@/lib/terminal-theme' @@ -53,15 +54,16 @@ export function AgentTerminalPreview({ ptyId }: { ptyId: string }): React.JSX.El const containerRef = useRef(null) const settings = useAppStore((state) => state.settings) const systemPrefersDark = useSystemPrefersDark() - const terminalTheme = useMemo(() => { + const { terminalTheme, terminalMode } = useMemo(() => { if (!settings) { - return null + return { terminalTheme: null, terminalMode: 'dark' as const } } const appearance = resolveEffectiveTerminalAppearance(settings, systemPrefersDark) - return composeActiveTerminalTheme( + const theme = composeActiveTerminalTheme( appearance.theme ?? getBuiltinTheme(appearance.themeName), settings ) + return { terminalTheme: theme, terminalMode: appearance.mode } }, [settings, systemPrefersDark]) // A null snapshot means no serializer knows this pty (it died or was never // spawned this session) — say so instead of painting a silent blank terminal. @@ -271,6 +273,10 @@ export function AgentTerminalPreview({ ptyId }: { ptyId: string }): React.JSX.El cols: clamp(snap.cols ?? FALLBACK_COLS, 2, 500), rows: clamp(snap.rows ?? FALLBACK_ROWS, 2, 200), theme: terminalTheme ?? undefined, + minimumContrastRatio: resolveTerminalMinimumContrastRatio( + terminalTheme?.background, + terminalMode + ), scrollback: 1000 }) try { @@ -399,7 +405,7 @@ export function AgentTerminalPreview({ ptyId }: { ptyId: string }): React.JSX.El void window.api.terminalPreview.unsubscribe(ptyId) terminal?.dispose() } - }, [ptyId, terminalTheme]) + }, [ptyId, terminalTheme, terminalMode]) return ( // Why: a size FIXED by the viewport (not shrink-to-fit) + overflow-hidden diff --git a/src/renderer/src/components/editor/CombinedDiffViewer.tsx b/src/renderer/src/components/editor/CombinedDiffViewer.tsx index c8999e17e11..4920a510dc4 100644 --- a/src/renderer/src/components/editor/CombinedDiffViewer.tsx +++ b/src/renderer/src/components/editor/CombinedDiffViewer.tsx @@ -75,6 +75,12 @@ import { getInitialCombinedDiffSectionLoadIndices } from './combined-diff-initia import { removeDiffSectionMeasuredHeight } from './diff-section-height-cache' import { createCombinedDiffLoadScheduler } from './combined-diff-load-scheduler' import { combinedDiffSectionsMatchEntryMetadata } from './combined-diff-section-cache-match' +import { + COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES, + getCombinedDiffViewedSectionKeys, + retainCombinedDiffSectionText, + retainCombinedDiffViewStateText +} from './combined-diff-text-retention' import { beginCombinedDiffScrollbarDrag, type CombinedDiffScrollbarDragCleanup @@ -294,9 +300,28 @@ export default function CombinedDiffViewer({ const loadedIndicesRef = useRef>(new Set()) const loadingIndicesRef = useRef>(new Set()) const sectionsRef = useRef([]) + const protectedSectionKeysRef = useRef>(new Set()) const generationRef = useRef(0) const loadSectionRef = useRef<(index: number) => Promise>(async () => {}) const retrySectionRef = useRef<(index: number) => void>(() => {}) + const applySectionTextRetention = useCallback( + (nextSections: DiffSection[], additionallyProtectedKey?: string): DiffSection[] => { + const protectedSectionKeys = new Set(protectedSectionKeysRef.current) + if (additionallyProtectedKey) { + protectedSectionKeys.add(additionallyProtectedKey) + } + const retained = retainCombinedDiffSectionText({ + sections: nextSections, + loadedIndices: loadedIndicesRef.current, + protectedSectionKeys + }) + for (const index of retained.evictedIndices) { + loadedIndicesRef.current.delete(index) + } + return retained.sections + }, + [] + ) const updateCombinedDiffScrollbar = useCallback(() => { const container = scrollContainerRef.current if (!container || container.scrollHeight <= container.clientHeight + 1) { @@ -689,7 +714,7 @@ export default function CombinedDiffViewer({ const storedResult = getStoredTextDiffResult(result, largeDiffRenderLimit) loadedIndicesRef.current.add(index) setSections((prev) => { - return prev.map((s, i) => + const nextSections = prev.map((s, i) => i === index ? { ...s, @@ -702,6 +727,7 @@ export default function CombinedDiffViewer({ } : s ) + return applySectionTextRetention(nextSections, nextSections[index]?.key) }) }, // eslint-disable-next-line react-hooks/exhaustive-deps @@ -719,7 +745,8 @@ export default function CombinedDiffViewer({ isBranchMode, isCommitMode, renderableBranchEntries, - uncommittedEntries + uncommittedEntries, + applySectionTextRetention ] ) loadSectionRef.current = loadSectionNow @@ -987,8 +1014,20 @@ export default function CombinedDiffViewer({ // Why: the tree highlight belongs to one entry set; reset now so it can't flash on another before an Effect would. setActiveTreeSectionState({ entrySignature, key: null }) } + const protectedSectionKeys = [ + ...virtualizer + .getVirtualItems() + .map((item) => sections[item.index]?.key) + .filter((key): key is string => key !== undefined), + ...(activeTreeSectionKey ? [activeTreeSectionKey] : []) + ] + protectedSectionKeysRef.current = new Set(protectedSectionKeys) + const protectedSectionSignature = protectedSectionKeys.join('\0') + useLayoutEffect(() => { + setSections((current) => applySectionTextRetention(current)) + }, [applySectionTextRetention, protectedSectionSignature, sections]) const viewedSectionKeys = React.useMemo( - () => new Set(sections.filter((section) => !section.loading).map((section) => section.key)), + () => getCombinedDiffViewedSectionKeys(sections), [sections] ) const handleTreeNavigate = useCallback( @@ -1203,8 +1242,8 @@ export default function CombinedDiffViewer({ content ) setSectionHeights((prev) => removeDiffSectionMeasuredHeight(prev, index)) - setSections((prev) => - prev.map((s, i) => { + setSections((prev) => { + const nextSections = prev.map((s, i) => { if (i !== index) { return s } @@ -1234,12 +1273,20 @@ export default function CombinedDiffViewer({ largeDiffRenderLimit: nextLargeDiffRenderLimit } }) - ) + return applySectionTextRetention(nextSections, nextSections[index]?.key) + }) } catch (err) { console.error('Save failed:', err) } }, - [file.filePath, file.operationProvenance, file.runtimeEnvironmentId, file.worktreeId, sections] + [ + applySectionTextRetention, + file.filePath, + file.operationProvenance, + file.runtimeEnvironmentId, + file.worktreeId, + sections + ] ) const handleSectionSaveRef = useRef(handleSectionSave) @@ -1251,17 +1298,23 @@ export default function CombinedDiffViewer({ } const preservedScrollTop = combinedDiffScrollTopCache.get(viewStateKey) ?? scrollContainerRef.current?.scrollTop ?? 0 - setWithLRU(combinedDiffViewStateCache, viewStateKey, { - entrySignature, - gitStatusSignature: combinedGitStatusSignature, - sections, - sectionHeights, - loadedIndices: Array.from(loadedIndicesRef.current).filter( - (index) => !sections[index]?.loading - ), - scrollTop: preservedScrollTop, - sideBySide - }) + setWithLRU( + combinedDiffViewStateCache, + viewStateKey, + { + entrySignature, + gitStatusSignature: combinedGitStatusSignature, + sections, + sectionHeights, + loadedIndices: Array.from(loadedIndicesRef.current).filter( + (index) => !sections[index]?.loading + ), + scrollTop: preservedScrollTop, + sideBySide + }, + COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES + ) + retainCombinedDiffViewStateText(combinedDiffViewStateCache) }, [ combinedGitStatusSignature, entries.length, diff --git a/src/renderer/src/components/editor/CsvViewer.tsx b/src/renderer/src/components/editor/CsvViewer.tsx index e424a98c938..d042fd1e31e 100644 --- a/src/renderer/src/components/editor/CsvViewer.tsx +++ b/src/renderer/src/components/editor/CsvViewer.tsx @@ -1,6 +1,6 @@ import React, { useMemo, useRef } from 'react' import { useVirtualizer } from '@tanstack/react-virtual' -import { detectCsvDelimiter, parseCsv } from './csv-parse' +import { CSV_PARSE_LIMITS, detectCsvDelimiter, parseCsv } from './csv-parse' import { translate } from '@/i18n/i18n' type CsvViewerProps = { @@ -14,6 +14,40 @@ const MIN_COL_PX = 80 const MAX_COL_PX = 320 const ROW_NUMBER_COL_PX = 48 const CHAR_PX = 7 +const numberFormatter = new Intl.NumberFormat() + +function CsvLimitFallback({ filePath }: { filePath: string }): React.JSX.Element { + return ( +
+
+
+ {translate( + 'auto.components.editor.CsvViewer.2b65aa2913', + 'This CSV is too large to display safely.' + )} +
+
{filePath}
+
+ {translate( + 'auto.components.editor.CsvViewer.d5e8a2b453', + 'Table view is limited to {{rows}} rows, {{columns}} columns, and {{cells}} cells.', + { + rows: numberFormatter.format(CSV_PARSE_LIMITS.rows), + columns: numberFormatter.format(CSV_PARSE_LIMITS.columnsPerRow), + cells: numberFormatter.format(CSV_PARSE_LIMITS.cells) + } + )} +
+
+ {translate( + 'auto.components.editor.CsvViewer.59cfb175a2', + 'Switch to source mode to inspect the raw text.' + )} +
+
+
+ ) +} // Why: CsvViewer is the table counterpart to source-mode Monaco for .csv/.tsv // files. Row virtualization via @tanstack/react-virtual keeps large files @@ -89,6 +123,10 @@ export default function CsvViewer({ content, filePath }: CsvViewerProps): React. getItemKey: (index) => index }) + if (parsed.limitExceeded) { + return + } + if (parsed.rows.length === 0) { return (
diff --git a/src/renderer/src/components/editor/EditorContent.tsx b/src/renderer/src/components/editor/EditorContent.tsx index 270622fdaf3..027c05f0ffd 100644 --- a/src/renderer/src/components/editor/EditorContent.tsx +++ b/src/renderer/src/components/editor/EditorContent.tsx @@ -316,6 +316,7 @@ export function EditorContent({ onContentChange={activeFile.readOnly === true ? noopEditorContentChange : handleContentChange} onSave={activeFile.readOnly === true ? noopEditorSave : isMarkdown ? md.mdSave : handleSave} worktreeId={activeFile.worktreeId} + runtimeEnvironmentId={activeFile.runtimeEnvironmentId} markdownAnnotationsEnabled={markdownAnnotationsEnabled && isMarkdown} conflictDecorationsEnabled={activeFile.conflict?.conflictStatus === 'unresolved'} revealLine={ diff --git a/src/renderer/src/components/editor/ImageViewer.test.tsx b/src/renderer/src/components/editor/ImageViewer.test.tsx index 1639a638de1..222d26701eb 100644 --- a/src/renderer/src/components/editor/ImageViewer.test.tsx +++ b/src/renderer/src/components/editor/ImageViewer.test.tsx @@ -144,6 +144,16 @@ async function renderExpandedImageViewer(content: string): Promise { ) } +function pngBase64(width: number): string { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(1, 20) + return bytes.toString('base64') +} + describe('ImageViewer preview source retry', () => { beforeEach(() => { reactHookRuntime.states = [] @@ -152,8 +162,8 @@ describe('ImageViewer preview source retry', () => { }) it('retries an earlier failed source after a later source loads successfully', async () => { - const failedContent = 'failed-source' - const loadedContent = 'loaded-source' + const failedContent = pngBase64(1) + const loadedContent = pngBase64(2) const firstRender = await renderExpandedImageViewer(failedContent) const firstImage = findPreviewImage(firstRender) @@ -176,4 +186,11 @@ describe('ImageViewer preview source retry', () => { const retryImage = findPreviewImage(retryRender) expect(retryImage.props.src).toBe(`data:image/png;base64,${failedContent}`) }) + + it('shows a failure instead of loading an unsafe raster forever', async () => { + const rendered = await renderExpandedImageViewer(pngBase64(32_769)) + + expect(findElementsByType(rendered, 'Image')).toHaveLength(1) + expect(findElementsByType(rendered, 'img')).toHaveLength(0) + }) }) diff --git a/src/renderer/src/components/editor/ImageViewer.tsx b/src/renderer/src/components/editor/ImageViewer.tsx index 88b57b092b6..07f49750b2d 100644 --- a/src/renderer/src/components/editor/ImageViewer.tsx +++ b/src/renderer/src/components/editor/ImageViewer.tsx @@ -63,7 +63,9 @@ export default function ImageViewer({ () => buildImageDataUri(mimeType, cleanedContent), [cleanedContent, mimeType] ) - const imageError = previewSrc !== null && failedPreviewSrc === previewSrc + const imageError = + (previewSrc === null && cleanedContent.length > 0) || + (previewSrc !== null && failedPreviewSrc === previewSrc) const estimatedSize = useMemo(() => { const bytes = Math.floor((cleanedContent.length * 3) / 4) if (bytes < 1024) { diff --git a/src/renderer/src/components/editor/IpynbViewer.tsx b/src/renderer/src/components/editor/IpynbViewer.tsx index bbf7ded32ed..8ed2d5902f3 100644 --- a/src/renderer/src/components/editor/IpynbViewer.tsx +++ b/src/renderer/src/components/editor/IpynbViewer.tsx @@ -75,6 +75,7 @@ import { type IpynbOutputItem } from './ipynb-parse' import { translate } from '@/i18n/i18n' +import { buildImageDataUri } from '../../../../shared/image-data-uri' type IpynbViewerProps = { content: string @@ -149,7 +150,7 @@ function dataUriForImage(item: IpynbOutputItem): string | null { if (item.mime === 'image/svg+xml') { return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(valueToText(item.value))}` } - return `data:${item.mime};base64,${value}` + return buildImageDataUri(item.mime, value) } function NotebookCellHeader({ diff --git a/src/renderer/src/components/editor/MonacoEditor.tsx b/src/renderer/src/components/editor/MonacoEditor.tsx index c6996ad341e..7f37014fe46 100644 --- a/src/renderer/src/components/editor/MonacoEditor.tsx +++ b/src/renderer/src/components/editor/MonacoEditor.tsx @@ -81,6 +81,7 @@ type MonacoEditorProps = { revealMatchLength?: number markdownDocuments?: MarkdownDocument[] worktreeId?: string + runtimeEnvironmentId?: string | null markdownAnnotationsEnabled?: boolean conflictDecorationsEnabled?: boolean readOnly?: boolean @@ -106,6 +107,7 @@ export default function MonacoEditor({ revealMatchLength, markdownDocuments, worktreeId, + runtimeEnvironmentId, markdownAnnotationsEnabled = false, conflictDecorationsEnabled = false, readOnly = false, @@ -204,11 +206,15 @@ export default function MonacoEditor({ return } if (language === 'markdown' && markdownDocuments) { - setMarkdownDocCompletionDocuments(modelKey, markdownDocuments) + setMarkdownDocCompletionDocuments( + modelKey, + JSON.stringify([runtimeEnvironmentId ?? '', worktreeId ?? modelKey]), + markdownDocuments + ) } else { clearMarkdownDocCompletionDocuments(modelKey) } - }, [language, markdownDocuments]) + }, [language, markdownDocuments, runtimeEnvironmentId, worktreeId]) const shouldShowMarkdownAnnotations = markdownAnnotationsEnabled && language === 'markdown' && Boolean(worktreeId) diff --git a/src/renderer/src/components/editor/base64-byte-decoder.test.ts b/src/renderer/src/components/editor/base64-byte-decoder.test.ts new file mode 100644 index 00000000000..373f6942997 --- /dev/null +++ b/src/renderer/src/components/editor/base64-byte-decoder.test.ts @@ -0,0 +1,13 @@ +// @vitest-environment happy-dom + +import { describe, expect, it } from 'vitest' +import { decodeBase64Bytes } from './base64-byte-decoder' + +describe('decodeBase64Bytes', () => { + it('decodes large and whitespace-delimited input without changing bytes', () => { + const expected = new Uint8Array(40_000).map((_, index) => index % 251) + const encoded = btoa(String.fromCharCode(...expected)).replace(/.{100}/g, '$&\n') + + expect(decodeBase64Bytes(encoded)).toEqual(expected) + }) +}) diff --git a/src/renderer/src/components/editor/base64-byte-decoder.ts b/src/renderer/src/components/editor/base64-byte-decoder.ts new file mode 100644 index 00000000000..364c33e327e --- /dev/null +++ b/src/renderer/src/components/editor/base64-byte-decoder.ts @@ -0,0 +1,18 @@ +const BASE64_DECODE_CHUNK_CHARS = 32 * 1024 + +export function decodeBase64Bytes(base64: string): Uint8Array { + const normalized = /\s/.test(base64) ? base64.replace(/\s/g, '') : base64 + const padding = normalized.endsWith('==') ? 2 : normalized.endsWith('=') ? 1 : 0 + const bytes = new Uint8Array(Math.floor((normalized.length * 3) / 4) - padding) + let targetOffset = 0 + for (let sourceOffset = 0; sourceOffset < normalized.length; ) { + const sourceEnd = Math.min(sourceOffset + BASE64_DECODE_CHUNK_CHARS, normalized.length) + const binary = atob(normalized.slice(sourceOffset, sourceEnd)) + for (let index = 0; index < binary.length; index += 1) { + bytes[targetOffset] = binary.charCodeAt(index) + targetOffset += 1 + } + sourceOffset = sourceEnd + } + return bytes +} diff --git a/src/renderer/src/components/editor/combined-diff-load-scheduler.test.ts b/src/renderer/src/components/editor/combined-diff-load-scheduler.test.ts index a982d6b692c..302a76cbdb6 100644 --- a/src/renderer/src/components/editor/combined-diff-load-scheduler.test.ts +++ b/src/renderer/src/components/editor/combined-diff-load-scheduler.test.ts @@ -183,6 +183,70 @@ describe('combined diff load scheduler', () => { expect(started).toEqual([1, 3]) }) + it('counts stale in-flight loads against concurrency across repeated resets', async () => { + const blockers = new Map([1, 2, 3, 5].map((index) => [index, deferred()])) + const started: number[] = [] + let active = 0 + let maxActive = 0 + const scheduler = createCombinedDiffLoadScheduler({ + maxConcurrent: 2, + schedule: (callback) => callback(), + loadSection: async (index) => { + started.push(index) + active += 1 + maxActive = Math.max(maxActive, active) + await blockers.get(index)!.promise + active -= 1 + } + }) + + scheduler.request(1) + scheduler.request(2) + scheduler.reset() + scheduler.request(3) + scheduler.request(4) + expect(started).toEqual([1, 2]) + + blockers.get(1)!.resolve() + await flushMicrotasks() + expect(started).toEqual([1, 2, 3]) + + scheduler.reset() + scheduler.request(5) + blockers.get(2)!.resolve() + await flushMicrotasks() + expect(started).toEqual([1, 2, 3, 5]) + expect(maxActive).toBe(2) + + blockers.get(3)!.resolve() + blockers.get(5)!.resolve() + await flushMicrotasks() + }) + + it('keeps a rerequest deduped while the previous load settles', async () => { + const first = deferred() + const second = deferred() + const started: number[] = [] + const scheduler = createCombinedDiffLoadScheduler({ + schedule: (callback) => callback(), + loadSection: async (index) => { + started.push(index) + await (started.length === 1 ? first.promise : second.promise) + } + }) + + scheduler.request(4) + scheduler.rerequest(4) + scheduler.request(4) + first.resolve() + await flushMicrotasks() + scheduler.request(4) + + expect(started).toEqual([4, 4]) + second.resolve() + await flushMicrotasks() + }) + it('revives after dispose when reset for a StrictMode remount', async () => { const started: number[] = [] const scheduler = createCombinedDiffLoadScheduler({ diff --git a/src/renderer/src/components/editor/combined-diff-load-scheduler.ts b/src/renderer/src/components/editor/combined-diff-load-scheduler.ts index 61c480dd716..45a344b7c6b 100644 --- a/src/renderer/src/components/editor/combined-diff-load-scheduler.ts +++ b/src/renderer/src/components/editor/combined-diff-load-scheduler.ts @@ -5,6 +5,11 @@ export type CombinedDiffLoadScheduler = { dispose: () => void } +type PendingLoad = { + index: number + requestId: number +} + export function createCombinedDiffLoadScheduler({ loadSection, schedule = (callback) => queueMicrotask(callback), @@ -16,11 +21,12 @@ export function createCombinedDiffLoadScheduler({ schedule?: (callback: () => void) => void maxConcurrent?: number }): CombinedDiffLoadScheduler { - const pending: number[] = [] - const queued = new Set() + const pending: PendingLoad[] = [] + const queuedRequestByIndex = new Map() let active = 0 let disposed = false let version = 0 + let nextRequestId = 0 const drain = (drainVersion: number): void => { if (disposed || drainVersion !== version) { @@ -28,29 +34,33 @@ export function createCombinedDiffLoadScheduler({ } while (active < maxConcurrent) { - const nextIndex = pending.shift() - if (nextIndex === undefined) { + const next = pending.shift() + if (!next) { return } active += 1 - void loadSection(nextIndex).finally(() => { - queued.delete(nextIndex) - if (disposed || drainVersion !== version) { + void loadSection(next.index).finally(() => { + active = Math.max(0, active - 1) + if (queuedRequestByIndex.get(next.index) === next.requestId) { + queuedRequestByIndex.delete(next.index) + } + if (disposed) { return } - active = Math.max(0, active - 1) - schedule(() => drain(drainVersion)) + const currentVersion = version + schedule(() => drain(currentVersion)) }) } } const enqueue = (index: number): void => { - if (disposed || queued.has(index)) { + if (disposed || queuedRequestByIndex.has(index)) { return } - queued.add(index) - pending.push(index) + const requestId = ++nextRequestId + queuedRequestByIndex.set(index, requestId) + pending.push({ index, requestId }) const requestVersion = version schedule(() => drain(requestVersion)) } @@ -63,8 +73,9 @@ export function createCombinedDiffLoadScheduler({ if (disposed) { return } - queued.delete(index) - const pendingIndex = pending.indexOf(index) + const requestId = queuedRequestByIndex.get(index) + queuedRequestByIndex.delete(index) + const pendingIndex = pending.findIndex((load) => load.requestId === requestId) if (pendingIndex !== -1) { pending.splice(pendingIndex, 1) } @@ -73,14 +84,14 @@ export function createCombinedDiffLoadScheduler({ reset() { disposed = false version += 1 + // Why: don't reset `active`; stale loads still consume memory and I/O until they settle. pending.length = 0 - queued.clear() - active = 0 + queuedRequestByIndex.clear() }, dispose() { disposed = true pending.length = 0 - queued.clear() + queuedRequestByIndex.clear() } } } diff --git a/src/renderer/src/components/editor/combined-diff-text-retention.test.ts b/src/renderer/src/components/editor/combined-diff-text-retention.test.ts new file mode 100644 index 00000000000..99243dc238f --- /dev/null +++ b/src/renderer/src/components/editor/combined-diff-text-retention.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest' +import { MAX_RENDERED_DIFF_COMBINED_CHARACTERS } from '../../../../shared/large-diff-render-limit' +import type { DiffSection } from './diff-section-types' +import { + COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES, + MAX_RETAINED_COMBINED_DIFF_TEXT_BYTES, + MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS, + getCombinedDiffViewedSectionKeys, + inspectCombinedDiffTextRetention, + retainCombinedDiffSectionText, + retainCombinedDiffViewStateText +} from './combined-diff-text-retention' + +function textSection(key: string, content: string): DiffSection { + return { + key, + path: `${key}.ts`, + status: 'modified', + originalContent: content, + modifiedContent: '', + collapsed: false, + loading: false, + dirty: false, + diffResult: { + kind: 'text', + originalContent: content, + modifiedContent: '', + originalIsBinary: false, + modifiedIsBinary: false + }, + largeDiffRenderLimit: null + } +} + +describe('combined diff text retention', () => { + it('bounds many individually renderable files by evicting the oldest offscreen bodies', () => { + const justUnderPerFileLimit = 'x'.repeat(MAX_RENDERED_DIFF_COMBINED_CHARACTERS - 1) + const sections = Array.from({ length: 10 }, (_, index) => + textSection(`section-${index}`, justUnderPerFileLimit) + ) + const loadedIndices = sections.map((_, index) => index) + + const result = retainCombinedDiffSectionText({ + sections, + loadedIndices, + protectedSectionKeys: new Set(['section-9']) + }) + + const inspection = inspectCombinedDiffTextRetention(result.sections) + expect(inspection.characters).toBeLessThanOrEqual(MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS) + expect(inspection.approximateBytes).toBeLessThanOrEqual(MAX_RETAINED_COMBINED_DIFF_TEXT_BYTES) + expect(result.evictedIndices[0]).toBe(0) + expect(result.sections[9].originalContent).toBe(justUnderPerFileLimit) + expect(result.loadedIndices).not.toContain(result.evictedIndices[0]) + for (const index of result.evictedIndices) { + expect(result.sections[index]).toMatchObject({ + originalContent: '', + modifiedContent: '', + loading: true, + diffResult: null + }) + } + }) + + it('makes an evicted section reloadable and retains its refetched body when revisited', () => { + const firstPass = retainCombinedDiffSectionText({ + sections: [textSection('old', '12345678'), textSection('active', 'abcdefgh')], + loadedIndices: [0, 1], + protectedSectionKeys: new Set(['active']), + maxCharacters: 10 + }) + + expect(firstPass.evictedIndices).toEqual([0]) + expect(firstPass.loadedIndices).toEqual([1]) + expect(getCombinedDiffViewedSectionKeys(firstPass.sections)).toEqual(new Set(['old', 'active'])) + + const refetched = firstPass.sections.map((section, index) => + index === 0 ? textSection('old', 'refetched') : section + ) + const secondPass = retainCombinedDiffSectionText({ + sections: refetched, + loadedIndices: [...firstPass.loadedIndices, 0], + protectedSectionKeys: new Set(['old']), + maxCharacters: 10 + }) + + expect(secondPass.sections[0].originalContent).toBe('refetched') + expect(secondPass.loadedIndices).toContain(0) + expect(secondPass.evictedIndices).toEqual([1]) + }) + + it('keeps the full view-state LRU within one aggregate text budget', () => { + const body = 'x'.repeat(Math.floor(MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS / 2)) + const viewStates = new Map( + Array.from({ length: COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES }, (_, index) => [ + `view-${index}`, + { sections: [textSection(`section-${index}`, body)], loadedIndices: [0] } + ]) + ) + + const inspection = retainCombinedDiffViewStateText(viewStates) + + expect(inspection.characters).toBeLessThanOrEqual(MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS) + expect(viewStates.get('view-0')?.sections[0].originalContent).toBe('') + expect(viewStates.get('view-19')?.sections[0].originalContent).toBe(body) + }) + + it('never evicts active, loading, or unsaved section text', () => { + const dirty = { ...textSection('dirty', 'dirty text'), dirty: true } + const loading = { ...textSection('loading', 'loading text'), loading: true } + const result = retainCombinedDiffSectionText({ + sections: [ + textSection('old', 'old text'), + dirty, + loading, + textSection('active', 'active text') + ], + loadedIndices: [0, 1, 2, 3], + protectedSectionKeys: new Set(['active']), + maxCharacters: 0 + }) + + expect(result.evictedIndices).toEqual([0]) + expect(result.sections[1].originalContent).toBe('dirty text') + expect(result.sections[2].originalContent).toBe('loading text') + expect(result.sections[3].originalContent).toBe('active text') + }) +}) diff --git a/src/renderer/src/components/editor/combined-diff-text-retention.ts b/src/renderer/src/components/editor/combined-diff-text-retention.ts new file mode 100644 index 00000000000..137eec85c2c --- /dev/null +++ b/src/renderer/src/components/editor/combined-diff-text-retention.ts @@ -0,0 +1,205 @@ +import type { DiffSection } from './diff-section-types' + +export const COMBINED_DIFF_TEXT_BYTES_PER_CHARACTER = 2 +// Why: bound raw UTF-16 text before Monaco creates its own models; inactive +// snapshots split the same budget and may refetch an oversized file on revisit. +export const MAX_RETAINED_COMBINED_DIFF_TEXT_BYTES = 24 * 1024 * 1024 +export const MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS = Math.floor( + MAX_RETAINED_COMBINED_DIFF_TEXT_BYTES / COMBINED_DIFF_TEXT_BYTES_PER_CHARACTER +) +export const COMBINED_DIFF_VIEW_STATE_CACHE_MAX_ENTRIES = 20 + +export type CombinedDiffTextRetentionInspection = { + characters: number + approximateBytes: number + sectionsWithText: number +} + +export type CombinedDiffTextRetentionResult = CombinedDiffTextRetentionInspection & { + sections: DiffSection[] + loadedIndices: number[] + evictedIndices: number[] +} + +type CombinedDiffTextViewState = { + sections: DiffSection[] + loadedIndices: number[] +} + +function getSectionTextCharacters(section: DiffSection): number { + if (section.diffResult?.kind !== 'text') { + return 0 + } + return section.originalContent.length + section.modifiedContent.length +} + +export function inspectCombinedDiffTextRetention( + sections: readonly DiffSection[] +): CombinedDiffTextRetentionInspection { + let characters = 0 + let sectionsWithText = 0 + for (const section of sections) { + const sectionCharacters = getSectionTextCharacters(section) + if (sectionCharacters === 0) { + continue + } + characters += sectionCharacters + sectionsWithText += 1 + } + return { + characters, + approximateBytes: characters * COMBINED_DIFF_TEXT_BYTES_PER_CHARACTER, + sectionsWithText + } +} + +function isReloadableSection( + section: DiffSection, + protectedSectionKeys: ReadonlySet +): boolean { + return ( + getSectionTextCharacters(section) > 0 && + !section.loading && + !section.dirty && + !section.error && + !protectedSectionKeys.has(section.key) + ) +} + +function releaseSectionText(section: DiffSection): DiffSection { + return { + ...section, + originalContent: '', + modifiedContent: '', + diffResult: null, + loading: true, + contentGeneration: (section.contentGeneration ?? 0) + 1, + textEvictedForRetention: true + } +} + +export function getCombinedDiffViewedSectionKeys( + sections: readonly DiffSection[] +): ReadonlySet { + return new Set( + sections + .filter((section) => !section.loading || section.textEvictedForRetention === true) + .map((section) => section.key) + ) +} + +export function retainCombinedDiffSectionText({ + sections, + loadedIndices, + protectedSectionKeys = new Set(), + maxCharacters = MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS +}: { + sections: DiffSection[] + loadedIndices: Iterable + protectedSectionKeys?: ReadonlySet + maxCharacters?: number +}): CombinedDiffTextRetentionResult { + const inspection = inspectCombinedDiffTextRetention(sections) + const loadedIndexOrder = Array.from(loadedIndices) + if (inspection.characters <= maxCharacters) { + return { + ...inspection, + sections, + loadedIndices: loadedIndexOrder, + evictedIndices: [] + } + } + + // Why: loaded indices preserve load order; a refetched section rejoins at the tail. + const orderedCandidates = loadedIndexOrder.filter((index) => { + const section = sections[index] + return section !== undefined && isReloadableSection(section, protectedSectionKeys) + }) + const orderedCandidateSet = new Set(orderedCandidates) + for (let index = 0; index < sections.length; index += 1) { + const section = sections[index] + if ( + !orderedCandidateSet.has(index) && + section !== undefined && + isReloadableSection(section, protectedSectionKeys) + ) { + orderedCandidates.push(index) + } + } + + let retainedCharacters = inspection.characters + let retainedSections = sections + const evictedIndices: number[] = [] + for (const index of orderedCandidates) { + if (retainedCharacters <= maxCharacters) { + break + } + const section = retainedSections[index] + const sectionCharacters = getSectionTextCharacters(section) + if (sectionCharacters === 0) { + continue + } + if (retainedSections === sections) { + retainedSections = [...sections] + } + retainedSections[index] = releaseSectionText(section) + retainedCharacters -= sectionCharacters + evictedIndices.push(index) + } + + const evictedIndexSet = new Set(evictedIndices) + return { + characters: retainedCharacters, + approximateBytes: retainedCharacters * COMBINED_DIFF_TEXT_BYTES_PER_CHARACTER, + sectionsWithText: inspection.sectionsWithText - evictedIndices.length, + sections: retainedSections, + loadedIndices: loadedIndexOrder.filter((index) => !evictedIndexSet.has(index)), + evictedIndices + } +} + +export function inspectCombinedDiffViewStateTextRetention( + viewStates: Iterable +): CombinedDiffTextRetentionInspection { + let characters = 0 + let sectionsWithText = 0 + for (const viewState of viewStates) { + const inspection = inspectCombinedDiffTextRetention(viewState.sections) + characters += inspection.characters + sectionsWithText += inspection.sectionsWithText + } + return { + characters, + approximateBytes: characters * COMBINED_DIFF_TEXT_BYTES_PER_CHARACTER, + sectionsWithText + } +} + +export function retainCombinedDiffViewStateText( + viewStates: Map, + maxCharacters = MAX_RETAINED_COMBINED_DIFF_TEXT_CHARACTERS +): CombinedDiffTextRetentionInspection { + let retainedCharacters = inspectCombinedDiffViewStateTextRetention(viewStates.values()).characters + // Why: view-state Maps are LRUs, so iteration releases the oldest inactive view first. + for (const [key, viewState] of viewStates) { + if (retainedCharacters <= maxCharacters) { + break + } + const viewStateCharacters = inspectCombinedDiffTextRetention(viewState.sections).characters + const retained = retainCombinedDiffSectionText({ + sections: viewState.sections, + loadedIndices: viewState.loadedIndices, + maxCharacters: Math.max(0, viewStateCharacters - (retainedCharacters - maxCharacters)) + }) + if (retained.evictedIndices.length === 0) { + continue + } + viewStates.set(key, { + ...viewState, + sections: retained.sections, + loadedIndices: retained.loadedIndices + }) + retainedCharacters -= viewStateCharacters - retained.characters + } + return inspectCombinedDiffViewStateTextRetention(viewStates.values()) +} diff --git a/src/renderer/src/components/editor/csv-parse.test.ts b/src/renderer/src/components/editor/csv-parse.test.ts index 267be8b1028..34581559c61 100644 --- a/src/renderer/src/components/editor/csv-parse.test.ts +++ b/src/renderer/src/components/editor/csv-parse.test.ts @@ -1,5 +1,15 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { CSV_DELIMITER_SNIFF_SCAN_CODE_UNITS, detectCsvDelimiter, parseCsv } from './csv-parse' +import { + CSV_DELIMITER_SNIFF_SCAN_CODE_UNITS, + CSV_PARSE_LIMITS, + type CsvParseLimits, + detectCsvDelimiter, + parseCsv +} from './csv-parse' + +function parseLimits(overrides: Partial): CsvParseLimits { + return { ...CSV_PARSE_LIMITS, ...overrides } +} afterEach(() => { vi.restoreAllMocks() @@ -68,8 +78,82 @@ describe('parseCsv', () => { const { rows } = parseCsv('""') expect(rows).toEqual([['']]) }) + + it.each([ + { + name: 'source code units', + source: 'a,b\n', + limits: parseLimits({ sourceCodeUnits: 3 }), + reason: 'source-code-units' + }, + { + name: 'rows', + source: 'a\nb\nc', + limits: parseLimits({ rows: 2 }), + reason: 'rows' + }, + { + name: 'columns per row', + source: 'a,b,c', + limits: parseLimits({ columnsPerRow: 2 }), + reason: 'columns-per-row' + }, + { + name: 'total cells', + source: 'a,b\nc,d,e', + limits: parseLimits({ cells: 4 }), + reason: 'cells' + }, + { + name: 'retained cell text', + source: 'a,bcd', + limits: parseLimits({ retainedCellCodeUnits: 3 }), + reason: 'retained-cell-code-units' + } + ])('fails closed when $name exceeds its memory limit', ({ source, limits, reason }) => { + expect(parseCsv(source, ',', limits)).toEqual({ + rows: [], + maxColumns: 0, + limitExceeded: { reason, limit: limits[reasonToLimitKey(reason)] } + }) + }) + + it('preserves exact output at every parser boundary', () => { + const source = 'a,b\nc,d' + const limits = parseLimits({ + sourceCodeUnits: source.length, + rows: 2, + columnsPerRow: 2, + cells: 4, + retainedCellCodeUnits: 4 + }) + + expect(parseCsv(source, ',', limits)).toEqual({ + rows: [ + ['a', 'b'], + ['c', 'd'] + ], + maxColumns: 2, + limitExceeded: null + }) + }) }) +function reasonToLimitKey(reason: string): keyof CsvParseLimits { + switch (reason) { + case 'source-code-units': + return 'sourceCodeUnits' + case 'rows': + return 'rows' + case 'columns-per-row': + return 'columnsPerRow' + case 'cells': + return 'cells' + default: + return 'retainedCellCodeUnits' + } +} + describe('detectCsvDelimiter', () => { it('uses tab for .tsv files regardless of content', () => { expect(detectCsvDelimiter('data.tsv', 'a,b,c')).toBe('\t') diff --git a/src/renderer/src/components/editor/csv-parse.ts b/src/renderer/src/components/editor/csv-parse.ts index 009e9646320..095b76779f7 100644 --- a/src/renderer/src/components/editor/csv-parse.ts +++ b/src/renderer/src/components/editor/csv-parse.ts @@ -1,9 +1,37 @@ export type CsvParseResult = { rows: string[][] maxColumns: number + limitExceeded: CsvParseLimitExceeded | null } export const CSV_DELIMITER_SNIFF_SCAN_CODE_UNITS = 64 * 1024 +export const CSV_PARSE_LIMITS = { + sourceCodeUnits: 50 * 1024 * 1024, + rows: 250_000, + columnsPerRow: 1024, + cells: 1_000_000, + retainedCellCodeUnits: 50 * 1024 * 1024 +} as const + +export type CsvParseLimitReason = + | 'source-code-units' + | 'rows' + | 'columns-per-row' + | 'cells' + | 'retained-cell-code-units' + +export type CsvParseLimitExceeded = { + reason: CsvParseLimitReason + limit: number +} + +export type CsvParseLimits = Readonly<{ + sourceCodeUnits: number + rows: number + columnsPerRow: number + cells: number + retainedCellCodeUnits: number +}> const LINE_FEED_CODE_UNIT = 10 const CARRIAGE_RETURN_CODE_UNIT = 13 @@ -12,7 +40,21 @@ const CARRIAGE_RETURN_CODE_UNIT = 13 // A hand-rolled parser avoids pulling a new dependency (papaparse) for what is // a small, well-specified grammar. Inline state machine keeps the hot path // allocation-light for large files. -export function parseCsv(source: string, delimiter: string = ','): CsvParseResult { +export function parseCsv( + source: string, + delimiter: string = ',', + limits: CsvParseLimits = CSV_PARSE_LIMITS +): CsvParseResult { + const limited = (reason: CsvParseLimitReason, limit: number): CsvParseResult => ({ + rows: [], + maxColumns: 0, + limitExceeded: { reason, limit } + }) + + if (source.length > limits.sourceCodeUnits) { + return limited('source-code-units', limits.sourceCodeUnits) + } + // Why: strip a leading UTF-8 BOM (U+FEFF). Excel and other spreadsheet tools // prepend a BOM to exported CSVs; without this, the BOM contaminates the // first header cell and breaks column-name lookups for downstream consumers. @@ -25,6 +67,8 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul let field = '' let inQuotes = false let maxColumns = 0 + let cellCount = 0 + let retainedCellCodeUnits = 0 // Why: track whether the current record has produced any content — including // a quoted-but-empty field like `""`. Without this flag, the EOF flush // condition `field.length > 0 || row.length > 0` would drop a record whose @@ -32,18 +76,41 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul // delimiter/newline ever pushed it onto the row. let recordHasContent = false - const pushField = (): void => { + const pushField = (): CsvParseResult | null => { + if (row.length >= limits.columnsPerRow) { + return limited('columns-per-row', limits.columnsPerRow) + } + if (cellCount >= limits.cells) { + return limited('cells', limits.cells) + } row.push(field) + cellCount += 1 field = '' + return null } - const pushRow = (): void => { - pushField() + const pushRow = (): CsvParseResult | null => { + if (rows.length >= limits.rows) { + return limited('rows', limits.rows) + } + const fieldLimit = pushField() + if (fieldLimit) { + return fieldLimit + } if (row.length > maxColumns) { maxColumns = row.length } rows.push(row) row = [] recordHasContent = false + return null + } + const appendToField = (value: string): CsvParseResult | null => { + if (retainedCellCodeUnits + value.length > limits.retainedCellCodeUnits) { + return limited('retained-cell-code-units', limits.retainedCellCodeUnits) + } + field += value + retainedCellCodeUnits += value.length + return null } for (let i = 0; i < source.length; i += 1) { @@ -52,13 +119,19 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul if (inQuotes) { if (ch === '"') { if (source[i + 1] === '"') { - field += '"' + const retainedLimit = appendToField('"') + if (retainedLimit) { + return retainedLimit + } i += 1 } else { inQuotes = false } } else { - field += ch + const retainedLimit = appendToField(ch) + if (retainedLimit) { + return retainedLimit + } } continue } @@ -71,7 +144,10 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul continue } if (ch === delimiter) { - pushField() + const fieldLimit = pushField() + if (fieldLimit) { + return fieldLimit + } recordHasContent = true continue } @@ -79,14 +155,23 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul if (source[i + 1] === '\n') { i += 1 } - pushRow() + const rowLimit = pushRow() + if (rowLimit) { + return rowLimit + } continue } if (ch === '\n') { - pushRow() + const rowLimit = pushRow() + if (rowLimit) { + return rowLimit + } continue } - field += ch + const retainedLimit = appendToField(ch) + if (retainedLimit) { + return retainedLimit + } recordHasContent = true } @@ -97,10 +182,13 @@ export function parseCsv(source: string, delimiter: string = ','): CsvParseResul // empty and nothing has been pushed to `row`, but the record is real and // must not be dropped. if (field.length > 0 || row.length > 0 || recordHasContent) { - pushRow() + const rowLimit = pushRow() + if (rowLimit) { + return rowLimit + } } - return { rows, maxColumns } + return { rows, maxColumns, limitExceeded: null } } export function detectCsvDelimiter(filePath: string, content: string): string { diff --git a/src/renderer/src/components/editor/diff-section-types.ts b/src/renderer/src/components/editor/diff-section-types.ts index 0c5d659f4be..0d23aafe0a5 100644 --- a/src/renderer/src/components/editor/diff-section-types.ts +++ b/src/renderer/src/components/editor/diff-section-types.ts @@ -20,4 +20,6 @@ export type DiffSection = { // Why: combined sections keep Monaco models by path; bump on reload so // refetched git content does not replay through keepCurrent* model reuse. contentGeneration?: number + // Why: releasing a reviewed body must not make its file-tree entry appear unviewed. + textEvictedForRetention?: boolean } diff --git a/src/renderer/src/components/editor/editor-autosave-controller.test.ts b/src/renderer/src/components/editor/editor-autosave-controller.test.ts index 734170badf1..131270c7e1c 100644 --- a/src/renderer/src/components/editor/editor-autosave-controller.test.ts +++ b/src/renderer/src/components/editor/editor-autosave-controller.test.ts @@ -7,7 +7,10 @@ import { ORCA_EDITOR_SAVE_DIRTY_FILES_EVENT } from '../../../../shared/editor-save-events' import { requestEditorFileSave, requestEditorSaveQuiesce } from './editor-autosave' -import { attachEditorAutosaveController } from './editor-autosave-controller' +import { + attachEditorAutosaveController, + EDITOR_BULK_SAVE_CONCURRENCY +} from './editor-autosave-controller' import { registerPendingEditorFlush } from './editor-pending-flush' import { __clearSelfWriteRegistryForTests, hasRecentSelfWrite } from './editor-self-write-registry' import { @@ -198,6 +201,70 @@ describe('attachEditorAutosaveController', () => { } }) + it.each([ + ['at the limit', EDITOR_BULK_SAVE_CONCURRENCY], + ['above the limit', EDITOR_BULK_SAVE_CONCURRENCY + 1] + ])('bounds dirty-file writes %s', async (_, count) => { + let active = 0 + let peak = 0 + const releases: (() => void)[] = [] + const writeFile = vi.fn(async () => { + active++ + peak = Math.max(peak, active) + await new Promise((resolve) => releases.push(resolve)) + active-- + }) + const eventTarget = new EventTarget() + vi.stubGlobal('window', { + addEventListener: eventTarget.addEventListener.bind(eventTarget), + removeEventListener: eventTarget.removeEventListener.bind(eventTarget), + dispatchEvent: eventTarget.dispatchEvent.bind(eventTarget), + setTimeout: globalThis.setTimeout.bind(globalThis), + clearTimeout: globalThis.clearTimeout.bind(globalThis), + api: { fs: { writeFile } } + } satisfies WindowStub) + const store = createEditorStore() + store.setState({ + settings: { editorAutoSave: false, editorAutoSaveDelayMs: 1000 } as never + }) + for (let index = 0; index < count; index++) { + const filePath = `/repo/file-${index}.ts` + store.getState().openFile({ + filePath, + relativePath: `file-${index}.ts`, + worktreeId: 'wt-1', + language: 'typescript', + mode: 'edit' + }) + store.getState().setEditorDraft(filePath, `draft ${index}`) + store.getState().markFileDirty(filePath, true) + } + + const cleanup = attachEditorAutosaveController(store) + try { + const save = requestDirtyFileSave() + for ( + let turn = 0; + turn < 10 && writeFile.mock.calls.length < Math.min(count, EDITOR_BULK_SAVE_CONCURRENCY); + turn++ + ) { + await Promise.resolve() + } + expect(writeFile).toHaveBeenCalledTimes(Math.min(count, EDITOR_BULK_SAVE_CONCURRENCY)) + if (count > EDITOR_BULK_SAVE_CONCURRENCY) { + releases.shift()?.() + await vi.waitFor(() => expect(writeFile).toHaveBeenCalledTimes(count)) + } + releases.splice(0).forEach((release) => release()) + await save + + expect(peak).toBe(Math.min(count, EDITOR_BULK_SAVE_CONCURRENCY)) + expect(store.getState().openFiles.filter((file) => file.isDirty)).toEqual([]) + } finally { + cleanup() + } + }) + it('saves folder workspace files through the path-specific SSH connection', async () => { const writeFile = vi.fn().mockResolvedValue(undefined) const eventTarget = new EventTarget() diff --git a/src/renderer/src/components/editor/editor-autosave-controller.ts b/src/renderer/src/components/editor/editor-autosave-controller.ts index 3326d93492d..ccf3e6261f5 100644 --- a/src/renderer/src/components/editor/editor-autosave-controller.ts +++ b/src/renderer/src/components/editor/editor-autosave-controller.ts @@ -43,8 +43,22 @@ import { type EditorPrepareHotExitDetail, type EditorSaveDirtyFilesDetail } from '../../../../shared/editor-save-events' +import { + mapSettledWithConcurrency, + mapWithConcurrency +} from '../../../../shared/map-with-concurrency' type AppStoreApi = Pick, 'getState' | 'subscribe'> +export const EDITOR_BULK_SAVE_CONCURRENCY = 4 + +function throwFirstRejected(results: readonly PromiseSettledResult[]): void { + const failure = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failure) { + throw failure.reason + } +} export function attachEditorAutosaveController(store: AppStoreApi): () => void { const autoSaveTimers = new Map() @@ -253,15 +267,18 @@ export function attachEditorAutosaveController(store: AppStoreApi): () => void { return } - await Promise.all( - dirtyFiles.map(async (file) => { + const saves = await mapSettledWithConcurrency( + dirtyFiles, + EDITOR_BULK_SAVE_CONCURRENCY, + async (file) => { const content = getLatestWritableContent(file) if (content === null) { throw new Error(`Missing editor buffer for ${file.relativePath}`) } await queueSave(file, content) - }) + } ) + throwFirstRejected(saves) detail.resolve() } catch (error) { detail.reject(String((error as Error)?.message ?? error)) @@ -278,7 +295,9 @@ export function attachEditorAutosaveController(store: AppStoreApi): () => void { detail.claim() const initiallyDirtyFiles = store.getState().openFiles.filter((file) => file.isDirty) - await Promise.all(initiallyDirtyFiles.map((file) => quiesceFileSave(file.id))) + await mapWithConcurrency(initiallyDirtyFiles, EDITOR_BULK_SAVE_CONCURRENCY, (file) => + quiesceFileSave(file.id) + ) const state = store.getState() const dirtyFiles = state.openFiles.filter((file) => file.isDirty) @@ -369,7 +388,9 @@ export function attachEditorAutosaveController(store: AppStoreApi): () => void { ? store.getState().openFiles.filter((file) => file.id === detail.fileId) : getOpenFilesForExternalFileChange(store.getState().openFiles, detail) - await Promise.all(matchingFiles.map((file) => quiesceFileSave(file.id))) + await mapWithConcurrency(matchingFiles, EDITOR_BULK_SAVE_CONCURRENCY, (file) => + quiesceFileSave(file.id) + ) detail.resolve() } diff --git a/src/renderer/src/components/editor/editor-pending-flush.test.ts b/src/renderer/src/components/editor/editor-pending-flush.test.ts new file mode 100644 index 00000000000..b50b1996501 --- /dev/null +++ b/src/renderer/src/components/editor/editor-pending-flush.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest' +import { flushPendingEditorChange, registerPendingEditorFlush } from './editor-pending-flush' + +describe('editor pending flush registry', () => { + it('releases a registered callback through its lifecycle disposer', () => { + const flush = vi.fn() + const unregister = registerPendingEditorFlush('/repo/file.md', flush) + + flushPendingEditorChange('/repo/file.md') + unregister() + flushPendingEditorChange('/repo/file.md') + + expect(flush).toHaveBeenCalledTimes(1) + }) + + it('does not let an older disposer remove a replacement callback', () => { + const olderFlush = vi.fn() + const newerFlush = vi.fn() + const unregisterOlder = registerPendingEditorFlush('/repo/file.md', olderFlush) + const unregisterNewer = registerPendingEditorFlush('/repo/file.md', newerFlush) + + unregisterOlder() + flushPendingEditorChange('/repo/file.md') + unregisterNewer() + flushPendingEditorChange('/repo/file.md') + + expect(olderFlush).not.toHaveBeenCalled() + expect(newerFlush).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/renderer/src/components/editor/ipynb-json-admission.test.ts b/src/renderer/src/components/editor/ipynb-json-admission.test.ts new file mode 100644 index 00000000000..354cf2002f2 --- /dev/null +++ b/src/renderer/src/components/editor/ipynb-json-admission.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { + assertIpynbJsonWithinMemoryLimits, + assertIpynbShapeWithinMemoryLimits, + IPYNB_MEMORY_LIMITS, + type IpynbMemoryLimits +} from './ipynb-json-admission' + +function limits(overrides: Partial): IpynbMemoryLimits { + return { ...IPYNB_MEMORY_LIMITS, ...overrides } +} + +describe('notebook memory admission', () => { + it('preserves JSON at exact source, structure, and nesting boundaries', () => { + const content = '{"cells":[]}' + expect(() => + assertIpynbJsonWithinMemoryLimits( + content, + limits({ sourceCodeUnits: content.length, structuralTokens: 5, nestingDepth: 2 }) + ) + ).not.toThrow() + }) + + it.each([ + ['source size', '{"cells":[]}', { sourceCodeUnits: 11 }, 'source size'], + ['JSON structure', '{"cells":[]}', { structuralTokens: 4 }, 'JSON structure'], + ['JSON nesting', '{"cells":[]}', { nestingDepth: 1 }, 'JSON nesting'] + ] as const)('rejects %s one step past its limit', (_name, content, override, message) => { + expect(() => assertIpynbJsonWithinMemoryLimits(content, limits(override))).toThrow(message) + }) + + it('ignores structural characters inside strings', () => { + const content = '{"cells":[{"source":"[{,:}]"}]}' + expect(() => + assertIpynbJsonWithinMemoryLimits(content, limits({ structuralTokens: 9 })) + ).not.toThrow() + }) + + it.each([ + ['cells', [{}, {}], { cells: 1 }, 'cell'], + ['outputs', [{ outputs: [{}, {}] }], { outputs: 1 }, 'output'], + [ + 'display items', + [{ outputs: [{ data: { one: 1, two: 2 } }] }], + { displayItems: 1 }, + 'display item' + ], + ['multiline parts', [{ source: ['one', 'two'] }], { multilineParts: 1 }, 'fragment'] + ] as const)('rejects too many %s before render projection', (_name, cells, override, message) => { + expect(() => assertIpynbShapeWithinMemoryLimits(cells, limits(override))).toThrow(message) + }) +}) diff --git a/src/renderer/src/components/editor/ipynb-json-admission.ts b/src/renderer/src/components/editor/ipynb-json-admission.ts new file mode 100644 index 00000000000..606b7162179 --- /dev/null +++ b/src/renderer/src/components/editor/ipynb-json-admission.ts @@ -0,0 +1,94 @@ +import { assertJsonTextStructureWithinLimits } from '../../../../shared/json-text-structure-limit' + +export const IPYNB_MEMORY_LIMITS = { + sourceCodeUnits: 50 * 1024 * 1024, + structuralTokens: 500_000, + nestingDepth: 256, + cells: 1000, + outputs: 10_000, + displayItems: 20_000, + multilineParts: 100_000 +} as const + +export type IpynbMemoryLimits = Readonly<{ + sourceCodeUnits: number + structuralTokens: number + nestingDepth: number + cells: number + outputs: number + displayItems: number + multilineParts: number +}> + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function assertIpynbJsonWithinMemoryLimits( + content: string, + limits: IpynbMemoryLimits = IPYNB_MEMORY_LIMITS +): void { + if (content.length > limits.sourceCodeUnits) { + throw new Error('Notebook exceeds the safe source size limit') + } + + assertJsonTextStructureWithinLimits(content, { + structuralTokens: limits.structuralTokens, + nestingDepth: limits.nestingDepth + }) +} + +export function assertIpynbShapeWithinMemoryLimits( + cells: readonly unknown[], + limits: IpynbMemoryLimits = IPYNB_MEMORY_LIMITS +): void { + if (cells.length > limits.cells) { + throw new Error('Notebook exceeds the safe cell limit') + } + + let outputCount = 0 + let displayItemCount = 0 + let multilinePartCount = 0 + const claimMultilineParts = (value: unknown): void => { + if (!Array.isArray(value)) { + return + } + multilinePartCount += value.length + if (multilinePartCount > limits.multilineParts) { + throw new Error('Notebook exceeds the safe multiline fragment limit') + } + } + + for (const cell of cells) { + if (!isRecord(cell)) { + continue + } + claimMultilineParts(cell.source) + if (!Array.isArray(cell.outputs)) { + continue + } + outputCount += cell.outputs.length + if (outputCount > limits.outputs) { + throw new Error('Notebook exceeds the safe output limit') + } + for (const output of cell.outputs) { + if (!isRecord(output)) { + continue + } + claimMultilineParts(output.text) + claimMultilineParts(output.traceback) + if (!isRecord(output.data)) { + continue + } + for (const key in output.data) { + if (!Object.prototype.hasOwnProperty.call(output.data, key)) { + continue + } + displayItemCount += 1 + if (displayItemCount > limits.displayItems) { + throw new Error('Notebook exceeds the safe display item limit') + } + } + } + } +} diff --git a/src/renderer/src/components/editor/ipynb-parse.test.ts b/src/renderer/src/components/editor/ipynb-parse.test.ts index 8178bed2355..e7835526c2e 100644 --- a/src/renderer/src/components/editor/ipynb-parse.test.ts +++ b/src/renderer/src/components/editor/ipynb-parse.test.ts @@ -11,6 +11,7 @@ import { updateIpynbCellSource, updateIpynbCellSources } from './ipynb-parse' +import { IPYNB_MEMORY_LIMITS } from './ipynb-json-admission' afterEach(() => { vi.restoreAllMocks() @@ -89,6 +90,18 @@ describe('ipynb parsing', () => { expect(() => parseIpynb('{}')).toThrow('Notebook is missing a cells array') }) + it('rejects renderer-amplifying cell counts before projecting cells', () => { + const content = JSON.stringify({ + cells: Array.from({ length: IPYNB_MEMORY_LIMITS.cells + 1 }, () => ({ + cell_type: 'raw', + metadata: {}, + source: [] + })) + }) + + expect(() => parseIpynb(content)).toThrow('safe cell limit') + }) + it('serializes cell source edits while preserving notebook metadata', () => { const content = JSON.stringify({ nbformat: 4, diff --git a/src/renderer/src/components/editor/ipynb-parse.ts b/src/renderer/src/components/editor/ipynb-parse.ts index b10aa5f2df5..a5ca9e80525 100644 --- a/src/renderer/src/components/editor/ipynb-parse.ts +++ b/src/renderer/src/components/editor/ipynb-parse.ts @@ -2,6 +2,11 @@ in one module makes nbformat preservation easier to audit while the notebook editor model is still small. */ import { createBrowserUuid } from '@/lib/browser-uuid' +import { + assertIpynbJsonWithinMemoryLimits, + assertIpynbShapeWithinMemoryLimits, + IPYNB_MEMORY_LIMITS +} from './ipynb-json-admission' export type IpynbCellKind = 'code' | 'markdown' | 'raw' @@ -65,12 +70,12 @@ function isRecord(value: unknown): value is Record { export function concatIpynbMultilineString(value: unknown): string { if (Array.isArray(value)) { - let result = '' + const parts: string[] = [] for (let i = 0; i < value.length; i += 1) { const item = String(value[i] ?? '') - result += i < value.length - 1 && !item.endsWith('\n') ? `${item}\n` : item + parts.push(i < value.length - 1 && !item.endsWith('\n') ? `${item}\n` : item) } - return result.replace(/\r\n/g, '\n') + return parts.join('').replace(/\r\n/g, '\n') } return String(value ?? '').replace(/\r\n/g, '\n') } @@ -183,16 +188,10 @@ function parseCell(rawCell: unknown, fallbackLanguage: string): IpynbCell | null } export function parseIpynb(content: string): ParsedIpynb { - const parsed = JSON.parse(content) as unknown - if (!isRecord(parsed)) { - throw new Error('Notebook root must be a JSON object') - } - if (!Array.isArray(parsed.cells)) { - throw new Error('Notebook is missing a cells array') - } + const parsed = parseNotebookRoot(content) const language = getPreferredLanguage(parsed) - const cells = parsed.cells + const cells = (parsed.cells as unknown[]) .map((cell) => parseCell(cell, language)) .filter((cell): cell is IpynbCell => cell !== null) @@ -208,6 +207,9 @@ export function parseIpynb(content: string): ParsedIpynb { } function splitIpynbSource(source: string): string[] { + if (source.length > IPYNB_MEMORY_LIMITS.sourceCodeUnits) { + throw new Error('Notebook cell source exceeds the safe size limit') + } if (!source) { return [] } @@ -218,15 +220,22 @@ function splitIpynbSource(source: string): string[] { continue } lines.push(source.slice(lineStart, index + 1)) + if (lines.length > IPYNB_MEMORY_LIMITS.multilineParts) { + throw new Error('Notebook cell source exceeds the safe line limit') + } lineStart = index + 1 } if (lineStart < source.length) { lines.push(source.slice(lineStart)) + if (lines.length > IPYNB_MEMORY_LIMITS.multilineParts) { + throw new Error('Notebook cell source exceeds the safe line limit') + } } return lines } function parseNotebookRoot(content: string): Record { + assertIpynbJsonWithinMemoryLimits(content) const parsed = JSON.parse(content) as unknown if (!isRecord(parsed)) { throw new Error('Notebook root must be a JSON object') @@ -234,6 +243,7 @@ function parseNotebookRoot(content: string): Record { if (!Array.isArray(parsed.cells)) { throw new Error('Notebook is missing a cells array') } + assertIpynbShapeWithinMemoryLimits(parsed.cells) return parsed } @@ -246,7 +256,9 @@ function ensureCell(root: Record, index: number): Record): string { - return `${JSON.stringify(root, null, 1)}\n` + const serialized = `${JSON.stringify(root, null, 1)}\n` + assertIpynbJsonWithinMemoryLimits(serialized) + return serialized } export function updateIpynbCellSource(content: string, index: number, source: string): string { @@ -300,6 +312,9 @@ export function insertIpynbCell( ): string { const root = parseNotebookRoot(content) const cells = root.cells as unknown[] + if (cells.length >= IPYNB_MEMORY_LIMITS.cells) { + throw new Error('Notebook exceeds the safe cell limit') + } const nextCell: Record = { cell_type: kind, id: createBrowserUuid(), diff --git a/src/renderer/src/components/editor/local-image-blob-retention.test.ts b/src/renderer/src/components/editor/local-image-blob-retention.test.ts new file mode 100644 index 00000000000..b16e2dea103 --- /dev/null +++ b/src/renderer/src/components/editor/local-image-blob-retention.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from 'vitest' +import { LocalImageBlobRetention, MAX_LOCAL_IMAGE_BLOB_BYTES } from './local-image-blob-retention' + +describe('LocalImageBlobRetention', () => { + it('evicts oldest blobs until aggregate retained bytes are bounded', () => { + const revoke = vi.fn() + const retention = new LocalImageBlobRetention(revoke) + + retention.set('first', { url: 'blob:first', bytes: MAX_LOCAL_IMAGE_BLOB_BYTES - 1 }) + retention.set('second', { url: 'blob:second', bytes: 2 }) + + expect(retention.has('first')).toBe(false) + expect(retention.get('second')).toBe('blob:second') + expect(retention.retainedBytes).toBe(2) + expect(revoke).toHaveBeenCalledWith('blob:first') + }) +}) diff --git a/src/renderer/src/components/editor/local-image-blob-retention.ts b/src/renderer/src/components/editor/local-image-blob-retention.ts new file mode 100644 index 00000000000..479bfdec4b4 --- /dev/null +++ b/src/renderer/src/components/editor/local-image-blob-retention.ts @@ -0,0 +1,61 @@ +export const MAX_LOCAL_IMAGE_BLOB_ENTRIES = 100 +export const MAX_LOCAL_IMAGE_BLOB_BYTES = 128 * 1024 * 1024 + +export type RetainedLocalImageBlob = { + url: string + bytes: number +} + +export class LocalImageBlobRetention { + private readonly entries = new Map() + private retained = 0 + + constructor(private readonly revoke: (url: string) => void) {} + + get(key: string): string | undefined { + return this.entries.get(key)?.url + } + + has(key: string): boolean { + return this.entries.has(key) + } + + set(key: string, entry: RetainedLocalImageBlob): void { + const previous = this.entries.get(key) + if (previous) { + this.entries.delete(key) + this.retained -= previous.bytes + if (previous.url !== entry.url) { + this.revoke(previous.url) + } + } + this.entries.set(key, entry) + this.retained += entry.bytes + while ( + this.entries.size > MAX_LOCAL_IMAGE_BLOB_ENTRIES || + this.retained > MAX_LOCAL_IMAGE_BLOB_BYTES + ) { + const oldestKey = this.entries.keys().next().value + if (oldestKey === undefined) { + break + } + const oldest = this.entries.get(oldestKey) + this.entries.delete(oldestKey) + this.retained -= oldest?.bytes ?? 0 + if (oldest) { + this.revoke(oldest.url) + } + } + } + + clear(): RetainedLocalImageBlob[] { + const stale = Array.from(this.entries.values()) + this.entries.clear() + this.retained = 0 + return stale + } + + get retainedBytes(): number { + return this.retained + } +} diff --git a/src/renderer/src/components/editor/local-image-load-admission.test.ts b/src/renderer/src/components/editor/local-image-load-admission.test.ts new file mode 100644 index 00000000000..c6ec0416412 --- /dev/null +++ b/src/renderer/src/components/editor/local-image-load-admission.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from 'vitest' +import { + LocalImageLoadAdmission, + MAX_ACTIVE_LOCAL_IMAGE_LOADS, + MAX_ADMITTED_LOCAL_IMAGE_LOADS +} from './local-image-load-admission' + +async function flushMicrotasks(): Promise { + for (let index = 0; index < 6; index += 1) { + await Promise.resolve() + } +} + +describe('LocalImageLoadAdmission', () => { + it('starts only the bounded number of image reads concurrently', async () => { + const admission = new LocalImageLoadAdmission() + const releases: (() => void)[] = [] + const task = vi.fn( + () => + new Promise((resolve) => { + releases.push(() => resolve(releases.length)) + }) + ) + + const loads = Array.from({ length: MAX_ACTIVE_LOCAL_IMAGE_LOADS + 1 }, () => + admission.admit(task) + ) + await flushMicrotasks() + expect(task).toHaveBeenCalledTimes(MAX_ACTIVE_LOCAL_IMAGE_LOADS) + + releases[0]?.() + await flushMicrotasks() + expect(task).toHaveBeenCalledTimes(MAX_ACTIVE_LOCAL_IMAGE_LOADS + 1) + + for (const release of releases) { + release() + } + await Promise.all(loads) + }) + + it('rejects excess admission and can release every queued load', async () => { + const admission = new LocalImageLoadAdmission() + const releases: (() => void)[] = [] + const admitted = Array.from({ length: MAX_ADMITTED_LOCAL_IMAGE_LOADS }, () => + admission.admit( + () => + new Promise((resolve) => { + releases.push(resolve) + }) + ) + ) + + await flushMicrotasks() + expect(admission.admit(() => Promise.resolve())).toBeNull() + admission.clearPending() + for (const release of releases) { + release() + } + await Promise.all(admitted) + }) +}) diff --git a/src/renderer/src/components/editor/local-image-load-admission.ts b/src/renderer/src/components/editor/local-image-load-admission.ts new file mode 100644 index 00000000000..0e60b1bc034 --- /dev/null +++ b/src/renderer/src/components/editor/local-image-load-admission.ts @@ -0,0 +1,62 @@ +export const MAX_ACTIVE_LOCAL_IMAGE_LOADS = 2 +export const MAX_ADMITTED_LOCAL_IMAGE_LOADS = 100 + +type PendingLoad = { + start: () => void + cancel: () => void +} + +export class LocalImageLoadAdmission { + private active = 0 + private readonly pending: PendingLoad[] = [] + + admit(task: () => Promise): Promise | null { + if (this.active + this.pending.length >= MAX_ADMITTED_LOCAL_IMAGE_LOADS) { + return null + } + let resolveResult!: (value: T | null) => void + let rejectResult!: (error: unknown) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + const pendingLoad = { + start: () => { + this.active += 1 + let operation: Promise + try { + operation = task() + } catch (error) { + this.active -= 1 + rejectResult(error) + this.drain() + return + } + void operation.then(resolveResult, rejectResult).finally(() => { + this.active -= 1 + this.drain() + }) + }, + cancel: () => resolveResult(null) + } + this.pending.push(pendingLoad) + this.drain() + return result + } + + clearPending(): void { + for (const load of this.pending.splice(0)) { + load.cancel() + } + } + + private drain(): void { + while (this.active < MAX_ACTIVE_LOCAL_IMAGE_LOADS) { + const next = this.pending.shift() + if (!next) { + return + } + next.start() + } + } +} diff --git a/src/renderer/src/components/editor/markdown-doc-links.test.ts b/src/renderer/src/components/editor/markdown-doc-links.test.ts index d5cfaade6e7..0c4b5c00912 100644 --- a/src/renderer/src/components/editor/markdown-doc-links.test.ts +++ b/src/renderer/src/components/editor/markdown-doc-links.test.ts @@ -71,6 +71,10 @@ describe('splitMarkdownDocLinkText', () => { }) describe('resolveMarkdownDocLink', () => { + it('shares one immutable index for consumers of the same document snapshot', () => { + expect(createMarkdownDocumentIndex(documents)).toBe(createMarkdownDocumentIndex(documents)) + }) + it('resolves basename links', () => { const result = resolveMarkdownDocLink('setup-guide', createMarkdownDocumentIndex(documents)) expect(result.status).toBe('resolved') diff --git a/src/renderer/src/components/editor/markdown-doc-links.ts b/src/renderer/src/components/editor/markdown-doc-links.ts index 82cd7133b38..37c8ce93836 100644 --- a/src/renderer/src/components/editor/markdown-doc-links.ts +++ b/src/renderer/src/components/editor/markdown-doc-links.ts @@ -101,7 +101,13 @@ function resolveMatches(matches: MarkdownDocument[] | undefined): MarkdownDocLin : { status: 'ambiguous', matches } } +const markdownDocumentIndexes = new WeakMap() + export function createMarkdownDocumentIndex(documents: MarkdownDocument[]): MarkdownDocumentIndex { + const cached = markdownDocumentIndexes.get(documents) + if (cached) { + return cached + } const byName = new Map() const byRelativePath = new Map() const byRelativePathWithoutExtension = new Map() @@ -116,7 +122,9 @@ export function createMarkdownDocumentIndex(documents: MarkdownDocument[]): Mark ) } - return { byName, byRelativePath, byRelativePathWithoutExtension } + const index = { byName, byRelativePath, byRelativePathWithoutExtension } + markdownDocumentIndexes.set(documents, index) + return index } export function resolveMarkdownDocLink( diff --git a/src/renderer/src/components/editor/markdown-document-list-request.test.ts b/src/renderer/src/components/editor/markdown-document-list-request.test.ts index 085b8599fb8..96e6bd1c528 100644 --- a/src/renderer/src/components/editor/markdown-document-list-request.test.ts +++ b/src/renderer/src/components/editor/markdown-document-list-request.test.ts @@ -3,8 +3,10 @@ import type { MarkdownDocument } from '../../../../shared/types' import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client' import { getMarkdownDocumentListRequestKey, + MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT, requestSharedMarkdownDocumentList } from './markdown-document-list-request' +import { MarkdownDocumentListingCapacityError } from '../../../../shared/markdown-document-listing-limits' function context(overrides: Partial = {}): RuntimeFileOperationArgs { return { @@ -173,4 +175,29 @@ describe('shared Markdown document list requests', () => { ) expect(load).toHaveBeenCalledTimes(2) }) + + it('rejects the first distinct scan beyond the global in-flight cap', async () => { + const pending = Array.from({ length: MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT }, () => + deferred() + ) + const load = vi.fn((_context: RuntimeFileOperationArgs, rootPath: string) => { + const index = Number(rootPath.slice('/route-'.length)) + return pending[index].promise + }) + const requests = pending.map((_entry, index) => + requestSharedMarkdownDocumentList(context(), `/route-${index}`, {}, load) + ) + + await expect( + requestSharedMarkdownDocumentList(context(), '/one-too-many', {}, load) + ).rejects.toBeInstanceOf(MarkdownDocumentListingCapacityError) + expect(load).toHaveBeenCalledTimes(MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT) + + for (const entry of pending) { + entry.resolve([]) + } + await expect(Promise.all(requests)).resolves.toEqual( + Array.from({ length: MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT }, () => []) + ) + }) }) diff --git a/src/renderer/src/components/editor/markdown-document-list-request.ts b/src/renderer/src/components/editor/markdown-document-list-request.ts index 7e9cc0f32b3..9d55a502637 100644 --- a/src/renderer/src/components/editor/markdown-document-list-request.ts +++ b/src/renderer/src/components/editor/markdown-document-list-request.ts @@ -3,6 +3,7 @@ import { listRuntimeMarkdownDocuments, type RuntimeFileOperationArgs } from '@/runtime/runtime-file-client' +import { MarkdownDocumentListingCapacityError } from '../../../../shared/markdown-document-listing-limits' type MarkdownDocumentListLoader = ( context: RuntimeFileOperationArgs, @@ -19,7 +20,9 @@ type InFlightMarkdownDocumentList = { } const MARKDOWN_DOCUMENT_LIST_JOIN_WINDOW_MS = 30_000 +export const MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT = 16 const inFlightMarkdownDocumentLists = new Map() +let activeMarkdownDocumentListLoads = 0 export function getMarkdownDocumentListRequestKey( context: RuntimeFileOperationArgs, @@ -54,10 +57,22 @@ export function requestSharedMarkdownDocumentList( if (existing && !options.requireFresh) { return existing.request } + if (activeMarkdownDocumentListLoads >= MARKDOWN_DOCUMENT_LIST_MAX_IN_FLIGHT) { + return Promise.reject(new MarkdownDocumentListingCapacityError()) + } // Why: split Markdown panes mount together and otherwise launch identical // whole-worktree local/SSH scans; mutation refreshes bypass older snapshots. - const request = load(context, rootPath).finally(() => { + activeMarkdownDocumentListLoads += 1 + let loaded: Promise + try { + loaded = load(context, rootPath) + } catch (error) { + activeMarkdownDocumentListLoads -= 1 + return Promise.reject(error) + } + const request = loaded.finally(() => { + activeMarkdownDocumentListLoads -= 1 if (inFlightMarkdownDocumentLists.get(key)?.request === request) { inFlightMarkdownDocumentLists.delete(key) } diff --git a/src/renderer/src/components/editor/markdown-document-worktree-retention.test.ts b/src/renderer/src/components/editor/markdown-document-worktree-retention.test.ts new file mode 100644 index 00000000000..50e22b39ccf --- /dev/null +++ b/src/renderer/src/components/editor/markdown-document-worktree-retention.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' +import type { MarkdownDocument } from '../../../../shared/types' +import { + retainMarkdownDocumentWorktreeSnapshot, + type MarkdownDocumentWorktreeSnapshot +} from './markdown-document-worktree-retention' + +function documents(name: string): MarkdownDocument[] { + return [ + { + filePath: `/repo/${name}.md`, + relativePath: `${name}.md`, + basename: `${name}.md`, + name + } + ] +} + +describe('Markdown document worktree retention', () => { + it('preserves recent under-limit worktrees and refreshes their LRU position', () => { + let snapshots = new Map() + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'a', documents('a'), { + maxSnapshots: 2, + maxRetainedBytes: 10_000 + }) + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'b', documents('b'), { + maxSnapshots: 2, + maxRetainedBytes: 10_000 + }) + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'a', documents('a-new'), { + maxSnapshots: 2, + maxRetainedBytes: 10_000 + }) + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'c', documents('c'), { + maxSnapshots: 2, + maxRetainedBytes: 10_000 + }) + + expect(Array.from(snapshots.keys())).toEqual(['a', 'c']) + expect(snapshots.get('a')?.documents[0]?.name).toBe('a-new') + }) + + it('evicts oldest snapshots when their aggregate byte budget is exceeded', () => { + let snapshots = new Map() + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'a', documents('a'), { + maxSnapshots: 10, + maxRetainedBytes: 10_000 + }) + const oneSnapshotBytes = snapshots.get('a')?.retainedBytes ?? 0 + snapshots = retainMarkdownDocumentWorktreeSnapshot(snapshots, 'b', documents('b'), { + maxSnapshots: 10, + maxRetainedBytes: oneSnapshotBytes + }) + + expect(Array.from(snapshots.keys())).toEqual(['b']) + }) +}) diff --git a/src/renderer/src/components/editor/markdown-document-worktree-retention.ts b/src/renderer/src/components/editor/markdown-document-worktree-retention.ts new file mode 100644 index 00000000000..4b3db84647f --- /dev/null +++ b/src/renderer/src/components/editor/markdown-document-worktree-retention.ts @@ -0,0 +1,41 @@ +import type { MarkdownDocument } from '../../../../shared/types' +import { assertMarkdownDocumentsWithinLimit } from '../../../../shared/markdown-document-listing-limits' + +export const MARKDOWN_DOCUMENT_WORKTREE_MAX_SNAPSHOTS = 8 +export const MARKDOWN_DOCUMENT_WORKTREE_MAX_RETAINED_BYTES = 32 * 1024 * 1024 + +export type MarkdownDocumentWorktreeSnapshot = { + documents: MarkdownDocument[] + retainedBytes: number +} + +export function retainMarkdownDocumentWorktreeSnapshot( + previous: ReadonlyMap, + worktreeId: string, + documents: MarkdownDocument[], + limits: { maxSnapshots: number; maxRetainedBytes: number } = { + maxSnapshots: MARKDOWN_DOCUMENT_WORKTREE_MAX_SNAPSHOTS, + maxRetainedBytes: MARKDOWN_DOCUMENT_WORKTREE_MAX_RETAINED_BYTES + } +): Map { + const next = new Map(previous) + next.delete(worktreeId) + next.set(worktreeId, { + documents, + retainedBytes: assertMarkdownDocumentsWithinLimit(documents) + }) + + let retainedBytes = 0 + for (const snapshot of next.values()) { + retainedBytes += snapshot.retainedBytes + } + while (next.size > limits.maxSnapshots || retainedBytes > limits.maxRetainedBytes) { + const oldestKey = next.keys().next().value + if (typeof oldestKey !== 'string') { + break + } + retainedBytes -= next.get(oldestKey)?.retainedBytes ?? 0 + next.delete(oldestKey) + } + return next +} diff --git a/src/renderer/src/components/editor/markdown-export-extract.test.ts b/src/renderer/src/components/editor/markdown-export-extract.test.ts index f10a8289c44..9494b10accf 100644 --- a/src/renderer/src/components/editor/markdown-export-extract.test.ts +++ b/src/renderer/src/components/editor/markdown-export-extract.test.ts @@ -1,7 +1,7 @@ // @vitest-environment happy-dom import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { getActiveMarkdownExportPayload } from './markdown-export-extract' +import { getActiveMarkdownExportPayload, inlineBlobImageSources } from './markdown-export-extract' vi.mock('@/store', () => ({ useAppStore: { @@ -14,10 +14,11 @@ describe('getActiveMarkdownExportPayload', () => { vi.clearAllMocks() vi.stubGlobal( 'fetch', - vi.fn().mockResolvedValue({ - ok: true, - blob: async () => new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }) - }) + vi + .fn() + .mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { headers: { 'content-type': 'image/png' } }) + ) ) const { useAppStore } = await import('@/store') vi.mocked(useAppStore.getState).mockReturnValue({ @@ -66,4 +67,30 @@ describe('getActiveMarkdownExportPayload', () => { }) ).rejects.toThrow('Failed to inline image for PDF export') }) + + it('rejects streamed image bytes before the rendered fragment can exceed its cap', async () => { + let cancelled = false + vi.mocked(fetch).mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(12)) + controller.enqueue(new Uint8Array(12)) + }, + cancel() { + cancelled = true + } + }), + { headers: { 'content-type': 'image/png' } } + ) + ) + const root = document.createElement('div') + root.innerHTML = '' + const prefixOnly = ''.length + + await expect(inlineBlobImageSources(root, prefixOnly + 16)).rejects.toThrow( + 'HTML export exceeds the PDF memory limit' + ) + expect(cancelled).toBe(true) + }) }) diff --git a/src/renderer/src/components/editor/markdown-export-extract.ts b/src/renderer/src/components/editor/markdown-export-extract.ts index a0ec5ebb2fe..8b5a8985073 100644 --- a/src/renderer/src/components/editor/markdown-export-extract.ts +++ b/src/renderer/src/components/editor/markdown-export-extract.ts @@ -1,5 +1,15 @@ import { useAppStore } from '@/store' import { detectLanguage } from '@/lib/language-detect' +import { + assertHtmlToPdfInputWithinMemoryLimit, + HTML_TO_PDF_MAX_INPUT_BYTES, + HTML_TO_PDF_MEMORY_LIMIT_ERROR +} from '../../../../shared/html-to-pdf-memory-limit' +import { + FetchResponseBodyTooLargeError, + readFetchResponseBytesWithinLimit +} from '../../../../shared/fetch-response-body' +import { measureUtf8ByteLength } from '../../../../shared/utf8-byte-limits' import { buildMarkdownExportHtml } from './markdown-export-html' export type MarkdownExportPayload = { @@ -65,6 +75,15 @@ export async function getActiveMarkdownExportPayload({ return null } + const title = basenameWithoutExt(activeFile.relativePath || activeFile.filePath) + const wrapper = buildMarkdownExportHtml({ title, renderedHtml: '' }) + const wrapperBytes = measureUtf8ByteLength(wrapper, { + stopAfterBytes: HTML_TO_PDF_MAX_INPUT_BYTES + }) + if (wrapperBytes.exceededLimit) { + throw new Error(HTML_TO_PDF_MEMORY_LIMIT_ERROR) + } + const clone = subtree.cloneNode(true) as Element for (const selector of UI_ONLY_SELECTORS) { for (const node of clone.querySelectorAll(selector)) { @@ -73,51 +92,77 @@ export async function getActiveMarkdownExportPayload({ } // Why: local-image previews use renderer-scoped blob URLs; the hidden PDF // window cannot dereference them, so embed the bytes before export. - await inlineBlobImageSources(clone) + await inlineBlobImageSources(clone, HTML_TO_PDF_MAX_INPUT_BYTES - wrapperBytes.byteLength) const renderedHtml = clone.innerHTML.trim() if (!renderedHtml) { return null } - const title = basenameWithoutExt(activeFile.relativePath || activeFile.filePath) const html = buildMarkdownExportHtml({ title, renderedHtml }) + assertHtmlToPdfInputWithinMemoryLimit(html) return { title, html } } -async function inlineBlobImageSources(root: Element): Promise { - const images = Array.from(root.querySelectorAll('img[src^="blob:"]')) - await Promise.all( - images.map(async (image) => { - const src = image.getAttribute('src') - if (!src) { - return - } - image.setAttribute('src', await readBlobImageAsDataUrl(src)) - }) - ) +export async function inlineBlobImageSources( + root: Element, + maxFragmentBytes: number +): Promise { + for (const image of root.querySelectorAll('img[src^="blob:"]')) { + const src = image.getAttribute('src') + if (!src) { + continue + } + await inlineBlobImageSource(root, image, src, maxFragmentBytes) + } } -async function readBlobImageAsDataUrl(src: string): Promise { +async function inlineBlobImageSource( + root: Element, + image: HTMLImageElement, + src: string, + maxFragmentBytes: number +): Promise { try { const response = await fetch(src) if (!response.ok) { throw new Error('Unable to fetch blob image') } - const blob = await response.blob() - const bytes = new Uint8Array(await blob.arrayBuffer()) - return `data:${blob.type || 'application/octet-stream'};base64,${bytesToBase64(bytes)}` + const mediaType = normalizedMediaType(response.headers.get('content-type')) + const prefix = `data:${mediaType};base64,` + image.setAttribute('src', prefix) + const prefixMeasurement = measureUtf8ByteLength(root.innerHTML, { + stopAfterBytes: maxFragmentBytes + }) + if (prefixMeasurement.exceededLimit) { + throw new Error(HTML_TO_PDF_MEMORY_LIMIT_ERROR) + } + const availableBase64Characters = maxFragmentBytes - prefixMeasurement.byteLength + const maxImageBytes = Math.floor(availableBase64Characters / 4) * 3 + const bytes = await readFetchResponseBytesWithinLimit(response, maxImageBytes) + image.setAttribute('src', `${prefix}${bytesToBase64(bytes)}`) } catch (error) { + if ( + error instanceof FetchResponseBodyTooLargeError || + (error instanceof Error && error.message === HTML_TO_PDF_MEMORY_LIMIT_ERROR) + ) { + throw new Error(HTML_TO_PDF_MEMORY_LIMIT_ERROR) + } const message = error instanceof Error ? error.message : String(error) throw new Error(`Failed to inline image for PDF export: ${message}`) } } +function normalizedMediaType(value: string | null): string { + return new Blob([], { type: value ?? '' }).type || 'application/octet-stream' +} + function bytesToBase64(bytes: Uint8Array): string { - let binary = '' - const chunkSize = 0x8000 + const chunks: string[] = [] + const chunkSize = 3 * 8192 for (let index = 0; index < bytes.length; index += chunkSize) { - binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize)) + const binary = String.fromCharCode(...bytes.subarray(index, index + chunkSize)) + chunks.push(btoa(binary)) } - return btoa(binary) + return chunks.join('') } diff --git a/src/renderer/src/components/editor/monaco-markdown-doc-completions.test.ts b/src/renderer/src/components/editor/monaco-markdown-doc-completions.test.ts new file mode 100644 index 00000000000..a13470cb515 --- /dev/null +++ b/src/renderer/src/components/editor/monaco-markdown-doc-completions.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type { MarkdownDocument } from '../../../../shared/types' +import { assertMarkdownDocumentsWithinLimit } from '../../../../shared/markdown-document-listing-limits' +import { + clearMarkdownDocCompletionDocuments, + getMarkdownCompletionRetentionForTests, + MARKDOWN_COMPLETION_MAX_MODELS, + MARKDOWN_COMPLETION_MAX_SCOPES, + resetMarkdownCompletionRetentionForTests, + setMarkdownDocCompletionDocuments +} from './monaco-markdown-doc-completions' + +function documents(scope: string): MarkdownDocument[] { + return [ + { + filePath: `/repo/${scope}.md`, + relativePath: `${scope}.md`, + basename: `${scope}.md`, + name: scope + } + ] +} + +afterEach(() => { + resetMarkdownCompletionRetentionForTests() +}) + +describe('Monaco Markdown completion retention', () => { + it('stores one document snapshot for every mounted model in the same worktree scope', () => { + setMarkdownDocCompletionDocuments('model-a', 'worktree-a', documents('old')) + const freshDocuments = documents('fresh') + setMarkdownDocCompletionDocuments('model-b', 'worktree-a', freshDocuments) + + expect(getMarkdownCompletionRetentionForTests()).toEqual({ + models: 2, + scopes: 1, + retainedBytes: assertMarkdownDocumentsWithinLimit(freshDocuments) + }) + + clearMarkdownDocCompletionDocuments('model-a') + expect(getMarkdownCompletionRetentionForTests()).toMatchObject({ models: 1, scopes: 1 }) + clearMarkdownDocCompletionDocuments('model-b') + expect(getMarkdownCompletionRetentionForTests()).toEqual({ + models: 0, + scopes: 0, + retainedBytes: 0 + }) + }) + + it('evicts oldest scopes and model associations at their exact caps', () => { + for (let index = 0; index <= MARKDOWN_COMPLETION_MAX_SCOPES; index += 1) { + setMarkdownDocCompletionDocuments(`model-${index}`, `scope-${index}`, documents(`${index}`)) + } + expect(getMarkdownCompletionRetentionForTests()).toMatchObject({ + models: MARKDOWN_COMPLETION_MAX_SCOPES, + scopes: MARKDOWN_COMPLETION_MAX_SCOPES + }) + + resetMarkdownCompletionRetentionForTests() + for (let index = 0; index <= MARKDOWN_COMPLETION_MAX_MODELS; index += 1) { + setMarkdownDocCompletionDocuments(`model-${index}`, 'shared-scope', documents('shared')) + } + expect(getMarkdownCompletionRetentionForTests()).toMatchObject({ + models: MARKDOWN_COMPLETION_MAX_MODELS, + scopes: 1 + }) + }) +}) diff --git a/src/renderer/src/components/editor/monaco-markdown-doc-completions.ts b/src/renderer/src/components/editor/monaco-markdown-doc-completions.ts index 882b011e9b3..d457004553f 100644 --- a/src/renderer/src/components/editor/monaco-markdown-doc-completions.ts +++ b/src/renderer/src/components/editor/monaco-markdown-doc-completions.ts @@ -5,12 +5,72 @@ import { getMarkdownDocCompletionContext, getMarkdownDocCompletionDocuments } from './markdown-doc-completions' +import { assertMarkdownDocumentsWithinLimit } from '../../../../shared/markdown-document-listing-limits' type MonacoApi = Parameters[1] +type CompletionScope = { + documents: MarkdownDocument[] + modelKeys: Set + retainedBytes: number +} + +export const MARKDOWN_COMPLETION_MAX_MODELS = 256 +export const MARKDOWN_COMPLETION_MAX_SCOPES = 32 +export const MARKDOWN_COMPLETION_MAX_RETAINED_BYTES = 64 * 1024 * 1024 + let provider: IDisposable | null = null let providerMonaco: MonacoApi | null = null -const documentsByModel = new Map() +const scopeKeyByModel = new Map() +const completionScopes = new Map() +let retainedBytes = 0 + +function deleteCompletionScope(scopeKey: string, scope: CompletionScope): void { + completionScopes.delete(scopeKey) + retainedBytes -= scope.retainedBytes + for (const modelKey of scope.modelKeys) { + scopeKeyByModel.delete(modelKey) + } +} + +function removeModel(modelKey: string): void { + const scopeKey = scopeKeyByModel.get(modelKey) + if (!scopeKey) { + return + } + scopeKeyByModel.delete(modelKey) + const scope = completionScopes.get(scopeKey) + scope?.modelKeys.delete(modelKey) + if (scope && scope.modelKeys.size === 0) { + deleteCompletionScope(scopeKey, scope) + } +} + +function enforceCompletionRetentionLimits(): void { + while (scopeKeyByModel.size > MARKDOWN_COMPLETION_MAX_MODELS) { + const oldestModelKey = scopeKeyByModel.keys().next().value + if (typeof oldestModelKey !== 'string') { + break + } + removeModel(oldestModelKey) + } + while ( + completionScopes.size > MARKDOWN_COMPLETION_MAX_SCOPES || + retainedBytes > MARKDOWN_COMPLETION_MAX_RETAINED_BYTES + ) { + const oldest = completionScopes.entries().next().value + if (!oldest) { + break + } + deleteCompletionScope(oldest[0], oldest[1]) + } +} + +function clearCompletionRetention(): void { + scopeKeyByModel.clear() + completionScopes.clear() + retainedBytes = 0 +} export function ensureMarkdownDocCompletionProvider(monaco: MonacoApi): void { // Why: if Monaco was torn down and re-created (e.g. window reload), the old @@ -21,7 +81,7 @@ export function ensureMarkdownDocCompletionProvider(monaco: MonacoApi): void { } if (provider) { provider.dispose() - documentsByModel.clear() + clearCompletionRetention() } providerMonaco = monaco @@ -34,7 +94,8 @@ export function ensureMarkdownDocCompletionProvider(monaco: MonacoApi): void { return { suggestions: [] } } - const documents = documentsByModel.get(model.uri.toString()) ?? [] + const scopeKey = scopeKeyByModel.get(model.uri.toString()) + const documents = scopeKey ? (completionScopes.get(scopeKey)?.documents ?? []) : [] const suffix = line.slice(position.column - 1) const range = { startLineNumber: position.lineNumber, @@ -60,11 +121,49 @@ export function ensureMarkdownDocCompletionProvider(monaco: MonacoApi): void { export function setMarkdownDocCompletionDocuments( modelKey: string, + scopeKey: string, documents: MarkdownDocument[] ): void { - documentsByModel.set(modelKey, documents) + removeModel(modelKey) + let nextRetainedBytes: number + try { + nextRetainedBytes = assertMarkdownDocumentsWithinLimit(documents) + } catch { + return + } + + let scope = completionScopes.get(scopeKey) + if (scope) { + completionScopes.delete(scopeKey) + retainedBytes -= scope.retainedBytes + scope.documents = documents + scope.retainedBytes = nextRetainedBytes + } else { + scope = { documents, modelKeys: new Set(), retainedBytes: nextRetainedBytes } + } + scope.modelKeys.add(modelKey) + completionScopes.set(scopeKey, scope) + scopeKeyByModel.set(modelKey, scopeKey) + retainedBytes += nextRetainedBytes + enforceCompletionRetentionLimits() } export function clearMarkdownDocCompletionDocuments(modelKey: string): void { - documentsByModel.delete(modelKey) + removeModel(modelKey) +} + +export function getMarkdownCompletionRetentionForTests(): { + models: number + scopes: number + retainedBytes: number +} { + return { + models: scopeKeyByModel.size, + scopes: completionScopes.size, + retainedBytes + } +} + +export function resetMarkdownCompletionRetentionForTests(): void { + clearCompletionRetention() } diff --git a/src/renderer/src/components/editor/rich-markdown-local-image.test.ts b/src/renderer/src/components/editor/rich-markdown-local-image.test.ts index e343e918619..0d4a629a6bb 100644 --- a/src/renderer/src/components/editor/rich-markdown-local-image.test.ts +++ b/src/renderer/src/components/editor/rich-markdown-local-image.test.ts @@ -22,7 +22,7 @@ describe('rich markdown local images', () => { ...globalThis.window.api, fs: { readFile: vi.fn().mockResolvedValue({ - content: 'AA==', + content: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB', isBinary: true, mimeType: 'image/png' }) diff --git a/src/renderer/src/components/editor/useLocalImageSrc.test.ts b/src/renderer/src/components/editor/useLocalImageSrc.test.ts index 98907f2709a..b8aa8221bff 100644 --- a/src/renderer/src/components/editor/useLocalImageSrc.test.ts +++ b/src/renderer/src/components/editor/useLocalImageSrc.test.ts @@ -31,7 +31,17 @@ function deferred(): { return { promise, reject, resolve } } -function binaryPreview(content = 'AA=='): PreviewResult { +function pngBase64(width = 1, height = 1): string { + const bytes = Buffer.alloc(24) + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes) + bytes.writeUInt32BE(13, 8) + bytes.write('IHDR', 12, 'ascii') + bytes.writeUInt32BE(width, 16) + bytes.writeUInt32BE(height, 20) + return bytes.toString('base64') +} + +function binaryPreview(content = pngBase64()): PreviewResult { return { content, isBinary: true, mimeType: 'image/png' } } @@ -151,6 +161,13 @@ describe('loadLocalImageSrc', () => { ).resolves.toBeNull() }) + it('rejects an inline raster dimension bomb before assigning it to an image', async () => { + const src = `data:image/png;base64,${pngBase64(32_769, 1)}` + + await expect(loadLocalImageSrc(src, '/repo/docs/readme.md')).resolves.toBeNull() + expect(URL.createObjectURL).not.toHaveBeenCalled() + }) + it('suppresses a stale pending completion after cache invalidation', async () => { const firstRead = deferred() const readFile = vi @@ -186,9 +203,9 @@ describe('loadLocalImageSrc', () => { invalidateLocalImageSrcCacheForTests() const newerLoad = loadLocalImageSrc('diagram.png', '/repo/docs/readme.md') - secondRead.resolve(binaryPreview('AQ==')) + secondRead.resolve(binaryPreview(pngBase64(2, 1))) await expect(newerLoad).resolves.toBe('blob:newer') - firstRead.resolve(binaryPreview('Ag==')) + firstRead.resolve(binaryPreview(pngBase64(3, 1))) await expect(staleLoad).resolves.toBeNull() await expect(loadLocalImageSrc('diagram.png', '/repo/docs/readme.md')).resolves.toBe( 'blob:newer' @@ -197,6 +214,14 @@ describe('loadLocalImageSrc', () => { expect(URL.revokeObjectURL).not.toHaveBeenCalledWith('blob:newer') }) + it('rejects an oversized raster header before creating a blob URL', async () => { + const readFile = vi.fn().mockResolvedValue(binaryPreview(pngBase64(32_769, 1))) + setReadFile(readFile) + + await expect(loadLocalImageSrc('bomb.png', '/repo/docs/readme.md')).resolves.toBeNull() + expect(URL.createObjectURL).not.toHaveBeenCalled() + }) + it('keeps runtime owners in separate image cache entries', async () => { const readFile = vi.fn().mockResolvedValue(binaryPreview()) vi.spyOn(URL, 'createObjectURL') diff --git a/src/renderer/src/components/editor/useLocalImageSrc.ts b/src/renderer/src/components/editor/useLocalImageSrc.ts index d99c55913ff..3eeaad65dc3 100644 --- a/src/renderer/src/components/editor/useLocalImageSrc.ts +++ b/src/renderer/src/components/editor/useLocalImageSrc.ts @@ -2,15 +2,24 @@ import { useEffect, useState } from 'react' import { resolveImageAbsolutePath } from './markdown-preview-links' import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client' import { readRuntimeFilePreview } from '@/runtime/runtime-file-client' +import { + LocalImageBlobRetention, + MAX_LOCAL_IMAGE_BLOB_BYTES, + type RetainedLocalImageBlob +} from './local-image-blob-retention' +import { LocalImageLoadAdmission } from './local-image-load-admission' +import { decodeBase64Bytes } from './base64-byte-decoder' +import { assertRasterImagePreviewWithinLimits } from '../../../../shared/raster-image-preview-limits' +import { validateRasterImageDataUri } from '../../../../shared/image-data-uri' // Why: the renderer is served from http://localhost in dev mode, so file:// // URLs in tags are blocked by cross-origin restrictions. Loading images // via the existing fs.readFile IPC and converting to blob URLs bypasses this // limitation and works identically in both dev and production modes. -const BLOB_URL_CACHE_MAX_SIZE = 100 -const blobUrlCache = new Map() +const blobUrlCache = new LocalImageBlobRetention((url) => URL.revokeObjectURL(url)) const inFlightBlobUrlLoads = new Map>() +const imageLoadAdmission = new LocalImageLoadAdmission() export function getLocalImageCacheKey( absolutePath: string, @@ -31,53 +40,45 @@ export function getLocalImageCacheKey( // the cache grows without bound and leaks memory. We evict the oldest entry // (Map iteration order is insertion order) and revoke its blob URL so the // browser can free the underlying data. -function cacheBlobUrl(key: string, url: string): void { - const previousUrl = blobUrlCache.get(key) - if (previousUrl !== undefined) { - blobUrlCache.delete(key) - if (previousUrl !== url) { - // Why: cache replacements must release the superseded Blob even when - // they come from rare stale state or future loader changes. - URL.revokeObjectURL(previousUrl) - } - } - blobUrlCache.set(key, url) - if (blobUrlCache.size > BLOB_URL_CACHE_MAX_SIZE) { - const oldest = blobUrlCache.keys().next().value - if (oldest !== undefined) { - const oldUrl = blobUrlCache.get(oldest) - blobUrlCache.delete(oldest) - if (oldUrl) { - URL.revokeObjectURL(oldUrl) - } - } - } +function cacheBlobUrl(key: string, entry: RetainedLocalImageBlob): void { + blobUrlCache.set(key, entry) } const cacheListeners = new Set<() => void>() let cacheGeneration = 0 -const pendingBlobUrlRevocations = new Set() +const pendingBlobUrlRevocations = new Map() +let pendingBlobUrlRevocationBytes = 0 let pendingBlobUrlRevocationTimer: ReturnType | null = null -function base64ToBlobUrl(base64: string, mimeType: string): string { - const binary = atob(base64.replace(/\s/g, '')) - const bytes = new Uint8Array(binary.length) - for (let i = 0; i < binary.length; i += 1) { - bytes[i] = binary.charCodeAt(i) - } - return URL.createObjectURL(new Blob([bytes], { type: mimeType })) +function base64ToBlobUrl(base64: string, mimeType: string): RetainedLocalImageBlob { + const bytes = decodeBase64Bytes(base64) + assertRasterImagePreviewWithinLimits(bytes, mimeType) + return { url: URL.createObjectURL(new Blob([bytes], { type: mimeType })), bytes: bytes.length } } function revokePendingBlobUrls(): void { pendingBlobUrlRevocationTimer = null - for (const url of pendingBlobUrlRevocations) { + for (const url of pendingBlobUrlRevocations.keys()) { URL.revokeObjectURL(url) } pendingBlobUrlRevocations.clear() + pendingBlobUrlRevocationBytes = 0 } -function scheduleBlobUrlRevocation(urls: string[]): void { - for (const url of urls) { - pendingBlobUrlRevocations.add(url) +function scheduleBlobUrlRevocation(entries: RetainedLocalImageBlob[]): void { + for (const { url, bytes } of entries) { + const previousBytes = pendingBlobUrlRevocations.get(url) ?? 0 + pendingBlobUrlRevocations.set(url, bytes) + pendingBlobUrlRevocationBytes += bytes - previousBytes + } + while (pendingBlobUrlRevocationBytes > MAX_LOCAL_IMAGE_BLOB_BYTES) { + const oldest = pendingBlobUrlRevocations.entries().next().value + if (!oldest) { + break + } + const [url, bytes] = oldest + pendingBlobUrlRevocations.delete(url) + pendingBlobUrlRevocationBytes -= bytes + URL.revokeObjectURL(url) } if (pendingBlobUrlRevocationTimer !== null || pendingBlobUrlRevocations.size === 0) { return @@ -92,9 +93,9 @@ function scheduleBlobUrlRevocation(urls: string[]): void { // display the old data while the fresh IPC load completes, avoiding a visible // flash. The 30-second window is generous enough for even slow IPC reads. function invalidateImageCache(): void { - const staleUrls = Array.from(blobUrlCache.values()) - blobUrlCache.clear() + const staleUrls = blobUrlCache.clear() inFlightBlobUrlLoads.clear() + imageLoadAdmission.clearPending() cacheGeneration += 1 for (const listener of cacheListeners) { listener() @@ -109,6 +110,7 @@ function invalidateImageCache(): void { } function disposeImageCacheModuleState(): void { + cacheGeneration += 1 if (typeof window !== 'undefined') { window.removeEventListener('focus', invalidateImageCache) } @@ -117,10 +119,10 @@ function disposeImageCacheModuleState(): void { pendingBlobUrlRevocationTimer = null } revokePendingBlobUrls() - for (const url of blobUrlCache.values()) { + for (const { url } of blobUrlCache.clear()) { URL.revokeObjectURL(url) } - blobUrlCache.clear() + imageLoadAdmission.clearPending() inFlightBlobUrlLoads.clear() cacheListeners.clear() } @@ -146,13 +148,13 @@ export function onImageCacheInvalidated(listener: () => void): () => void { } } -function isExternalUrl(src: string): boolean { - return ( - src.startsWith('http://') || - src.startsWith('https://') || - src.startsWith('data:') || - src.startsWith('blob:') - ) +function resolveExternalImageUrl(src: string): string | null { + if (src.startsWith('data:')) { + return validateRasterImageDataUri(src) + } + return src.startsWith('http://') || src.startsWith('https://') || src.startsWith('blob:') + ? src + : null } /** @@ -177,8 +179,9 @@ export function useLocalImageSrc( if (!rawSrc) { return undefined } - if (isExternalUrl(rawSrc)) { - return rawSrc + const externalSrc = resolveExternalImageUrl(rawSrc) + if (externalSrc) { + return externalSrc } const absolutePath = resolveImageAbsolutePath(rawSrc, filePath) if (absolutePath) { @@ -196,8 +199,9 @@ export function useLocalImageSrc( return } - if (isExternalUrl(rawSrc)) { - setDisplaySrc(rawSrc) + const externalSrc = resolveExternalImageUrl(rawSrc) + if (externalSrc) { + setDisplaySrc(externalSrc) return } @@ -247,13 +251,9 @@ export async function loadLocalImageSrc( connectionId?: string | null, runtimeContext?: Omit & { connectionId?: string | null } ): Promise { - if ( - rawSrc.startsWith('http://') || - rawSrc.startsWith('https://') || - rawSrc.startsWith('data:') || - rawSrc.startsWith('blob:') - ) { - return rawSrc + const externalSrc = resolveExternalImageUrl(rawSrc) + if (externalSrc) { + return externalSrc } const absolutePath = resolveImageAbsolutePath(rawSrc, filePath) @@ -287,20 +287,26 @@ export function loadLocalImageAbsolutePath( } const readGeneration = cacheGeneration - const loadPromise = readImagePreview(absolutePath, connectionId, runtimeContext) + const admitted = imageLoadAdmission.admit(() => + readImagePreview(absolutePath, connectionId, runtimeContext) + ) + if (!admitted) { + return Promise.resolve(null) + } + const loadPromise = admitted .then((result) => { - if (!result.isBinary || !result.content || cacheGeneration !== readGeneration) { + if (!result?.isBinary || !result.content || cacheGeneration !== readGeneration) { // Why: local image paths must stay behind IPC/runtime authorization; // handing raw file: or relative paths back to Chromium can escape it. return null } - const url = base64ToBlobUrl(result.content, result.mimeType ?? 'image/png') + const entry = base64ToBlobUrl(result.content, result.mimeType ?? 'image/png') if (cacheGeneration !== readGeneration) { - URL.revokeObjectURL(url) + URL.revokeObjectURL(entry.url) return null } - cacheBlobUrl(cacheKey, url) - return url + cacheBlobUrl(cacheKey, entry) + return entry.url }) .catch(() => null) .finally(() => { @@ -318,13 +324,14 @@ export function resetLocalImageSrcStateForTests(): void { pendingBlobUrlRevocationTimer = null } revokePendingBlobUrls() - for (const url of blobUrlCache.values()) { + for (const { url } of blobUrlCache.clear()) { URL.revokeObjectURL(url) } - blobUrlCache.clear() + imageLoadAdmission.clearPending() inFlightBlobUrlLoads.clear() - cacheGeneration = 0 + cacheGeneration += 1 pendingBlobUrlRevocations.clear() + pendingBlobUrlRevocationBytes = 0 cacheListeners.clear() } diff --git a/src/renderer/src/components/editor/useMarkdownDocuments.test.tsx b/src/renderer/src/components/editor/useMarkdownDocuments.test.tsx new file mode 100644 index 00000000000..b0b41dd9799 --- /dev/null +++ b/src/renderer/src/components/editor/useMarkdownDocuments.test.tsx @@ -0,0 +1,96 @@ +// @vitest-environment happy-dom +import { act, renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { MarkdownDocument } from '../../../../shared/types' +import { MarkdownDocumentListingCapacityError } from '../../../../shared/markdown-document-listing-limits' +import type { OpenFile } from '@/store/slices/editor' +import { useMarkdownDocuments } from './useMarkdownDocuments' + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + openFile: vi.fn(), + openMarkdownPreview: vi.fn(), + toastError: vi.fn() +})) + +vi.mock('sonner', () => ({ toast: { error: mocks.toastError } })) +vi.mock('@/lib/connection-context', () => ({ getConnectionId: () => null })) +vi.mock('@/runtime/runtime-file-client', () => ({ statRuntimePath: vi.fn() })) +vi.mock('@/runtime/runtime-rpc-client', () => ({ + settingsForRuntimeOwner: () => ({ activeRuntimeEnvironmentId: null }) +})) +vi.mock('./markdown-document-list-request', () => ({ + requestSharedMarkdownDocumentList: mocks.list +})) +vi.mock('./markdown-document-worktree-path-selector', () => ({ + selectMarkdownDocumentWorktreePath: (_state: unknown, worktreeId: string) => `/repo/${worktreeId}` +})) +vi.mock('@/store', () => { + const state = { + settings: {}, + openFile: mocks.openFile, + openMarkdownPreview: mocks.openMarkdownPreview + } + const useAppStore = Object.assign( + (selector: (value: typeof state) => unknown) => selector(state), + { getState: () => state } + ) + return { useAppStore } +}) + +function file(worktreeId: string): OpenFile { + return { + id: `/repo/${worktreeId}/README.md`, + filePath: `/repo/${worktreeId}/README.md`, + relativePath: 'README.md', + worktreeId, + language: 'markdown', + isDirty: false, + mode: 'edit' + } +} + +function deferred(): { + promise: Promise + reject: (error: Error) => void + resolve: (value: T) => void +} { + let reject!: (error: Error) => void + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, reject, resolve } +} + +afterEach(() => { + vi.clearAllMocks() + vi.restoreAllMocks() +}) + +describe('useMarkdownDocuments', () => { + it('does not toast when a superseded worktree scan later exceeds capacity', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const first = deferred() + const second = deferred() + mocks.list.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise) + + const hook = renderHook( + ({ activeFile }) => useMarkdownDocuments(activeFile, true, 'source', vi.fn()), + { initialProps: { activeFile: file('first') } } + ) + await waitFor(() => expect(mocks.list).toHaveBeenCalledTimes(1)) + + hook.rerender({ activeFile: file('second') }) + await waitFor(() => expect(mocks.list).toHaveBeenCalledTimes(2)) + + await act(async () => { + first.reject(new MarkdownDocumentListingCapacityError()) + second.resolve([]) + await Promise.allSettled([first.promise, second.promise]) + }) + + expect(mocks.toastError).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/components/editor/useMarkdownDocuments.ts b/src/renderer/src/components/editor/useMarkdownDocuments.ts index 5e8e5a17fb8..842f826a62a 100644 --- a/src/renderer/src/components/editor/useMarkdownDocuments.ts +++ b/src/renderer/src/components/editor/useMarkdownDocuments.ts @@ -1,5 +1,10 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { toast } from 'sonner' import type { MarkdownDocument } from '../../../../shared/types' +import { + isMarkdownDocumentListingCapacityError, + MARKDOWN_DOCUMENT_LISTING_ERROR_MESSAGE +} from '../../../../shared/markdown-document-listing-limits' import { useAppStore } from '@/store' import { getConnectionId } from '@/lib/connection-context' import { statRuntimePath } from '@/runtime/runtime-file-client' @@ -12,6 +17,10 @@ import { } from './markdown-doc-links' import { selectMarkdownDocumentWorktreePath } from './markdown-document-worktree-path-selector' import { requestSharedMarkdownDocumentList } from './markdown-document-list-request' +import { + retainMarkdownDocumentWorktreeSnapshot, + type MarkdownDocumentWorktreeSnapshot +} from './markdown-document-worktree-retention' type OpenMarkdownDocumentOptions = { anchor?: string | null @@ -60,8 +69,8 @@ export function useMarkdownDocuments( const openFile = useAppStore((s) => s.openFile) const openMarkdownPreview = useAppStore((s) => s.openMarkdownPreview) const [markdownDocumentsByWorktree, setMarkdownDocumentsByWorktree] = useState< - Record - >({}) + Map + >(() => new Map()) const requestRef = useRef(0) const connectionId = getConnectionId(worktreeId) @@ -91,17 +100,20 @@ export function useMarkdownDocuments( if (requestRef.current !== requestId) { return } - setMarkdownDocumentsByWorktree((prev) => ({ - ...prev, - [worktreeId]: documents - })) + setMarkdownDocumentsByWorktree((prev) => + retainMarkdownDocumentWorktreeSnapshot(prev, worktreeId, documents) + ) } catch (err) { console.error('Failed to list markdown documents:', err) if (requestRef.current === requestId) { - setMarkdownDocumentsByWorktree((prev) => ({ - ...prev, - [worktreeId]: [] - })) + if (isMarkdownDocumentListingCapacityError(err)) { + toast.error(MARKDOWN_DOCUMENT_LISTING_ERROR_MESSAGE, { + id: `markdown-document-listing-capacity:${worktreeId}` + }) + } + setMarkdownDocumentsByWorktree((prev) => + retainMarkdownDocumentWorktreeSnapshot(prev, worktreeId, []) + ) } } }, @@ -182,7 +194,7 @@ export function useMarkdownDocuments( }, [activeFile.id, isMarkdown, viewMode, refreshMarkdownDocuments]) const markdownDocuments = useMemo( - () => (worktreeId ? (markdownDocumentsByWorktree[worktreeId] ?? []) : []), + () => (worktreeId ? (markdownDocumentsByWorktree.get(worktreeId)?.documents ?? []) : []), [worktreeId, markdownDocumentsByWorktree] ) diff --git a/src/renderer/src/components/github-item-dialog-source-boundary.test.ts b/src/renderer/src/components/github-item-dialog-source-boundary.test.ts index 6eaab34ec3e..697a28397f6 100644 --- a/src/renderer/src/components/github-item-dialog-source-boundary.test.ts +++ b/src/renderer/src/components/github-item-dialog-source-boundary.test.ts @@ -81,24 +81,25 @@ describe('GitHubItemDialog source host boundaries', () => { it('uses source-aware details routing and cache identity', () => { const source = componentSource('GitHubItemDialog.tsx') + const cacheSource = componentSource('../lib/github-work-item-details-cache.ts') const cacheKeySection = sourceBetween( - source, - 'function getWorkItemDetailsCacheKey', - 'function touchWorkItemDetailsCache' + cacheSource, + 'export function getWorkItemDetailsCacheKey', + 'export function getWorkItemDetailsCacheEntry' ) const matchInvalidationSection = sourceBetween( - source, - 'function invalidateWorkItemDetailsCacheByMatch', - 'function patchCachedPRFileViewedState' + cacheSource, + 'export function invalidateWorkItemDetailsCacheByMatch', + 'export function clearWorkItemDetailsCacheForTests' ) expect(source).toContain('lookupGitHubWorkItemDetailsForSource({') expect(source).toContain('sourceContext,') expect(cacheKeySection).toContain('sourceCacheScope') expect(source).toContain('getTaskSourceCacheScope(sourceContext)') - expect(matchInvalidationSection).toContain( - 'if (removed) {\n workItemDetailsCacheGeneration += 1' - ) + expect(source).toContain('useWorkItemDetailsCacheEntry(detailsCacheKey)') + expect(source).not.toContain('new Map') + expect(matchInvalidationSection).toContain('cacheGeneration += 1') }) it('treats null details as unavailable while preserving empty detail payloads', () => { @@ -115,6 +116,8 @@ describe('GitHubItemDialog source host boundaries', () => { expect(resultSection).toContain('} else if (result === null) {') expect(resultSection).toContain('error: WORK_ITEM_DETAILS_UNAVAILABLE_MESSAGE') expect(resultSection).toContain('details: result') + expect(resultSection).toContain('getWorkItemDetailsCacheGeneration() !== launchedAtGeneration') + expect(resultSection).toContain('prev?.pending !== inflight') }) it('routes PR file viewed mutations through the task source context', () => { @@ -160,6 +163,7 @@ describe('GitHubItemDialog source host boundaries', () => { it('routes PR file contents and runtime viewed invalidations through the task source context', () => { const source = componentSource('GitHubItemDialog.tsx') + const cacheSource = componentSource('../lib/github-work-item-details-cache.ts') const fileContentsSection = sourceBetween( source, 'function loadPRFileContents', @@ -170,7 +174,11 @@ describe('GitHubItemDialog source host boundaries', () => { 'function getPRFileContentCacheKey', 'function loadPRFileContents' ) - const listenerSection = sourceBetween(source, 'let workItemMutatedUnsub', '// Why: bounded LRU') + const listenerSection = sourceBetween( + cacheSource, + 'let workItemMutatedUnsub', + "if (typeof import.meta !== 'undefined'" + ) expect(fileContentsCacheKeySection).toContain( 'source:${getTaskSourceCacheScope(args.sourceContext)}' @@ -182,6 +190,7 @@ describe('GitHubItemDialog source host boundaries', () => { expect(fileContentsSection).toContain('sourceContext: args.sourceContext') expect(fileContentsSection).toContain('sourceContext,') expect(listenerSection).toContain('onGitHubWorkItemDetailsCacheMutation') + expect(listenerSection).toContain('invalidateWorkItemDetailsCacheByMatch') expect(source).toContain('emitGitHubWorkItemDetailsCacheMutation(args)') expect(source).toContain('options.local !== false') expect(source).toContain('notifyWorkItemMutated({') diff --git a/src/renderer/src/components/mobile/MobileHero.test.tsx b/src/renderer/src/components/mobile/MobileHero.test.tsx index 8e1d237a00d..cf816081ac7 100644 --- a/src/renderer/src/components/mobile/MobileHero.test.tsx +++ b/src/renderer/src/components/mobile/MobileHero.test.tsx @@ -139,7 +139,7 @@ describe('HeroFlow height', () => { relayDegraded: true }) const notice = screen.getByTestId('relay-degraded-notice') - expect(notice).toHaveTextContent('only works on your local network') + expect(notice).toHaveTextContent('only works on your LAN or Tailscale') // Why: wrap-capable text item inside the fixed QR track (#9700); bare text // nodes in a flex row cannot shrink below max-content and overflow the track. expect(notice.querySelector('.min-w-0')).not.toBeNull() diff --git a/src/renderer/src/components/mobile/MobileHero.tsx b/src/renderer/src/components/mobile/MobileHero.tsx index cdb49f7c743..e9fdc7154d2 100644 --- a/src/renderer/src/components/mobile/MobileHero.tsx +++ b/src/renderer/src/components/mobile/MobileHero.tsx @@ -279,7 +279,7 @@ export function HeroFlow({ onClick={onRegeneratePairing} // Why: signed-out Anywhere can't serve Relay; disabling avoids // minting a local-only QR under the Relay label. Sign in or pick - // Local network (shown in the path options above) to enable it. + // LAN (shown in the path options above) to enable it. disabled={pairLoading || !canGeneratePairing} > {pairLoading @@ -298,7 +298,7 @@ export function HeroFlow({ {translate( 'auto.components.mobile.MobileHero.relayDegradedNotice', - 'Relay couldn’t be reached — this code only works on your local network.' + 'Relay couldn’t be reached — this code only works on your LAN or Tailscale.' )}

diff --git a/src/renderer/src/components/mobile/MobilePage.test.tsx b/src/renderer/src/components/mobile/MobilePage.test.tsx index 23d9feb286b..b8fca95666d 100644 --- a/src/renderer/src/components/mobile/MobilePage.test.tsx +++ b/src/renderer/src/components/mobile/MobilePage.test.tsx @@ -66,7 +66,7 @@ vi.mock('./MobilePageContent', () => ({ Orca Relay